diff --git a/.github/workflows/accuracy_performance.yml b/.github/workflows/accuracy_performance.yml index 83235342..f21880ee 100644 --- a/.github/workflows/accuracy_performance.yml +++ b/.github/workflows/accuracy_performance.yml @@ -12,7 +12,6 @@ on: paths: - 'asap-query-engine/**' - 'asap-planner-rs/**' - - 'asap-summary-ingest/**' - 'asap-quickstart/**' - '.github/workflows/accuracy_performance.yml' - 'benchmarks/**' @@ -67,15 +66,6 @@ jobs: cache-from: type=registry,ref=ghcr.io/projectasap/asap-planner-rs:buildcache cache-to: type=registry,ref=ghcr.io/projectasap/asap-planner-rs:buildcache,mode=max - - name: Build and push asap-summary-ingest - uses: docker/build-push-action@v6 - with: - context: asap-summary-ingest - file: asap-summary-ingest/Dockerfile - push: true - tags: ghcr.io/projectasap/asap-summary-ingest:${{ steps.tag.outputs.value }} - build-args: BASE_IMAGE=ghcr.io/projectasap/asap-base:latest - - name: Build and push asap-query-engine uses: docker/build-push-action@v6 with: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d928369f..af7f3eec 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -10,7 +10,6 @@ on: paths: - 'asap-common/installation/**' - 'asap-planner-rs/**' - - 'asap-summary-ingest/**' - 'asap-query-engine/**' - 'asap-tools/queriers/prometheus-client/**' - 'asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/**' @@ -89,22 +88,6 @@ jobs: cache-to: ${{ (startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch') && 'type=registry,ref=ghcr.io/projectasap/asap-planner-rs:buildcache,mode=max' || '' }} - # --- Summary Ingest (Python, depends on base) --- - - name: Build summary-ingest image - run: | - docker build \ - -t asap-summary-ingest:local \ - -f asap-summary-ingest/Dockerfile \ - asap-summary-ingest - - - name: Push summary-ingest image - if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' - run: | - docker tag asap-summary-ingest:local ghcr.io/projectasap/asap-summary-ingest:${{ steps.tag.outputs.value }} - docker tag asap-summary-ingest:local ghcr.io/projectasap/asap-summary-ingest:latest - docker push ghcr.io/projectasap/asap-summary-ingest:${{ steps.tag.outputs.value }} - docker push ghcr.io/projectasap/asap-summary-ingest:latest - # --- Prometheus Client (Python, depends on base) --- - name: Build prometheus-client image run: | diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 0590d12e..50fc271e 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -4,7 +4,6 @@ on: push: branches: [ main ] paths: - - 'asap-summary-ingest/**' - 'asap-tools/queriers/prometheus-client/**' - 'asap-tools/**' - 'asap-tools/data-sources/prometheus-exporters/**' @@ -15,7 +14,6 @@ on: types: [opened, synchronize, reopened, ready_for_review] branches: [ main ] paths: - - 'asap-summary-ingest/**' - 'asap-tools/queriers/prometheus-client/**' - 'asap-tools/**' - 'asap-tools/data-sources/prometheus-exporters/**' @@ -37,7 +35,6 @@ jobs: if: github.event_name != 'pull_request' || github.event.pull_request.draft == false runs-on: ubuntu-latest outputs: - summary_ingest: ${{ steps.filter.outputs.summary_ingest }} prometheus_client: ${{ steps.filter.outputs.prometheus_client }} utilities: ${{ steps.filter.outputs.utilities }} prometheus_exporters: ${{ steps.filter.outputs.prometheus_exporters }} @@ -48,9 +45,6 @@ jobs: id: filter with: filters: | - summary_ingest: - - 'asap-summary-ingest/**' - - 'asap-common/dependencies/py/**' prometheus_client: - 'asap-tools/queriers/prometheus-client/**' - 'asap-common/dependencies/py/**' @@ -64,35 +58,6 @@ jobs: - 'asap-tools/execution-utilities/**' - 'asap-common/dependencies/py/**' - test-summary-ingest: - needs: detect-changes - if: needs.detect-changes.outputs.summary_ingest == 'true' - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install black==24.8.0 flake8==6.1.0 - if [ -f asap-summary-ingest/requirements.txt ]; then pip install -r asap-summary-ingest/requirements.txt; fi - - name: Check formatting with Black - working-directory: asap-summary-ingest - run: black --check --diff . - - name: Lint with flake8 - working-directory: asap-summary-ingest - run: | - # Stop the build if there are Python syntax errors or undefined names - flake8 . --config=../.flake8 --count --select=E9,F63,F7,F82 --show-source --statistics - # Exit-zero treats all errors as warnings - flake8 . --config=../.flake8 --count --exit-zero --max-complexity=10 --statistics - test-prometheus-client: needs: detect-changes if: needs.detect-changes.outputs.prometheus_client == 'true' diff --git a/benchmarks/docker-compose.yml b/benchmarks/docker-compose.yml index fea9921a..11203649 100644 --- a/benchmarks/docker-compose.yml +++ b/benchmarks/docker-compose.yml @@ -13,8 +13,5 @@ services: asap-planner-rs: image: ghcr.io/projectasap/asap-planner-rs:${ASAP_IMAGE_TAG} - asap-summary-ingest: - image: ghcr.io/projectasap/asap-summary-ingest:${ASAP_IMAGE_TAG} - queryengine: image: ghcr.io/projectasap/asap-query-engine:${ASAP_IMAGE_TAG} diff --git a/benchmarks/scripts/ingest_wait.sh b/benchmarks/scripts/ingest_wait.sh index 8b12d3a1..7b6a1962 100644 --- a/benchmarks/scripts/ingest_wait.sh +++ b/benchmarks/scripts/ingest_wait.sh @@ -1,72 +1,18 @@ #!/usr/bin/env bash set -euo pipefail -# ingest_wait.sh — waits for the asap-demo Arroyo pipeline to reach RUNNING -# state, then sleeps to allow sketches to accumulate before verifying that the -# query engine has ingested data. +# ingest_wait.sh — waits for the query engine to accumulate sketches before +# benchmarking. With the precompute engine, the query engine begins computing +# immediately on startup, so we just sleep and then verify data is present. -ARROYO_URL="http://localhost:5115/api/v1/pipelines" QE_URL="http://localhost:8088/api/v1/query" -PIPELINE_NAME="asap-demo" -MAX_PIPELINE_WAIT=600 # seconds — Arroyo must compile Rust UDFs; allow extra time -ACCUMULATE_SLEEP=90 # seconds after pipeline is running -SLEEP=5 +ACCUMULATE_SLEEP=90 # seconds for sketches to accumulate -# ── 1. Wait for asap-demo pipeline to reach RUNNING ───────────────────────── -echo "[ingest_wait] Waiting for Arroyo pipeline '${PIPELINE_NAME}' to reach RUNNING state ..." -elapsed=0 -while true; do - state=$(curl -sf --max-time 10 "${ARROYO_URL}" 2>/dev/null \ - | python3 -c " -import sys, json -try: - data = json.load(sys.stdin) - # 'data' key may be null when no pipelines exist; use 'or []' to handle that - pipelines = data if isinstance(data, list) else (data.get('data') or []) - for p in pipelines: - name = str(p.get('name') or p.get('id') or '') - # Normalise hyphens/underscores so 'asap-demo' matches 'asap_demo' - if '${PIPELINE_NAME}'.replace('-', '_') in name.replace('-', '_'): - state = p.get('state') - stop = p.get('stop', '') - # Arroyo signals a running pipeline via state=null and stop='none' - if state is None and stop == 'none': - print('Running') - elif state is not None: - print(str(state)) - else: - print('stopped') - break - else: - # No matching pipeline found yet — print nothing so caller retries - pass -except Exception: - pass -" 2>/dev/null || true) - - if [ "${state}" = "Running" ] || [ "${state}" = "RUNNING" ] || [ "${state}" = "running" ]; then - echo "[ingest_wait] Pipeline '${PIPELINE_NAME}' is RUNNING (${elapsed}s elapsed)" - break - fi - - if [ "${elapsed}" -ge "${MAX_PIPELINE_WAIT}" ]; then - echo "[ingest_wait] ERROR: Pipeline '${PIPELINE_NAME}' did not reach RUNNING within ${MAX_PIPELINE_WAIT}s (last state: '${state:-unknown}')" >&2 - # Dump pipeline list for diagnosis - echo "[ingest_wait] Current Arroyo pipeline list:" >&2 - curl -sf --max-time 10 "${ARROYO_URL}" 2>/dev/null | python3 -m json.tool 2>/dev/null >&2 || true - exit 1 - fi - - echo "[ingest_wait] Pipeline state: '${state:-not found}' — retrying in ${SLEEP}s (${elapsed}s elapsed) ..." - sleep "${SLEEP}" - elapsed=$(( elapsed + SLEEP )) -done - -# ── 2. Allow sketches to accumulate ───────────────────────────────────────── -echo "[ingest_wait] Pipeline running. Sleeping ${ACCUMULATE_SLEEP}s for sketches to accumulate ..." +# ── 1. Allow sketches to accumulate ───────────────────────────────────────── +echo "[ingest_wait] Sleeping ${ACCUMULATE_SLEEP}s for sketches to accumulate ..." sleep "${ACCUMULATE_SLEEP}" -# ── 3. Verify query engine has data ───────────────────────────────────────── +# ── 2. Verify query engine has data ───────────────────────────────────────── echo "[ingest_wait] Verifying query engine has data ..." response=$(curl -sf --max-time 10 \ "${QE_URL}?query=avg%28sensor_reading%29" 2>/dev/null || true) diff --git a/benchmarks/scripts/wait_for_stack.sh b/benchmarks/scripts/wait_for_stack.sh index 672aa6d9..0522b2d2 100644 --- a/benchmarks/scripts/wait_for_stack.sh +++ b/benchmarks/scripts/wait_for_stack.sh @@ -28,7 +28,6 @@ wait_for_url() { } wait_for_url "Prometheus" "http://localhost:9090/-/healthy" -wait_for_url "Arroyo API" "http://localhost:5115/api/v1/pipelines" wait_for_url "QueryEngine" "http://localhost:8088/api/v1/query?query=vector(1)" echo "[wait_for_stack] All services are healthy."