Add end to end tests

.github/workflows/test_end_to_end.yml

@@ -0,0 +1,102 @@
+name: End to End Data Pipeline Test
+
+on:
+  push:
+    branches:
+      - main
+      - add_end_to_end_tests
+
+  workflow_dispatch:
+    inputs:
+      debug_enabled:
+        description: "Run the build with tmate debugging enabled"
+        required: false
+        default: false
+jobs:
+  build-and-run:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: cache inputs and outputs
+        uses: actions/cache@v5
+        with:
+          key: data-cache-${{ github.run_id }}
+          restore-keys: |
+            data-cache-
+          path: |
+            tests/integration/data
+            tests/integration/fastpath
+
+      - name: change permission of fastpath cache for container read/write
+        run: |
+          chmod 0777 -R tests/integration/fastpath/lib/cache
+
+      - name: Setup tmate session
+        uses: mxschmitt/action-tmate@v3
+        if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled }}
+        with:
+          limit-access-to-actor: true
+
+      - name: Run Docker Compose (detached)
+        run: |
+          cd tests/integration
+          docker compose up -d
+          docker compose ps --all
+          docker compose logs --no-color --timestamps --tail=200 &
+
+      - name: Wait for verifier to start
+        run: |
+          cd tests/integration
+          for i in $(seq 1 60); do
+            if docker ps --format "{{.Names}}" | grep -q 'integration-verify'; then
+              echo "verifier started"
+              exit 0
+            fi
+            sleep 10
+          done
+          echo "verifier did not start within timeout" >&2
+          docker ps -a
+          exit 1
+
+      - name: Stream verifier logs until it exits
+        run: |
+          cd tests/integration
+          # Start streaming logs; docker logs --follow will exit when the container stops.
+          docker logs --follow integration-verify-1 & LOG_PID=$!
+          # Wait for verifier container to stop, with a timeout (360*5s = 30 minutes).
+          for i in $(seq 1 360); do
+            if ! docker ps --format "{{.Names}}" | grep -q 'integration-verify'; then
+              echo "verifier exited"
+              break
+            fi
+            sleep 5
+          done
+          # If still running after timeout, show debugging info and fail.
+          if docker ps --format "{{.Names}}" | grep -q 'integration-verify'; then
+            echo "verifier did not finish within timeout" >&2
+            docker ps -a
+            docker logs integration-verify-1 || true
+            if ps -p $LOG_PID > /dev/null; then
+              kill $LOG_PID || true
+            fi
+            exit 1
+          fi
+          # Wait for the logs follower to finish and show final logs if any.
+          if ps -p $LOG_PID > /dev/null; then
+            wait $LOG_PID || true
+          fi
+          docker ps -a --filter name=verify --format "Container: {{.Names}} Status: {{.Status}}"
+          docker logs integration-verify-1 || true
+          EXIT_CODE=$(docker inspect --format='{{.State.ExitCode}}' integration-verify-1)
+          if [ "$EXIT_CODE" -ne 0 ]; then
+            echo "Container exited with code $EXIT_CODE"
+            exit $EXIT_CODE
+          fi
+
+      - name: Shutdown remaining services
+        run: |
+          cd tests/integration
+          docker compose down --remove-orphans

Dockerfile

@@ -0,0 +1,15 @@
+# Use a specific Python image version (if compatible)
+FROM python:3.11
+
+# Set the working directory
+WORKDIR /app
+
+# Copy oonidata and oonipipeline source files into the container
+COPY oonidata ./oonidata
+COPY oonipipeline ./oonipipeline
+
+# Install dependencies for both projects
+RUN pip install ./oonidata && pip install ./oonipipeline
+
+# Set the default command for the container
+CMD ["/bin/bash"]

oonipipeline/pyproject.toml

@@ -69,6 +69,9 @@ path = ".venv/"
 [tool.hatch.version]
 path = "src/oonipipeline/__about__.py"
 
+[project.scripts]
+oonipipeline = "oonipipeline.main:cli"
+
 [tool.hatch.envs.default.scripts]
 oonipipeline = "python -m oonipipeline.main {args}"
 dataviz = "uvicorn oonipipeline.dataviz.main:app {args}"

tests/integration/Dockerfile

@@ -0,0 +1,14 @@
+# Use a specific Python image version (if compatible)
+FROM python:3.11
+
+# Set the working directory
+WORKDIR /app
+
+# Copy oonidata and oonipipeline source files into the container
+COPY ./tests ./app
+
+# Install dependencies for both projects
+RUN pip install pytest httpx jwt
+
+# Set the default command for the container
+CMD ["/bin/bash"]

tests/integration/clickhouse_init.sql

@@ -0,0 +1,82 @@
+CREATE TABLE ooni.fingerprints_dns
+(
+    `name` String,
+    `scope` Enum8('nat' = 1, 'isp' = 2, 'prod' = 3, 'inst' = 4, 'vbw' = 5, 'fp' = 6),
+    `other_names` String,
+    `location_found` String,
+    `pattern_type` Enum8('full' = 1, 'prefix' = 2, 'contains' = 3, 'regexp' = 4),
+    `pattern` String,
+    `confidence_no_fp` UInt8,
+    `expected_countries` String,
+    `source` String,
+    `exp_url` String,
+    `notes` String
+)
+ENGINE = EmbeddedRocksDB
+PRIMARY KEY name;
+
+CREATE TABLE ooni.fingerprints_http
+(
+    `name` String,
+    `scope` Enum8('nat' = 1, 'isp' = 2, 'prod' = 3, 'inst' = 4, 'vbw' = 5, 'fp' = 6, 'injb' = 7, 'prov' = 8),
+    `other_names` String,
+    `location_found` String,
+    `pattern_type` Enum8('full' = 1, 'prefix' = 2, 'contains' = 3, 'regexp' = 4),
+    `pattern` String,
+    `confidence_no_fp` UInt8,
+    `expected_countries` String,
+    `source` String,
+    `exp_url` String,
+    `notes` String
+)
+ENGINE = EmbeddedRocksDB
+PRIMARY KEY name;
+
+CREATE TABLE ooni.fastpath
+(
+    `measurement_uid` String,
+    `report_id` String,
+    `input` String,
+    `probe_cc` LowCardinality(String),
+    `probe_asn` Int32,
+    `test_name` LowCardinality(String),
+    `test_start_time` DateTime,
+    `measurement_start_time` DateTime,
+    `filename` String,
+    `scores` String,
+    `platform` String,
+    `anomaly` String,
+    `confirmed` String,
+    `msm_failure` String,
+    `domain` String,
+    `software_name` String,
+    `software_version` String,
+    `control_failure` String,
+    `blocking_general` Float32,
+    `is_ssl_expected` Int8,
+    `page_len` Int32,
+    `page_len_ratio` Float32,
+    `server_cc` String,
+    `server_asn` Int8,
+    `server_as_name` String,
+    `update_time` DateTime64(3) MATERIALIZED now64(),
+    `test_version` String,
+    `architecture` String,
+    `engine_name` LowCardinality(String),
+    `engine_version` String,
+    `test_runtime` Float32,
+    `blocking_type` String,
+    `test_helper_address` LowCardinality(String),
+    `test_helper_type` LowCardinality(String),
+    `ooni_run_link_id` Nullable(UInt64),
+    `is_verified` Int8 DEFAULT 0,
+    `nym` String DEFAULT '',
+    `zkp_request` String DEFAULT '',
+    `age_range` String DEFAULT '',
+    `msm_range` String DEFAULT '',
+    INDEX fastpath_rid_idx report_id TYPE minmax GRANULARITY 1,
+    INDEX measurement_uid_idx measurement_uid TYPE minmax GRANULARITY 8
+)
+ENGINE = ReplacingMergeTree(update_time)
+ORDER BY (measurement_start_time, report_id, input, measurement_uid)
+SETTINGS index_granularity = 8192;

tests/integration/data/.gitignore

@@ -0,0 +1,2 @@
+/*
+!.gitignore

tests/integration/docker-compose.yml

@@ -0,0 +1,121 @@
+version: '3.8'
+
+services:
+  downloader:
+    build:
+      context: ../../
+      dockerfile: Dockerfile
+    image: oonidata_image
+    volumes:
+      - ./data:/data:Z
+    working_dir: /data
+    command: >
+      bash -c "oonidata sync --output-dir . --probe-cc IT,IR --start-day 2026-01-01 --end-day 2026-01-10 &&
+      oonipipeline run --create-tables --probe-cc IT,IR --workflow-name observations --start-at 2026-01-01 --end-at 2026-01-10"
+    depends_on:
+      clickhouse:
+        condition: service_healthy
+    environment:
+      CLICKHOUSE_URL: "http://testuser:testuser@clickhouse:9000/ooni"
+
+  clickhouse:
+    image: docker.io/clickhouse/clickhouse-server:latest
+    ports:
+      - "8123:8123"  # HTTP interface
+      - "9000:9000"  # Native interface
+    volumes:
+      - ./clickhouse_init.sql:/docker-entrypoint-initdb.d/init.sql:Z
+    environment:
+      CLICKHOUSE_USER: "testuser"
+      CLICKHOUSE_PASSWORD: "testuser"
+      CLICKHOUSE_DB: "ooni"
+    healthcheck:
+      test: ["CMD", "clickhouse-client", "-u", "testuser", "--password", "testuser", "-q", "SELECT 1"]
+      interval: 5s
+      timeout: 3s
+      retries: 10
+      start_period: 10s
+
+  postgres:
+    image: docker.io/library/postgres:latest
+    ports:
+      - "5432:5432"
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: ooni
+
+  fastpath:
+    image: docker.io/ooni/fastpath:latest
+    volumes:
+      - ./fastpath/etc:/etc/ooni:Z
+      - ./fastpath/lib:/var/lib/fastpath:Z
+    command: [ "./run_fastpath", "--debug", "--clickhouse-url",
+               "clickhouse://testuser:testuser@clickhouse:9000/ooni",
+               "--keep-s3-cache", "--write-to-disk", "--stdout", "--ccs", "IT,IR",
+               "--start-day", "2026-01-01", "--end-day", "2026-01-02", "--noapi" ]
+    depends_on:
+      clickhouse:
+        condition: service_healthy
+      postgres:
+        condition: service_started
+
+  valkey:
+    image: docker.io/valkey/valkey:latest
+    ports:
+      - "6379:6379"
+    #volumes:
+    #  - valkey_data:/data
+    command: valkey-server --appendonly yes
+    restart: unless-stopped
+
+  api:
+    image: docker.io/ooni/api-oonimeasurements:latest
+    environment:
+      VALKEY_URL: valkey://valkey:6379
+      POSTGRESQL_URL: "postgresql://postgres:postgres@postgres:5432/ooni"
+      JWT_ENCRYPTION_KEY: "0123456789abcdef"
+      PROMETHEUS_METRICS_PASSWORD: "testme"
+      CLICKHOUSE_URL: "http://testuser:testuser@clickhouse:9000/ooni"
+      ACCOUNT_ID_HASHING_KEY: "0123456789abcdef"
+      RATE_LIMITS: "1000/minute;400000/day;200000/7day"
+
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:80/health"]
+      interval: 5s
+      timeout: 3s
+      retries: 10
+      start_period: 10s
+
+    ports:
+      - "8080:80"
+
+    depends_on:
+      clickhouse:
+        condition: service_healthy
+      postgres:
+        condition: service_started
+      valkey:
+        condition: service_started
+      downloader:
+        condition: service_completed_successfully
+      fastpath:
+        condition: service_completed_successfully
+
+  # replace this with a pytest suite that verifies results
+  verify:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: verifier
+
+    working_dir: /app
+    volumes:
+      - ./tests:/app:Z
+    environment:
+      API_URL: http://api:80
+    command: >
+      pytest -v
+    depends_on:
+      api:
+        condition: service_healthy

tests/integration/fastpath/etc/fastpath.conf

@@ -0,0 +1,11 @@
+[DEFAULT]
+# Collector hostnames, comma separated
+collectors = localhost
+clickhouse_url = clickhouse://testuser:testuser@clickhouse:9000/ooni
+# S3 access credentials
+# Currently unused
+s3_access_key =
+s3_secret_key =
+
+debug = true
+msmt_spool_dir = /data/

tests/integration/fastpath/lib/cache/.gitignore

@@ -0,0 +1,2 @@
+/*
+!.gitignore