From d1b3566607680353185bd2ebf90b4cca78744fd6 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Wed, 11 Mar 2026 13:35:33 +0100
Subject: [PATCH 01/59] add end-to-end tests with docker-compose.yml

---
 .github/workflows/test_end_to_end.yml | 58 +++++++++++++++++++++++++++
 Dockerfile                            | 15 +++++++
 tests/integration/data/.gitignore     |  1 +
 tests/integration/docker-compose.yml  | 46 +++++++++++++++++++++
 4 files changed, 120 insertions(+)
 create mode 100644 .github/workflows/test_end_to_end.yml
 create mode 100644 Dockerfile
 create mode 100644 tests/integration/data/.gitignore
 create mode 100644 tests/integration/docker-compose.yml

diff --git a/.github/workflows/test_end_to_end.yml b/.github/workflows/test_end_to_end.yml
new file mode 100644
index 00000000..e905c2b0
--- /dev/null
+++ b/.github/workflows/test_end_to_end.yml
@@ -0,0 +1,58 @@
+name: End to End Data Pipeline Test
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build-and-run:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: cache inputs and outputs
+        uses: actions/cache@v5
+        with:
+          key: data-cache-${{ github.run_id }}
+          restore-keys: |
+            data-cache-
+          path: |
+            tests/integration/data
+
+      - name: Install Podman
+        run: |
+          sudo apt update
+          sudo apt install -y podman podman-compose podman-docker
+
+      - name: Run Docker Compose with Podman
+        run: |
+          cd tests/integration && podman-compose up -d
+
+      - name: Wait for Downlaoder Container to Exit
+        run: |
+          # Replace `your_container_name` with the name of your container
+          while [ $(podman ps -q -f name=oonidata-pipeline | wc -l) -gt 0 ]; do
+            echo "Container is still running..."
+            sleep 5 # Wait for 5 seconds before checking again
+          done
+          echo "Container has exited."
+
+      - name: Query and verify data from API
+        run: |
+          response=$(curl -s -o response.json -w "%{http_code}" http://localhost:8000/api/v1/measurements?since=2026-01-01T00%3A00%3A00&until=2026-01-02T00%3A00%3A00&order_by=measurement_start_time&order=DESC&offset=0&limit=100)
+          
+          if [ "$response" -eq 200 ]; then
+            echo "HTTP 200 OK: Data retrieved successfully."
+            cat response.json # Optional: Process or display the retrieved data
+          else
+            echo "Failed to retrieve data. HTTP Status Code: $response"
+            exit 1
+          fi
+
+      - name: Shutdown
+        run: |
+          cd tests/integration && podman-compose down
+
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 00000000..6f0f1cae
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,15 @@
+# Use a specific Python image version (if compatible)
+FROM python:3.11
+
+# Set the working directory
+WORKDIR /app
+
+# Copy oonidata and oonipipeline source files into the container
+COPY oonidata ./oonidata
+COPY oonipipeline ./oonipipeline
+
+# Install dependencies for both projects
+RUN pip install ./oonidata && pip install ./oonipipeline
+
+# Set the default command for the container
+CMD ["/bin/bash"]
diff --git a/tests/integration/data/.gitignore b/tests/integration/data/.gitignore
new file mode 100644
index 00000000..f935021a
--- /dev/null
+++ b/tests/integration/data/.gitignore
@@ -0,0 +1 @@
+!.gitignore
diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml
new file mode 100644
index 00000000..e00902b6
--- /dev/null
+++ b/tests/integration/docker-compose.yml
@@ -0,0 +1,46 @@
+version: '3.8'
+
+services:
+  downloader:
+    build:
+      context: ../../
+      dockerfile: Dockerfile
+    image: oonidata_image
+    container_name: oonidata-pipeline
+    volumes:
+      - ./data:/data:Z
+    working_dir: /data
+    command: >
+      bash -c "oonidata sync --output-dir . --probe-cc IT --start-day 2026-01-01 --end-day 2026-01-02 --test-name webconnectivity &&
+      oonipipeline run --create-tables --probe-cc IT --test-name signal --workflow-name observations --start-at 2026-01-01 --end-at 2026-01-02
+    depends_on:
+      - clickhouse
+    environment:
+      CLICKHOUSE_URL: "http://testuser:testuser@clickhouse:9000/ooni"
+
+  api:
+    container_name: oonidata-api
+    image: oonidata_image
+    command: "uvicorn --log-level debug --port 8000 --host 0.0.0.0 oonipipeline.api.main:app"
+    depends_on:
+      - clickhouse
+    ports:
+      - "8000:8000"  # HTTP interface
+    environment:
+      CLICKHOUSE_URL: "http://testuser:testuser@clickhouse:9000/ooni"
+
+  clickhouse:
+    image: clickhouse/clickhouse-server:latest
+    container_name: clickhouse_server
+    ports:
+      - "8123:8123"  # HTTP interface
+      - "9000:9000"  # Native interface
+    #volumes:
+    #  - clickhouse_data:/var/lib/clickhouse
+    environment:
+      CLICKHOUSE_USER: "testuser"
+      CLICKHOUSE_PASSWORD: "testuser"
+      CLICKHOUSE_DB: "ooni"
+
+volumes:
+  clickhouse_data:

From 19870b3884fde4b3b76237dda142a97db4a0adf6 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Wed, 11 Mar 2026 13:38:31 +0100
Subject: [PATCH 02/59] run workflow on this branch

---
 .github/workflows/test_end_to_end.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_end_to_end.yml b/.github/workflows/test_end_to_end.yml
index e905c2b0..7d5506b4 100644
--- a/.github/workflows/test_end_to_end.yml
+++ b/.github/workflows/test_end_to_end.yml
@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - main
+      - add_end_to_end_tests
 
 jobs:
   build-and-run:

From 66366ecdc7cdc48101644da323b2ac14f039fb80 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Wed, 11 Mar 2026 13:38:51 +0100
Subject: [PATCH 03/59] build oonipipeline as command

---
 oonipipeline/pyproject.toml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/oonipipeline/pyproject.toml b/oonipipeline/pyproject.toml
index 1c8f92aa..a619df00 100644
--- a/oonipipeline/pyproject.toml
+++ b/oonipipeline/pyproject.toml
@@ -69,6 +69,9 @@ path = ".venv/"
 [tool.hatch.version]
 path = "src/oonipipeline/__about__.py"
 
+[project.scripts]
+oonipipeline = "oonipipeline.main:cli"
+
 [tool.hatch.envs.default.scripts]
 oonipipeline = "python -m oonipipeline.main {args}"
 dataviz = "uvicorn oonipipeline.dataviz.main:app {args}"

From 303b276e72abb93b9a4b4d6cc8ead05b9bf30ef0 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Wed, 18 Mar 2026 13:02:07 +0100
Subject: [PATCH 04/59] fix typo

---
 .github/workflows/test_end_to_end.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_end_to_end.yml b/.github/workflows/test_end_to_end.yml
index 7d5506b4..ffef3911 100644
--- a/.github/workflows/test_end_to_end.yml
+++ b/.github/workflows/test_end_to_end.yml
@@ -32,7 +32,7 @@ jobs:
         run: |
           cd tests/integration && podman-compose up -d
 
-      - name: Wait for Downlaoder Container to Exit
+      - name: Wait for Downloader Container to Exit
         run: |
           # Replace `your_container_name` with the name of your container
           while [ $(podman ps -q -f name=oonidata-pipeline | wc -l) -gt 0 ]; do

From 97629dc074ad464f79767c0855ed601b3a25c3f5 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Wed, 18 Mar 2026 13:02:38 +0100
Subject: [PATCH 05/59] add missing quote

---
 tests/integration/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml
index e00902b6..a793116b 100644
--- a/tests/integration/docker-compose.yml
+++ b/tests/integration/docker-compose.yml
@@ -12,7 +12,7 @@ services:
     working_dir: /data
     command: >
       bash -c "oonidata sync --output-dir . --probe-cc IT --start-day 2026-01-01 --end-day 2026-01-02 --test-name webconnectivity &&
-      oonipipeline run --create-tables --probe-cc IT --test-name signal --workflow-name observations --start-at 2026-01-01 --end-at 2026-01-02
+      oonipipeline run --create-tables --probe-cc IT --test-name signal --workflow-name observations --start-at 2026-01-01 --end-at 2026-01-02"
     depends_on:
       - clickhouse
     environment:

From 543856dec58174f027fd685946e347a676fdf7b3 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Wed, 18 Mar 2026 13:02:55 +0100
Subject: [PATCH 06/59] remove deprecated API endpoint

---
 .github/workflows/test_end_to_end.yml | 13 -------------
 tests/integration/docker-compose.yml  | 11 -----------
 2 files changed, 24 deletions(-)

diff --git a/.github/workflows/test_end_to_end.yml b/.github/workflows/test_end_to_end.yml
index ffef3911..1efca92f 100644
--- a/.github/workflows/test_end_to_end.yml
+++ b/.github/workflows/test_end_to_end.yml
@@ -41,19 +41,6 @@ jobs:
           done
           echo "Container has exited."
 
-      - name: Query and verify data from API
-        run: |
-          response=$(curl -s -o response.json -w "%{http_code}" http://localhost:8000/api/v1/measurements?since=2026-01-01T00%3A00%3A00&until=2026-01-02T00%3A00%3A00&order_by=measurement_start_time&order=DESC&offset=0&limit=100)
-          
-          if [ "$response" -eq 200 ]; then
-            echo "HTTP 200 OK: Data retrieved successfully."
-            cat response.json # Optional: Process or display the retrieved data
-          else
-            echo "Failed to retrieve data. HTTP Status Code: $response"
-            exit 1
-          fi
-
       - name: Shutdown
         run: |
           cd tests/integration && podman-compose down
-
diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml
index a793116b..cb8cd532 100644
--- a/tests/integration/docker-compose.yml
+++ b/tests/integration/docker-compose.yml
@@ -18,17 +18,6 @@ services:
     environment:
       CLICKHOUSE_URL: "http://testuser:testuser@clickhouse:9000/ooni"
 
-  api:
-    container_name: oonidata-api
-    image: oonidata_image
-    command: "uvicorn --log-level debug --port 8000 --host 0.0.0.0 oonipipeline.api.main:app"
-    depends_on:
-      - clickhouse
-    ports:
-      - "8000:8000"  # HTTP interface
-    environment:
-      CLICKHOUSE_URL: "http://testuser:testuser@clickhouse:9000/ooni"
-
   clickhouse:
     image: clickhouse/clickhouse-server:latest
     container_name: clickhouse_server

From 7d8120accbdc1344e6e89089eca11e01bf229984 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Wed, 18 Mar 2026 13:04:00 +0100
Subject: [PATCH 07/59] add clickhouse_init.sql

---
 tests/integration/clickhouse_init.sql | 33 +++++++++++++++++++++++++++
 tests/integration/docker-compose.yml  |  4 ++--
 2 files changed, 35 insertions(+), 2 deletions(-)
 create mode 100644 tests/integration/clickhouse_init.sql

diff --git a/tests/integration/clickhouse_init.sql b/tests/integration/clickhouse_init.sql
new file mode 100644
index 00000000..ea8fc8c6
--- /dev/null
+++ b/tests/integration/clickhouse_init.sql
@@ -0,0 +1,33 @@
+CREATE TABLE ooni.fingerprints_dns
+(
+    `name` String,
+    `scope` Enum8('nat' = 1, 'isp' = 2, 'prod' = 3, 'inst' = 4, 'vbw' = 5, 'fp' = 6),
+    `other_names` String,
+    `location_found` String,
+    `pattern_type` Enum8('full' = 1, 'prefix' = 2, 'contains' = 3, 'regexp' = 4),
+    `pattern` String,
+    `confidence_no_fp` UInt8,
+    `expected_countries` String,
+    `source` String,
+    `exp_url` String,
+    `notes` String
+)
+ENGINE = EmbeddedRocksDB
+PRIMARY KEY name;
+
+CREATE TABLE ooni.fingerprints_http
+(
+    `name` String,
+    `scope` Enum8('nat' = 1, 'isp' = 2, 'prod' = 3, 'inst' = 4, 'vbw' = 5, 'fp' = 6, 'injb' = 7, 'prov' = 8),
+    `other_names` String,
+    `location_found` String,
+    `pattern_type` Enum8('full' = 1, 'prefix' = 2, 'contains' = 3, 'regexp' = 4),
+    `pattern` String,
+    `confidence_no_fp` UInt8,
+    `expected_countries` String,
+    `source` String,
+    `exp_url` String,
+    `notes` String
+)
+ENGINE = EmbeddedRocksDB
+PRIMARY KEY name;
diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml
index cb8cd532..0d076387 100644
--- a/tests/integration/docker-compose.yml
+++ b/tests/integration/docker-compose.yml
@@ -24,8 +24,8 @@ services:
     ports:
       - "8123:8123"  # HTTP interface
       - "9000:9000"  # Native interface
-    #volumes:
-    #  - clickhouse_data:/var/lib/clickhouse
+    volumes:
+      - ./clickhouse_init.sql:/docker-entrypoint-initdb.d/init.sql:Z
     environment:
       CLICKHOUSE_USER: "testuser"
       CLICKHOUSE_PASSWORD: "testuser"

From 3e3664362d0febe36bbd66aa0db3ff3e7b0b304c Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Wed, 18 Mar 2026 13:04:23 +0100
Subject: [PATCH 08/59] run fastpath_feeder

---
 tests/integration/docker-compose.yml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml
index 0d076387..e4cf250b 100644
--- a/tests/integration/docker-compose.yml
+++ b/tests/integration/docker-compose.yml
@@ -31,5 +31,17 @@ services:
       CLICKHOUSE_PASSWORD: "testuser"
       CLICKHOUSE_DB: "ooni"
 
+  fastpath:
+    image: ooni/fastpath:latest
+    container_name: fastpath_feeder
+    volumes:
+      - ./data:/data
+      - ./fastpath:/etc/ooni:Z
+    command: >
+      bash -c "sleep 10 && ./run_fastpath --debug --clickhouse-url "clickhouse://testuser:testuser@clickhouse:9000/ooni" --stdout"
+    depends_on:
+      - clickhouse
+      - downloader
+
 volumes:
   clickhouse_data:

From 52b5db358786bd56fda94ca7b51b2cbbb69048af Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Mon, 23 Mar 2026 14:54:51 +0100
Subject: [PATCH 09/59] integ: postgres, valkey, and api-oonimeasurements

add these services to docker-compose.yml
---
 tests/integration/docker-compose.yml | 40 ++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml
index e4cf250b..1069d6e2 100644
--- a/tests/integration/docker-compose.yml
+++ b/tests/integration/docker-compose.yml
@@ -31,6 +31,16 @@ services:
       CLICKHOUSE_PASSWORD: "testuser"
       CLICKHOUSE_DB: "ooni"
 
+  postgres:
+    image: docker.io/library/postgres:latest
+    container_name: postgres_db
+    ports:
+      - "5432:5432"
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: ooni
+
   fastpath:
     image: ooni/fastpath:latest
     container_name: fastpath_feeder
@@ -41,7 +51,37 @@ services:
       bash -c "sleep 10 && ./run_fastpath --debug --clickhouse-url "clickhouse://testuser:testuser@clickhouse:9000/ooni" --stdout"
     depends_on:
       - clickhouse
+      - postgres
       - downloader
 
+  valkey:
+    image: valkey/valkey:latest
+    container_name: valkey_cache
+    ports:
+      - "6379:6379"
+    #volumes:
+    #  - valkey_data:/data
+    command: valkey-server --appendonly yes
+    restart: unless-stopped
+
+  api:
+    image: ooni/api-oonimeasurements:latest
+    container_name: oonimeasurements
+    environment:
+      VALKEY_URL: redis://valkey:6379
+      POSTGRESQL_URL: "postgresql://postgres:postgres@postgres:5432/ooni"
+      JWT_ENCRYPTION_KEY: "0123456789abcdef"
+      PROMETHEUS_METRICS_PASSWORD: "testme"
+      CLICKHOUSE_URL: "http://testuser:testuser@clickhouse:9000/ooni"
+      ACCOUNT_ID_HASHING_KEY: "0123456789abcdef"
+    ports:
+      - "8080:80"
+
+    depends_on:
+      - clickhouse
+      - postgres
+      - valkey
+
 volumes:
   clickhouse_data:
+  valkey-data:

From f2f0b96b477fa0b9efb4d92585a349a5a4986a93 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Mon, 23 Mar 2026 17:27:58 +0100
Subject: [PATCH 10/59] fix VALKEY_URL

---
 tests/integration/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml
index 1069d6e2..ce79ce7b 100644
--- a/tests/integration/docker-compose.yml
+++ b/tests/integration/docker-compose.yml
@@ -68,7 +68,7 @@ services:
     image: ooni/api-oonimeasurements:latest
     container_name: oonimeasurements
     environment:
-      VALKEY_URL: redis://valkey:6379
+      VALKEY_URL: valkey://valkey:6379
       POSTGRESQL_URL: "postgresql://postgres:postgres@postgres:5432/ooni"
       JWT_ENCRYPTION_KEY: "0123456789abcdef"
       PROMETHEUS_METRICS_PASSWORD: "testme"

From 32249776bd7c854c1b49877899e8f204d5931d2e Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Tue, 24 Mar 2026 14:45:02 +0100
Subject: [PATCH 11/59] use healthcheck and depends_on: condition to stage
 services

---
 tests/integration/docker-compose.yml | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml
index ce79ce7b..0dd61020 100644
--- a/tests/integration/docker-compose.yml
+++ b/tests/integration/docker-compose.yml
@@ -14,7 +14,8 @@ services:
       bash -c "oonidata sync --output-dir . --probe-cc IT --start-day 2026-01-01 --end-day 2026-01-02 --test-name webconnectivity &&
       oonipipeline run --create-tables --probe-cc IT --test-name signal --workflow-name observations --start-at 2026-01-01 --end-at 2026-01-02"
     depends_on:
-      - clickhouse
+      clickhouse:
+        condition: service_healthy
     environment:
       CLICKHOUSE_URL: "http://testuser:testuser@clickhouse:9000/ooni"
 
@@ -30,6 +31,12 @@ services:
       CLICKHOUSE_USER: "testuser"
       CLICKHOUSE_PASSWORD: "testuser"
       CLICKHOUSE_DB: "ooni"
+    healthcheck:
+      test: ["CMD", "clickhouse-client", "-u", "testuser", "--password", "testuser", "-q", "SELECT 1"]
+      interval: 5s
+      timeout: 3s
+      retries: 10
+      start_period: 10s
 
   postgres:
     image: docker.io/library/postgres:latest
@@ -50,9 +57,12 @@ services:
     command: >
       bash -c "sleep 10 && ./run_fastpath --debug --clickhouse-url "clickhouse://testuser:testuser@clickhouse:9000/ooni" --stdout"
     depends_on:
-      - clickhouse
-      - postgres
-      - downloader
+      clickhouse:
+        condition: service_healthy
+      postgres:
+        condition: service_started
+      downloader:
+        condition: service_completed_successfully
 
   valkey:
     image: valkey/valkey:latest

From 7609b139e1c38ffeee156065788359b79af77971 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Tue, 24 Mar 2026 14:45:28 +0100
Subject: [PATCH 12/59] use CLICKHOUSE_URL in environment

---
 tests/integration/docker-compose.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml
index 0dd61020..6411fa28 100644
--- a/tests/integration/docker-compose.yml
+++ b/tests/integration/docker-compose.yml
@@ -54,8 +54,10 @@ services:
     volumes:
       - ./data:/data
       - ./fastpath:/etc/ooni:Z
+    environment:
+      CLICKHOUSE_URL: "http://testuser:testuser@clickhouse:9000/ooni"
     command: >
-      bash -c "sleep 10 && ./run_fastpath --debug --clickhouse-url "clickhouse://testuser:testuser@clickhouse:9000/ooni" --stdout"
+      bash -c "./run_fastpath --debug --clickhouse-url $CLICKHOUSE_URL  --stdout"
     depends_on:
       clickhouse:
         condition: service_healthy

From faaf2d80ff4f22cf93e0f8ac49eabaad52ca3f26 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Tue, 24 Mar 2026 14:47:01 +0100
Subject: [PATCH 13/59] add healthcheck to api

---
 tests/integration/docker-compose.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml
index 6411fa28..8ea65244 100644
--- a/tests/integration/docker-compose.yml
+++ b/tests/integration/docker-compose.yml
@@ -86,6 +86,14 @@ services:
       PROMETHEUS_METRICS_PASSWORD: "testme"
       CLICKHOUSE_URL: "http://testuser:testuser@clickhouse:9000/ooni"
       ACCOUNT_ID_HASHING_KEY: "0123456789abcdef"
+
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:80/health"]
+      interval: 5s
+      timeout: 3s
+      retries: 10
+      start_period: 10s
+
     ports:
       - "8080:80"
 

From c14ba81ffb07e44ad0ce7dd9a78f31c30e67bd88 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Tue, 24 Mar 2026 14:47:21 +0100
Subject: [PATCH 14/59] set higher rate limits

---
 tests/integration/docker-compose.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml
index 8ea65244..e4a2bdc8 100644
--- a/tests/integration/docker-compose.yml
+++ b/tests/integration/docker-compose.yml
@@ -86,6 +86,7 @@ services:
       PROMETHEUS_METRICS_PASSWORD: "testme"
       CLICKHOUSE_URL: "http://testuser:testuser@clickhouse:9000/ooni"
       ACCOUNT_ID_HASHING_KEY: "0123456789abcdef"
+      RATE_LIMITS: "1000/minute;400000/day;200000/7day"
 
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:80/health"]

From 30b675e8488a561d810721905b309dafbe5d1e06 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Tue, 24 Mar 2026 14:48:08 +0100
Subject: [PATCH 15/59] use depends_on: condition: to stage starting
 oonimeasurements

---
 tests/integration/docker-compose.yml | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml
index e4a2bdc8..81ff496f 100644
--- a/tests/integration/docker-compose.yml
+++ b/tests/integration/docker-compose.yml
@@ -99,9 +99,14 @@ services:
       - "8080:80"
 
     depends_on:
-      - clickhouse
-      - postgres
-      - valkey
+      clickhouse:
+        condition: service_healthy
+      postgres:
+        condition: service_started
+      valkey:
+        condition: service_started
+      downloader:
+        condition: service_completed_successfully
 
 volumes:
   clickhouse_data:

From 7f50e11479738616f92f1518eb803ed874c765df Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Tue, 24 Mar 2026 14:48:56 +0100
Subject: [PATCH 16/59] add verify to run oonimeasurements tests against api

---
 tests/integration/Dockerfile         | 14 ++++++++++++++
 tests/integration/docker-compose.yml | 21 ++++++++++++++++++---
 2 files changed, 32 insertions(+), 3 deletions(-)
 create mode 100644 tests/integration/Dockerfile

diff --git a/tests/integration/Dockerfile b/tests/integration/Dockerfile
new file mode 100644
index 00000000..e5bc6d16
--- /dev/null
+++ b/tests/integration/Dockerfile
@@ -0,0 +1,14 @@
+# Use a specific Python image version (if compatible)
+FROM python:3.11
+
+# Set the working directory
+WORKDIR /app
+
+# Copy oonidata and oonipipeline source files into the container
+COPY ./tests ./app
+
+# Install dependencies for both projects
+RUN pip install pytest httpx jwt
+
+# Set the default command for the container
+CMD ["/bin/bash"]
diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml
index 81ff496f..f64e50c4 100644
--- a/tests/integration/docker-compose.yml
+++ b/tests/integration/docker-compose.yml
@@ -108,6 +108,21 @@ services:
       downloader:
         condition: service_completed_successfully
 
-volumes:
-  clickhouse_data:
-  valkey-data:
+  # replace this with a pytest suite that verifies results
+  verify:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: verifier
+    container_name: verifier
+
+    working_dir: /app
+    volumes:
+      - ./tests:/app:Z
+    environment:
+      API_URL: http://api:80
+    command: >
+      pytest -v
+    depends_on:
+      api:
+        condition: service_healthy

From a2aec7bb0bdbfc219abfba89e35cab1333352d1c Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Tue, 24 Mar 2026 14:49:26 +0100
Subject: [PATCH 17/59] add api tests

---
 tests/integration/tests/conftest.py           | 107 +++++++++++
 .../tests/test_oonidata_aggregate_analysis.py | 145 +++++++++++++++
 .../test_oonidata_aggregate_observations.py   |  94 ++++++++++
 .../tests/test_oonidata_list_analysis.py      | 165 +++++++++++++++++
 .../tests/test_oonidata_list_observations.py  | 167 ++++++++++++++++++
 5 files changed, 678 insertions(+)
 create mode 100644 tests/integration/tests/conftest.py
 create mode 100644 tests/integration/tests/test_oonidata_aggregate_analysis.py
 create mode 100644 tests/integration/tests/test_oonidata_aggregate_observations.py
 create mode 100644 tests/integration/tests/test_oonidata_list_analysis.py
 create mode 100644 tests/integration/tests/test_oonidata_list_observations.py

diff --git a/tests/integration/tests/conftest.py b/tests/integration/tests/conftest.py
new file mode 100644
index 00000000..537fcb60
--- /dev/null
+++ b/tests/integration/tests/conftest.py
@@ -0,0 +1,107 @@
+import time
+import jwt
+import pytest
+import httpx
+from typing import Optional, Any
+import os
+
+
+class APIClient:
+    def __init__(self, base_url: str, timeout: float = 10.0, **kwargs: Any):
+        """
+        Initialize the API client with a base URL.
+
+        Args:
+            base_url: The base URL for all requests (e.g., "http://localhost:8000")
+            timeout: Request timeout in seconds (default: 10.0)
+            **kwargs: Additional arguments passed to httpx.Client
+        """
+        self.base_url = base_url.rstrip("/")
+        self.client = httpx.Client(base_url=self.base_url, timeout=timeout, **kwargs)
+
+    def get(self, path: str, **kwargs: Any) -> httpx.Response:
+        """GET request"""
+        return self.client.get(path, **kwargs)
+
+    def post(self, path: str, **kwargs: Any) -> httpx.Response:
+        """POST request"""
+        return self.client.post(path, **kwargs)
+
+    def put(self, path: str, **kwargs: Any) -> httpx.Response:
+        """PUT request"""
+        return self.client.put(path, **kwargs)
+
+    def patch(self, path: str, **kwargs: Any) -> httpx.Response:
+        """PATCH request"""
+        return self.client.patch(path, **kwargs)
+
+    def delete(self, path: str, **kwargs: Any) -> httpx.Response:
+        """DELETE request"""
+        return self.client.delete(path, **kwargs)
+
+    def head(self, path: str, **kwargs: Any) -> httpx.Response:
+        """HEAD request"""
+        return self.client.head(path, **kwargs)
+
+    def close(self) -> None:
+        """Close the client connection"""
+        self.client.close()
+
+    def __enter__(self):
+        """Context manager entry"""
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Context manager exit"""
+        self.close()
+
+@pytest.fixture
+def client():
+    yield APIClient(os.environ.get("API_URL"))
+
+def create_jwt(payload: dict) -> str:
+    return jwt.encode(payload, "super_secure", algorithm="HS256")
+
+
+def create_session_token(account_id: str, role: str) -> str:
+    now = int(time.time())
+    payload = {
+        "nbf": now,
+        "iat": now,
+        "exp": now + 10 * 86400,
+        "aud": "user_auth",
+        "account_id": account_id,
+        "login_time": None,
+        "role": role,
+    }
+    return create_jwt(payload)
+
+
+@pytest.fixture
+def client_with_user_role(client):
+    jwt_token = create_session_token("0" * 16, "user")
+    client.headers = {"Authorization": f"Bearer {jwt_token}"}
+    yield client
+
+
+@pytest.fixture
+def client_with_admin_role(client):
+    jwt_token = create_session_token("0" * 16, "admin")
+    client.headers = {"Authorization": f"Bearer {jwt_token}"}
+    yield client
+
+
+@pytest.fixture
+def params_since_and_until_with_two_days():
+    return set_since_and_until_params(since="2024-11-01", until="2024-11-02")
+
+
+@pytest.fixture
+def params_since_and_until_with_ten_days():
+    return set_since_and_until_params(since="2024-11-01", until="2024-11-10")
+
+
+def set_since_and_until_params(since, until):
+    params = {"since": since, "until": until}
+
+    return params
diff --git a/tests/integration/tests/test_oonidata_aggregate_analysis.py b/tests/integration/tests/test_oonidata_aggregate_analysis.py
new file mode 100644
index 00000000..593f4573
--- /dev/null
+++ b/tests/integration/tests/test_oonidata_aggregate_analysis.py
@@ -0,0 +1,145 @@
+import pytest
+import os
+
+route = "/api/v1/aggregation/analysis"
+since = "2024-11-01"
+until = "2024-11-10"
+
+
+def test_oonidata_aggregation_analysis(client):
+    response = client.get(route)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) == 0
+
+
+def test_oonidata_aggregation_analysis_with_since_and_until(
+    client, params_since_and_until_with_two_days
+):
+    response = client.get(route, params=params_since_and_until_with_two_days)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) > 0
+
+    for result in json["results"]:
+        assert "domain" in result, result
+
+
+@pytest.mark.parametrize(
+    "filter_param, filter_value",
+    [
+        ("domain", "zh.wikipedia.org"),
+        ("probe_cc", "IR"),
+        ("probe_asn", 45758),
+        ("test_name", "whatsapp"),
+        ("input", "stun://stun.voys.nl:3478"),
+    ],
+)
+def test_oonidata_aggregation_analysis_with_filters(
+    client, filter_param, filter_value, params_since_and_until_with_ten_days
+):
+    params = params_since_and_until_with_ten_days
+    params[filter_param] = filter_value
+
+    response = client.get(route, params=params)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) > 0
+    for result in json["results"]:
+        assert result[filter_param] == filter_value, result
+
+
+def test_oonidata_aggregation_analysis_filtering_by_probe_asn_as_a_string_with_since_and_until(
+    client, params_since_and_until_with_ten_days
+):
+    params = params_since_and_until_with_ten_days
+    probe_asn = 45758
+    params["probe_asn"] = "AS" + str(probe_asn)
+
+    response = client.get(route, params=params)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) > 0
+    for result in json["results"]:
+        assert result["probe_asn"] == probe_asn, result
+
+
+@pytest.mark.parametrize(
+    "field",
+    [
+        "measurement_start_day",
+        "domain",
+        "probe_cc",
+        "probe_asn",
+        "test_name",
+        "input",
+    ],
+)
+def test_oonidata_aggregation_analysis_with_axis_x(
+    client, field, params_since_and_until_with_ten_days
+):
+    params = params_since_and_until_with_ten_days
+    params["axis_x"] = field
+
+    response = client.get(route, params=params)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) > 0
+    for result in json["results"]:
+        assert result[field] is not None, result
+
+
+@pytest.mark.parametrize(
+    "field",
+    [
+        "measurement_start_day",
+        "domain",
+        "probe_cc",
+        "probe_asn",
+        "test_name",
+        "input",
+    ],
+)
+def test_oonidata_aggregation_analysis_axis_y(
+    client, field, params_since_and_until_with_ten_days
+):
+    params = params_since_and_until_with_ten_days
+    params["axis_y"] = field
+
+    response = client.get(route, params=params)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) > 0
+    for result in json["results"]:
+        assert result[field] is not None, result
+
+
+@pytest.mark.parametrize(
+    "time_grain, total",
+    [
+        ("hour", 216),
+        ("day", 9),
+        ("week", 2),
+        ("month", 1),
+        ("year", 1),
+        ("auto", 9),
+    ],
+)
+def test_oonidata_aggregation_analysis_time_grain(
+    client, time_grain, total, params_since_and_until_with_ten_days
+):
+    params = params_since_and_until_with_ten_days
+    params["group_by"] = "timestamp"
+    params["time_grain"] = time_grain
+
+    response = client.get(route, params=params)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) == total
diff --git a/tests/integration/tests/test_oonidata_aggregate_observations.py b/tests/integration/tests/test_oonidata_aggregate_observations.py
new file mode 100644
index 00000000..f4824d20
--- /dev/null
+++ b/tests/integration/tests/test_oonidata_aggregate_observations.py
@@ -0,0 +1,94 @@
+import pytest
+
+route = "/api/v1/aggregation/observations"
+
+
+def test_oonidata_aggregation_observations(client):
+    response = client.get(route)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) == 0
+
+
+def test_oonidata_aggregation_observations_with_since_and_until(
+    client, params_since_and_until_with_two_days
+):
+    response = client.get(route, params=params_since_and_until_with_two_days)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) > 0
+
+    for result in json["results"]:
+        assert "observation_count" in result, result
+        assert "failure" in result, result
+
+
+@pytest.mark.parametrize(
+    "filter_name, filter_value",
+    [
+        ("probe_cc", "IT"),
+        ("probe_asn", 45758),
+        ("probe_asn", [45758, 5650]),
+        ("test_name", "whatsapp"),
+        ("hostname", "www.on-instant.com"),
+        ("ip", "64.233.190.139"),
+    ],
+)
+def test_oonidata_aggregation_observations_with_filters(
+    client, filter_name, filter_value, params_since_and_until_with_ten_days
+):
+    params = params_since_and_until_with_ten_days
+    params[filter_name] = filter_value
+
+    response = client.get(route, params=params)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) > 0
+    for result in json["results"]:
+        if isinstance(filter_value, list):
+            assert result[filter_name] in filter_value, result
+        else:
+            assert result[filter_name] == filter_value, result
+
+
+@pytest.mark.parametrize(
+    "time_grain, total",
+    [
+        ("hour", 215),
+        ("day", 9),
+        ("week", 2),
+        ("month", 1),
+        ("year", 1),
+        ("auto", 9),
+    ],
+)
+def test_oonidata_aggregation_observations_time_grain(
+    client, time_grain, total, params_since_and_until_with_ten_days
+):
+    params = params_since_and_until_with_ten_days
+    params["group_by"] = "timestamp"
+    params["time_grain"] = time_grain
+
+    response = client.get(route, params=params)
+
+    json = response.json()
+    assert len(json["results"]) == total
+
+
+def test_oonidata_aggregation_observations_groupby_failure(
+    client, params_since_and_until_with_two_days
+):
+    params = params_since_and_until_with_two_days
+    params["group_by"] = ["failure", "timestamp"]
+
+    response = client.get(route, params=params)
+
+    json = response.json()
+    assert len(json["results"]) == 24
+    first_result = json["results"][0]
+    assert "failure" in first_result.keys()
+    assert "timestamp" in first_result.keys()
+    assert "observation_count" in first_result.keys()
diff --git a/tests/integration/tests/test_oonidata_list_analysis.py b/tests/integration/tests/test_oonidata_list_analysis.py
new file mode 100644
index 00000000..1f89b539
--- /dev/null
+++ b/tests/integration/tests/test_oonidata_list_analysis.py
@@ -0,0 +1,165 @@
+import pytest
+
+route = "/api/v1/analysis"
+since = "2024-11-01"
+until = "2024-11-02"
+
+
+def test_oonidata_list_analysis(client):
+    response = client.get(route)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) == 0
+
+
+def test_oonidata_list_analysis_with_since_and_until(
+    client, params_since_and_until_with_two_days
+):
+    response = client.get(route, params=params_since_and_until_with_two_days)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) > 0
+    for result in json["results"]:
+        assert "test_name" in result, result
+        assert "probe_cc" in result, result
+
+
+@pytest.mark.parametrize(
+    "filter_param, filter_value",
+    [
+        (
+            "measurement_uid",
+            "20241101233756.866609_TH_webconnectivity_1bf55fb5699c39ec",
+        ),
+        ("probe_asn", 45758),
+        ("probe_cc", "IT"),
+        ("test_name", "web_connectivity"),
+    ],
+)
+def test_oonidata_list_analysis_with_filters(
+    client, filter_param, filter_value, params_since_and_until_with_two_days
+):
+    params = params_since_and_until_with_two_days
+    params[filter_param] = filter_value
+
+    response = client.get(route, params=params)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) > 0
+    for result in json["results"]:
+        assert result[filter_param] == filter_value, result
+
+
+def test_oonidata_list_analysis_filtering_by_probe_asn_as_a_string_with_since_and_until(
+    client, params_since_and_until_with_two_days
+):
+    params = params_since_and_until_with_two_days
+    probe_asn = 45758
+    params["probe_asn"] = "AS" + str(probe_asn)
+
+    response = client.get(route, params=params)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) > 0
+    for result in json["results"]:
+        assert result["probe_asn"] == probe_asn, result
+
+
+def test_oonidata_list_analysis_order_default(
+    client, params_since_and_until_with_two_days
+):
+    response = client.get(route, params=params_since_and_until_with_two_days)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) > 0
+    for i in range(1, len(json["results"])):
+        assert "measurement_start_time" in json["results"][i], json["results"][i]
+        previous_date = json["results"][i - 1]["measurement_start_time"]
+        current_date = json["results"][i]["measurement_start_time"]
+        assert (
+            previous_date >= current_date
+        ), f"The dates are not ordered: {previous_date} < {current_date}"
+
+
+def test_oonidata_list_analysis_order_asc(client, params_since_and_until_with_two_days):
+    params = params_since_and_until_with_two_days
+    params["order"] = "ASC"
+
+    response = client.get(route, params=params)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) > 0
+    for i in range(1, len(json["results"])):
+        assert "measurement_start_time" in json["results"][i], json["results"][i]
+        previous_date = json["results"][i - 1]["measurement_start_time"]
+        current_date = json["results"][i]["measurement_start_time"]
+        assert (
+            previous_date <= current_date
+        ), f"The dates are not ordered: {previous_date} > {current_date}"
+
+
+@pytest.mark.parametrize(
+    "field, order",
+    [
+        ("input", "asc"),
+        ("probe_cc", "asc"),
+        ("probe_asn", "asc"),
+        ("test_name", "asc"),
+        ("input", "desc"),
+        ("probe_cc", "desc"),
+        ("probe_asn", "desc"),
+        ("test_name", "desc"),
+    ],
+)
+def test_oonidata_list_analysis_order_by_field(
+    client, field, order, params_since_and_until_with_two_days
+):
+    params = params_since_and_until_with_two_days
+    params["order_by"] = field
+    params["order"] = order
+
+    response = client.get(route, params=params)
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) > 0
+    for i in range(1, len(json["results"])):
+        assert field in json["results"][i], json["results"][i]
+        previous = json["results"][i - 1][field]
+        current = json["results"][i][field]
+        if order == "asc":
+            assert (
+                previous <= current
+            ), f"The {field} values are not ordered in ascending order: {previous} > {current}"
+        else:
+            assert (
+                previous >= current
+            ), f"The {field} values are not ordered in descending order: {previous} < {current}"
+
+
+def test_oonidata_list_analysis_limit_by_default(
+    client, params_since_and_until_with_two_days
+):
+    response = client.get(route, params=params_since_and_until_with_two_days)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) == 100
+
+
+def test_oonidata_list_analysis_with_limit_and_offset(
+    client, params_since_and_until_with_two_days
+):
+    params = params_since_and_until_with_two_days
+    params["limit"] = 10
+
+    response = client.get(route, params=params)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) == 10
diff --git a/tests/integration/tests/test_oonidata_list_observations.py b/tests/integration/tests/test_oonidata_list_observations.py
new file mode 100644
index 00000000..de7f2c19
--- /dev/null
+++ b/tests/integration/tests/test_oonidata_list_observations.py
@@ -0,0 +1,167 @@
+import pytest
+
+route = "/api/v1/observations"
+
+
+def test_oonidata_list_observations(client):
+    response = client.get(route)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) == 0
+
+
+def test_oonidata_list_observations_with_since_and_until(
+    client, params_since_and_until_with_two_days
+):
+    response = client.get(route, params=params_since_and_until_with_two_days)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) > 0
+    for result in json["results"]:
+        assert "test_name" in result, result
+        assert "probe_cc" in result, result
+
+
+@pytest.mark.parametrize(
+    "filter_name, filter_value",
+    [
+        ("report_id", "20241101T233351Z_webconnectivity_DE_3209_n1_I7QVY7IdnaSfYmsb"),
+        ("probe_asn", 45758),
+        ("probe_cc", "IT"),
+        ("software_name", "ooniprobe-cli"),
+        ("software_version", "3.20.0"),
+        ("test_name", "web_connectivity"),
+        ("test_version", "0.4.3"),
+        ("engine_version", "3.20.0"),
+    ],
+)
+def test_oonidata_list_observations_with_filters(
+    client, filter_name, filter_value, params_since_and_until_with_two_days
+):
+    params = params_since_and_until_with_two_days
+    params[filter_name] = filter_value
+
+    response = client.get(route, params=params)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) > 0
+    for result in json["results"]:
+        assert result[filter_name] == filter_value, result
+
+
+def test_oonidata_list_observations_filtering_by_probe_asn_as_a_string_with_since_and_until(
+    client, params_since_and_until_with_two_days
+):
+    params = params_since_and_until_with_two_days
+    probe_asn = 45758
+    params["probe_asn"] = "AS" + str(probe_asn)
+
+    response = client.get(route, params=params)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) > 0
+    for result in json["results"]:
+        assert result["probe_asn"] == probe_asn, result
+
+
+def test_oonidata_list_observations_order_default(
+    client, params_since_and_until_with_two_days
+):
+    response = client.get(route, params=params_since_and_until_with_two_days)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) > 0
+    for i in range(1, len(json["results"])):
+        assert "measurement_start_time" in json["results"][i], json["results"][i]
+        previous_date = json["results"][i - 1]["measurement_start_time"]
+        current_date = json["results"][i]["measurement_start_time"]
+        assert (
+            previous_date >= current_date
+        ), f"The dates are not ordered: {previous_date} < {current_date}"
+
+
+def test_oonidata_list_observations_order_asc(
+    client, params_since_and_until_with_two_days
+):
+    params = params_since_and_until_with_two_days
+    params["order"] = "ASC"
+
+    response = client.get(route, params=params)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) > 0
+    for i in range(1, len(json["results"])):
+        assert "measurement_start_time" in json["results"][i], json["results"][i]
+        previous_date = json["results"][i - 1]["measurement_start_time"]
+        current_date = json["results"][i]["measurement_start_time"]
+        assert (
+            previous_date <= current_date
+        ), f"The dates are not ordered: {previous_date} > {current_date}"
+
+
+@pytest.mark.parametrize(
+    "field, order",
+    [
+        ("input", "asc"),
+        ("probe_cc", "asc"),
+        ("probe_asn", "asc"),
+        ("test_name", "asc"),
+        ("input", "desc"),
+        ("probe_cc", "desc"),
+        ("probe_asn", "desc"),
+        ("test_name", "desc"),
+    ],
+)
+def test_oonidata_list_observations_order_by_field(
+    client, field, order, params_since_and_until_with_two_days
+):
+    params = params_since_and_until_with_two_days
+    params["order_by"] = field
+    params["order"] = order
+
+    response = client.get(route, params=params)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) > 0
+    for i in range(1, len(json["results"])):
+        assert field in json["results"][i], json["results"][i]
+        previous = json["results"][i - 1][field]
+        current = json["results"][i][field]
+        if order == "asc":
+            assert (
+                previous <= current
+            ), f"The {field} values are not ordered in ascending order: {previous} > {current}"
+        else:
+            assert (
+                previous >= current
+            ), f"The {field} values are not ordered in descending order: {previous} < {current}"
+
+
+def test_oonidata_list_observations_limit_by_default(
+    client, params_since_and_until_with_two_days
+):
+    response = client.get(route, params=params_since_and_until_with_two_days)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) == 100
+
+
+def test_oonidata_list_observations_with_limit_and_offset(
+    client, params_since_and_until_with_two_days
+):
+    params = params_since_and_until_with_two_days
+    params["limit"] = 10
+
+    response = client.get(route, params=params)
+
+    json = response.json()
+    assert isinstance(json["results"], list), json
+    assert len(json["results"]) == 10

From 077e234bd258c359592f8e3133205ec356323246 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Tue, 24 Mar 2026 14:55:49 +0100
Subject: [PATCH 18/59] add fastpath table to init

---
 tests/integration/clickhouse_init.sql | 49 +++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/tests/integration/clickhouse_init.sql b/tests/integration/clickhouse_init.sql
index ea8fc8c6..9ecc86b9 100644
--- a/tests/integration/clickhouse_init.sql
+++ b/tests/integration/clickhouse_init.sql
@@ -31,3 +31,52 @@ CREATE TABLE ooni.fingerprints_http
 )
 ENGINE = EmbeddedRocksDB
 PRIMARY KEY name;
+
+CREATE TABLE ooni.fastpath
+(
+    `measurement_uid` String,
+    `report_id` String,
+    `input` String,
+    `probe_cc` LowCardinality(String),
+    `probe_asn` Int32,
+    `test_name` LowCardinality(String),
+    `test_start_time` DateTime,
+    `measurement_start_time` DateTime,
+    `filename` String,
+    `scores` String,
+    `platform` String,
+    `anomaly` String,
+    `confirmed` String,
+    `msm_failure` String,
+    `domain` String,
+    `software_name` String,
+    `software_version` String,
+    `control_failure` String,
+    `blocking_general` Float32,
+    `is_ssl_expected` Int8,
+    `page_len` Int32,
+    `page_len_ratio` Float32,
+    `server_cc` String,
+    `server_asn` Int8,
+    `server_as_name` String,
+    `update_time` DateTime64(3) MATERIALIZED now64(),
+    `test_version` String,
+    `architecture` String,
+    `engine_name` LowCardinality(String),
+    `engine_version` String,
+    `test_runtime` Float32,
+    `blocking_type` String,
+    `test_helper_address` LowCardinality(String),
+    `test_helper_type` LowCardinality(String),
+    `ooni_run_link_id` Nullable(UInt64),
+    `is_verified` Int8 DEFAULT 0,
+    `nym` String DEFAULT '',
+    `zkp_request` String DEFAULT '',
+    `age_range` String DEFAULT '',
+    `msm_range` String DEFAULT '',
+    INDEX fastpath_rid_idx report_id TYPE minmax GRANULARITY 1,
+    INDEX measurement_uid_idx measurement_uid TYPE minmax GRANULARITY 8
+)
+ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{cluster}/tables/ooni/fastpath/{shard}', '{replica}', update_time)
+ORDER BY (measurement_start_time, report_id, input, measurement_uid)
+SETTINGS index_granularity = 8192;

From 05c0dfcd1efa40c1d1915306c6d6ac0ea83568aa Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Tue, 24 Mar 2026 14:57:15 +0100
Subject: [PATCH 19/59] use date range of downloaded data

FIXME: parametize this
---
 tests/integration/tests/test_oonidata_aggregate_analysis.py | 4 ++--
 tests/integration/tests/test_oonidata_list_analysis.py      | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/integration/tests/test_oonidata_aggregate_analysis.py b/tests/integration/tests/test_oonidata_aggregate_analysis.py
index 593f4573..23762bda 100644
--- a/tests/integration/tests/test_oonidata_aggregate_analysis.py
+++ b/tests/integration/tests/test_oonidata_aggregate_analysis.py
@@ -2,8 +2,8 @@
 import os
 
 route = "/api/v1/aggregation/analysis"
-since = "2024-11-01"
-until = "2024-11-10"
+since = "2026-01-01"
+until = "2026-01-02"
 
 
 def test_oonidata_aggregation_analysis(client):
diff --git a/tests/integration/tests/test_oonidata_list_analysis.py b/tests/integration/tests/test_oonidata_list_analysis.py
index 1f89b539..1be6fce1 100644
--- a/tests/integration/tests/test_oonidata_list_analysis.py
+++ b/tests/integration/tests/test_oonidata_list_analysis.py
@@ -1,8 +1,8 @@
 import pytest
 
 route = "/api/v1/analysis"
-since = "2024-11-01"
-until = "2024-11-02"
+since = "2026-01-01"
+until = "2026-01-02"
 
 
 def test_oonidata_list_analysis(client):

From 4a60f1a4ae05512d503cdbf7e59cc545c0ef0645 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Tue, 24 Mar 2026 15:02:24 +0100
Subject: [PATCH 20/59] use right engine

---
 tests/integration/clickhouse_init.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/clickhouse_init.sql b/tests/integration/clickhouse_init.sql
index 9ecc86b9..36a30d1f 100644
--- a/tests/integration/clickhouse_init.sql
+++ b/tests/integration/clickhouse_init.sql
@@ -77,6 +77,6 @@ CREATE TABLE ooni.fastpath
     INDEX fastpath_rid_idx report_id TYPE minmax GRANULARITY 1,
     INDEX measurement_uid_idx measurement_uid TYPE minmax GRANULARITY 8
 )
-ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{cluster}/tables/ooni/fastpath/{shard}', '{replica}', update_time)
+ENGINE = ReplacingMergeTree(update_time)
 ORDER BY (measurement_start_time, report_id, input, measurement_uid)
 SETTINGS index_granularity = 8192;

From dd04a6343ee5c770749571b8812c7443d64f2c14 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Tue, 24 Mar 2026 15:44:19 +0100
Subject: [PATCH 21/59] fix date range in conftest.py

---
 tests/integration/tests/conftest.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/tests/conftest.py b/tests/integration/tests/conftest.py
index 537fcb60..36b7ca69 100644
--- a/tests/integration/tests/conftest.py
+++ b/tests/integration/tests/conftest.py
@@ -93,12 +93,12 @@ def client_with_admin_role(client):
 
 @pytest.fixture
 def params_since_and_until_with_two_days():
-    return set_since_and_until_params(since="2024-11-01", until="2024-11-02")
+    return set_since_and_until_params(since="2026-01-01", until="2026-01-02")
 
 
 @pytest.fixture
 def params_since_and_until_with_ten_days():
-    return set_since_and_until_params(since="2024-11-01", until="2024-11-10")
+    return set_since_and_until_params(since="2026-01-01", until="2026-01-02")
 
 
 def set_since_and_until_params(since, until):

From ce707a6ed25ac5bfa56ba2077d1552d2e09cff67 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Mon, 30 Mar 2026 10:59:28 +0200
Subject: [PATCH 22/59] wait until fastpath has completed successfully

---
 tests/integration/docker-compose.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml
index f64e50c4..6607ff4f 100644
--- a/tests/integration/docker-compose.yml
+++ b/tests/integration/docker-compose.yml
@@ -107,6 +107,8 @@ services:
         condition: service_started
       downloader:
         condition: service_completed_successfully
+      fastpath:
+        condition: service_completed_successfully
 
   # replace this with a pytest suite that verifies results
   verify:

From b61ebbdf82e50eb66f6bebabcef5ba90c7780b90 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Tue, 31 Mar 2026 10:31:50 +0200
Subject: [PATCH 23/59] extend date range

---
 tests/integration/docker-compose.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml
index 6607ff4f..e40465fd 100644
--- a/tests/integration/docker-compose.yml
+++ b/tests/integration/docker-compose.yml
@@ -11,8 +11,8 @@ services:
       - ./data:/data:Z
     working_dir: /data
     command: >
-      bash -c "oonidata sync --output-dir . --probe-cc IT --start-day 2026-01-01 --end-day 2026-01-02 --test-name webconnectivity &&
-      oonipipeline run --create-tables --probe-cc IT --test-name signal --workflow-name observations --start-at 2026-01-01 --end-at 2026-01-02"
+      bash -c "oonidata sync --output-dir . --probe-cc IT,IR --start-day 2026-01-01 --end-day 2026-01-10 &&
+      oonipipeline run --create-tables --probe-cc IT,IR --workflow-name observations --start-at 2026-01-01 --end-at 2026-01-10"
     depends_on:
       clickhouse:
         condition: service_healthy

From af700278af30c2e8cbbeb8151c870a030a28a9a2 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Tue, 31 Mar 2026 10:32:48 +0200
Subject: [PATCH 24/59] add /var/lib/fastpath to fastpath volumes

---
 tests/integration/docker-compose.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml
index e40465fd..0ef9b1df 100644
--- a/tests/integration/docker-compose.yml
+++ b/tests/integration/docker-compose.yml
@@ -52,8 +52,8 @@ services:
     image: ooni/fastpath:latest
     container_name: fastpath_feeder
     volumes:
-      - ./data:/data
-      - ./fastpath:/etc/ooni:Z
+      - ./fastpath/etc:/etc/ooni:Z
+      - ./fastpath/lib:/var/lib/fastpath:Z
     environment:
       CLICKHOUSE_URL: "http://testuser:testuser@clickhouse:9000/ooni"
     command: >

From 9b980a9c7882214366695a7ee4596718ab7cb870 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Tue, 31 Mar 2026 10:33:39 +0200
Subject: [PATCH 25/59] fix since and until date range diff --git
 a/tests/integration/tests/conftest.py b/tests/integration/tests/conftest.py
 index 36b7ca6..8ca007e 100644 --- a/tests/integration/tests/conftest.py +++
 b/tests/integration/tests/conftest.py @@ -98,7 +98,7 @@ def
 params_since_and_until_with_two_days():

 @pytest.fixture
 def params_since_and_until_with_ten_days():
-    return set_since_and_until_params(since="2026-01-01", until="2026-01-02")
+    return set_since_and_until_params(since="2026-01-01", until="2026-01-10")

 def set_since_and_until_params(since, until):
diff --git a/tests/integration/tests/test_oonidata_aggregate_analysis.py b/tests/integration/tests/test_oonidata_aggregate_analysis.py
index 23762bd..6bad311 100644
--- a/tests/integration/tests/test_oonidata_aggregate_analysis.py
+++ b/tests/integration/tests/test_oonidata_aggregate_analysis.py
@@ -3,7 +3,7 @@ import os

 route = "/api/v1/aggregation/analysis"
 since = "2026-01-01"
-until = "2026-01-02"
+until = "2026-01-10"

 def test_oonidata_aggregation_analysis(client):
diff --git a/tests/integration/tests/test_oonidata_list_analysis.py b/tests/integration/tests/test_oonidata_list_analysis.py
index 1be6fce..64af92e 100644
--- a/tests/integration/tests/test_oonidata_list_analysis.py
+++ b/tests/integration/tests/test_oonidata_list_analysis.py
@@ -2,7 +2,7 @@ import pytest

 route = "/api/v1/analysis"
 since = "2026-01-01"
-until = "2026-01-02"
+until = "2026-01-10"

 def test_oonidata_list_analysis(client):
---
 tests/integration/tests/conftest.py                         | 2 +-
 tests/integration/tests/test_oonidata_aggregate_analysis.py | 2 +-
 tests/integration/tests/test_oonidata_list_analysis.py      | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/integration/tests/conftest.py b/tests/integration/tests/conftest.py
index 36b7ca69..8ca007eb 100644
--- a/tests/integration/tests/conftest.py
+++ b/tests/integration/tests/conftest.py
@@ -98,7 +98,7 @@ def params_since_and_until_with_two_days():
 
 @pytest.fixture
 def params_since_and_until_with_ten_days():
-    return set_since_and_until_params(since="2026-01-01", until="2026-01-02")
+    return set_since_and_until_params(since="2026-01-01", until="2026-01-10")
 
 
 def set_since_and_until_params(since, until):
diff --git a/tests/integration/tests/test_oonidata_aggregate_analysis.py b/tests/integration/tests/test_oonidata_aggregate_analysis.py
index 23762bda..6bad3116 100644
--- a/tests/integration/tests/test_oonidata_aggregate_analysis.py
+++ b/tests/integration/tests/test_oonidata_aggregate_analysis.py
@@ -3,7 +3,7 @@
 
 route = "/api/v1/aggregation/analysis"
 since = "2026-01-01"
-until = "2026-01-02"
+until = "2026-01-10"
 
 
 def test_oonidata_aggregation_analysis(client):
diff --git a/tests/integration/tests/test_oonidata_list_analysis.py b/tests/integration/tests/test_oonidata_list_analysis.py
index 1be6fce1..64af92e7 100644
--- a/tests/integration/tests/test_oonidata_list_analysis.py
+++ b/tests/integration/tests/test_oonidata_list_analysis.py
@@ -2,7 +2,7 @@
 
 route = "/api/v1/analysis"
 since = "2026-01-01"
-until = "2026-01-02"
+until = "2026-01-10"
 
 
 def test_oonidata_list_analysis(client):

From ebe4334a0a0e11a5ff4e2c2c68e410f8dfb961e9 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Tue, 31 Mar 2026 15:19:56 +0200
Subject: [PATCH 26/59] set start-day and end-day rnage in run_fastpath ---
 a/tests/integration/docker-compose.yml +++
 b/tests/integration/docker-compose.yml @@ -57,7 +57,7 @@ services:     
 environment:        CLICKHOUSE_URL:
 "http://testuser:testuser@clickhouse:9000/ooni"      command: > -      bash
 -c "./run_fastpath --debug --clickhouse-url $CLICKHOUSE_URL  --stdout" +     
 bash -c "./run_fastpath --debug --clickhouse-url $CLICKHOUSE_URL
 --keep-s3-cache --write-to-disk --stdout --start-day 2026-01-01 --end-day
 2026-01-10 --noapi"      depends_on:        clickhouse:          condition:
 service_healthy

---
 tests/integration/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml
index 0ef9b1df..384e28e8 100644
--- a/tests/integration/docker-compose.yml
+++ b/tests/integration/docker-compose.yml
@@ -57,7 +57,7 @@ services:
     environment:
       CLICKHOUSE_URL: "http://testuser:testuser@clickhouse:9000/ooni"
     command: >
-      bash -c "./run_fastpath --debug --clickhouse-url $CLICKHOUSE_URL  --stdout"
+      bash -c "./run_fastpath --debug --clickhouse-url $CLICKHOUSE_URL --keep-s3-cache --write-to-disk --stdout --start-day 2026-01-01 --end-day 2026-01-10 --noapi"
     depends_on:
       clickhouse:
         condition: service_healthy

From e4bd778212984d38224ccff1c78c9f11f7cce89d Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Tue, 31 Mar 2026 15:20:41 +0200
Subject: [PATCH 27/59] update tests to use reports in dataset

---
 .../tests/test_oonidata_aggregate_analysis.py        |  2 +-
 .../tests/test_oonidata_aggregate_observations.py    |  4 ++--
 .../integration/tests/test_oonidata_list_analysis.py |  6 +++---
 .../tests/test_oonidata_list_observations.py         | 12 ++++++------
 4 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/tests/integration/tests/test_oonidata_aggregate_analysis.py b/tests/integration/tests/test_oonidata_aggregate_analysis.py
index 6bad3116..f2824f99 100644
--- a/tests/integration/tests/test_oonidata_aggregate_analysis.py
+++ b/tests/integration/tests/test_oonidata_aggregate_analysis.py
@@ -32,7 +32,7 @@ def test_oonidata_aggregation_analysis_with_since_and_until(
     [
         ("domain", "zh.wikipedia.org"),
         ("probe_cc", "IR"),
-        ("probe_asn", 45758),
+        ("probe_asn", 1267),
         ("test_name", "whatsapp"),
         ("input", "stun://stun.voys.nl:3478"),
     ],
diff --git a/tests/integration/tests/test_oonidata_aggregate_observations.py b/tests/integration/tests/test_oonidata_aggregate_observations.py
index f4824d20..3ee6c1e3 100644
--- a/tests/integration/tests/test_oonidata_aggregate_observations.py
+++ b/tests/integration/tests/test_oonidata_aggregate_observations.py
@@ -29,8 +29,8 @@ def test_oonidata_aggregation_observations_with_since_and_until(
     "filter_name, filter_value",
     [
         ("probe_cc", "IT"),
-        ("probe_asn", 45758),
-        ("probe_asn", [45758, 5650]),
+        ("probe_asn", 1267),
+        ("probe_asn", [1267, 5650]),
         ("test_name", "whatsapp"),
         ("hostname", "www.on-instant.com"),
         ("ip", "64.233.190.139"),
diff --git a/tests/integration/tests/test_oonidata_list_analysis.py b/tests/integration/tests/test_oonidata_list_analysis.py
index 64af92e7..de6b8777 100644
--- a/tests/integration/tests/test_oonidata_list_analysis.py
+++ b/tests/integration/tests/test_oonidata_list_analysis.py
@@ -31,9 +31,9 @@ def test_oonidata_list_analysis_with_since_and_until(
     [
         (
             "measurement_uid",
-            "20241101233756.866609_TH_webconnectivity_1bf55fb5699c39ec",
+            "20260102000000.504120_IT_webconnectivity_9de2bdd30c705f7a",
         ),
-        ("probe_asn", 45758),
+        ("probe_asn", 1267),
         ("probe_cc", "IT"),
         ("test_name", "web_connectivity"),
     ],
@@ -57,7 +57,7 @@ def test_oonidata_list_analysis_filtering_by_probe_asn_as_a_string_with_since_an
     client, params_since_and_until_with_two_days
 ):
     params = params_since_and_until_with_two_days
-    probe_asn = 45758
+    probe_asn = 1267
     params["probe_asn"] = "AS" + str(probe_asn)
 
     response = client.get(route, params=params)
diff --git a/tests/integration/tests/test_oonidata_list_observations.py b/tests/integration/tests/test_oonidata_list_observations.py
index de7f2c19..7d2d520e 100644
--- a/tests/integration/tests/test_oonidata_list_observations.py
+++ b/tests/integration/tests/test_oonidata_list_observations.py
@@ -27,14 +27,14 @@ def test_oonidata_list_observations_with_since_and_until(
 @pytest.mark.parametrize(
     "filter_name, filter_value",
     [
-        ("report_id", "20241101T233351Z_webconnectivity_DE_3209_n1_I7QVY7IdnaSfYmsb"),
-        ("probe_asn", 45758),
+        ("report_id", "20260101T235954Z_webconnectivity_IT_1267_n4_ozfKmtNGTeLyiVbE"),
+        ("probe_asn", 1267),
         ("probe_cc", "IT"),
-        ("software_name", "ooniprobe-cli"),
-        ("software_version", "3.20.0"),
+        ("software_name", "ooniprobe-android-unattended"),
+        ("software_version", "5.3.0"),
         ("test_name", "web_connectivity"),
         ("test_version", "0.4.3"),
-        ("engine_version", "3.20.0"),
+        ("engine_version", "3.28.0"),
     ],
 )
 def test_oonidata_list_observations_with_filters(
@@ -56,7 +56,7 @@ def test_oonidata_list_observations_filtering_by_probe_asn_as_a_string_with_sinc
     client, params_since_and_until_with_two_days
 ):
     params = params_since_and_until_with_two_days
-    probe_asn = 45758
+    probe_asn = 1267
     params["probe_asn"] = "AS" + str(probe_asn)
 
     response = client.get(route, params=params)

From be374cedecc1047435b38c76467999e337b97b3f Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Tue, 7 Apr 2026 13:31:13 +0200
Subject: [PATCH 28/59] add fastpath.conf

---
 tests/integration/fastpath/etc/fastpath.conf | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 tests/integration/fastpath/etc/fastpath.conf

diff --git a/tests/integration/fastpath/etc/fastpath.conf b/tests/integration/fastpath/etc/fastpath.conf
new file mode 100644
index 00000000..86b25d76
--- /dev/null
+++ b/tests/integration/fastpath/etc/fastpath.conf
@@ -0,0 +1,11 @@
+[DEFAULT]
+# Collector hostnames, comma separated
+collectors = localhost
+clickhouse_url = clickhouse://testuser:testuser@clickhouse:9000/ooni
+# S3 access credentials
+# Currently unused
+s3_access_key =
+s3_secret_key =
+
+debug = true
+msmt_spool_dir = /data/

From 40838c7632b0712c6bee16fa2970778d0fe7021f Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Tue, 7 Apr 2026 14:11:21 +0200
Subject: [PATCH 29/59] fastpath: filter --ccs IR,IT

---
 tests/integration/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml
index 384e28e8..a10cb8a4 100644
--- a/tests/integration/docker-compose.yml
+++ b/tests/integration/docker-compose.yml
@@ -57,7 +57,7 @@ services:
     environment:
       CLICKHOUSE_URL: "http://testuser:testuser@clickhouse:9000/ooni"
     command: >
-      bash -c "./run_fastpath --debug --clickhouse-url $CLICKHOUSE_URL --keep-s3-cache --write-to-disk --stdout --start-day 2026-01-01 --end-day 2026-01-10 --noapi"
+      bash -c "./run_fastpath --debug --clickhouse-url $CLICKHOUSE_URL --keep-s3-cache --write-to-disk --stdout --ccs IT,IR --start-day 2026-01-01 --end-day 2026-01-10 --noapi"
     depends_on:
       clickhouse:
         condition: service_healthy

From 7b2d0892490cdc3bfeb49117592d7df1a89650db Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Tue, 7 Apr 2026 14:18:16 +0200
Subject: [PATCH 30/59] add .gitignore to cache dir

---
 tests/integration/data/.gitignore               | 1 +
 tests/integration/fastpath/lib/cache/.gitignore | 2 ++
 2 files changed, 3 insertions(+)
 create mode 100644 tests/integration/fastpath/lib/cache/.gitignore

diff --git a/tests/integration/data/.gitignore b/tests/integration/data/.gitignore
index f935021a..78d91016 100644
--- a/tests/integration/data/.gitignore
+++ b/tests/integration/data/.gitignore
@@ -1 +1,2 @@
+/*
 !.gitignore
diff --git a/tests/integration/fastpath/lib/cache/.gitignore b/tests/integration/fastpath/lib/cache/.gitignore
new file mode 100644
index 00000000..78d91016
--- /dev/null
+++ b/tests/integration/fastpath/lib/cache/.gitignore
@@ -0,0 +1,2 @@
+/*
+!.gitignore

From 8c32ed3d00e0af07b1298c51d48fa4fbab741b43 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Tue, 7 Apr 2026 14:55:43 +0200
Subject: [PATCH 31/59] add valid probe_asn

---
 tests/integration/tests/test_oonidata_aggregate_analysis.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/tests/test_oonidata_aggregate_analysis.py b/tests/integration/tests/test_oonidata_aggregate_analysis.py
index f2824f99..95941741 100644
--- a/tests/integration/tests/test_oonidata_aggregate_analysis.py
+++ b/tests/integration/tests/test_oonidata_aggregate_analysis.py
@@ -56,7 +56,7 @@ def test_oonidata_aggregation_analysis_filtering_by_probe_asn_as_a_string_with_s
     client, params_since_and_until_with_ten_days
 ):
     params = params_since_and_until_with_ten_days
-    probe_asn = 45758
+    probe_asn = 12874
     params["probe_asn"] = "AS" + str(probe_asn)
 
     response = client.get(route, params=params)

From a89adb704eb1c482c923bb925c1e0e356dac74b4 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Tue, 7 Apr 2026 15:53:37 +0200
Subject: [PATCH 32/59] use valid whatsapp endpoint

---
 .../integration/tests/test_oonidata_aggregate_observations.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/tests/test_oonidata_aggregate_observations.py b/tests/integration/tests/test_oonidata_aggregate_observations.py
index 3ee6c1e3..eed9ba5f 100644
--- a/tests/integration/tests/test_oonidata_aggregate_observations.py
+++ b/tests/integration/tests/test_oonidata_aggregate_observations.py
@@ -32,8 +32,8 @@ def test_oonidata_aggregation_observations_with_since_and_until(
         ("probe_asn", 1267),
         ("probe_asn", [1267, 5650]),
         ("test_name", "whatsapp"),
-        ("hostname", "www.on-instant.com"),
-        ("ip", "64.233.190.139"),
+        ("hostname", "e7.whatsapp.net"),
+        ("ip", "15.197.206.217"),
     ],
 )
 def test_oonidata_aggregation_observations_with_filters(

From 9ed9f7ee9d98e22c0d076c516add3f5061354909 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Wed, 8 Apr 2026 08:25:55 +0200
Subject: [PATCH 33/59] adjust expected test result values

FIXME: These tests should be improved so that they work for any input
data
---
 .../integration/tests/test_oonidata_aggregate_observations.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/tests/test_oonidata_aggregate_observations.py b/tests/integration/tests/test_oonidata_aggregate_observations.py
index eed9ba5f..c9695561 100644
--- a/tests/integration/tests/test_oonidata_aggregate_observations.py
+++ b/tests/integration/tests/test_oonidata_aggregate_observations.py
@@ -57,7 +57,7 @@ def test_oonidata_aggregation_observations_with_filters(
 @pytest.mark.parametrize(
     "time_grain, total",
     [
-        ("hour", 215),
+        ("hour", 216),
         ("day", 9),
         ("week", 2),
         ("month", 1),
@@ -87,7 +87,7 @@ def test_oonidata_aggregation_observations_groupby_failure(
     response = client.get(route, params=params)
 
     json = response.json()
-    assert len(json["results"]) == 24
+    assert len(json["results"]) > 10
     first_result = json["results"][0]
     assert "failure" in first_result.keys()
     assert "timestamp" in first_result.keys()

From 0c6568daea428c48bed5ac4a0eae1778453ec8b2 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Wed, 8 Apr 2026 11:01:43 +0200
Subject: [PATCH 34/59] modify github workflow to wait for verifier to start
 and complete

---
 .github/workflows/test_end_to_end.yml | 58 ++++++++++++++++++++++-----
 1 file changed, 48 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/test_end_to_end.yml b/.github/workflows/test_end_to_end.yml
index 1efca92f..32b4f04d 100644
--- a/.github/workflows/test_end_to_end.yml
+++ b/.github/workflows/test_end_to_end.yml
@@ -22,25 +22,63 @@ jobs:
             data-cache-
           path: |
             tests/integration/data
+            tests/integration/fastpath
 
       - name: Install Podman
         run: |
           sudo apt update
           sudo apt install -y podman podman-compose podman-docker
 
-      - name: Run Docker Compose with Podman
+      - name: Run Docker Compose with Podman (detached)
         run: |
-          cd tests/integration && podman-compose up -d
+          cd tests/integration
+          podman-compose up -d
 
-      - name: Wait for Downloader Container to Exit
+      - name: Wait for verifier to start
         run: |
-          # Replace `your_container_name` with the name of your container
-          while [ $(podman ps -q -f name=oonidata-pipeline | wc -l) -gt 0 ]; do
-            echo "Container is still running..."
-            sleep 5 # Wait for 5 seconds before checking again
+          cd tests/integration
+          for i in $(seq 1 60); do
+            if podman ps --format "{{.Names}}" | grep -q '^verifier$'; then
+              echo "verifier started"
+              exit 0
+            fi
+            sleep 10
           done
-          echo "Container has exited."
+          echo "verifier did not start within timeout" >&2
+          podman ps -a
+          exit 1
 
-      - name: Shutdown
+      - name: Stream verifier logs until it exits
         run: |
-          cd tests/integration && podman-compose down
+          cd tests/integration
+          # Start streaming logs; podman logs --follow will exit when the container stops.
+          podman logs --follow verifier & LOG_PID=$!
+          # Wait for verifier container to stop, with a timeout (360*5s = 30 minutes).
+          for i in $(seq 1 360); do
+            if ! podman ps --format "{{.Names}}" | grep -q '^verifier$'; then
+              echo "verifier exited"
+              break
+            fi
+            sleep 5
+          done
+          # If still running after timeout, show debugging info and fail.
+          if podman ps --format "{{.Names}}" | grep -q '^verifier$'; then
+            echo "verifier did not finish within timeout" >&2
+            podman ps -a
+            podman logs verifier || true
+            if ps -p $LOG_PID > /dev/null; then
+              kill $LOG_PID || true
+            fi
+            exit 1
+          fi
+          # Wait for the logs follower to finish and show final logs if any.
+          if ps -p $LOG_PID > /dev/null; then
+            wait $LOG_PID || true
+          fi
+          podman ps -a --filter name=verifier --format "Container: {{.Names}} Status: {{.Status}}"
+          podman logs verifier || true
+
+      - name: Shutdown remaining services
+        run: |
+          cd tests/integration
+          podman-compose down --remove-orphans

From 34fd0ba52c64dda37fe27aeb296dcfb025ea5698 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Wed, 8 Apr 2026 11:19:58 +0200
Subject: [PATCH 35/59] give full path to container location

---
 tests/integration/docker-compose.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml
index a10cb8a4..c2c01680 100644
--- a/tests/integration/docker-compose.yml
+++ b/tests/integration/docker-compose.yml
@@ -20,7 +20,7 @@ services:
       CLICKHOUSE_URL: "http://testuser:testuser@clickhouse:9000/ooni"
 
   clickhouse:
-    image: clickhouse/clickhouse-server:latest
+    image: docker.io/clickhouse/clickhouse-server:latest
     container_name: clickhouse_server
     ports:
       - "8123:8123"  # HTTP interface
@@ -49,7 +49,7 @@ services:
       POSTGRES_DB: ooni
 
   fastpath:
-    image: ooni/fastpath:latest
+    image: docker.io/ooni/fastpath:latest
     container_name: fastpath_feeder
     volumes:
       - ./fastpath/etc:/etc/ooni:Z
@@ -67,7 +67,7 @@ services:
         condition: service_completed_successfully
 
   valkey:
-    image: valkey/valkey:latest
+    image: docker.io/valkey/valkey:latest
     container_name: valkey_cache
     ports:
       - "6379:6379"
@@ -77,7 +77,7 @@ services:
     restart: unless-stopped
 
   api:
-    image: ooni/api-oonimeasurements:latest
+    image: docker.io/ooni/api-oonimeasurements:latest
     container_name: oonimeasurements
     environment:
       VALKEY_URL: valkey://valkey:6379

From 7ad75c6adb36adcc7a46934437a407bfa054d39a Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Wed, 8 Apr 2026 11:36:48 +0200
Subject: [PATCH 36/59] remove container_name

---
 tests/integration/docker-compose.yml | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml
index c2c01680..ea0dfcca 100644
--- a/tests/integration/docker-compose.yml
+++ b/tests/integration/docker-compose.yml
@@ -6,7 +6,6 @@ services:
       context: ../../
       dockerfile: Dockerfile
     image: oonidata_image
-    container_name: oonidata-pipeline
     volumes:
       - ./data:/data:Z
     working_dir: /data
@@ -21,7 +20,6 @@ services:
 
   clickhouse:
     image: docker.io/clickhouse/clickhouse-server:latest
-    container_name: clickhouse_server
     ports:
       - "8123:8123"  # HTTP interface
       - "9000:9000"  # Native interface
@@ -40,7 +38,6 @@ services:
 
   postgres:
     image: docker.io/library/postgres:latest
-    container_name: postgres_db
     ports:
       - "5432:5432"
     environment:
@@ -50,7 +47,6 @@ services:
 
   fastpath:
     image: docker.io/ooni/fastpath:latest
-    container_name: fastpath_feeder
     volumes:
       - ./fastpath/etc:/etc/ooni:Z
       - ./fastpath/lib:/var/lib/fastpath:Z
@@ -68,7 +64,6 @@ services:
 
   valkey:
     image: docker.io/valkey/valkey:latest
-    container_name: valkey_cache
     ports:
       - "6379:6379"
     #volumes:
@@ -78,7 +73,6 @@ services:
 
   api:
     image: docker.io/ooni/api-oonimeasurements:latest
-    container_name: oonimeasurements
     environment:
       VALKEY_URL: valkey://valkey:6379
       POSTGRESQL_URL: "postgresql://postgres:postgres@postgres:5432/ooni"
@@ -116,7 +110,6 @@ services:
       context: .
       dockerfile: Dockerfile
     image: verifier
-    container_name: verifier
 
     working_dir: /app
     volumes:

From 37bbfc94528c69dcd22e47cc392e70a4080d564f Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Wed, 8 Apr 2026 11:46:43 +0200
Subject: [PATCH 37/59] test; why is this broken in github actions...

---
 .github/workflows/test_end_to_end.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_end_to_end.yml b/.github/workflows/test_end_to_end.yml
index 32b4f04d..6889eaf8 100644
--- a/.github/workflows/test_end_to_end.yml
+++ b/.github/workflows/test_end_to_end.yml
@@ -32,7 +32,7 @@ jobs:
       - name: Run Docker Compose with Podman (detached)
         run: |
           cd tests/integration
-          podman-compose up -d
+          podman compose up -d
 
       - name: Wait for verifier to start
         run: |

From 132197451df531c0bde75bf11949d6776d353057 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Wed, 8 Apr 2026 11:49:52 +0200
Subject: [PATCH 38/59] remove podman-docker package

---
 .github/workflows/test_end_to_end.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_end_to_end.yml b/.github/workflows/test_end_to_end.yml
index 6889eaf8..e872419d 100644
--- a/.github/workflows/test_end_to_end.yml
+++ b/.github/workflows/test_end_to_end.yml
@@ -27,7 +27,7 @@ jobs:
       - name: Install Podman
         run: |
           sudo apt update
-          sudo apt install -y podman podman-compose podman-docker
+          sudo apt install -y podman podman-compose
 
       - name: Run Docker Compose with Podman (detached)
         run: |

From b468593ad8e4c5894ce6cbb0633000582a32dab3 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Wed, 8 Apr 2026 11:59:54 +0200
Subject: [PATCH 39/59] start podman service

---
 .github/workflows/test_end_to_end.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_end_to_end.yml b/.github/workflows/test_end_to_end.yml
index e872419d..1785279d 100644
--- a/.github/workflows/test_end_to_end.yml
+++ b/.github/workflows/test_end_to_end.yml
@@ -32,6 +32,7 @@ jobs:
       - name: Run Docker Compose with Podman (detached)
         run: |
           cd tests/integration
+          podman machine init --now
           podman compose up -d
 
       - name: Wait for verifier to start

From bab0dfd5de4914668eac16071338fd232aefcf81 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Wed, 8 Apr 2026 12:09:50 +0200
Subject: [PATCH 40/59] use docker on github actions

---
 .github/workflows/test_end_to_end.yml | 32 +++++++++++----------------
 1 file changed, 13 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/test_end_to_end.yml b/.github/workflows/test_end_to_end.yml
index 1785279d..9c0f365f 100644
--- a/.github/workflows/test_end_to_end.yml
+++ b/.github/workflows/test_end_to_end.yml
@@ -24,49 +24,43 @@ jobs:
             tests/integration/data
             tests/integration/fastpath
 
-      - name: Install Podman
-        run: |
-          sudo apt update
-          sudo apt install -y podman podman-compose
-
-      - name: Run Docker Compose with Podman (detached)
+      - name: Run Docker Compose (detached)
         run: |
           cd tests/integration
-          podman machine init --now
-          podman compose up -d
+          docker compose up -d
 
       - name: Wait for verifier to start
         run: |
           cd tests/integration
           for i in $(seq 1 60); do
-            if podman ps --format "{{.Names}}" | grep -q '^verifier$'; then
+            if docker ps --format "{{.Names}}" | grep -q '^verifier$'; then
               echo "verifier started"
               exit 0
             fi
             sleep 10
           done
           echo "verifier did not start within timeout" >&2
-          podman ps -a
+          docker ps -a
           exit 1
 
       - name: Stream verifier logs until it exits
         run: |
           cd tests/integration
-          # Start streaming logs; podman logs --follow will exit when the container stops.
-          podman logs --follow verifier & LOG_PID=$!
+          # Start streaming logs; docker logs --follow will exit when the container stops.
+          docker logs --follow verifier & LOG_PID=$!
           # Wait for verifier container to stop, with a timeout (360*5s = 30 minutes).
           for i in $(seq 1 360); do
-            if ! podman ps --format "{{.Names}}" | grep -q '^verifier$'; then
+            if ! docker ps --format "{{.Names}}" | grep -q '^verifier$'; then
               echo "verifier exited"
               break
             fi
             sleep 5
           done
           # If still running after timeout, show debugging info and fail.
-          if podman ps --format "{{.Names}}" | grep -q '^verifier$'; then
+          if docker ps --format "{{.Names}}" | grep -q '^verifier$'; then
             echo "verifier did not finish within timeout" >&2
-            podman ps -a
-            podman logs verifier || true
+            docker ps -a
+            docker logs verifier || true
             if ps -p $LOG_PID > /dev/null; then
               kill $LOG_PID || true
             fi
@@ -76,10 +70,10 @@ jobs:
           if ps -p $LOG_PID > /dev/null; then
             wait $LOG_PID || true
           fi
-          podman ps -a --filter name=verifier --format "Container: {{.Names}} Status: {{.Status}}"
-          podman logs verifier || true
+          docker ps -a --filter name=verifier --format "Container: {{.Names}} Status: {{.Status}}"
+          docker logs verifier || true
 
       - name: Shutdown remaining services
         run: |
           cd tests/integration
-          podman-compose down --remove-orphans
+          docker compose down --remove-orphans

From 8044a425a7f4c284bc66a9b9de28cf37134450f2 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Wed, 8 Apr 2026 13:13:25 +0200
Subject: [PATCH 41/59] try to debug github-actions hanging

---
 .github/workflows/test_end_to_end.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test_end_to_end.yml b/.github/workflows/test_end_to_end.yml
index 9c0f365f..ce005c59 100644
--- a/.github/workflows/test_end_to_end.yml
+++ b/.github/workflows/test_end_to_end.yml
@@ -28,6 +28,8 @@ jobs:
         run: |
           cd tests/integration
           docker compose up -d
+          docker compose ps --all
+          docker compose logs --no-color --timestamps --tail=200 &
 
       - name: Wait for verifier to start
         run: |

From 5bf516f5614a1e14909896dd8ea6818f7569e26a Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Wed, 8 Apr 2026 13:19:58 +0200
Subject: [PATCH 42/59] try to get debug output...

---
 .github/workflows/test_end_to_end.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_end_to_end.yml b/.github/workflows/test_end_to_end.yml
index ce005c59..eddc92fa 100644
--- a/.github/workflows/test_end_to_end.yml
+++ b/.github/workflows/test_end_to_end.yml
@@ -27,7 +27,7 @@ jobs:
       - name: Run Docker Compose (detached)
         run: |
           cd tests/integration
-          docker compose up -d
+          docker compose up &
           docker compose ps --all
           docker compose logs --no-color --timestamps --tail=200 &
 

From 7f3076b3f878daf084d9a8291e4e17d49f746c41 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Wed, 8 Apr 2026 13:21:29 +0200
Subject: [PATCH 43/59] ...

---
 .github/workflows/test_end_to_end.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_end_to_end.yml b/.github/workflows/test_end_to_end.yml
index eddc92fa..923e8350 100644
--- a/.github/workflows/test_end_to_end.yml
+++ b/.github/workflows/test_end_to_end.yml
@@ -27,7 +27,7 @@ jobs:
       - name: Run Docker Compose (detached)
         run: |
           cd tests/integration
-          docker compose up &
+          docker compose up
           docker compose ps --all
           docker compose logs --no-color --timestamps --tail=200 &
 

From ca9155f44e168069aa8060ca929e211b92d3c8e2 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Wed, 8 Apr 2026 13:55:25 +0200
Subject: [PATCH 44/59] Revert "use docker on github actions"

This reverts commit bab0dfd5de4914668eac16071338fd232aefcf81.
---
 .github/workflows/test_end_to_end.yml | 35 +++++++++++++++------------
 1 file changed, 20 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/test_end_to_end.yml b/.github/workflows/test_end_to_end.yml
index 923e8350..fea58474 100644
--- a/.github/workflows/test_end_to_end.yml
+++ b/.github/workflows/test_end_to_end.yml
@@ -24,45 +24,50 @@ jobs:
             tests/integration/data
             tests/integration/fastpath
 
-      - name: Run Docker Compose (detached)
+      - name: Install Podman
+        run: |
+          sudo apt update
+          sudo apt install -y podman podman-compose
+
+      - name: Run Docker Compose with Podman (detached)
         run: |
           cd tests/integration
-          docker compose up
-          docker compose ps --all
-          docker compose logs --no-color --timestamps --tail=200 &
+          export PODMAN_HOST=unix:///run/user/$(id -u)/podman/podman.sock
+          podman system service -t0 &
+          podman compose up -d
 
       - name: Wait for verifier to start
         run: |
           cd tests/integration
           for i in $(seq 1 60); do
-            if docker ps --format "{{.Names}}" | grep -q '^verifier$'; then
+            if podman ps --format "{{.Names}}" | grep -q '^verifier$'; then
               echo "verifier started"
               exit 0
             fi
             sleep 10
           done
           echo "verifier did not start within timeout" >&2
-          docker ps -a
+          podman ps -a
           exit 1
 
       - name: Stream verifier logs until it exits
         run: |
           cd tests/integration
-          # Start streaming logs; docker logs --follow will exit when the container stops.
-          docker logs --follow verifier & LOG_PID=$!
+          # Start streaming logs; podman logs --follow will exit when the container stops.
+          podman logs --follow verifier & LOG_PID=$!
           # Wait for verifier container to stop, with a timeout (360*5s = 30 minutes).
           for i in $(seq 1 360); do
-            if ! docker ps --format "{{.Names}}" | grep -q '^verifier$'; then
+            if ! podman ps --format "{{.Names}}" | grep -q '^verifier$'; then
               echo "verifier exited"
               break
             fi
             sleep 5
           done
           # If still running after timeout, show debugging info and fail.
-          if docker ps --format "{{.Names}}" | grep -q '^verifier$'; then
+          if podman ps --format "{{.Names}}" | grep -q '^verifier$'; then
             echo "verifier did not finish within timeout" >&2
-            docker ps -a
-            docker logs verifier || true
+            podman ps -a
+            podman logs verifier || true
             if ps -p $LOG_PID > /dev/null; then
               kill $LOG_PID || true
             fi
@@ -72,10 +77,10 @@ jobs:
           if ps -p $LOG_PID > /dev/null; then
             wait $LOG_PID || true
           fi
-          docker ps -a --filter name=verifier --format "Container: {{.Names}} Status: {{.Status}}"
-          docker logs verifier || true
+          podman ps -a --filter name=verifier --format "Container: {{.Names}} Status: {{.Status}}"
+          podman logs verifier || true
 
       - name: Shutdown remaining services
         run: |
           cd tests/integration
-          docker compose down --remove-orphans
+          podman-compose down --remove-orphans

From b85325edd1eee524d7019e15bc75b4fdb28ed10d Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Wed, 8 Apr 2026 13:58:24 +0200
Subject: [PATCH 45/59] try to debug github actions hanging

---
 .github/workflows/test_end_to_end.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_end_to_end.yml b/.github/workflows/test_end_to_end.yml
index fea58474..37f46330 100644
--- a/.github/workflows/test_end_to_end.yml
+++ b/.github/workflows/test_end_to_end.yml
@@ -34,7 +34,7 @@ jobs:
           cd tests/integration
           export PODMAN_HOST=unix:///run/user/$(id -u)/podman/podman.sock
           podman system service -t0 &
-          podman compose up -d
+          podman compose up
 
       - name: Wait for verifier to start
         run: |

From 30de67644b915e6a6bd871efada0cbd728fb89bd Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Wed, 8 Apr 2026 15:12:23 +0200
Subject: [PATCH 46/59] Reapply "use docker on github actions"

This reverts commit ca9155f44e168069aa8060ca929e211b92d3c8e2.

Try to use docker directly again and obtain debug logging
---
 .github/workflows/test_end_to_end.yml | 35 ++++++++++++---------------
 1 file changed, 15 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/test_end_to_end.yml b/.github/workflows/test_end_to_end.yml
index 37f46330..923e8350 100644
--- a/.github/workflows/test_end_to_end.yml
+++ b/.github/workflows/test_end_to_end.yml
@@ -24,50 +24,45 @@ jobs:
             tests/integration/data
             tests/integration/fastpath
 
-      - name: Install Podman
-        run: |
-          sudo apt update
-          sudo apt install -y podman podman-compose
-
-      - name: Run Docker Compose with Podman (detached)
+      - name: Run Docker Compose (detached)
         run: |
           cd tests/integration
-          export PODMAN_HOST=unix:///run/user/$(id -u)/podman/podman.sock
-          podman system service -t0 &
-          podman compose up
+          docker compose up
+          docker compose ps --all
+          docker compose logs --no-color --timestamps --tail=200 &
 
       - name: Wait for verifier to start
         run: |
           cd tests/integration
           for i in $(seq 1 60); do
-            if podman ps --format "{{.Names}}" | grep -q '^verifier$'; then
+            if docker ps --format "{{.Names}}" | grep -q '^verifier$'; then
               echo "verifier started"
               exit 0
             fi
             sleep 10
           done
           echo "verifier did not start within timeout" >&2
-          podman ps -a
+          docker ps -a
           exit 1
 
       - name: Stream verifier logs until it exits
         run: |
           cd tests/integration
-          # Start streaming logs; podman logs --follow will exit when the container stops.
-          podman logs --follow verifier & LOG_PID=$!
+          # Start streaming logs; docker logs --follow will exit when the container stops.
+          docker logs --follow verifier & LOG_PID=$!
           # Wait for verifier container to stop, with a timeout (360*5s = 30 minutes).
           for i in $(seq 1 360); do
-            if ! podman ps --format "{{.Names}}" | grep -q '^verifier$'; then
+            if ! docker ps --format "{{.Names}}" | grep -q '^verifier$'; then
               echo "verifier exited"
               break
             fi
             sleep 5
           done
           # If still running after timeout, show debugging info and fail.
-          if podman ps --format "{{.Names}}" | grep -q '^verifier$'; then
+          if docker ps --format "{{.Names}}" | grep -q '^verifier$'; then
             echo "verifier did not finish within timeout" >&2
-            podman ps -a
-            podman logs verifier || true
+            docker ps -a
+            docker logs verifier || true
             if ps -p $LOG_PID > /dev/null; then
               kill $LOG_PID || true
             fi
@@ -77,10 +72,10 @@ jobs:
           if ps -p $LOG_PID > /dev/null; then
             wait $LOG_PID || true
           fi
-          podman ps -a --filter name=verifier --format "Container: {{.Names}} Status: {{.Status}}"
-          podman logs verifier || true
+          docker ps -a --filter name=verifier --format "Container: {{.Names}} Status: {{.Status}}"
+          docker logs verifier || true
 
       - name: Shutdown remaining services
         run: |
           cd tests/integration
-          podman-compose down --remove-orphans
+          docker compose down --remove-orphans

From 7bee89a8c74e61c498a4476f8a667d9fcb7f87d8 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Wed, 8 Apr 2026 15:35:59 +0200
Subject: [PATCH 47/59] add tmate debugging

---
 .github/workflows/test_end_to_end.yml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/.github/workflows/test_end_to_end.yml b/.github/workflows/test_end_to_end.yml
index 923e8350..47b38c60 100644
--- a/.github/workflows/test_end_to_end.yml
+++ b/.github/workflows/test_end_to_end.yml
@@ -6,6 +6,12 @@ on:
       - main
       - add_end_to_end_tests
 
+  workflow_dispatch:
+    inputs:
+      debug_enabled:
+        description: "Run the build with tmate debugging enabled"
+        required: false
+        default: false
 jobs:
   build-and-run:
     runs-on: ubuntu-latest
@@ -24,6 +30,12 @@ jobs:
             tests/integration/data
             tests/integration/fastpath
 
+      - name: Setup tmate session
+        uses: mxschmitt/action-tmate@v3
+        if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled }}
+        with:
+          limit-access-to-actor: true
+
       - name: Run Docker Compose (detached)
         run: |
           cd tests/integration

From 7633a200df4d74c6005f8ca2e0ad1e50e6e95bb4 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Wed, 8 Apr 2026 16:37:44 +0200
Subject: [PATCH 48/59] remove downloader as dependency from fastpath

fastpath doesn't seem to injest these from disk and this seems to create
a double dependency of the downloader service, so it gets invoked twice
---
 tests/integration/docker-compose.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml
index ea0dfcca..c651041a 100644
--- a/tests/integration/docker-compose.yml
+++ b/tests/integration/docker-compose.yml
@@ -59,8 +59,6 @@ services:
         condition: service_healthy
       postgres:
         condition: service_started
-      downloader:
-        condition: service_completed_successfully
 
   valkey:
     image: docker.io/valkey/valkey:latest

From e5cd6e3992654a5f2725f8fd59954b65db32e412 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Thu, 9 Apr 2026 13:14:01 +0200
Subject: [PATCH 49/59] convert fastpath command to exec form

variable expansion works slightly differently in docker compose than
podman compose; so the CLICKHOSUE_URL was not being expanded correctly
---
 tests/integration/docker-compose.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml
index c651041a..888d3e59 100644
--- a/tests/integration/docker-compose.yml
+++ b/tests/integration/docker-compose.yml
@@ -52,8 +52,9 @@ services:
       - ./fastpath/lib:/var/lib/fastpath:Z
     environment:
       CLICKHOUSE_URL: "http://testuser:testuser@clickhouse:9000/ooni"
-    command: >
-      bash -c "./run_fastpath --debug --clickhouse-url $CLICKHOUSE_URL --keep-s3-cache --write-to-disk --stdout --ccs IT,IR --start-day 2026-01-01 --end-day 2026-01-10 --noapi"
+    command: [ "./run_fastpath", "--debug", "--clickhouse-url", "$CLICKHOUSE_URL",
+               "--keep-s3-cache", "--write-to-disk", "--stdout", "--ccs", "IT,IR",
+               "--start-day", "2026-01-01", "--end-day", "2026-01-02", "--noapi" ]
     depends_on:
       clickhouse:
         condition: service_healthy

From 90d1ed7cfc8a9c71b2576dcead551955479bc7ae Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Thu, 9 Apr 2026 13:57:05 +0200
Subject: [PATCH 50/59] make fastpath cache world writable for volume mapping

---
 .github/workflows/test_end_to_end.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/test_end_to_end.yml b/.github/workflows/test_end_to_end.yml
index 47b38c60..a54b4463 100644
--- a/.github/workflows/test_end_to_end.yml
+++ b/.github/workflows/test_end_to_end.yml
@@ -30,6 +30,10 @@ jobs:
             tests/integration/data
             tests/integration/fastpath
 
+      - name: change permission of fastpath cache for container read/write
+        run: |
+          chmod 0777 tests/integration/fastpath/lib/cache
+
       - name: Setup tmate session
         uses: mxschmitt/action-tmate@v3
         if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled }}

From 00f2e3d516f64a98681c1f086e0fabb2ca186aad Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Thu, 9 Apr 2026 15:34:54 +0200
Subject: [PATCH 51/59] fix CLICKHOUSE_URL

---
 tests/integration/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml
index 888d3e59..167e0581 100644
--- a/tests/integration/docker-compose.yml
+++ b/tests/integration/docker-compose.yml
@@ -51,7 +51,7 @@ services:
       - ./fastpath/etc:/etc/ooni:Z
       - ./fastpath/lib:/var/lib/fastpath:Z
     environment:
-      CLICKHOUSE_URL: "http://testuser:testuser@clickhouse:9000/ooni"
+      CLICKHOUSE_URL: "clickhouse://testuser:testuser@clickhouse:9000/ooni"
     command: [ "./run_fastpath", "--debug", "--clickhouse-url", "$CLICKHOUSE_URL",
                "--keep-s3-cache", "--write-to-disk", "--stdout", "--ccs", "IT,IR",
                "--start-day", "2026-01-01", "--end-day", "2026-01-02", "--noapi" ]

From eb52b750fe1ed0f6bc129b2194263b4ae48fe630 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Thu, 9 Apr 2026 15:35:24 +0200
Subject: [PATCH 52/59] decrease fetched data

---
 tests/integration/docker-compose.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml
index 167e0581..9f01d7e8 100644
--- a/tests/integration/docker-compose.yml
+++ b/tests/integration/docker-compose.yml
@@ -10,8 +10,8 @@ services:
       - ./data:/data:Z
     working_dir: /data
     command: >
-      bash -c "oonidata sync --output-dir . --probe-cc IT,IR --start-day 2026-01-01 --end-day 2026-01-10 &&
-      oonipipeline run --create-tables --probe-cc IT,IR --workflow-name observations --start-at 2026-01-01 --end-at 2026-01-10"
+      bash -c "oonidata sync --output-dir . --probe-cc IT,IR --start-day 2026-01-01 --end-day 2026-01-02 &&
+      oonipipeline run --create-tables --probe-cc IT,IR --workflow-name observations --start-at 2026-01-01 --end-at 2026-01-02"
     depends_on:
       clickhouse:
         condition: service_healthy

From 14670dcc4218eb99c625feea0ad9a0a2675d98b4 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Thu, 9 Apr 2026 15:42:41 +0200
Subject: [PATCH 53/59] fix lack of env expansion in docker

---
 tests/integration/docker-compose.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml
index 9f01d7e8..fb984e19 100644
--- a/tests/integration/docker-compose.yml
+++ b/tests/integration/docker-compose.yml
@@ -50,9 +50,8 @@ services:
     volumes:
       - ./fastpath/etc:/etc/ooni:Z
       - ./fastpath/lib:/var/lib/fastpath:Z
-    environment:
-      CLICKHOUSE_URL: "clickhouse://testuser:testuser@clickhouse:9000/ooni"
-    command: [ "./run_fastpath", "--debug", "--clickhouse-url", "$CLICKHOUSE_URL",
+    command: [ "./run_fastpath", "--debug", "--clickhouse-url",
+               "clickhouse://testuser:testuser@clickhouse:9000/ooni",
                "--keep-s3-cache", "--write-to-disk", "--stdout", "--ccs", "IT,IR",
                "--start-day", "2026-01-01", "--end-day", "2026-01-02", "--noapi" ]
     depends_on:

From e83f62e7f4c21a856cd32c2f7f9a5af614ed43b4 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Thu, 9 Apr 2026 17:22:57 +0200
Subject: [PATCH 54/59] start docker compose daemonized

---
 .github/workflows/test_end_to_end.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_end_to_end.yml b/.github/workflows/test_end_to_end.yml
index a54b4463..545f43f7 100644
--- a/.github/workflows/test_end_to_end.yml
+++ b/.github/workflows/test_end_to_end.yml
@@ -43,7 +43,7 @@ jobs:
       - name: Run Docker Compose (detached)
         run: |
           cd tests/integration
-          docker compose up
+          docker compose up -d
           docker compose ps --all
           docker compose logs --no-color --timestamps --tail=200 &
 

From 4b2f5af5f984eeb50391a8c76f62061df0249453 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Thu, 9 Apr 2026 17:55:51 +0200
Subject: [PATCH 55/59] match verifier name

---
 .github/workflows/test_end_to_end.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_end_to_end.yml b/.github/workflows/test_end_to_end.yml
index 545f43f7..21833e8d 100644
--- a/.github/workflows/test_end_to_end.yml
+++ b/.github/workflows/test_end_to_end.yml
@@ -51,7 +51,7 @@ jobs:
         run: |
           cd tests/integration
           for i in $(seq 1 60); do
-            if docker ps --format "{{.Names}}" | grep -q '^verifier$'; then
+            if docker ps --format "{{.Names}}" | grep -q 'integration-verify'; then
               echo "verifier started"
               exit 0
             fi
@@ -68,7 +68,7 @@ jobs:
           docker logs --follow verifier & LOG_PID=$!
           # Wait for verifier container to stop, with a timeout (360*5s = 30 minutes).
           for i in $(seq 1 360); do
-            if ! docker ps --format "{{.Names}}" | grep -q '^verifier$'; then
+            if ! docker ps --format "{{.Names}}" | grep -q 'integration-verify'; then
               echo "verifier exited"
               break
             fi

From cb6073583180a16229eca3c9d1eeef5c211c6541 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Thu, 9 Apr 2026 18:14:34 +0200
Subject: [PATCH 56/59] fix container name

---
 .github/workflows/test_end_to_end.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test_end_to_end.yml b/.github/workflows/test_end_to_end.yml
index 21833e8d..7344d752 100644
--- a/.github/workflows/test_end_to_end.yml
+++ b/.github/workflows/test_end_to_end.yml
@@ -65,7 +65,7 @@ jobs:
         run: |
           cd tests/integration
           # Start streaming logs; docker logs --follow will exit when the container stops.
-          docker logs --follow verifier & LOG_PID=$!
+          docker logs --follow integration-verify-1 & LOG_PID=$!
           # Wait for verifier container to stop, with a timeout (360*5s = 30 minutes).
           for i in $(seq 1 360); do
             if ! docker ps --format "{{.Names}}" | grep -q 'integration-verify'; then
@@ -75,10 +75,10 @@ jobs:
             sleep 5
           done
           # If still running after timeout, show debugging info and fail.
-          if docker ps --format "{{.Names}}" | grep -q '^verifier$'; then
+          if docker ps --format "{{.Names}}" | grep -q 'integration-verify'; then
             echo "verifier did not finish within timeout" >&2
             docker ps -a
-            docker logs verifier || true
+            docker logs integration-verify-1 || true
             if ps -p $LOG_PID > /dev/null; then
               kill $LOG_PID || true
             fi
@@ -88,8 +88,8 @@ jobs:
           if ps -p $LOG_PID > /dev/null; then
             wait $LOG_PID || true
           fi
-          docker ps -a --filter name=verifier --format "Container: {{.Names}} Status: {{.Status}}"
-          docker logs verifier || true
+          docker ps -a --filter name=verify --format "Container: {{.Names}} Status: {{.Status}}"
+          docker logs integration-verify-1 || true
 
       - name: Shutdown remaining services
         run: |

From 3cf400e9c40b6535665dbfb3b3c4a1cdecfa27bb Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Fri, 10 Apr 2026 10:31:51 +0200
Subject: [PATCH 57/59] recursively make entire restored cache entries
 read/write

---
 .github/workflows/test_end_to_end.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_end_to_end.yml b/.github/workflows/test_end_to_end.yml
index 7344d752..db9dbc3e 100644
--- a/.github/workflows/test_end_to_end.yml
+++ b/.github/workflows/test_end_to_end.yml
@@ -32,7 +32,7 @@ jobs:
 
       - name: change permission of fastpath cache for container read/write
         run: |
-          chmod 0777 tests/integration/fastpath/lib/cache
+          chmod 0777 -R tests/integration/fastpath/lib/cache
 
       - name: Setup tmate session
         uses: mxschmitt/action-tmate@v3

From 0191160f90b8a005d207a8d9f4ba6982eec2397d Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Fri, 10 Apr 2026 10:50:12 +0200
Subject: [PATCH 58/59] Revert "decrease fetched data"

This reverts commit eb52b750fe1ed0f6bc129b2194263b4ae48fe630.
---
 tests/integration/docker-compose.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/docker-compose.yml b/tests/integration/docker-compose.yml
index fb984e19..54fd6595 100644
--- a/tests/integration/docker-compose.yml
+++ b/tests/integration/docker-compose.yml
@@ -10,8 +10,8 @@ services:
       - ./data:/data:Z
     working_dir: /data
     command: >
-      bash -c "oonidata sync --output-dir . --probe-cc IT,IR --start-day 2026-01-01 --end-day 2026-01-02 &&
-      oonipipeline run --create-tables --probe-cc IT,IR --workflow-name observations --start-at 2026-01-01 --end-at 2026-01-02"
+      bash -c "oonidata sync --output-dir . --probe-cc IT,IR --start-day 2026-01-01 --end-day 2026-01-10 &&
+      oonipipeline run --create-tables --probe-cc IT,IR --workflow-name observations --start-at 2026-01-01 --end-at 2026-01-10"
     depends_on:
       clickhouse:
         condition: service_healthy

From b791f4f0001f6d9cf31e24f53997cd5dcfff3708 Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Fri, 10 Apr 2026 10:52:47 +0200
Subject: [PATCH 59/59] fail test if verifier exited

---
 .github/workflows/test_end_to_end.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/test_end_to_end.yml b/.github/workflows/test_end_to_end.yml
index db9dbc3e..2c0b41dd 100644
--- a/.github/workflows/test_end_to_end.yml
+++ b/.github/workflows/test_end_to_end.yml
@@ -90,6 +90,11 @@ jobs:
           fi
           docker ps -a --filter name=verify --format "Container: {{.Names}} Status: {{.Status}}"
           docker logs integration-verify-1 || true
+          EXIT_CODE=$(docker inspect --format='{{.State.ExitCode}}' integration-verify-1)
+          if [ "$EXIT_CODE" -ne 0 ]; then
+            echo "Container exited with code $EXIT_CODE"
+            exit $EXIT_CODE
+          fi
 
       - name: Shutdown remaining services
         run: |