SeaweedFS integration

gateway/.envs/example/minio.env

@@ -1,13 +1,12 @@
 # -------------------------------------------------------
 # ====================== LOCAL ENV ======================
+# DEPRECATED ::: see sfs.env for a SeaweedFS setup that replaces MinIO.
 # MINIO Config
-MINIO_ROOT_USER=minioadmin
-MINIO_ROOT_PASSWORD=<SAME AS AWS_SECRET_ACCESS_KEY>
+MINIO_ACCESS_KEY_ID=minioadmin
 MINIO_ENDPOINT_URL=minio:9000
-MINIO_STORAGE_USE_HTTPS=false # prod: true
-
-# AWS S3 Config
-AWS_ACCESS_KEY_ID=minioadmin
-AWS_SECRET_ACCESS_KEY=<SAME AS MINIO_ROOT_PASSWORD>
-AWS_STORAGE_BUCKET_NAME=spectrumx
-AWS_S3_ENDPOINT_URL=http://minio:9000
+MINIO_ROOT_PASSWORD=<GENERATED MINIO ROOT PASSWORD>
+MINIO_ROOT_USER=minioadmin
+MINIO_S3_ENDPOINT_URL=http://minio:9000
+MINIO_SECRET_ACCESS_KEY=<SAME AS MINIO_ROOT_PASSWORD>
+MINIO_STORAGE_BUCKET_NAME=spectrumx
+MINIO_STORAGE_USE_HTTPS=false

gateway/.envs/example/sfs.env

@@ -0,0 +1,17 @@
+# SeaweedFS S3-compatible storage — see seaweedfs/compose.yaml
+# credentials are configured via `weed shell s3.configure` on the SFS cluster
+SFS_ACCESS_KEY_ID=admin
+SFS_SECRET_ACCESS_KEY=admin
+SFS_STORAGE_BUCKET_NAME=spectrumx
+SFS_S3_ENDPOINT_URL=http://sds-gateway-local-sfs-s3:8333
+SFS_STORAGE_USE_HTTPS=false
+SFS_ENDPOINT_URL=sds-gateway-local-sfs-s3:8333
+
+# Enables writes to both storage backends (SFS as primary, MinIO as fallback)
+OBJECT_STORE_WRITE_BOTH_ENABLED=false
+
+# Enables MinIO reads as fallback. Set to false if MinIO is not running
+OBJECT_STORE_READ_FALLBACK_TO_MINIO_ENABLED=false
+
+# Requires successful writes to both SFS and MinIO backends
+OBJECT_STORE_DUAL_WRITE_STRICT=false

gateway/compose.ci.yaml

@@ -45,8 +45,6 @@ services:
                 condition: service_healthy
             redis:
                 condition: service_healthy
-            minio:
-                condition: service_healthy
         volumes:
             - sds-gateway-ci-uv-cache:/opt/uv-cache/
             - sds-gateway-ci-uv-venv-app:/opt/uv-venv/
@@ -74,7 +72,8 @@ services:
             # - ./staticfiles/:/app/staticfiles/:z  # used in prod only
         env_file:
             - ./.envs/ci/django.env
-            - ./.envs/ci/minio.env
+            - ./.envs/ci/minio.env # legacy — kept during migration
+            - ./.envs/ci/sfs.env # SeaweedFS S3 — see seaweedfs/compose.yaml
             - ./.envs/ci/postgres.env
             - ./.envs/ci/opensearch.env
         # remember /entrypoint runs first
@@ -124,7 +123,18 @@ services:
                   selinux: z
         networks:
             - sds-network-ci
+        healthcheck:
+            test:
+                [
+                    "CMD-SHELL",
+                    "wget -q -O /dev/null http://localhost/ || exit 1",
+                ]
+            interval: 30s
+            timeout: 5s
+            retries: 5
+            start_period: 10s
 
+    # DEPRECATED: kept during migration for data transfer. Remove after migration complete.
     minio:
         # main file storage for sds
         # minio uses rolling upgrades that are non-disruptive, so we can target latest
@@ -264,14 +274,25 @@ services:
                   selinux: z
         env_file:
             - ./.envs/ci/django.env
-            - ./.envs/ci/minio.env
+            - ./.envs/ci/minio.env # legacy — kept during migration
+            - ./.envs/ci/sfs.env # SeaweedFS S3 — see seaweedfs/compose.yaml
             - ./.envs/ci/postgres.env
             - ./.envs/ci/opensearch.env
         command: "/worker-start"
         networks:
             - sds-gateway-ci-minio-net
             - sds-gateway-ci-opensearch-net
             - sds-network-ci
+        healthcheck:
+            test:
+                [
+                    "CMD-SHELL",
+                    'uv run celery -A config.celery_app inspect ping -d "celery@$$HOSTNAME" | grep -q "OK"',
+                ]
+            interval: 30s
+            timeout: 30s
+            retries: 5
+            start_period: 30s
 
     celery-beat:
         # Celery Beat scheduler for periodic tasks
@@ -309,14 +330,25 @@ services:
                   selinux: z
         env_file:
             - ./.envs/ci/django.env
-            - ./.envs/ci/minio.env
+            - ./.envs/ci/minio.env # legacy — kept during migration
+            - ./.envs/ci/sfs.env # SeaweedFS S3 — see seaweedfs/compose.yaml
             - ./.envs/ci/postgres.env
             - ./.envs/ci/opensearch.env
         command: "/beat-start"
         networks:
             - sds-gateway-ci-minio-net
             - sds-gateway-ci-opensearch-net
             - sds-network-ci
+        healthcheck:
+            test:
+                [
+                    "CMD-SHELL",
+                    'uv run python -c "import pathlib,sys; ok=any((b\"beat\" in data) and ((b\"celery\" in data) or (b\"watchfiles\" in data)) for data in (path.read_bytes() for path in pathlib.Path(\"/proc\").glob(\"[0-9]*/cmdline\"))); sys.exit(0 if ok else 1)"',
+                ]
+            interval: 30s
+            timeout: 30s
+            retries: 5
+            start_period: 30s
 
     celery-flower:
         # Celery monitoring and administration tool
@@ -354,7 +386,8 @@ services:
                   selinux: z
         env_file:
             - ./.envs/ci/django.env
-            - ./.envs/ci/minio.env
+            - ./.envs/ci/minio.env # legacy — kept during migration
+            - ./.envs/ci/sfs.env # SeaweedFS S3 — see seaweedfs/compose.yaml
             - ./.envs/ci/postgres.env
             - ./.envs/ci/opensearch.env
         command: "/flower-start"
@@ -364,6 +397,16 @@ services:
             - sds-gateway-ci-minio-net
             - sds-gateway-ci-opensearch-net
             - sds-network-ci
+        healthcheck:
+            test:
+                [
+                    "CMD-SHELL",
+                    'uv run python -c "import sys,urllib.request; urllib.request.urlopen(\"http://127.0.0.1:5555/\", timeout=5); sys.exit(0)"',
+                ]
+            interval: 30s
+            timeout: 30s
+            retries: 5
+            start_period: 30s
 
     # ==========================
     # local development services
@@ -395,6 +438,16 @@ services:
                 - action: sync
                   path: ./
                   target: /app/
+        healthcheck:
+            test:
+                [
+                    "CMD-SHELL",
+                    'node -e "const http=require(\"http\"); const req=http.get(\"http://127.0.0.1:3000\", res => process.exit(res.statusCode < 500 ? 0 : 1)); req.on(\"error\", () => process.exit(1)); req.setTimeout(5000, () => { req.destroy(); process.exit(1); });"',
+                ]
+            interval: 30s
+            timeout: 10s
+            retries: 5
+            start_period: 45s
 
     mailhog:
         # email testing service for local development
@@ -405,3 +458,13 @@ services:
             - "8025:8025" # Web UI
         networks:
             - sds-network-ci
+        healthcheck:
+            test:
+                [
+                    "CMD-SHELL",
+                    "wget -q -O /dev/null http://localhost:8025/api/v2/messages || exit 1",
+                ]
+            interval: 30s
+            timeout: 5s
+            retries: 5
+            start_period: 10s

gateway/compose.local.yaml

@@ -23,10 +23,16 @@ networks:
     # for safety, all gateway local networks start with "sds-gateway-local-"
     sds-gateway-local-minio-net:
         driver: bridge
+        name: sds-gateway-local-minio-net
     sds-gateway-local-opensearch-net:
         driver: bridge
+        name: sds-gateway-local-opensearch-net
+    sds-gateway-local-postgres-net:
+        driver: bridge
+        name: sds-gateway-local-postgres-net
     sds-network-local:
-        # external: true # make it external if running with traefik on this machine
+        # externally defined in traefik and/or in the seaweedfs compose file
+        external: true
         # should match traefik's network name
         name: sds-network-local
         driver: bridge
@@ -45,8 +51,6 @@ services:
                 condition: service_healthy
             redis:
                 condition: service_healthy
-            minio:
-                condition: service_healthy
         volumes:
             - sds-gateway-local-uv-cache:/opt/uv-cache/
             - sds-gateway-local-uv-venv-app:/opt/uv-venv/
@@ -75,15 +79,17 @@ services:
         env_file:
             - ./.envs/local/django.env
             - ./.envs/local/minio.env
+            - ./.envs/local/sfs.env
             - ./.envs/local/postgres.env
             - ./.envs/local/opensearch.env
         # remember /entrypoint runs first
         command: "/start"
         ports:
             - "8000:8000" # make sure this port matches traefik's config, if used
         networks:
-            - sds-gateway-local-minio-net
             - sds-gateway-local-opensearch-net
+            - sds-gateway-local-minio-net # TODO: deprecated, remove after migration complete
+            - sds-gateway-local-postgres-net
             - sds-network-local
         healthcheck:
             test: ["CMD-SHELL", "curl -f http://localhost:8000/ || exit 1"]
@@ -124,12 +130,20 @@ services:
                   selinux: z
         networks:
             - sds-network-local
+        healthcheck:
+            test:
+                [
+                    "CMD-SHELL",
+                    "wget -q -O /dev/null http://localhost/healthz || exit 1",
+                ]
+            interval: 30s
+            timeout: 5s
+            retries: 5
+            start_period: 10s
 
+    # TODO: DEPRECATED: being replaced by SeaweedFS. Keep running during migration.
+    # Remove after data migration is complete — see docs/minio-to-sfs-migration.md
     minio:
-        # main file storage for sds
-        # minio uses rolling upgrades that are non-disruptive, so we can target latest
-        # For more information on how to upgrade MinIO deployment, refer to the MinIO documentation:
-        # https://min.io/docs/minio/container/operations/install-deploy-manage/upgrade-minio-deployment.html
         image: minio/minio:latest
         container_name: sds-gateway-local-minio
         volumes:
@@ -200,7 +214,7 @@ services:
         env_file:
             - ./.envs/local/postgres.env
         networks:
-            - sds-gateway-local-minio-net
+            - sds-gateway-local-postgres-net
         healthcheck:
             test:
                 [
@@ -265,13 +279,26 @@ services:
         env_file:
             - ./.envs/local/django.env
             - ./.envs/local/minio.env
+            - ./.envs/local/sfs.env
             - ./.envs/local/postgres.env
             - ./.envs/local/opensearch.env
         command: "/worker-start"
         networks:
-            - sds-gateway-local-minio-net
+            # additional networks are used for health checks
             - sds-gateway-local-opensearch-net
+            - sds-gateway-local-postgres-net
+            - sds-gateway-local-minio-net
             - sds-network-local
+        healthcheck:
+            test:
+                [
+                    "CMD-SHELL",
+                    'uv run celery -A config.celery_app inspect ping -d "celery@$$HOSTNAME" | grep -q "OK"',
+                ]
+            interval: 30s
+            timeout: 30s
+            retries: 5
+            start_period: 30s
 
     celery-beat:
         # Celery Beat scheduler for periodic tasks
@@ -310,13 +337,24 @@ services:
         env_file:
             - ./.envs/local/django.env
             - ./.envs/local/minio.env
+            - ./.envs/local/sfs.env
             - ./.envs/local/postgres.env
             - ./.envs/local/opensearch.env
         command: "/beat-start"
         networks:
-            - sds-gateway-local-minio-net
             - sds-gateway-local-opensearch-net
+            - sds-gateway-local-postgres-net
             - sds-network-local
+        healthcheck:
+            test:
+                [
+                    "CMD-SHELL",
+                    'uv run python -c "import pathlib,sys; ok=any((b\"beat\" in data) and ((b\"celery\" in data) or (b\"watchfiles\" in data)) for data in (path.read_bytes() for path in pathlib.Path(\"/proc\").glob(\"[0-9]*/cmdline\"))); sys.exit(0 if ok else 1)"',
+                ]
+            interval: 30s
+            timeout: 30s
+            retries: 5
+            start_period: 30s
 
     celery-flower:
         # Celery monitoring and administration tool
@@ -355,15 +393,26 @@ services:
         env_file:
             - ./.envs/local/django.env
             - ./.envs/local/minio.env
+            - ./.envs/local/sfs.env
             - ./.envs/local/postgres.env
             - ./.envs/local/opensearch.env
         command: "/flower-start"
         ports:
             - "5555:5555" # Flower web interface
         networks:
-            - sds-gateway-local-minio-net
             - sds-gateway-local-opensearch-net
+            - sds-gateway-local-postgres-net
             - sds-network-local
+        healthcheck:
+            test:
+                [
+                    "CMD-SHELL",
+                    'curl -f --header "Authorization: Basic $(echo -n "$$CELERY_FLOWER_USER:$$CELERY_FLOWER_PASSWORD" | base64)" http://localhost:5555/api/workers || exit 1',
+                ]
+            interval: 30s
+            timeout: 30s
+            retries: 5
+            start_period: 30s
 
     # ==========================
     # local development services
@@ -395,13 +444,33 @@ services:
                 - action: sync
                   path: ./
                   target: /app/
+        healthcheck:
+            test:
+                [
+                    "CMD-SHELL",
+                    'node -e "const http=require(\"http\"); const req=http.get(\"http://127.0.0.1:3000\", res => process.exit(res.statusCode < 500 ? 0 : 1)); req.on(\"error\", () => process.exit(1)); req.setTimeout(5000, () => { req.destroy(); process.exit(1); });"',
+                ]
+            interval: 30s
+            timeout: 10s
+            retries: 5
+            start_period: 45s
 
     mailhog:
         # email testing service for local development
-        image: mailhog/mailhog:latest
+        image: docker.io/mailhog/mailhog:latest
         container_name: sds-gateway-local-mailhog
         ports:
             - "1025:1025" # SMTP server
             - "8025:8025" # Web UI
         networks:
             - sds-network-local
+        healthcheck:
+            test:
+                [
+                    "CMD-SHELL",
+                    "wget -q -O /dev/null http://localhost:8025/api/v2/messages || exit 1",
+                ]
+            interval: 30s
+            timeout: 5s
+            retries: 5
+            start_period: 10s