diff --git a/apps/appsets/project-understack.yaml b/apps/appsets/project-understack.yaml index 49f280341..c3187360a 100644 --- a/apps/appsets/project-understack.yaml +++ b/apps/appsets/project-understack.yaml @@ -15,6 +15,8 @@ spec: server: '*' - namespace: 'argo-events' server: '*' + - namespace: 'cdn' + server: '*' - namespace: 'cert-manager' server: '*' - namespace: 'dex' diff --git a/apps/site/understack-cdn.yaml b/apps/site/understack-cdn.yaml new file mode 100644 index 000000000..8d1e72ba1 --- /dev/null +++ b/apps/site/understack-cdn.yaml @@ -0,0 +1,7 @@ +--- +component: understack-cdn +sources: + - ref: understack + path: 'components/understack-cdn' + - ref: deploy + path: '{{.name}}/manifests/understack-cdn' diff --git a/components/understack-cdn/README.md b/components/understack-cdn/README.md new file mode 100644 index 000000000..f21eb8659 --- /dev/null +++ b/components/understack-cdn/README.md @@ -0,0 +1,69 @@ +# Poor-man's CDN for serving firmware images + +Images are stored in Object Store + +Caching reverse-proxies at each fabric will fetch the images from Object Store +and make them available via HTTPS. + +This allows a device to access firmware images via an HTTPS request to a +cluster-local tendot IP address. + +## Proxy configuration + +The proxy edge service caches files locally on a persistent volume. + +Nginx configuration contains: +- the service address for rook-ceph +- the name of our bucket + +All files are proxied to that object bucket. Anonymous credentials are used, +therefore we need to make the files in our bucket readable by anonymous if they +are to be accessible via HTTP. + +## Uploading file to object storage + +Our credentials and bucket info is in a secret and a configmap both named after +the bucketclaim: + +``` sh +KEY_ID=`kubectl -n understack-cdn get secrets firmware-images -o jsonpath='{.data.AWS_ACCESS_KEY_ID}' | base64 -d` +KEY=`kubectl -n understack-cdn get secrets firmware-images -o jsonpath='{.data.AWS_SECRET_ACCESS_KEY}' | base64 -d` +``` + +I was able to manage the bucket using the minio CLI client called "mc". + +I was testing this without direct access to the object store because there was +no ingress for it at the time of writing. Therefore I configured a port forward +so I could upload files from my laptop. I also had to mess with DNS resolution +because RGW is looking at the "host" header: + +``` sh +kubectl -n rook-ceph port-forward svc/rook-ceph-rgw-ceph-objectstore 8081:80 & +echo "127.0.0.1 rook-ceph-rgw-ceph-objectstore.rook-ceph.svc" | sudo tee -a /etc/hosts +mc alias set myrook http://rook-ceph-rgw-ceph-objectstore.rook-ceph.svc:8081 $KEY_ID $KEY +mc anonymous set download myrook/firmware-images +mc cp DELL/R7615/BIOS_H3TGJ_WN64_1.15.3.EXE myrook/firmware-images/DELL/R7615/ +mc anonymous set download myrook/firmware-images/DELL/R7615/BIOS_H3TGJ_WN64_1.15.3.EXE +``` + +## Testing with curl + +curl https://cdn.dev.undercloud.rackspace.net/firmware-images/DELL/R7615/BIOS_H3TGJ_WN64_1.15.3.EXE | shasum + +## See nginx logs to check that it is Caching + +``` sh +⇒ kubectl -n understack-cdn logs deployments/cdn-edge +Defaulted container "nginx" out of: nginx, cache-dir-init (init) +/docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt to perform configuration +/docker-entrypoint.sh: Looking for shell scripts in /docker-entrypoint.d/ +/docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh +10-listen-on-ipv6-by-default.sh: info: can not modify /etc/nginx/conf.d/default.conf (read-only file system?) +/docker-entrypoint.sh: Sourcing /docker-entrypoint.d/15-local-resolvers.envsh +/docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-templates.sh +/docker-entrypoint.sh: Launching /docker-entrypoint.d/30-tune-worker-processes.sh +/docker-entrypoint.sh: Configuration complete; ready for start up +10.64.49.118 - - [26/Feb/2026:12:36:47 +0000] "GET /DELL/R7615/BIOS_H3TGJ_WN64_1.15.3.EXE HTTP/1.1" 200 2523429 "-" "curl/8.14.1" "10.64.50.136" cache=EXPIRED +10.64.49.118 - - [26/Feb/2026:12:36:56 +0000] "GET /DELL/R7615/BIOS_H3TGJ_WN64_1.15.3.EXE HTTP/1.1" 200 32591328 "-" "curl/8.14.1" "10.64.50.136" cache=HIT +10.64.49.118 - - [26/Feb/2026:12:45:18 +0000] "GET /DELL/R7615/BIOS_H3TGJ_WN64_1.15.3.EXE HTTP/1.1" 200 32591328 "-" "curl/8.14.1" "10.64.50.136" cache=HIT +``` diff --git a/components/understack-cdn/deployment.yaml b/components/understack-cdn/deployment.yaml new file mode 100644 index 000000000..8a7425c21 --- /dev/null +++ b/components/understack-cdn/deployment.yaml @@ -0,0 +1,91 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cdn-edge + namespace: understack-cdn + labels: + app: cdn-edge +spec: + replicas: 1 + selector: + matchLabels: + app: cdn-edge + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + template: + metadata: + labels: + app: cdn-edge + spec: + # Init container to set correct permissions on cache dir + initContainers: + - name: cache-dir-init + image: busybox:1.36 + command: ["sh", "-c", "mkdir -p /var/cache/nginx/cdn && chown -R 101:101 /var/cache/nginx"] + volumeMounts: + - name: nginx-cache + mountPath: /var/cache/nginx + + containers: + - name: nginx + image: nginx:1.27-alpine + ports: + - containerPort: 8080 + name: http + + resources: + requests: + cpu: "250m" + memory: "256Mi" + limits: + cpu: "2" + memory: "1Gi" + + volumeMounts: + - name: nginx-config + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + - name: nginx-config + mountPath: /etc/nginx/conf.d/default.conf + subPath: default.conf + - name: nginx-cache + mountPath: /var/cache/nginx + + livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 10 + + readinessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 3 + periodSeconds: 5 + + # Graceful shutdown — let in-flight transfers complete + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "sleep 5 && nginx -s quit"] + + securityContext: + runAsNonRoot: true + runAsUser: 101 # nginx user in nginx:alpine + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + + volumes: + - name: nginx-config + configMap: + name: nginx-config + - name: nginx-cache + persistentVolumeClaim: + claimName: nginx-cache + + terminationGracePeriodSeconds: 30 diff --git a/components/understack-cdn/kustomization.yaml b/components/understack-cdn/kustomization.yaml new file mode 100644 index 000000000..5177e7a1e --- /dev/null +++ b/components/understack-cdn/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- namespace.yaml +- deployment.yaml +- service.yaml +- nginx-config.yaml +- pvc.yaml +- object-bucket-claim.yaml diff --git a/components/understack-cdn/namespace.yaml b/components/understack-cdn/namespace.yaml new file mode 100644 index 000000000..6d758312d --- /dev/null +++ b/components/understack-cdn/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: understack-cdn diff --git a/components/understack-cdn/nginx-config.yaml b/components/understack-cdn/nginx-config.yaml new file mode 100644 index 000000000..ba67aced5 --- /dev/null +++ b/components/understack-cdn/nginx-config.yaml @@ -0,0 +1,145 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx-config + namespace: understack-cdn +data: + nginx.conf: | + worker_processes auto; + error_log /var/log/nginx/error.log warn; + pid /var/cache/nginx/nginx.pid; + + # Tune for large file serving + worker_rlimit_nofile 65535; + + events { + worker_connections 4096; + use epoll; + multi_accept on; + } + + http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for" ' + 'cache=$upstream_cache_status'; + + access_log /var/log/nginx/access.log main; + + # Large file optimisations + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + + # Proxy cache zone configuration: + # keys_zone=cdn_cache:50m — 50MB for cache keys/metadata (~400k keys) + # max_size=50g — on-disk cache (adjust to your PVC size) + # inactive=30d — evict if not accessed in this time + # use_temp_path=off — write directly to cache dir (avoids extra copy) + proxy_cache_path /var/cache/nginx/cdn + levels=1:2 + keys_zone=cdn_cache:50m + max_size=5g + inactive=7d + use_temp_path=off; + + # Don't buffer large files to disk before sending — stream them + proxy_buffering on; + proxy_request_buffering off; + + # Increase timeouts for large file transfers + proxy_connect_timeout 10s; + proxy_send_timeout 300s; + proxy_read_timeout 300s; + send_timeout 300s; + + # Hide upstream headers we don't want to leak + proxy_hide_header x-amz-request-id; + proxy_hide_header x-amz-id-2; + + include /etc/nginx/conf.d/*.conf; + } + default.conf: | + # Upstream: your S3 S3 origin + # In production this points at your S3 service endpoint. + # Can also point at R2/S3 by changing the server and Host header. + upstream s3_origin { + server rook-ceph-rgw-ceph-objectstore.rook-ceph.svc:80; + keepalive 32; + } + + server { + listen 8080; + server_name _; + + # TLS — cert mounted from a k8s secret via ingress or directly + #ssl_certificate /etc/nginx/tls/tls.crt; + #ssl_certificate_key /etc/nginx/tls/tls.key; + #ssl_protocols TLSv1.2 TLSv1.3; + #ssl_ciphers HIGH:!aNULL:!MD5; + + proxy_cache cdn_cache; + proxy_cache_valid 200 206 7d; # Cache 200 and partial content + proxy_cache_valid 404 1m; # Don't cache 404s for long + proxy_cache_use_stale error timeout updating http_500 http_502 http_503; + proxy_cache_lock on; # Collapse simultaneous requests for the same file + proxy_cache_lock_timeout 10s; + + proxy_cache_key "$scheme$proxy_host$uri"; + + add_header X-Cache-Status $upstream_cache_status always; + add_header X-Served-By $hostname always; + + add_header X-Content-Type-Options nosniff always; + add_header X-Frame-Options DENY always; + + proxy_cache_revalidate on; + proxy_cache_bypass 0; + proxy_no_cache 0; + proxy_ignore_headers Cache-Control Expires Set-Cookie; + + location /firmware-images/ { + # Forward to Object Storage. + # S3-compatible API expects requests in the form: /bucket/key + # Our bucket is called firmware-images + proxy_pass http://s3_origin$request_uri; + + proxy_http_version 1.1; + proxy_set_header Connection ""; # keepalive to upstream + proxy_set_header Host rook-ceph-rgw-ceph-objectstore.rook-ceph.svc; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # Don't forward auth headers downstream + proxy_set_header Authorization ""; + + # Tell clients files are immutable — they should cache forever + add_header Cache-Control "public, max-age=31536000, immutable" always; + + # Support resumable downloads + proxy_force_ranges on; + + # Stream large files rather than buffering defeats the cache, so keep buffering on: + proxy_buffering on; + } + + # Health check endpoint (used by k8s liveness/readiness probes) + location /healthz { + access_log off; + return 200 "ok\n"; + add_header Content-Type text/plain; + } + + # Expose basic cache stats (restrict to internal) + location /nginx_status { + stub_status; + allow 10.0.0.0/8; + allow 172.16.0.0/12; + allow 192.168.0.0/16; + deny all; + } + } diff --git a/components/understack-cdn/object-bucket-claim.yaml b/components/understack-cdn/object-bucket-claim.yaml new file mode 100644 index 000000000..d2e8c5efd --- /dev/null +++ b/components/understack-cdn/object-bucket-claim.yaml @@ -0,0 +1,11 @@ +apiVersion: objectbucket.io/v1alpha1 +kind: ObjectBucketClaim +metadata: + name: firmware-images + namespace: understack-cdn +spec: + bucketName: firmware-images + storageClassName: ceph-bucket + additionalConfig: + maxObjects: "1000" + maxSize: "5G" diff --git a/components/understack-cdn/pvc.yaml b/components/understack-cdn/pvc.yaml new file mode 100644 index 000000000..ddba4a0bb --- /dev/null +++ b/components/understack-cdn/pvc.yaml @@ -0,0 +1,13 @@ +# Persistent volume for the Nginx cache. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nginx-cache + namespace: understack-cdn +spec: + accessModes: + - ReadWriteOnce + storageClassName: openebs-lvm + resources: + requests: + storage: 5Gi diff --git a/components/understack-cdn/service.yaml b/components/understack-cdn/service.yaml new file mode 100644 index 000000000..8c17ca4a5 --- /dev/null +++ b/components/understack-cdn/service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: cdn-edge + namespace: understack-cdn +spec: + selector: + app: cdn-edge + ports: + - name: http + port: 80 + targetPort: 8080 + type: ClusterIP