Add support for envoy gateway

Makefile

@@ -7,7 +7,7 @@ DOCKER_TAG            ?= $(USER)
 # default helm chart version must be 0.0.42 for local development (because 42 is the answer to the universe and everything)
 HELM_SEMVER           ?= 0.0.42
 # The list of helm charts needed on internal kubernetes testing environments
-CHARTS_INTEGRATION    := wire-server databases-ephemeral rabbitmq fake-aws ingress-nginx-controller nginx-ingress-services fluent-bit kibana restund k8ssandra-test-cluster wire-server-enterprise
+CHARTS_INTEGRATION    := wire-server databases-ephemeral rabbitmq fake-aws ingress-nginx-controller nginx-ingress-services wire-ingress fluent-bit kibana restund k8ssandra-test-cluster wire-server-enterprise
 # The list of helm charts to publish on S3
 # FUTUREWORK: after we "inline local subcharts",
 # (e.g. move charts/brig to charts/wire-server/brig)
@@ -18,7 +18,8 @@ fake-aws fake-aws-s3 fake-aws-sqs aws-ingress fluent-bit kibana backoffice		\
 calling-test demo-smtp elasticsearch-curator elasticsearch-external				\
 elasticsearch-ephemeral minio-external cassandra-external						\
 ingress-nginx-controller nginx-ingress-services reaper restund \
-k8ssandra-test-cluster ldap-scim-bridge wire-server-enterprise
+k8ssandra-test-cluster ldap-scim-bridge wire-server-enterprise \
+wire-ingress 
 KIND_CLUSTER_NAME     := wire-server
 HELM_PARALLELISM      ?= 1 # 1 for sequential tests; 6 for all-parallel tests
 PSQL_DB               ?= backendA

changelog.d/5-internal/WPB-23903

@@ -0,0 +1 @@
+New `wire-ingress` Helm chart — Gateway API / Envoy Gateway replacement for `nginx-ingress-services`. Not yet production-ready.

charts/backoffice/templates/tests/configmap.yaml

@@ -3,7 +3,7 @@ kind: ConfigMap
 metadata:
   name: "stern-integration"
   annotations:
-    "helm.sh/hook": post-install
+    "helm.sh/hook": post-install,post-upgrade
     "helm.sh/hook-delete-policy": before-hook-creation
 data:
   integration.yaml: |

charts/federator/templates/tests/configmap.yaml

@@ -3,7 +3,7 @@ kind: ConfigMap
 metadata:
   name: "federator-integration"
   annotations:
-    "helm.sh/hook": post-install
+    "helm.sh/hook": post-install,post-upgrade
     "helm.sh/hook-delete-policy": before-hook-creation
 data:
   integration.yaml: |
@@ -23,6 +23,6 @@ data:
       host: cargohold
       port: 8080
     nginxIngress:
-      host: federation-test-helper.{{ .Release.Namespace }}.svc.cluster.local
+      host: {{ .Values.tests.nginxIngressHost }}
       port: 443
-    originDomain: federation-test-helper.{{ .Release.Namespace }}.svc.cluster.local
+    originDomain: {{ .Values.tests.nginxIngressHost }}

charts/federator/values.yaml

@@ -57,6 +57,10 @@ podSecurityContext:
     type: RuntimeDefault
 
 tests:
+  # The host used for the nginxIngress endpoint and originDomain in the integration
+  # test config. Depends on the release name of the "wire-ingress" helm chart
+  # (see federation-test-helper.yaml in that chart).
+  nginxIngressHost: "set-me"
   config: {}
 #  config:
 #   uploadXml:

charts/integration/templates/_helpers.tpl

@@ -1,4 +1,48 @@
 
+{{/*
+Name of the Gateway resource for dynamic backends in envoy mode.
+*/}}
+{{- define "integration.getDynBackendsGatewayName" -}}
+{{- if .Values.envoy.gateway.name -}}
+{{ .Values.envoy.gateway.name }}
+{{- else -}}
+{{ .Release.Name }}-dynamic-backends
+{{- end -}}
+{{- end -}}
+
+{{/*
+Federation origin domain for a given namespace (used as originDomain in the config).
+Returns the SRV hostname that other backends use to reach this namespace's federator.
+Args: list $namespace $envoyEnabled $controllerNamespace
+*/}}
+{{- define "integration.federationOriginDomain" -}}
+{{- $namespace := index . 0 -}}
+{{- $envoyEnabled := index . 1 -}}
+{{- $controllerNs := index . 2 -}}
+{{- if $envoyEnabled -}}
+{{- printf "%s-fed.%s.svc.cluster.local" $namespace $controllerNs -}}
+{{- else -}}
+{{- printf "federation-test-helper.%s.svc.cluster.local" $namespace -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Domain for a dynamic backend. Returns the correct hostname depending on whether
+envoy mode is enabled.
+Args: list $dynamicBackend $namespace $envoyEnabled $controllerNamespace
+*/}}
+{{- define "integration.dynamicBackendDomain" -}}
+{{- $dynamicBackend := index . 0 -}}
+{{- $namespace := index . 1 -}}
+{{- $envoyEnabled := index . 2 -}}
+{{- $controllerNs := index . 3 -}}
+{{- if $envoyEnabled -}}
+{{- printf "%s-%s.%s.svc.cluster.local" $dynamicBackend.federatorExternalHostPrefix $namespace $controllerNs -}}
+{{- else -}}
+{{- printf "%s.%s.svc.cluster.local" $dynamicBackend.federatorExternalHostPrefix $namespace -}}
+{{- end -}}
+{{- end -}}
+
 {{/* Allow KubeVersion to be overridden. */}}
 {{- define "kubeVersion" -}}
   {{- default $.Capabilities.KubeVersion.Version $.Values.kubeVersionOverride -}}

charts/integration/templates/configmap.yaml

@@ -77,7 +77,7 @@ data:
       apiPort: 5380
       dohPort: 5381
 
-    originDomain: federation-test-helper.{{ .Release.Namespace }}.svc.cluster.local
+    originDomain: {{ include "integration.federationOriginDomain" (list .Release.Namespace .Values.envoy.enabled .Values.envoy.controllerNamespace) }}
 
     rabbitmq:
       host: rabbitmq
@@ -158,12 +158,12 @@ data:
 
       rabbitMqVHost: /
 
-      originDomain: federation-test-helper.{{ .Release.Namespace }}-fed2.svc.cluster.local
+      originDomain: {{ include "integration.federationOriginDomain" (list (printf "%s-fed2" .Release.Namespace) .Values.envoy.enabled .Values.envoy.controllerNamespace) }}
 
     dynamicBackends:
       {{- range $name, $dynamicBackend := .Values.config.dynamicBackends }}
       {{ $name }}:
-        domain: {{ $dynamicBackend.federatorExternalHostPrefix }}.{{ $.Release.Namespace }}.svc.cluster.local
+        domain: {{ include "integration.dynamicBackendDomain" (list $dynamicBackend $.Release.Namespace $.Values.envoy.enabled $.Values.envoy.controllerNamespace) }}
         federatorExternalPort: {{ $dynamicBackend.federatorExternalPort }}
         mlsPrivateKeyPaths:
           removal:

charts/integration/templates/envoy-gateway.yaml

@@ -0,0 +1,154 @@
+{{- if .Values.envoy.enabled }}
+{{- $gatewayName := include "integration.getDynBackendsGatewayName" . }}
+{{- $httpsPort := int .Values.envoy.gateway.listeners.https.port }}
+{{- $controllerNs := .Values.envoy.controllerNamespace }}
+{{- if lt $httpsPort 1024 }}
+{{- fail (printf "envoy.gateway.listeners.https.port is %d (privileged, <1024). Envoy Gateway remaps it to %d on the proxy pod. Set envoy.gateway.listeners.https.port to the actual container port (e.g. %d)." $httpsPort (add $httpsPort 10000) (add $httpsPort 10000)) }}
+{{- end }}
+---
+# EnvoyProxy configures the proxy deployment/service for the dynamic-backends Gateway.
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: EnvoyProxy
+metadata:
+  name: {{ $gatewayName }}
+spec:
+  provider:
+    type: Kubernetes
+    kubernetes:
+      envoyService:
+        # ClusterIP: no external load balancer needed for in-cluster integration tests.
+        type: ClusterIP
+---
+# Gateway for all dynamic backends. A single HTTPS listener covers all backend hostnames.
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: {{ $gatewayName }}
+spec:
+  gatewayClassName: {{ required "envoy.gateway.className is required when envoy.enabled is true" .Values.envoy.gateway.className | quote }}
+  infrastructure:
+    parametersRef:
+      group: gateway.envoyproxy.io
+      kind: EnvoyProxy
+      name: {{ $gatewayName | quote }}
+  listeners:
+    - name: https
+      port: {{ $httpsPort }}
+      protocol: HTTPS
+      tls:
+        mode: Terminate
+        certificateRefs:
+          - name: {{ .Values.envoy.federator.tls.secretName | quote }}
+            kind: Secret
+---
+# ClientTrafficPolicy enforces optional mTLS client cert validation on all dynamic-backend
+# connections (mirrors the nginx auth-tls-verify-client: "on" annotation).
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: ClientTrafficPolicy
+metadata:
+  name: {{ $gatewayName }}-mtls
+spec:
+  targetRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: {{ $gatewayName | quote }}
+      sectionName: https
+  tls:
+    clientValidation:
+      optional: true
+      caCertificateRefs:
+        - name: federator-ca
+          kind: ConfigMap
+---
+# EnvoyPatchPolicy strips the trailing dot that Kubernetes DNS appends to FQDNs.
+# Wire federator resolves targets via SRV and sends the raw FQDN (with trailing dot)
+# as the HTTP/2 :authority header; strip_trailing_host_dot normalises it before
+# Envoy's virtual-host matching, preventing route_not_found errors.
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: EnvoyPatchPolicy
+metadata:
+  name: {{ $gatewayName }}-strip-dot
+spec:
+  targetRef:
+    group: gateway.networking.k8s.io
+    kind: Gateway
+    name: {{ $gatewayName | quote }}
+  type: JSONPatch
+  jsonPatches:
+    - type: "type.googleapis.com/envoy.config.listener.v3.Listener"
+      # Single HTTPS listener → only one filter chain at index 0.
+      name: {{ printf "%s/%s/https" .Release.Namespace $gatewayName | quote }}
+      operation:
+        op: add
+        path: "/filter_chains/0/filters/0/typed_config/strip_trailing_host_dot"
+        value: true
+{{- range $name, $dynamicBackend := .Values.config.dynamicBackends }}
+{{- $httpRouteName := printf "%s-dynbackend-%s" $gatewayName $name }}
+{{- $svcDomain := printf "%s-%s.%s.svc.cluster.local" $dynamicBackend.federatorExternalHostPrefix $.Release.Namespace $controllerNs }}
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: {{ $httpRouteName }}
+spec:
+  parentRefs:
+    - name: {{ $gatewayName | quote }}
+      namespace: {{ $.Release.Namespace | quote }}
+      kind: Gateway
+      sectionName: https
+  hostnames:
+    - {{ $svcDomain | quote }}
+  rules:
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /
+      backendRefs:
+        - name: integration
+          port: {{ $dynamicBackend.federatorExternalPort }}
+          kind: Service
+---
+# EnvoyExtensionPolicy injects the mTLS client certificate as X-SSL-Certificate request
+# header, matching the nginx $ssl_client_escaped_cert behaviour expected by federator.
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: EnvoyExtensionPolicy
+metadata:
+  name: {{ $httpRouteName }}-cert-header
+spec:
+  targetRefs:
+    - group: gateway.networking.k8s.io
+      kind: HTTPRoute
+      name: {{ $httpRouteName }}
+  lua:
+    - type: Inline
+      inline: |
+        function envoy_on_request(request_handle)
+          local ssl = request_handle:connection():ssl()
+          if ssl ~= nil then
+            local cert = ssl:urlEncodedPemEncodedPeerCertificate()
+            if cert ~= nil and cert ~= "" then
+              request_handle:headers():add("X-SSL-Certificate", cert)
+            end
+          end
+        end
+---
+# ClusterIP service in {{ $controllerNs }} selects the Envoy proxy pods for this Gateway.
+# The service name determines the SRV record used by federation discovery:
+#   _wire-server-federator._tcp.{{ $svcDomain }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ $dynamicBackend.federatorExternalHostPrefix }}-{{ $.Release.Namespace }}
+  namespace: {{ $controllerNs }}
+spec:
+  type: ClusterIP
+  ports:
+  - name: wire-server-federator
+    port: 443
+    protocol: TCP
+    targetPort: {{ $httpsPort }}
+  selector:
+    gateway.envoyproxy.io/owning-gateway-name: {{ $gatewayName }}
+    gateway.envoyproxy.io/owning-gateway-namespace: {{ $.Release.Namespace }}
+{{- end }}
+{{- end }}

charts/integration/templates/ingress.yaml → charts/integration/templates/ingress-nginx.yaml

@@ -1,3 +1,5 @@
+{{- if not .Values.envoy.enabled }}
+{{- $newLabels := eq (include "integrationTestHelperNewLabels" .) "true" -}}
 {{- range $name, $dynamicBackend := .Values.config.dynamicBackends }}
 ---
 apiVersion: networking.k8s.io/v1
@@ -29,4 +31,25 @@ spec:
                 name: integration
                 port:
                   number: {{ $dynamicBackend.federatorExternalPort }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ $dynamicBackend.federatorExternalHostPrefix }}
+spec:
+  ports:
+  - name: wire-server-federator
+    port: 443
+    protocol: TCP
+    targetPort: https
+  selector:
+    {{- if $newLabels }}
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/name: ingress-nginx
+    {{- else }}
+    app: nginx-ingress
+    component: controller
+    {{- end }}
+  type: ClusterIP
+{{- end }}
 {{- end }}

charts/integration/templates/integration-integration.yaml

@@ -168,7 +168,7 @@ spec:
         integration-dynamic-backends-ses.sh {{ .Values.config.sesEndpointUrl }}
         integration-dynamic-backends-s3.sh {{ .Values.config.s3EndpointUrl }}
         {{- range $name, $dynamicBackend := .Values.config.dynamicBackends }}
-        integration-dynamic-backends-vhosts.sh {{ $.Values.config.rabbitmqPutVHostUrl }} {{ $dynamicBackend.federatorExternalHostPrefix}}.{{ $.Release.Namespace }}.svc.cluster.local
+        integration-dynamic-backends-vhosts.sh {{ $.Values.config.rabbitmqPutVHostUrl }} {{ include "integration.dynamicBackendDomain" (list $dynamicBackend $.Release.Namespace $.Values.envoy.enabled $.Values.envoy.controllerNamespace) }}
         {{- end }}
     resources:
       requests:

charts/integration/templates/service.yaml

@@ -1,4 +1,3 @@
-{{- $newLabels := eq (include "integrationTestHelperNewLabels" .) "true" -}}
 ---
 apiVersion: v1
 kind: Service
@@ -26,26 +25,3 @@ spec:
   selector:
     app: integration-integration
   type: ClusterIP
-
-{{- range $name, $dynamicBackend := .Values.config.dynamicBackends }}
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ $dynamicBackend.federatorExternalHostPrefix }}
-spec:
-  ports:
-  - name: wire-server-federator
-    port: 443
-    protocol: TCP
-    targetPort: https
-  selector:
-    {{- if $newLabels }}
-    app.kubernetes.io/component: controller
-    app.kubernetes.io/name: ingress-nginx
-    {{- else }}
-    app: nginx-ingress
-    component: controller
-    {{- end }}
-  type: ClusterIP
-{{- end }}

charts/integration/values.yaml

@@ -129,4 +129,27 @@ tls:
 ingress:
   class: nginx
 
+envoy:
+  # Set to true to deploy Gateway API resources instead of nginx Ingress objects
+  # for the dynamic backends. Requires an Envoy Gateway controller in the cluster.
+  enabled: false
+  # Namespace where the Envoy Gateway controller runs its proxy pods.
+  # Change only if you installed Envoy Gateway into a non-default namespace.
+  controllerNamespace: envoy-gateway-system
+  gateway:
+    # Name of the Gateway resource. Defaults to <release-name>-dynamic-backends if empty.
+    name: ""
+    # Name of the GatewayClass installed by the Envoy Gateway controller (e.g. "envoy").
+    className: ""
+    listeners:
+      https:
+        # Use a non-privileged port (>=1024) to avoid the +10000 container-port
+        # remapping applied by Envoy Gateway to privileged ports.
+        port: 10443
+  federator:
+    tls:
+      # Name of the TLS Secret presented by the Gateway for the dynamic-backend
+      # listeners. Must exist before deploying (created by the wire-ingress chart).
+      secretName: "federator-certificate-secret"
+
 secrets: {}