From b3a607bd342088262884fae158f8028db21438cf Mon Sep 17 00:00:00 2001 From: mohitrajain Date: Thu, 5 Mar 2026 12:14:23 +0100 Subject: [PATCH 01/13] Fixed: debug_logs.sh to log only the pods for default and cert-manager-ns namespace and limit log lines --- bin/debug_logs.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/debug_logs.sh b/bin/debug_logs.sh index 8a40701b3..6660cd22a 100755 --- a/bin/debug_logs.sh +++ b/bin/debug_logs.sh @@ -4,14 +4,14 @@ set -euo pipefail echo "Printing all pods status" kubectl get pods --all-namespaces echo "------------------------------------" -namespaces=$(kubectl get ns -o=jsonpath='{.items[*].metadata.name}') +namespaces="cert-manager-ns default" echo "Namespaces = $namespaces" for ns in $namespaces; do - pods=$(kubectl get pods --all-namespaces -o=jsonpath='{.items[*].metadata.name}') + pods=$(kubectl get pods -n $ns -o=jsonpath='{.items[*].metadata.name}') echo "Pods in namespace: $ns = $pods" for pod in $pods; do echo "Logs for pod: $pod" - kubectl logs --all-containers -n "$ns" "$pod" || true + kubectl logs --tail 30 --all-containers -n "$ns" "$pod" || true echo "Description for pod: $pod" kubectl describe pod -n "$ns" "$pod" || true echo "------------------------------------" From 2544ef59ac3fc6746c157911a941df66a3e2b676 Mon Sep 17 00:00:00 2001 From: mohitrajain Date: Thu, 5 Mar 2026 12:16:11 +0100 Subject: [PATCH 02/13] fix: rebase master --- bin/helm-operations.sh | 83 ++++++++++++++++++++++++++---------------- 1 file changed, 51 insertions(+), 32 deletions(-) diff --git a/bin/helm-operations.sh b/bin/helm-operations.sh index 1f9175579..ad82ff671 100755 --- a/bin/helm-operations.sh +++ b/bin/helm-operations.sh @@ -3,17 +3,33 @@ set -Eeo pipefail # Read values from environment variables with defaults -BASE_DIR="/wire-server-deploy" -TARGET_SYSTEM="example.dev" -CERT_MASTER_EMAIL="certmaster@${TARGET_SYSTEM}" +BASE_DIR="${BASE_DIR:-/wire-server-deploy}" +TARGET_SYSTEM="${TARGET_SYSTEM:-example.com}" +CERT_MASTER_EMAIL="certmaster@${CERT_MASTER_EMAIL}:-certmaster@${TARGET_SYSTEM}" + +# DEPLOY_CERT_MANAGER env variable to decide to check if cert_manager and nginx-ingress-services charts should get deployed +# default is set to TRUE to deploy it unless changed +DEPLOY_CERT_MANAGER="${DEPLOY_CERT_MANAGER:-TRUE}" + +# DUMP_LOGS_ON_FAIL to dump logs on failure +# it is false by default +DUMP_LOGS_ON_FAIL="${DUMP_LOGS_ON_FAIL:-FALSE}" # this IP should match the DNS A record value for TARGET_SYSTEM # assuming it to be the public address used by clients to reach public Address -HOST_IP="" +HOST_IP="${HOST_IP:-}" + if [ -z "$HOST_IP" ]; then HOST_IP=$(wget -qO- https://api.ipify.org) fi +function dump_debug_logs { + if [[ "$DUMP_LOGS_ON_FAIL" == "TRUE" ]]; then + $BASE_DIR/bin/debug_logs.sh + fi +} +trap dump_debug_logs ERR + # picking a node for calling traffic (3rd kube worker node) CALLING_NODE=$(kubectl get nodes --no-headers | tail -n 1 | awk '{print $1}') if [[ -z "$CALLING_NODE" ]]; then @@ -21,6 +37,20 @@ if [[ -z "$CALLING_NODE" ]]; then exit 1 fi +sync_pg_secrets() { + echo "Retrieving PostgreSQL password from databases-ephemeral for wire-server deployment..." + if kubectl get secret wire-postgresql-external-secret &>/dev/null; then + # Usage: sync-k8s-secret-to-wire-secrets.sh + "$BASE_DIR/bin/sync-k8s-secret-to-wire-secrets.sh" \ + wire-postgresql-external-secret password \ + "$BASE_DIR/values/wire-server/secrets.yaml" \ + .brig.secrets.pgPassword .galley.secrets.pgPassword + else + echo "⚠️ Warning: PostgreSQL secret 'wire-postgresql-secret' not found, skipping secret sync" + echo " Make sure databases-ephemeral chart is deployed before wire-server" + fi +} + # Creates values.yaml from prod-values.example.yaml and secrets.yaml from prod-secrets.example.yaml # Works on all chart directories in $BASE_DIR/values/ process_values() { @@ -136,23 +166,6 @@ deploy_charts() { helm_command+=" --values $secrets_file" fi - # handle wire-server to inject PostgreSQL password from databases-ephemeral - if [[ "$chart" == "wire-server" ]]; then - - echo "Retrieving PostgreSQL password from databases-ephemeral for wire-server deployment..." - if kubectl get secret wire-postgresql-external-secret &>/dev/null; then - # Usage: sync-k8s-secret-to-wire-secrets.sh - "$BASE_DIR/bin/sync-k8s-secret-to-wire-secrets.sh" \ - "wire-postgresql-external-secret" \ - "password" \ - "$BASE_DIR/values/wire-server/secrets.yaml" \ - .brig.secrets.pgPassword .galley.secrets.pgPassword .background-worker.secrets.pgPassword - else - echo "⚠️ Warning: PostgreSQL secret 'wire-postgresql-external-secret' not found, skipping secret sync" - echo " Make sure databases-ephemeral chart is deployed before wire-server" - fi - fi - echo "Deploying $chart as $helm_command" eval "$helm_command" done @@ -164,7 +177,7 @@ deploy_charts() { deploy_cert_manager() { kubectl get namespace cert-manager-ns || kubectl create namespace cert-manager-ns - helm upgrade --install -n cert-manager-ns cert-manager "$BASE_DIR/charts/cert-manager" --values "$BASE_DIR/values/cert-manager/values.yaml" + helm upgrade --install --wait --timeout=5m0s -n cert-manager-ns cert-manager "$BASE_DIR/charts/cert-manager" --values "$BASE_DIR/values/cert-manager/values.yaml" # display running pods kubectl get pods --sort-by=.metadata.creationTimestamp -n cert-manager-ns @@ -175,39 +188,45 @@ deploy_calling_services() { echo "Deploying sftd and coturn" # select the node to deploy sftd kubectl annotate node "$CALLING_NODE" wire.com/external-ip="$HOST_IP" --overwrite - helm upgrade --install sftd "$BASE_DIR/charts/sftd" --set "nodeSelector.kubernetes\\.io/hostname=$CALLING_NODE" --values "$BASE_DIR/values/sftd/values.yaml" + helm upgrade --install --wait --timeout=5m0s sftd "$BASE_DIR/charts/sftd" --set "nodeSelector.kubernetes\\.io/hostname=$CALLING_NODE" --values "$BASE_DIR/values/sftd/values.yaml" kubectl annotate node "$CALLING_NODE" wire.com/external-ip="$HOST_IP" --overwrite - helm upgrade --install coturn "$BASE_DIR/charts/coturn" --set "nodeSelector.kubernetes\\.io/hostname=$CALLING_NODE" --values "$BASE_DIR/values/coturn/values.yaml" --values "$BASE_DIR/values/coturn/secrets.yaml" + helm upgrade --install --wait --timeout=5m0s coturn "$BASE_DIR/charts/coturn" --set "nodeSelector.kubernetes\\.io/hostname=$CALLING_NODE" --values "$BASE_DIR/values/coturn/values.yaml" --values "$BASE_DIR/values/coturn/secrets.yaml" # display running pods post deploying all helm charts in default namespace kubectl get pods --sort-by=.metadata.creationTimestamp } main() { + # Create prod-values.example.yaml to values.yaml and take backup process_values "prod" "values" # Create prod-secrets.example.yaml to secrets.yaml and take backup process_values "prod" "secrets" +# Sync postgresql secret +sync_pg_secrets + # configure chart specific variables for each chart in values.yaml file configure_values # deploying with external datastores, useful for prod setup deploy_charts cassandra-external elasticsearch-external minio-external postgresql-external fake-aws smtp rabbitmq-external databases-ephemeral reaper wire-server webapp account-pages team-settings smallstep-accomp ingress-nginx-controller -# deploying cert manager to issue certs, by default letsencrypt-http01 issuer is configured -deploy_cert_manager +# deploying cert-manager only when the env var DEPLOY_CERT_MANAGER is set to TRUE +if [[ "$DEPLOY_CERT_MANAGER" == "TRUE" ]]; then + # deploying cert manager to issue certs, by default letsencrypt-http01 issuer is configured + deploy_cert_manager + + # nginx-ingress-services chart needs cert-manager to be deployed + deploy_charts nginx-ingress-services -# nginx-ingress-services chart needs cert-manager to be deployed -deploy_charts nginx-ingress-services + # print status of certs + kubectl get certificate +fi # deploying sft and coturn services -# not implemented yet deploy_calling_services - -# print status of certs -kubectl get certificate } main From 902d39a74766b299e99b4b7c49b829d66d28b7a8 Mon Sep 17 00:00:00 2001 From: mohitrajain Date: Thu, 5 Mar 2026 12:16:32 +0100 Subject: [PATCH 03/13] Rebase master --- bin/offline-deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/offline-deploy.sh b/bin/offline-deploy.sh index 61c7d3dfa..3bede967a 100755 --- a/bin/offline-deploy.sh +++ b/bin/offline-deploy.sh @@ -41,4 +41,4 @@ fi $DOCKER_RUN_BASE $SSH_MOUNT $WSD_CONTAINER ./bin/offline-cluster.sh -sudo docker run --network=host -v $PWD:/wire-server-deploy $WSD_CONTAINER ./bin/helm-operations.sh +sudo docker run --network=host -v $PWD:/wire-server-deploy $WSD_CONTAINER sh -c 'TARGET_SYSTEM="example.dev" CERT_MASTER_EMAIL="certmaster@example.dev" DEPLOY_CERT_MANAGER=TRUE DUMP_LOGS_ON_FAIL=TRUE ./bin/helm-operations.sh' From f3086560bc36c322b7afd119ee0be2e75c689a19 Mon Sep 17 00:00:00 2001 From: mohitrajain Date: Thu, 5 Mar 2026 12:16:53 +0100 Subject: [PATCH 04/13] Fixed: documentation for wiab-staging.md based on a user feedback --- offline/wiab-staging.md | 133 +++++++++++++++++++++++----------------- 1 file changed, 76 insertions(+), 57 deletions(-) diff --git a/offline/wiab-staging.md b/offline/wiab-staging.md index 1fa85aef9..4165e5711 100644 --- a/offline/wiab-staging.md +++ b/offline/wiab-staging.md @@ -90,6 +90,7 @@ We need the whole ansible directory as ansible-playbook uses some templates for **Option A: Download as ZIP** ```bash +# requirements: wget and unzip wget https://github.com/wireapp/wire-server-deploy/archive/refs/heads/master.zip unzip master.zip cd wire-server-deploy-master @@ -97,6 +98,7 @@ cd wire-server-deploy-master **Option B: Clone with Git** ```bash +# requirements: git git clone https://github.com/wireapp/wire-server-deploy.git cd wire-server-deploy ``` @@ -105,7 +107,7 @@ cd wire-server-deploy A sample inventory is available at [ansible/inventory/demo/wiab-staging.yml](https://github.com/wireapp/wire-server-deploy/blob/master/ansible/inventory/demo/wiab-staging.yml). -*Note: Replace example.com with your physical machine address where KVM is available and adjust other variables accordingly.* +*Note: Replace example.com with your physical machine address where KVM is available and adjust other variables like ansible_user and ansible_ssh_private_key_file. The SSH user for ansible `ansible_user` should have password-less `sudo` access. The physical host should be running Ubuntu 22.04.* **Step 3: Run the VM and network provision** @@ -140,7 +142,7 @@ Since the inventory is ready, please continue with the following steps: ### Helm Operations to install wire services and supporting helm charts -**Helm chart deployment (automated):** The script `bin/helm-operations.sh` will deploy the charts for you. It prepares `values.yaml`/`secrets.yaml`, customizes them for your domain/IPs, then runs Helm installs/upgrades in the correct order. +**Helm chart deployment (automated):** The script `bin/helm-operations.sh` will deploy the charts for you. It prepares `values.yaml`/`secrets.yaml`, customizes them for your domain/IPs, then runs Helm installs/upgrades in the correct order. Prepare the values before running it. **User-provided inputs (set these before running):** - `TARGET_SYSTEM`: your domain (e.g., `wire.example.com` or `example.dev`). @@ -149,11 +151,15 @@ Since the inventory is ready, please continue with the following steps: **TLS / certificate behavior (cert-manager vs. Bring Your Own):** - By default, `bin/helm-operations.sh` runs `deploy_cert_manager`, which installs cert-manager and configures a Let’s Encrypt (HTTP-01) issuer for the ingress charts. -- If you **do not** want Let’s Encrypt / cert-manager (for example, you are using **[Bring Your Own certificates](docs_ubuntu_22.04.md#acquiring--deploying-ssl-certificates)** or you cannot satisfy HTTP-01 requirements), disable this step by commenting out the `deploy_cert_manager` call inside `bin/helm-operations.sh`. - - After disabling cert-manager, ensure your ingress is configured with your own TLS secret(s) as described in the TLS documentation below. +- If you **do not** want Let’s Encrypt / cert-manager (for example, you are using **[Bring Your Own certificates](docs_ubuntu_22.04.md#acquiring--deploying-ssl-certificates)**), disable this step by passing env variable `DEPLOY_CERT_MANAGER=FALSE` when running `bin/helm-operations.sh`. + - When choosing `DEPLOY_CERT_MANAGER=FALSE`, ensure your ingress is configured with your own TLS secret(s) as described at [Acquiring / Deploying SSL Certificates](docs_ubuntu_22.04.md#acquiring--deploying-ssl-certificates). + - When choosing `DEPLOY_CERT_MANAGER=TRUE`, ensure if further network configuration is required by following [cert-manager behaviour in NAT / bridge environments](#cert-manager-behaviour-in-nat--bridge-environments). -**To run the automated helm chart deployment**: -`d ./bin/helm-operations.sh` +**To run the automated helm chart deployment with your variables**: +```bash +# example command - verify the variables before running it +d sh -c 'TARGET_SYSTEM="example.dev" CERT_MASTER_EMAIL="certmaster@example.dev" DEPLOY_CERT_MANAGER=TRUE ./bin/helm-operations.sh' +``` **Charts deployed by the script:** - External datastores and helpers: `cassandra-external`, `elasticsearch-external`, `minio-external`, `rabbitmq-external`, `databases-ephemeral`, `reaper`, `fake-aws`, `demo-smtp`. @@ -223,57 +229,70 @@ calling_node_ip=192.168.122.13 inf_wan=eth0 ``` -> **Note (cert-manager & hairpin NAT):** -> When cert-manager performs HTTP-01 self-checks inside the cluster, traffic can hairpin (Pod → Node → host public IP → DNAT → Node → Ingress). -> If your nftables rules DNAT in `PREROUTING` without a matching SNAT on `virbr0 → virbr0`, return packets may bypass the host and break conntrack, causing HTTP-01 timeouts, resulting in certificate verification failure. -> Additionally, strict `rp_filter` can drop asymmetric return packets. -> If cert-manager is deployed in a NAT/bridge (`virbr0`) environment, first verify whether certificate issuance is failing before applying hairpin handling. -> Check whether certificates are successfully issued: -> ```bash -> d kubectl get certificates -> ``` -> If certificates are not in `Ready=True` state, inspect cert-manager logs for HTTP-01 self-check or timeout errors: -> ```bash -> d kubectl logs -n cert-manager-ns -> ``` -> If you observe HTTP-01 challenge timeouts or self-check failures in a NAT/bridge environment, hairpin SNAT and relaxed reverse-path filtering handling may be required. - > - Relax reverse-path filtering to loose mode to allow asymmetric flows: - > ```bash - > sudo sysctl -w net.ipv4.conf.all.rp_filter=2 - > sudo sysctl -w net.ipv4.conf.virbr0.rp_filter=2 - > ``` - > These settings help conntrack reverse DNAT correctly and avoid drops during cert-manager’s HTTP-01 challenges in NAT/bridge (virbr0) environments. - > - > - Enable Hairpin SNAT (temporary for cert-manager HTTP-01): - > ```bash - > sudo nft insert rule ip nat POSTROUTING position 0 \ - > iifname "virbr0" oifname "virbr0" \ - > ip daddr 192.168.122.0/24 ct status dnat \ - > counter masquerade \ - > comment "wire-hairpin-dnat-virbr0" - > ``` - > This forces DNATed traffic that hairpins over the bridge to be masqueraded, ensuring return traffic flows back through the host and conntrack can correctly reverse the DNAT. - > Verify the rule was added: - > ```bash - > sudo nft list chain ip nat POSTROUTING - > ``` - > You should see a rule similar to: - > ``` - > iifname "virbr0" oifname "virbr0" ip daddr 192.168.122.0/24 ct status dnat counter masquerade # handle - > ``` - > - > - Remove the rule after certificates are issued - > ```bash - > d kubectl get certificates - > ``` - > - Once Let's Encrypt validation completes and certificates are issued, remove the temporary hairpin SNAT rule. Use the following pipeline to locate the rule handle and delete it safely: - > ```bash - > sudo nft list chain ip nat POSTROUTING | \ - > grep wire-hairpin-dnat-virbr0 | \ - > sed -E 's/.*handle ([0-9]+).*/\1/' | \ - > xargs -r -I {} sudo nft delete rule ip nat POSTROUTING handle {} - > ``` - +### cert-manager behaviour in NAT / bridge environments + +When cert-manager performs HTTP-01 self-checks inside the cluster, traffic can hairpin: + +- Pod → Node → host public IP → DNAT → Node → Ingress + +In NAT/bridge setups (for example, using `virbr0` on the host): + +- If nftables rules DNAT in `PREROUTING` without a matching SNAT on `virbr0 → virbr0`, return packets may bypass the host and break conntrack, causing HTTP-01 timeouts and certificate verification failures. +- Strict `rp_filter` can drop asymmetric return packets. + +Before changing anything, first verify whether certificate issuance is actually failing: + +1. Check whether certificates are successfully issued: + ```bash + d kubectl get certificates + ``` +2. If certificates are not in `Ready=True` state, inspect cert-manager logs for HTTP-01 self-check or timeout errors: + ```bash + d kubectl logs -n cert-manager-ns + ``` + +If you observe HTTP-01 challenge timeouts or self-check failures in a NAT/bridge environment, hairpin SNAT and relaxed reverse-path filtering handling may be required. One possible approach is: + +- Relax reverse-path filtering to loose mode to allow asymmetric flows: + ```bash + sudo sysctl -w net.ipv4.conf.all.rp_filter=2 + sudo sysctl -w net.ipv4.conf.virbr0.rp_filter=2 + ``` + These settings help conntrack reverse DNAT correctly and avoid drops during cert-manager’s HTTP-01 challenges in NAT/bridge (`virbr0`) environments. + +- Enable Hairpin SNAT (temporary for cert-manager HTTP-01): + ```bash + sudo nft insert rule ip nat POSTROUTING position 0 \ + iifname "virbr0" oifname "virbr0" \ + ip daddr 192.168.122.0/24 ct status dnat \ + counter masquerade \ + comment "wire-hairpin-dnat-virbr0" + ``` + This forces DNATed traffic that hairpins over the bridge to be masqueraded, ensuring return traffic flows back through the host and conntrack can correctly reverse the DNAT. + + Verify the rule was added: + ```bash + sudo nft list chain ip nat POSTROUTING + ``` + You should see a rule similar to: + ``` + iifname "virbr0" oifname "virbr0" ip daddr 192.168.122.0/24 ct status dnat counter masquerade # handle + ``` + +- Remove the rule after certificates are issued, confirm by running the following: + ```bash + d kubectl get certificates + ``` + + Once Let’s Encrypt validation completes and certificates are issued, remove the temporary hairpin SNAT rule. Use the following pipeline to locate the rule handle and delete it safely: + ```bash + sudo nft -a list chain ip nat POSTROUTING | \ + grep wire-hairpin-dnat-virbr0 | \ + sed -E 's/.*handle ([0-9]+).*/\1/' | \ + xargs -r -I {} sudo nft delete rule ip nat POSTROUTING handle {} + ``` + +For additional background on when hairpin NAT is required and how it relates to WIAB Dev and WIAB Staging, see [Hairpin networking for WIAB Dev and WIAB Staging](tls-certificates.md#hairpin-networking-for-wiab-dev-and-wiab-staging). ## Further Reading From 76bcf8cfaaf34aad597912bb956f4e7b4b6edded Mon Sep 17 00:00:00 2001 From: mohitrajain Date: Thu, 5 Mar 2026 12:17:20 +0100 Subject: [PATCH 05/13] Fixed: sftd helm chart values for joinCall component which fails to find hashbased images --- changelog.d/3-deploy-builds/minor-deploy-fixes | 8 ++++++++ values/sftd/demo-values.example.yaml | 4 ++++ values/sftd/prod-values.example.yaml | 10 ++++++++++ 3 files changed, 22 insertions(+) create mode 100644 changelog.d/3-deploy-builds/minor-deploy-fixes diff --git a/changelog.d/3-deploy-builds/minor-deploy-fixes b/changelog.d/3-deploy-builds/minor-deploy-fixes new file mode 100644 index 000000000..c27d2b844 --- /dev/null +++ b/changelog.d/3-deploy-builds/minor-deploy-fixes @@ -0,0 +1,8 @@ +Fixed: debug_logs.sh to log only the pods for default and cert-manager-ns namespace and limit log lines +Added: enabled debug_logs.sh on helm install failures (helm_operations.sh) with a flag DUMP_LOGS_ON_FAIL +Added: env vars to helm_operations.sh to improve UX while configuring variables +Fixed: sync_pg_secrets operation in helm_operations.sh and clean the deploy_charts logic +Added: wait and timeout on cert-manager and calling_services helm chart operations +Fixed: offline-cluster.sh to run helm-operations.sh using new env vars and with default DUMP_LOGS_ON_FAIL=TRUE +Fixed: documentation for wiab-staging.md based on a user feedback +Fixed: sftd helm chart values for joinCall component which fails to find hashbased images diff --git a/values/sftd/demo-values.example.yaml b/values/sftd/demo-values.example.yaml index 566db6bc4..91dc2c885 100644 --- a/values/sftd/demo-values.example.yaml +++ b/values/sftd/demo-values.example.yaml @@ -3,6 +3,10 @@ host: sftd.example.com replicaCount: 1 joinCall: replicaCount: 1 + image: + repository: docker.io/bitnamilegacy/nginx + pullPolicy: IfNotPresent + tag: "1.27.3-debian-12-r5" tls: issuerRef: name: letsencrypt-http01 diff --git a/values/sftd/prod-values.example.yaml b/values/sftd/prod-values.example.yaml index ac48178f3..1c2374f9e 100644 --- a/values/sftd/prod-values.example.yaml +++ b/values/sftd/prod-values.example.yaml @@ -9,6 +9,16 @@ tls: issuerRef: name: letsencrypt-http01 kind: ClusterIssuer + +joinCall: +# this value should be set to 3 when deployed in a full production DMZ manner +# replicaCount = 1 is to support the simple wiab-staging solution + replicaCount: 1 + image: + repository: docker.io/bitnamilegacy/nginx + pullPolicy: IfNotPresent + tag: "1.27.3-debian-12-r5" + # Uncomment to enable SFT to SFT communication for federated calls # multiSFT: # enabled: true From bf1b85b017b2b057e47bf7a9685ece593f31b52a Mon Sep 17 00:00:00 2001 From: mohitrajain Date: Thu, 5 Mar 2026 15:50:48 +0100 Subject: [PATCH 06/13] fix: update bash scripts for issues highlighted by linting and update the wiab-staging artifact hash --- ansible/inventory/demo/wiab-staging.yml | 2 +- bin/debug_logs.sh | 2 +- bin/helm-operations.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/inventory/demo/wiab-staging.yml b/ansible/inventory/demo/wiab-staging.yml index 7652ce731..3ddf4aa6f 100644 --- a/ansible/inventory/demo/wiab-staging.yml +++ b/ansible/inventory/demo/wiab-staging.yml @@ -6,4 +6,4 @@ wiab-staging: ansible_user: 'demo' ansible_ssh_private_key_file: "~/.ssh/id_ed25519" vars: - artifact_hash: f1f624256bdab0f9f76158c7f45e0618ee641237 + artifact_hash: 78a52d4686b6de91853d715fd08ef42fe8e2fd20 diff --git a/bin/debug_logs.sh b/bin/debug_logs.sh index 6660cd22a..3138e025f 100755 --- a/bin/debug_logs.sh +++ b/bin/debug_logs.sh @@ -7,7 +7,7 @@ echo "------------------------------------" namespaces="cert-manager-ns default" echo "Namespaces = $namespaces" for ns in $namespaces; do - pods=$(kubectl get pods -n $ns -o=jsonpath='{.items[*].metadata.name}') + pods=$(kubectl get pods -n "$ns" -o=jsonpath='{.items[*].metadata.name}') echo "Pods in namespace: $ns = $pods" for pod in $pods; do echo "Logs for pod: $pod" diff --git a/bin/helm-operations.sh b/bin/helm-operations.sh index ad82ff671..4baca7533 100755 --- a/bin/helm-operations.sh +++ b/bin/helm-operations.sh @@ -25,7 +25,7 @@ fi function dump_debug_logs { if [[ "$DUMP_LOGS_ON_FAIL" == "TRUE" ]]; then - $BASE_DIR/bin/debug_logs.sh + "$BASE_DIR"/bin/debug_logs.sh fi } trap dump_debug_logs ERR From e94d0ab101a8a43cf237e781ed5d75db8761ba90 Mon Sep 17 00:00:00 2001 From: mohitrajain Date: Fri, 6 Mar 2026 16:09:06 +0100 Subject: [PATCH 07/13] patched documentation further --- offline/wiab-staging.md | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/offline/wiab-staging.md b/offline/wiab-staging.md index 4165e5711..c16900751 100644 --- a/offline/wiab-staging.md +++ b/offline/wiab-staging.md @@ -107,7 +107,7 @@ cd wire-server-deploy A sample inventory is available at [ansible/inventory/demo/wiab-staging.yml](https://github.com/wireapp/wire-server-deploy/blob/master/ansible/inventory/demo/wiab-staging.yml). -*Note: Replace example.com with your physical machine address where KVM is available and adjust other variables like ansible_user and ansible_ssh_private_key_file. The SSH user for ansible `ansible_user` should have password-less `sudo` access. The physical host should be running Ubuntu 22.04.* +*Note: Replace example.com with your physical machine (adminhost) address where KVM is available and adjust other variables like ansible_user and ansible_ssh_private_key_file. The SSH user for ansible `ansible_user` should have password-less `sudo` access. The physical host should be running Ubuntu 22.04.* **Step 3: Run the VM and network provision** @@ -127,6 +127,8 @@ Ensure the inventory file `ansible/inventory/offline/inventory.yml` in the direc Since the inventory is ready, please continue with the following steps: +> **Note**: All next steps assume that the wire-server-deploy artifact has been downloaded on the `adminhost` (your physical machine) and extracted at `/home/ansible_user/wire-server-deploy`. All commands from here on will be issued from this directory on the `adminhost`, ssh on the node before proceeding. + ### Environment Setup - **[Making tooling available in your environment](docs_ubuntu_22.04.md#making-tooling-available-in-your-environment)** @@ -150,7 +152,7 @@ Since the inventory is ready, please continue with the following steps: - `HOST_IP`: public IP that matches your DNS A record (auto-detected if empty). **TLS / certificate behavior (cert-manager vs. Bring Your Own):** -- By default, `bin/helm-operations.sh` runs `deploy_cert_manager`, which installs cert-manager and configures a Let’s Encrypt (HTTP-01) issuer for the ingress charts. +- By default, `bin/helm-operations.sh` has `DEPLOY_CERT_MANAGER=TRUE`, which installs cert-manager and configures a Let’s Encrypt (HTTP-01) issuer for the ingress charts. - If you **do not** want Let’s Encrypt / cert-manager (for example, you are using **[Bring Your Own certificates](docs_ubuntu_22.04.md#acquiring--deploying-ssl-certificates)**), disable this step by passing env variable `DEPLOY_CERT_MANAGER=FALSE` when running `bin/helm-operations.sh`. - When choosing `DEPLOY_CERT_MANAGER=FALSE`, ensure your ingress is configured with your own TLS secret(s) as described at [Acquiring / Deploying SSL Certificates](docs_ubuntu_22.04.md#acquiring--deploying-ssl-certificates). - When choosing `DEPLOY_CERT_MANAGER=TRUE`, ensure if further network configuration is required by following [cert-manager behaviour in NAT / bridge environments](#cert-manager-behaviour-in-nat--bridge-environments). @@ -162,7 +164,7 @@ d sh -c 'TARGET_SYSTEM="example.dev" CERT_MASTER_EMAIL="certmaster@example.dev" ``` **Charts deployed by the script:** -- External datastores and helpers: `cassandra-external`, `elasticsearch-external`, `minio-external`, `rabbitmq-external`, `databases-ephemeral`, `reaper`, `fake-aws`, `demo-smtp`. +- External datastores and helpers: `cassandra-external`, `elasticsearch-external`, `minio-external`, `rabbitmq-external`,`postgresql-external`, `databases-ephemeral`, `reaper`, `fake-aws`, `demo-smtp`. - Wire services: `wire-server`, `webapp`, `account-pages`, `team-settings`, `smallstep-accomp`. - Ingress and certificates: `ingress-nginx-controller`, `cert-manager`, `nginx-ingress-services`. - Calling services: `sftd`, `coturn`. @@ -171,23 +173,17 @@ d sh -c 'TARGET_SYSTEM="example.dev" CERT_MASTER_EMAIL="certmaster@example.dev" - Creates `values.yaml` and `secrets.yaml` from `prod-values.example.yaml` and `prod-secrets.example.yaml` for each chart under `values/`. - Backs up any existing `values.yaml`/`secrets.yaml` before replacing them. -**Values configured by the script:** -- Replaces `example.com` with `TARGET_SYSTEM` in Wire and webapp hostnames. -- Enables cert-manager and sets `certmasterEmail` using `CERT_MASTER_EMAIL`. -- Sets SFTD hosts and switches issuer to `letsencrypt-http01`. -- Sets coturn listen/relay/external IPs using the calling node IP and `HOST_IP`. - *Note: The `bin/helm-operations.sh` script above deploys these charts; you do not need to run the Helm commands manually unless you want to customize or debug.* ## Network Traffic Configuration ### Bring traffic from the physical machine to Wire services in the k8s cluster -If you used the Ansible playbook earlier, nftables firewall rules are pre-configured to forward traffic. If you set up VMs manually with your own hypervisor, you must manually configure network traffic flow using nftables. +If you used the Ansible playbook earlier, nftables firewall rules are pre-configured to forward traffic. If you set up VMs manually with your own hypervisor, you must manually configure network traffic flow using nftables as descibed below. **Required Network Configuration:** -The physical machine must forward traffic from external clients to the Kubernetes cluster running Wire services. This involves: +The physical machine (adminhost) must forward traffic from external clients to the Kubernetes cluster running Wire services. This involves: 1. **HTTP/HTTPS Traffic (Ingress)** - Forward ports 80 and 443 to the nginx-ingress-controller running on a Kubernetes node - Port 80 (HTTP) → Kubernetes node port 31772 @@ -199,19 +195,20 @@ The physical machine must forward traffic from external clients to the Kubernete **Implementation:** -Use the detailed nftables rules in [../ansible/files/wiab_server_nftables.conf.j2](../ansible/files/wiab_server_nftables.conf.j2) as the template. The guide covers: +Use the detailed nftables rules in [../ansible/files/wiab_server_nftables.conf.j2](../ansible/files/wiab_server_nftables.conf.j2) as the template. The nftable configuration template covers: - Defining your network variables (Coturn IP, Kubernetes node IP, WAN interface) - Creating NAT rules for HTTP/HTTPS ingress traffic -- Setting up TURN protocol forwarding for Coturn -- Restarting nftables to apply changes +- Setting up TURN protocol forwarding for Coturn and traffic for SFTD + +*Note: If you have already ran the playbook wiab-staging-provision.yml then it is already be configured for you. Confirm it by checking if the wire endpoint `https://webapp.TARGET_SYSTEM` is reachable from public internet or your private network (in case of private network), but not from the adminhost itself.* -You can also apply these rules using the Ansible playbook, by following: +You can also apply these rules using the Ansible playbook against your adminhost, by following: ```bash ansible-playbook -i inventory.yml ansible/wiab-staging-nftables.yml ``` -*Note: If you ran the playbook wiab-staging-provision.yml then it might already be configured for you. Please confirm before running.* +You can run the above playbook from local system or where you have cloned/downloaded the [Wire server deploy ansible playbooks](#getting-the-ansible-playbooks). The inventory should define the following variables: @@ -227,6 +224,12 @@ calling_node_ip=192.168.122.13 # Host WAN interface name inf_wan=eth0 + +# These are the same as wiab-staging.yml +# user and ssh key for adminhost +ansible_user='demo' +ansible_ssh_private_key_file='~/.ssh/id_ed25519' + ``` ### cert-manager behaviour in NAT / bridge environments From 12775fab06439bdfbdadfc626ec7ab0c44df787e Mon Sep 17 00:00:00 2001 From: mohitrajain Date: Fri, 6 Mar 2026 16:09:44 +0100 Subject: [PATCH 08/13] updated artifact hash post fixing issues shown by linter --- ansible/inventory/demo/wiab-staging.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/inventory/demo/wiab-staging.yml b/ansible/inventory/demo/wiab-staging.yml index 3ddf4aa6f..cb95c01aa 100644 --- a/ansible/inventory/demo/wiab-staging.yml +++ b/ansible/inventory/demo/wiab-staging.yml @@ -6,4 +6,4 @@ wiab-staging: ansible_user: 'demo' ansible_ssh_private_key_file: "~/.ssh/id_ed25519" vars: - artifact_hash: 78a52d4686b6de91853d715fd08ef42fe8e2fd20 + artifact_hash: 82edf88d9193e9f7e0a62ee4b287fd0c7cebb1bd From 135901ad622c510d3ea201420da6dc8be79129c8 Mon Sep 17 00:00:00 2001 From: mohitrajain Date: Tue, 10 Mar 2026 14:15:42 +0100 Subject: [PATCH 09/13] fix: helm-operations.sh for sonarcloud exceptions and update wiab-staging based on review --- bin/helm-operations.sh | 5 ++++- offline/wiab-staging.md | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/bin/helm-operations.sh b/bin/helm-operations.sh index 4baca7533..eedc3adc0 100755 --- a/bin/helm-operations.sh +++ b/bin/helm-operations.sh @@ -7,7 +7,7 @@ BASE_DIR="${BASE_DIR:-/wire-server-deploy}" TARGET_SYSTEM="${TARGET_SYSTEM:-example.com}" CERT_MASTER_EMAIL="certmaster@${CERT_MASTER_EMAIL}:-certmaster@${TARGET_SYSTEM}" -# DEPLOY_CERT_MANAGER env variable to decide to check if cert_manager and nginx-ingress-services charts should get deployed +# DEPLOY_CERT_MANAGER env variable is used to decide if cert_manager and nginx-ingress-services charts should get deployed # default is set to TRUE to deploy it unless changed DEPLOY_CERT_MANAGER="${DEPLOY_CERT_MANAGER:-TRUE}" @@ -24,9 +24,11 @@ HOST_IP=$(wget -qO- https://api.ipify.org) fi function dump_debug_logs { + local exit_code=$? if [[ "$DUMP_LOGS_ON_FAIL" == "TRUE" ]]; then "$BASE_DIR"/bin/debug_logs.sh fi + return $exit_code } trap dump_debug_logs ERR @@ -49,6 +51,7 @@ sync_pg_secrets() { echo "⚠️ Warning: PostgreSQL secret 'wire-postgresql-secret' not found, skipping secret sync" echo " Make sure databases-ephemeral chart is deployed before wire-server" fi + return $? } # Creates values.yaml from prod-values.example.yaml and secrets.yaml from prod-secrets.example.yaml diff --git a/offline/wiab-staging.md b/offline/wiab-staging.md index c16900751..d3a3de4ea 100644 --- a/offline/wiab-staging.md +++ b/offline/wiab-staging.md @@ -153,7 +153,7 @@ Since the inventory is ready, please continue with the following steps: **TLS / certificate behavior (cert-manager vs. Bring Your Own):** - By default, `bin/helm-operations.sh` has `DEPLOY_CERT_MANAGER=TRUE`, which installs cert-manager and configures a Let’s Encrypt (HTTP-01) issuer for the ingress charts. -- If you **do not** want Let’s Encrypt / cert-manager (for example, you are using **[Bring Your Own certificates](docs_ubuntu_22.04.md#acquiring--deploying-ssl-certificates)**), disable this step by passing env variable `DEPLOY_CERT_MANAGER=FALSE` when running `bin/helm-operations.sh`. +- If you **do not** want Let’s Encrypt / cert-manager (for example, you are using **[Bring Your Own certificates](docs_ubuntu_22.04.md#acquiring--deploying-ssl-certificates)**), disable this step by passing the environment variable `DEPLOY_CERT_MANAGER=FALSE` when running `bin/helm-operations.sh`. - When choosing `DEPLOY_CERT_MANAGER=FALSE`, ensure your ingress is configured with your own TLS secret(s) as described at [Acquiring / Deploying SSL Certificates](docs_ubuntu_22.04.md#acquiring--deploying-ssl-certificates). - When choosing `DEPLOY_CERT_MANAGER=TRUE`, ensure if further network configuration is required by following [cert-manager behaviour in NAT / bridge environments](#cert-manager-behaviour-in-nat--bridge-environments). @@ -240,8 +240,8 @@ When cert-manager performs HTTP-01 self-checks inside the cluster, traffic can h In NAT/bridge setups (for example, using `virbr0` on the host): -- If nftables rules DNAT in `PREROUTING` without a matching SNAT on `virbr0 → virbr0`, return packets may bypass the host and break conntrack, causing HTTP-01 timeouts and certificate verification failures. -- Strict `rp_filter` can drop asymmetric return packets. +- If nftables DNAT rules exist in `PREROUTING` without a matching SNAT on `virbr0 → virbr0`, return packets may bypass the host and break conntrack, causing HTTP-01 timeouts and certificate verification failures. +- too strict of `rp_filter` settings can drop asymmetric return packets. Before changing anything, first verify whether certificate issuance is actually failing: From 2f256a077da76dbae5ea8697d65abc333e4c7413 Mon Sep 17 00:00:00 2001 From: mohitrajain Date: Tue, 10 Mar 2026 14:30:25 +0100 Subject: [PATCH 10/13] fix: update sync_pg_secrets function to also update .background-worker --- bin/helm-operations.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/helm-operations.sh b/bin/helm-operations.sh index eedc3adc0..9dbe49a58 100755 --- a/bin/helm-operations.sh +++ b/bin/helm-operations.sh @@ -46,7 +46,7 @@ sync_pg_secrets() { "$BASE_DIR/bin/sync-k8s-secret-to-wire-secrets.sh" \ wire-postgresql-external-secret password \ "$BASE_DIR/values/wire-server/secrets.yaml" \ - .brig.secrets.pgPassword .galley.secrets.pgPassword + .brig.secrets.pgPassword .galley.secrets.pgPassword .background-worker.secrets.pgPassword else echo "⚠️ Warning: PostgreSQL secret 'wire-postgresql-secret' not found, skipping secret sync" echo " Make sure databases-ephemeral chart is deployed before wire-server" From 7968165cbac21b48cfe8629afe3b7bbc17e6f6d1 Mon Sep 17 00:00:00 2001 From: mohit rajain Date: Tue, 24 Mar 2026 14:26:42 +0100 Subject: [PATCH 11/13] Wpb 23988 enable 5.25 (#879) * fix: wpb-23988 sync offline-secrets and prod-secrets.example.yaml and add comments * fix: wpb-23988 enable postgresql secret for background-worker inwiab-dev * fix: wpb-23988 sync wire-server helm chart values for wiab-dev from prod values * fix: wpb-23988 sync wire-server helm chart secrets for wiab-dev from prod values for 5.25 * fix: wpb-23988 add a changelog file * fix: wpb-23988 fix the changelog verification workflow to consider the non-master branches as well * fix: wpb-23988 remove the changelog trigger for every push operation * fix: wpb-23988 comment out empty mls secrets for wiab-dev * fix: wpb-23988 update wiab-stag artifact hash * fix: wpb-22988 fix minio service name * Update values/wire-server/demo-values.example.yaml Co-authored-by: Sukanta --------- Co-authored-by: Sukanta --- .github/workflows/changelog-verify.yml | 4 +- ansible/inventory/demo/wiab-staging.yml | 2 +- ansible/wiab-demo/wire_secrets.yml | 3 + bin/offline-secrets.sh | 3 +- .../3-deploy-builds/wiab-dev-5.25-fixes | 4 + values/wire-server/demo-secrets.example.yaml | 103 +++++++++------ values/wire-server/demo-values.example.yaml | 125 ++++++++++++++---- values/wire-server/prod-secrets.example.yaml | 29 ++-- 8 files changed, 193 insertions(+), 80 deletions(-) create mode 100644 changelog.d/3-deploy-builds/wiab-dev-5.25-fixes diff --git a/.github/workflows/changelog-verify.yml b/.github/workflows/changelog-verify.yml index 79441a420..164e6beb8 100644 --- a/.github/workflows/changelog-verify.yml +++ b/.github/workflows/changelog-verify.yml @@ -1,9 +1,7 @@ name: Changelog verification on: pull_request: - branches: [master] - push: - branches: [master] + branches: ["**"] permissions: contents: read diff --git a/ansible/inventory/demo/wiab-staging.yml b/ansible/inventory/demo/wiab-staging.yml index cb95c01aa..fb3ee33fd 100644 --- a/ansible/inventory/demo/wiab-staging.yml +++ b/ansible/inventory/demo/wiab-staging.yml @@ -6,4 +6,4 @@ wiab-staging: ansible_user: 'demo' ansible_ssh_private_key_file: "~/.ssh/id_ed25519" vars: - artifact_hash: 82edf88d9193e9f7e0a62ee4b287fd0c7cebb1bd + artifact_hash: 2200257f7a528f3a8157e8878fc7ee1c945594d1 diff --git a/ansible/wiab-demo/wire_secrets.yml b/ansible/wiab-demo/wire_secrets.yml index 8b1ef9681..4afd87b8f 100644 --- a/ansible/wiab-demo/wire_secrets.yml +++ b/ansible/wiab-demo/wire_secrets.yml @@ -418,6 +418,9 @@ galley: secrets: pgPassword: "{{ pgpassword }}" + background-worker: + secrets: + pgPassword: "{{ pgpassword }}" when: "'postgresql' in charts_to_deploy" - name: Update secrets in-place diff --git a/bin/offline-secrets.sh b/bin/offline-secrets.sh index 76756a1a6..bb8d1f990 100755 --- a/bin/offline-secrets.sh +++ b/bin/offline-secrets.sh @@ -51,8 +51,6 @@ brig: rabbitmq: username: guest password: guest - # These are only necessary if you wish to support sign up via SMS/calls - # And require accounts at twilio.com / nexmo.com cargohold: secrets: @@ -105,6 +103,7 @@ team-settings: configJson: "e30K" background-worker: secrets: + pgPassword: verysecurepassword rabbitmq: username: guest password: guest diff --git a/changelog.d/3-deploy-builds/wiab-dev-5.25-fixes b/changelog.d/3-deploy-builds/wiab-dev-5.25-fixes new file mode 100644 index 000000000..0b7a84bad --- /dev/null +++ b/changelog.d/3-deploy-builds/wiab-dev-5.25-fixes @@ -0,0 +1,4 @@ +Fixed: sync offline-secrets and prod-secrets.example.yaml and add comments +Added: enable postgresql secret for background-worker in wiab-dev +Fixed: sync wire-server helm chart values for wiab-dev from prod values for 5.25 +Fixed: sync wire-server helm chart secrets for wiab-dev from prod values for 5.25 diff --git a/values/wire-server/demo-secrets.example.yaml b/values/wire-server/demo-secrets.example.yaml index 7bc5ecad3..0780cc146 100644 --- a/values/wire-server/demo-secrets.example.yaml +++ b/values/wire-server/demo-secrets.example.yaml @@ -1,66 +1,93 @@ -# CHANGEME-DEMO: All values here should be changed/reviewed +# CHANGEME-DEV: All values here should be changed/reviewed +# check the ansible playbook ansible/wiab-demo/wire_secrets.yml on how these secrets are being randomly generated and rotated +# make sure that any secrets related to external services like AWS, giphy, youtube, spotify etc are being updated before running the random secret generation (ansible/wiab-demo/wire_secrets.yml) at demo-secrets.example.yaml and before deploying the helm charts using the playbook (ansible/wiab-demo/helm_install.yml) + +# The secrets for services like elasticsearch, postgresql, rabbitmq and AWS (fake) secretID and key are configured in their helm charts. The values passed to these charts can be modified at wire-server-deploy/service-name/demo-[values|secrets].example.yaml +# postgresql - https://github.com/wireapp/helm-charts/tree/dev/charts/postgresql +# elasticsearch - https://github.com/wireapp/wire-server/blob/develop/charts/elasticsearch-ephemeral +# rabbitMQ - https://github.com/wireapp/wire-server/tree/develop/charts/rabbitmq +# fake-aws - https://github.com/wireapp/wire-server/tree/develop/charts/fake-aws +# AWS - this needs to be checked with wire support if needs to use real AWS services + elasticsearch-index: secrets: elasticsearch: - username: elastic - password: changeme + username: "elastic" + password: "changeme" + brig: secrets: + pgPassword: verysecurepassword smtpPassword: dummyPassword zAuth: # generate zauth public/private keys with the 'zauth' executable from wire-server: - # ./dist/zauth -m gen-keypair -i 1 + # sudo docker run $ZAUTH_CONTAINER -m gen-keypair publicKeys: "" privateKeys: "" turn: # generate a high-entropy random string, e.g. using - # openssl rand -base64 64 | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 42 + # openssl rand -base64 64 | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 64 secret: CHANGEMEE6KHMJU1uDhhgvsVWoIyzmn3u3GHRoWjTp - # these only need to be changed if using real AWS services awsKeyId: dummykey awsSecretKey: dummysecret - # These are only necessary if you wish to support sign up via SMS/calls - # And require accounts at twilio.com / nexmo.com rabbitmq: username: wire-server password: verysecurepassword - # PostgreSQL password is synced with the wire-postgresql-secret from k8s cluster - # To extract the secret from an existing Kubernetes cluster: - # kubectl get secret wire-postgresql-secret -n postgresql -o jsonpath='{.data.password}' | base64 -d - pgPassword: dummyPassword # gets replaced by the actual secret elasticsearch: username: "elastic" password: "changeme" elasticsearchAdditional: username: "elastic" password: "changeme" -cannon: + +cargohold: secrets: + awsKeyId: dummykey + awsSecretKey: dummysecret rabbitmq: username: wire-server password: verysecurepassword -cargohold: +cannon: secrets: - # these only need to be changed if using real AWS services - awsKeyId: dummykey - awsSecretKey: dummysecret rabbitmq: username: wire-server password: verysecurepassword galley: secrets: - # these only need to be changed if using real AWS services - awsKeyId: dummykey - awsSecretKey: dummysecret - # PostgreSQL password is synced with the wire-postgresql-secret from k8s cluster - # To extract the secret from an existing Kubernetes cluster: - # kubectl get secret wire-postgresql-secret -n postgresql -o jsonpath='{.data.password}' | base64 -d - pgPassword: dummyPassword # gets replaced by the actual secret rabbitmq: username: wire-server password: verysecurepassword + pgPassword: verysecurepassword + # these only need to be changed if using real AWS services + awsKeyId: dummykey + awsSecretKey: dummysecret + + # Generate MLS private keys using openssl + # readonly MLS_KEY_INDENT=" " + # Keys need 10 spaces indent (5 levels deep: galley.secrets.mlsPrivateKeys.removal.keyname) + # generate_mls_key() { openssl genpkey "$@" 2>/dev/null | awk -v indent="$MLS_KEY_INDENT" '{printf "%s%s\n", indent, $0}'} + # mls_ed25519_key="$(generate_mls_key -algorithm ed25519)" + # mls_ecdsa_p256_key="$(generate_mls_key -algorithm ec -pkeyopt ec_paramgen_curve:P-256)" + # mls_ecdsa_p384_key="$(generate_mls_key -algorithm ec -pkeyopt ec_paramgen_curve:P-384)" + # mls_ecdsa_p521_key="$(generate_mls_key -algorithm ec -pkeyopt ec_paramgen_curve:P-521)" + + # this will get initialized from wire_secrets.yml playbook or can be generated using above logic + # mlsPrivateKeys: + # removal: + # ed25519: | + # -----BEGIN PRIVATE KEY----- + # -----END PRIVATE KEY----- + # ecdsa_secp256r1_sha256: | + # -----BEGIN PRIVATE KEY----- + # -----END PRIVATE KEY----- + # ecdsa_secp384r1_sha384: | + # -----BEGIN PRIVATE KEY----- + # -----END PRIVATE KEY----- + # ecdsa_secp521r1_sha512: | + # -----BEGIN PRIVATE KEY----- + # -----END PRIVATE KEY----- gundeck: secrets: @@ -71,19 +98,19 @@ gundeck: username: wire-server password: verysecurepassword -proxy: - secrets: - # If you desire proxying/previews for the following services, - # set 'tags.proxy: true' in demo-values.yaml, - # create accounts with them and fill in these values: - proxy_config: |- - secrets { - youtube = "..." - googlemaps = "..." - soundcloud = "..." - giphy = "..." - spotify = "Basic ..." - } +# proxy: +# secrets: +# # If you desire proxying/previews for the following services, +# # set 'tags.proxy: true' in demo-values.yaml, +# # create accounts with them and fill in these values: +# proxy_config: |- +# secrets { +# youtube = "..." +# googlemaps = "..." +# soundcloud = "..." +# giphy = "..." +# spotify = "Basic ..." +# } nginz: secrets: @@ -93,9 +120,11 @@ nginz: # only necessary in test environments (env="staging"). See charts/nginz/README.md basicAuth: ":" + # RabbitMQ credentials for background-worker. background-worker: secrets: + pgPassword: verysecurepassword rabbitmq: username: wire-server password: verysecurepassword diff --git a/values/wire-server/demo-values.example.yaml b/values/wire-server/demo-values.example.yaml index fed128a0a..230fd6d7d 100644 --- a/values/wire-server/demo-values.example.yaml +++ b/values/wire-server/demo-values.example.yaml @@ -1,3 +1,4 @@ +# CHANGEME-DEV: All values here should be changed/reviewed tags: proxy: false # enable if you want/need giphy/youtube/etc proxying legalhold: false # Enable if you need legalhold @@ -15,18 +16,21 @@ elasticsearch-index: host: elasticsearch-ephemeral cassandra: host: cassandra-ephemeral - brig: replicaCount: 1 # image: # tag: some-tag (only override if you want a newer/different version than what is in the chart) config: + multiSFT: + enabled: false # enable to turn on SFT to SFT communication for federated calls cassandra: host: cassandra-ephemeral elasticsearch: host: elasticsearch-ephemeral rabbitmq: - host: rabbitmq # name of the rabbitmq service, either `rabbitmq-external` or `rabbitmq` + # Default: rabbitmq-external (for production external RabbitMQ VMs) + # CI/Demo: Change to rabbitmq when using rabbitmq chart from databases-ephemeral + host: rabbitmq postgresql: host: postgresql # DNS name without protocol port: "5432" @@ -34,27 +38,29 @@ brig: dbname: wire-server useSES: false # Set to false if you want to hand out DynamoDB to store prekeys - randomPrekeys: true + randomPrekeys: true aws: # change if using real AWS region: "eu-west-1" sqsEndpoint: http://fake-aws-sqs:4568 - #dynamoDBEndpoint: http://fake-aws-dynamodb:4567 - + # dynamoDBEndpoint: http://fake-aws-dynamodb:4567 + # these must match the table names created on fake or real AWS services internalQueue: integration-brig-events-internal prekeyTable: integration-brig-prekeys externalUrls: nginz: https://nginz-https.example.com # change this - teamSettings: https://teams.example.com # change this (on unset if team settings are not used) + teamSettings: https://teams.example.com # change this (or unset if team settings are not used) teamCreatorWelcome: https://teams.example.com/login # change this teamMemberWelcome: https://wire.example.com/download # change this - enableFederation: false # Enable to use federation + enableFederation: false # Keep false unless federation is explicitly configured optSettings: - setFederationDomain: example.com # change this + setEnableMLS: false # Enable for MLS protocol use + setFederationDomain: example.com # change this per host deployment # Sync the domain with the 'host' variable in the sftd chart # Comment the next line (by adding '#' before it) if conference calling is not used setSftStaticUrl: "https://sftd.example.com:443" + # setSftListAllServers: "enabled" # Uncomment for Federation! # If set to true, creating new personal users or new teams on your instance from # outside your backend installation is disabled setRestrictUserCreation: false @@ -86,7 +92,9 @@ brig: deletionUrl: https://account.example.com/d/?key=${key}&code=${code} invitationUrl: https://account.example.com/i/${code} smtp: - host: smtp # change this if you want to use your own SMTP server + # Default: smtp (for CI/demo environments with demo-smtp chart) + # Production: Change to your actual SMTP server hostname + host: smtp port: 25 # change this connType: plain # change this. Possible values: plain|ssl|tls # proxy: @@ -127,12 +135,15 @@ cannon: # For demo mode only, we don't need to keep websocket connections open on chart upgrades drainTimeout: 10 config: - cassandra: + rabbitmq: + # Default: rabbitmq-external (for production external RabbitMQ VMs) + # CI/Demo: Change to rabbitmq when using rabbitmq chart from databases-ephemeral + host: rabbitmq + cassandra: host: cassandra-ephemeral metrics: serviceMonitor: enabled: true - cargohold: replicaCount: 1 # image: @@ -144,6 +155,7 @@ cargohold: s3Bucket: assets s3Endpoint: http://fake-aws-s3:9000 s3DownloadEndpoint: https://assets.example.com + enableFederation: false # Enable to use federation settings: federationDomain: example.com # change this # proxy: @@ -155,7 +167,6 @@ cargohold: metrics: serviceMonitor: enabled: true - galley: replicaCount: 1 # image: @@ -164,12 +175,18 @@ galley: cassandra: host: cassandra-ephemeral rabbitmq: - host: rabbitmq # name of the rabbitmq service, either `rabbitmq-external` or `rabbitmq` + # Default: rabbitmq-external (for production external RabbitMQ VMs) + # CI/Demo: Change to rabbitmq when using rabbitmq chart from databases-ephemeral + host: rabbitmq postgresql: host: postgresql # DNS name without protocol port: "5432" user: wire-server dbname: wire-server + # Explicitly set postgresMigration to cassandra for fresh deployments. + # This controls whether galley reads conversations from Cassandra or PostgreSQL. + postgresMigration: + conversation: cassandra enableFederation: false # Enable to use federation settings: # prefix URI used when inviting users to a conversation by link @@ -177,7 +194,33 @@ galley: federationDomain: example.com # change this # see #RefConfigOptions in `/docs/reference` (https://github.com/wireapp/wire-server/) featureFlags: + mls: # Keep disabled unless MLS is explicitly configured + defaults: + status: disabled + config: + protocolToggleUsers: [] + defaultProtocol: mls + allowedCipherSuites: [2] + defaultCipherSuite: 2 + supportedProtocols: [proteus, mls] + lockStatus: unlocked + mlsMigration: # Keep disabled unless MLS migration is explicitly configured + defaults: + status: disabled + config: + startTime: null + finalizeRegardlessAfter: null + usersThreshold: 100 + clientsThreshold: 100 + lockStatus: unlocked sso: disabled-by-default + # channels: # Uncomment to enable channels by default for all newly created teams + # defaults: + # status: enabled + # config: + # allowed_to_create_channels: team-members + # allowed_to_open_channels: team-members + # lockStatus: unlocked # NOTE: Change this to "disabled-by-default" for legalhold support # legalhold: disabled-by-default legalhold: disabled-permanently @@ -193,12 +236,17 @@ galley: metrics: serviceMonitor: enabled: true - gundeck: replicaCount: 1 # image: # tag: some-tag (only override if you want a newer/different version than what is in the chart) config: + redis: + host: databases-ephemeral-redis-ephemeral # Updated hostname as per your env, check the redis service name in the k8s cluster `d kubectl get svc -A | grep redis` + rabbitmq: + # Default: rabbitmq-external (for production external RabbitMQ VMs) + # CI/Demo: Change to rabbitmq when using rabbitmq chart from databases-ephemeral + host: rabbitmq cassandra: host: cassandra-ephemeral aws: @@ -218,7 +266,6 @@ gundeck: metrics: serviceMonitor: enabled: true - nginz: replicaCount: 1 config: @@ -281,19 +328,51 @@ spar: enabled: true # Only needed when legalhold is enabled -#legalhold: -# host: "legalhold.example.com" -# wireApiHost: "https://nginz-https.example.com" - +legalhold: + host: "legalhold.example.com" + wireApiHost: "https://nginz-https.example.com" + metrics: + serviceMonitor: + enabled: true # Only needed when federation is enabled -#federator: -# tls: -# useSharedFederatorSecret: true - +federator: + # config: + # optSettings: + # federationStrategy: + # allowedDomains: + # - example.com + tls: + useSharedFederatorSecret: true + # remoteCAContents: | # Uncomment and place the federating backends root CA certificates in chain (if there are multiple) + metrics: + serviceMonitor: + enabled: true background-worker: config: + federationDomain: example.com + # logLevel: Debug + rabbitmq: + # Default: rabbitmq-external (for production external RabbitMQ VMs) + # CI/Demo: Change to rabbitmq when using rabbitmq chart from databases-ephemeral + host: rabbitmq cassandra: host: cassandra-ephemeral + cassandraGalley: + host: cassandra-ephemeral + cassandraBrig: + host: cassandra-ephemeral + postgresql: + host: postgresql # DNS name without protocol + port: "5432" + user: wire-server + dbname: wire-server + # IMPORTANT: At Chart 5.25.0, the background-worker Helm chart incorrectly defaults + # postgresMigration.conversation to "postgresql" instead of "cassandra". + # This MUST be explicitly set to "cassandra" unless you have already migrated + # conversations to PostgreSQL. Without this override, User Group to Channel sync + # jobs will silently skip member synchronization. + postgresMigration: + conversation: cassandra # Enable for federation enableFederation: false metrics: diff --git a/values/wire-server/prod-secrets.example.yaml b/values/wire-server/prod-secrets.example.yaml index c60a0f434..fee8c28d6 100644 --- a/values/wire-server/prod-secrets.example.yaml +++ b/values/wire-server/prod-secrets.example.yaml @@ -1,30 +1,25 @@ # CHANGEME-PROD: All values here should be changed/reviewed +# check the script bin/offline-secrets.sh on how these secrets are being randomly generated. +# check the script bin/helm-operations.sh on how the secrets are being rotated and updated. +# make sure that any secrets related to external services like AWS, giphy, youtube, spotify etc are being updated post running the random secret generation by bin/offline-secrets.sh and before deploying the helm charts using the script bin/helm-operations.sh + brig: secrets: #retrieve postgresql password from Kubernetes Secret with "kubectl get secret wire-postgresql-external-secret -n default -o jsonpath='{.data.password}' | base64 -d" pgPassword: verysecurepassword - smtpPassword: dummyPassword zAuth: # generate zauth public/private keys with the 'zauth' executable from wire-server: - # ./dist/zauth -m gen-keypair -i 1 + # sudo docker run $ZAUTH_CONTAINER -m gen-keypair publicKeys: "" privateKeys: "" turn: # generate a high-entropy random string, e.g. using - # openssl rand -base64 64 | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 42 + # openssl rand -base64 64 | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 64 secret: CHANGEMEE6KHMJU1uDhhgvsVWoIyzmn3u3GHRoWjTp # these only need to be changed if using real AWS services awsKeyId: dummykey awsSecretKey: dummysecret - # These are only necessary if you wish to support sign up via SMS/calls - # And require accounts at twilio.com / nexmo.com - rabbitmq: - username: guest - password: guest - -cannon: - secrets: rabbitmq: username: guest password: guest @@ -39,16 +34,22 @@ cargohold: username: guest password: guest +cannon: + secrets: + rabbitmq: + username: guest + password: guest + galley: secrets: + rabbitmq: + username: guest + password: guest #retrieve postgresql password from Kubernetes Secret with "kubectl get secret wire-postgresql-external-secret -n default -o jsonpath='{.data.password}' | base64 -d" pgPassword: verysecurepassword # these only need to be changed if using real AWS services awsKeyId: dummykey awsSecretKey: dummysecret - rabbitmq: - username: guest - password: guest mlsPrivateKeys: removal: ed25519: | From c51ad4c6ce3e3cf844ee6c97d4fd6ed96e5fce8a Mon Sep 17 00:00:00 2001 From: mohit rajain Date: Fri, 27 Mar 2026 18:02:55 +0100 Subject: [PATCH 12/13] Update offline/wiab-staging.md Co-authored-by: Julia Longtin --- offline/wiab-staging.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offline/wiab-staging.md b/offline/wiab-staging.md index d3a3de4ea..e020b3d96 100644 --- a/offline/wiab-staging.md +++ b/offline/wiab-staging.md @@ -127,7 +127,7 @@ Ensure the inventory file `ansible/inventory/offline/inventory.yml` in the direc Since the inventory is ready, please continue with the following steps: -> **Note**: All next steps assume that the wire-server-deploy artifact has been downloaded on the `adminhost` (your physical machine) and extracted at `/home/ansible_user/wire-server-deploy`. All commands from here on will be issued from this directory on the `adminhost`, ssh on the node before proceeding. +> **Note**: All next steps assume that the wire-server-deploy artifact has been downloaded on the `adminhost` (your physical machine) and extracted at `/home/ansible_user/wire-server-deploy`. All commands from here on will be issued from this directory on the `adminhost`. Make sure you SSH into the node before proceeding. ### Environment Setup From 7e43c4c34f0a60ebd0addc7953f145866650b8b1 Mon Sep 17 00:00:00 2001 From: mohit rajain Date: Thu, 2 Apr 2026 16:18:21 +0200 Subject: [PATCH 13/13] Wpb 23988 fix internet access wiab stag (#881) * fix: wpb-23988 sync wire-server helm chart values for wiab-dev from prod values * fix: wpb-23988 fix the changelog verification workflow to consider the non-master branches as well * fix: wpb-23988 remove the changelog trigger for every push operation * add: wpb-23988 variable private_deployment with default true to disable SNAT on adminhost * fix: wpb-23988 cert_master_email env var * fix: wpb-23988 running wiab-staging-nftables.yaml playbook is explicit * fix: wpb-23988 wiab-staging.md documentation to add details about default SNAT access being denied and how to enable it * fix: wpb-23988 add changelog * fix: wpb-23988 update the wiab-staging.md to improve documentation around running commands * fix: wpb-23988 update the artifact hash * Apply suggestions from code review Co-authored-by: Julia Longtin * fix: wpb-24291 update the documentation for wiab-staging * fix wpb-24291: fix the minio endpoint for s3 endpoint * fix wpb-23988: fix the logic in wiab_server_nftables.conf.j2 and update the inventory comment * fix wpb-23988: upgrade documentation and small fixes for inventory * Wpb 23988 disable smallstep (#882) * fix: wpb-23988 update the wiab-staging.md to improve documentation around running commands * fix: wpb-23988 stop deploying smallstep by default in wiab-staging and wiab-dev * fix: wpb-23988 changelog * fix wpb-23988: replace demo-smtp with smtp * fix wpb-23988: push action on master branch only * fix wpb-23988: update documentation based on wire-docs pr101 --------- Co-authored-by: Julia Longtin --- .github/workflows/changelog-verify.yml | 2 + ansible/files/wiab_server_nftables.conf.j2 | 2 + ansible/inventory/demo/host.yml | 1 - ansible/inventory/demo/wiab-staging.yml | 4 +- ansible/wiab-staging-provision.yml | 3 +- bin/helm-operations.sh | 6 +- changelog.d/3-deploy-builds/disable-smallstep | 1 + .../wiab-stag-nftables-snat-fix | 5 + offline/wiab-staging.md | 264 +++++++++++++----- 9 files changed, 208 insertions(+), 80 deletions(-) create mode 100644 changelog.d/3-deploy-builds/disable-smallstep create mode 100644 changelog.d/3-deploy-builds/wiab-stag-nftables-snat-fix diff --git a/.github/workflows/changelog-verify.yml b/.github/workflows/changelog-verify.yml index 164e6beb8..13ef3fe19 100644 --- a/.github/workflows/changelog-verify.yml +++ b/.github/workflows/changelog-verify.yml @@ -2,6 +2,8 @@ name: Changelog verification on: pull_request: branches: ["**"] + push: + branches: ["master"] permissions: contents: read diff --git a/ansible/files/wiab_server_nftables.conf.j2 b/ansible/files/wiab_server_nftables.conf.j2 index 709c0e6c9..bf36bb6b2 100644 --- a/ansible/files/wiab_server_nftables.conf.j2 +++ b/ansible/files/wiab_server_nftables.conf.j2 @@ -67,7 +67,9 @@ table ip nat { chain POSTROUTING { type nat hook postrouting priority 100; oifname != docker0 ip saddr 172.17.0.0/16 counter masquerade +{% if not (private_deployment | default(true) | bool) %} oifname $INF_WAN counter masquerade comment "{{ wire_comment }} masquerade outgoing traffic" +{% endif %} } chain DOCKER { iifname docker0 counter return diff --git a/ansible/inventory/demo/host.yml b/ansible/inventory/demo/host.yml index 0caa01901..e1274101a 100644 --- a/ansible/inventory/demo/host.yml +++ b/ansible/inventory/demo/host.yml @@ -57,7 +57,6 @@ wiab: - databases-ephemeral - postgresql - reaper - - smallstep-accomp - wire-server - webapp - account-pages diff --git a/ansible/inventory/demo/wiab-staging.yml b/ansible/inventory/demo/wiab-staging.yml index fb3ee33fd..a7e907fec 100644 --- a/ansible/inventory/demo/wiab-staging.yml +++ b/ansible/inventory/demo/wiab-staging.yml @@ -6,4 +6,6 @@ wiab-staging: ansible_user: 'demo' ansible_ssh_private_key_file: "~/.ssh/id_ed25519" vars: - artifact_hash: 2200257f7a528f3a8157e8878fc7ee1c945594d1 + artifact_hash: 7da2319729ba792f91d7ccba4e026c21cd3a3691 + # when enabled, disable WAN SNAT/masquerading for VMs on the private network + private_deployment: true diff --git a/ansible/wiab-staging-provision.yml b/ansible/wiab-staging-provision.yml index 39c24027d..2ac5d2187 100644 --- a/ansible/wiab-staging-provision.yml +++ b/ansible/wiab-staging-provision.yml @@ -298,9 +298,8 @@ kubenode2_ip: "{{ kubenode_ip_result.results[1].stdout }}" kubenode3_ip: "{{ kubenode_ip_result.results[2].stdout }}" wire_comment: "wiab-stag" - tags: always - name: Configure nftables import_playbook: ./wiab-staging-nftables.yaml - tags: nftables + tags: [never, nftables] diff --git a/bin/helm-operations.sh b/bin/helm-operations.sh index 9dbe49a58..10c57fed2 100755 --- a/bin/helm-operations.sh +++ b/bin/helm-operations.sh @@ -5,7 +5,7 @@ set -Eeo pipefail # Read values from environment variables with defaults BASE_DIR="${BASE_DIR:-/wire-server-deploy}" TARGET_SYSTEM="${TARGET_SYSTEM:-example.com}" -CERT_MASTER_EMAIL="certmaster@${CERT_MASTER_EMAIL}:-certmaster@${TARGET_SYSTEM}" +CERT_MASTER_EMAIL="${CERT_MASTER_EMAIL:-certmaster@example.com}" # DEPLOY_CERT_MANAGER env variable is used to decide if cert_manager and nginx-ingress-services charts should get deployed # default is set to TRUE to deploy it unless changed @@ -60,7 +60,7 @@ process_values() { ENV=$1 TYPE=$2 - charts=(fake-aws smtp rabbitmq databases-ephemeral reaper wire-server webapp account-pages team-settings smallstep-accomp ingress-nginx-controller nginx-ingress-services coturn sftd cert-manager) + charts=(fake-aws smtp rabbitmq databases-ephemeral reaper wire-server webapp account-pages team-settings ingress-nginx-controller nginx-ingress-services coturn sftd cert-manager) if [[ "$ENV" != "prod" ]] || [[ -z "$TYPE" ]] ; then echo "Error: This function only supports prod deployments with TYPE as values or secrets. ENV must be 'prod', got: '$ENV' and '$TYPE'" @@ -214,7 +214,7 @@ sync_pg_secrets configure_values # deploying with external datastores, useful for prod setup -deploy_charts cassandra-external elasticsearch-external minio-external postgresql-external fake-aws smtp rabbitmq-external databases-ephemeral reaper wire-server webapp account-pages team-settings smallstep-accomp ingress-nginx-controller +deploy_charts cassandra-external elasticsearch-external minio-external postgresql-external fake-aws smtp rabbitmq-external databases-ephemeral reaper wire-server webapp account-pages team-settings ingress-nginx-controller # deploying cert-manager only when the env var DEPLOY_CERT_MANAGER is set to TRUE if [[ "$DEPLOY_CERT_MANAGER" == "TRUE" ]]; then diff --git a/changelog.d/3-deploy-builds/disable-smallstep b/changelog.d/3-deploy-builds/disable-smallstep new file mode 100644 index 000000000..2f3741837 --- /dev/null +++ b/changelog.d/3-deploy-builds/disable-smallstep @@ -0,0 +1 @@ +Fixed: stop deploying smallstep in wiab-staging and wiab-dev environments diff --git a/changelog.d/3-deploy-builds/wiab-stag-nftables-snat-fix b/changelog.d/3-deploy-builds/wiab-stag-nftables-snat-fix new file mode 100644 index 000000000..55ceb183e --- /dev/null +++ b/changelog.d/3-deploy-builds/wiab-stag-nftables-snat-fix @@ -0,0 +1,5 @@ +Added: variable private_deployment with default true to disable SNAT on adminhost +Fixed: cert_master_email env var parsing in helm-operations.sh +Fixed: made running wiab-staging-nftables.yaml playbook explicit +Added: wiab-staging.md documentation to add details about default SNAT access being denied and how to enable it +Added: wiab-staging.md network flow diagram diff --git a/offline/wiab-staging.md b/offline/wiab-staging.md index e020b3d96..5b37f53b2 100644 --- a/offline/wiab-staging.md +++ b/offline/wiab-staging.md @@ -1,6 +1,6 @@ # Scope -**Wire in a Box (WIAB) Staging** is a demo installation of Wire running on a single physical machine using KVM-based virtual machines. This setup replicates the multi-node production Wire architecture in a consolidated environment suitable for testing, evaluation, and learning about Wire's infrastructure—but **not for production use**. +**Wire in a Box (WIAB) Staging** is an installation of Wire running on a single physical machine using KVM-based virtual machines. This setup replicates the multi-node production Wire architecture in a consolidated environment suitable for testing, evaluation, and learning about Wire's infrastructure—but **not for production use**. The main use of this package is to verify that automation inside and outside of the wire product functions in the fashion you expect, before you run said automation in production. This will not test your network environment, load based behaviors, or the interface between wire and it's calling services when using a DMZ'd network configuration. **Important:** This is a sandbox environment. Data from a staging installation cannot be migrated to production. WIAB Staging is designed for experimentation, validation, and understanding Wire's deployment model. @@ -9,27 +9,26 @@ **Architecture Overview:** - Multiple VMs (7) are deployed to simulate production infrastructure with separate roles (Kubernetes, data services, asset storage) - All VMs share the same physical node and storage, creating a single failure domain -- [Calling services](https://docs.wire.com/latest/understand/overview.html#calling) will share the same k8s cluster as Wire services hence, all infrastructure will be DMZ (De-militarized zone). +- [Calling services](https://docs.wire.com/latest/understand/overview.html#calling) are deployed in the same Kubernetes cluster as Wire services. This setup does not implement a separate DMZ, and all components share the same network boundary, reducing the level of isolation compared to a production deployment. - This solution helps developers understand Wire's infrastructure requirements and test deployment processes **Resource Requirements:** -- One physical machine with hypervisor support: +- One physical machine (aka `adminhost`) with hypervisor support: - **Memory:** 55 GiB RAM - **Compute:** 29 vCPUs - **Storage:** 850 GB disk space (thin-provisioned) - - 7 VMs with [Ubuntu 22](https://releases.ubuntu.com/jammy/) as per (#VM-Provisioning) - **DNS Records**: - - a way to create DNS records for your domain name (e.g. wire.example.com) + - A method to create DNS records for your domain name (e.g. wire.example.com) - Find a detailed explanation at [How to set up DNS records](https://docs.wire.com/latest/how-to/install/demo-wiab.html#dns-requirements) - **SSL/TLS certificates**: - - a way to create SSL/TLS certificates for your domain name (to allow connecting via https://) + - A method to create SSL/TLS certificates for your domain name (to allow connecting via https://) - To ease out the process of managing certs, we recommend using [Let's Encrypt](https://letsencrypt.org/getting-started/) & [cert-manager](https://cert-manager.io/docs/tutorials/acme/http-validation/) - **Network**: No interference from UFW or other system specific firewalls, and IP forwarding enabled between network cards. An IP address reachable for ssh and which can act as entry point for Wire traffic. - **Wire-server-deploy artifact**: A tar bundle containing all the required bash scripts, deb packages, ansible playbooks, helm charts and docker images to help with the installation. Reach out to [Wire support](https://support.wire.com/) to get access to the latest stable Wire artifact. ## VM Provisioning -We would require 7 VMs as per the following details, you can choose to use your own hypervisor to manage the VMs or use our [Wiab staging ansible playbook](https://github.com/wireapp/wire-server-deploy/blob/master/ansible/wiab-staging-provision.yml) against your physical node to setup the VMs. +Our deployment will be into 7 VMs with [Ubuntu 22](https://releases.ubuntu.com/jammy/), shown in the below VM Archetecture and Resource Allocation table, You can choose to use your own hypervisor to manage the VMs or use our [Wiab staging ansible playbook](https://github.com/wireapp/wire-server-deploy/blob/master/ansible/wiab-staging-provision.yml) against your physical node to setup the VMs. **VM Architecture and Resource Allocation:** @@ -50,20 +49,32 @@ We would require 7 VMs as per the following details, you can choose to use your - **kubenodes (kubenode1, kubenode2, kubenode3):** Run the Kubernetes cluster and host Wire backend services - **datanodes (datanode1, datanode2, datanode3):** Run distributed data services: - - Cassandra (distributed database) - - PostgreSQL (operational database) - - Elasticsearch (search engine) - - Minio (S3-compatible object storage) - - RabbitMQ (message broker) + - Cassandra + - PostgreSQL + - Elasticsearch + - Minio + - RabbitMQ - **assethost:** Hosts static assets to be used by kubenodes and datanodes +### Internet access for VMs: + +In most cases, Wire Server components do not require internet access, except in the following situations: +- **External email services** – If your users’ email servers are hosted on the public internet (for example, user@gmail.com etc). +- **Mobile push notifications (FCM/APNS)** – Required to enable notifications for Android and Apple mobile devices. Wire uses [AWS services](https://docs.wire.com/latest/how-to/install/infrastructure-configuration.html#enable-push-notifications-using-the-public-appstore-playstore-mobile-wire-clients) to relay notifications to Firebase Cloud Messaging (FCM) and Apple Push Notification Service (APNS). +- **Third-party content previews** – If you want clients to display previews for services such as Giphy, Google, Spotify, or SoundCloud. Wire provides a proxy service for third-party content so clients do not communicate directly with these services, preventing exposure of IP addresses, cookies, or other metadata. +- **Federation with other Wire servers** – Required if your deployment needs to federate with another Wire server hosted on the public internet. + +> **Note:** Internet access is also required by the cert-manager pods (via Let's Encrypt) to issue TLS certificates when manual certificates are not used. +> +> This internet access is temporarily enabled as described in [cert-manager behaviour in NAT / bridge environments](#cert-manager-behaviour-in-nat--bridge-environments) to allow certificate issuance. Once the certificates are successfully issued by cert-manager, the internet access is removed from the VMs. + ## WIAB staging ansible playbook -The ansible playbook will perform the following operations for you: +The WIAB-staging ansible playbooks require internet access to be available on the target machine. Assuming it is available, these playbooks will perform the following steps automatically: **System Setup & Networking**: - Updates all system packages and installs required tools (git, curl, docker, qemu, libvirt, yq, etc.) - - Configures SSH, firewall (nftables), and user permissions (sudo, kvm, docker groups) + - Configures SSH and user permissions (sudo, kvm, docker groups) **wire-server-deploy Artifact & Ubuntu Cloud Image**: - Downloads wire-server-deploy static artifact and Ubuntu cloud image @@ -79,7 +90,6 @@ The ansible playbook will perform the following operations for you: - Generates inventory.yml with actual VM IPs replacing placeholders - Configures network interface variables for all k8s-nodes and datanodes - *Note: Skip the Ansible playbook step if you are managing VMs with your own hypervisor.* ### Getting started with Ansible playbook @@ -106,8 +116,9 @@ cd wire-server-deploy **Step 2: Configure your Ansible inventory for your physical machine** A sample inventory is available at [ansible/inventory/demo/wiab-staging.yml](https://github.com/wireapp/wire-server-deploy/blob/master/ansible/inventory/demo/wiab-staging.yml). +Replace example.com with your physical machine (`adminhost`) address where KVM is available and adjust other variables like `ansible_user` and `ansible_ssh_private_key_file`. The SSH user for ansible `ansible_user` should have password-less `sudo` access. The adminhost should be running Ubuntu 22.04. From here on, we would refer the physical machine as `adminhost`. -*Note: Replace example.com with your physical machine (adminhost) address where KVM is available and adjust other variables like ansible_user and ansible_ssh_private_key_file. The SSH user for ansible `ansible_user` should have password-less `sudo` access. The physical host should be running Ubuntu 22.04.* +The `private_deployment` variable determines whether the VMs created below will have internet access. When set to `true` (default value), no internet access is available to VMs. Check [Internet access for VMs](#internet-access-for-vms) to understand more about it. **Step 3: Run the VM and network provision** @@ -119,28 +130,45 @@ ansible-playbook -i ansible/inventory/demo/wiab-staging.yml ansible/wiab-staging ## Ensure secondary ansible inventory for VMs -Now you should have 7 VMs running on your physical machine. If you have used the ansible playbook, you should also have a directory `/home/ansible_user/wire-server-deploy` with all resources required for further deployment. If you didn't use the above playbook, download the `wire-server-deploy` artifact shared by Wire support and unarchieve (tar tgz) it. +Now you should have 7 VMs running on your `adminhost`. If you have used the ansible playbook, you should also have a directory `/home/ansible_user/wire-server-deploy` with all resources required for further deployment. If you didn't use the above playbook, download the `wire-server-deploy` artifact shared by Wire support and extract it with tar. Ensure the inventory file `ansible/inventory/offline/inventory.yml` in the directory `/home/ansible_user/wire-server-deploy` contains values corresponding to your VMs. If you have already used the [Ansible playbook above](#getting-started-with-ansible-playbook) to set up VMs, this file should have been prepared for you. +The purpose of secondary ansible inventory is to interact only with the VMs. All the operations concerning the secondary inventory are meant to install datastores and k8s services. + ## Next steps -Since the inventory is ready, please continue with the following steps: +Once the inventory is ready, please continue with the following steps: > **Note**: All next steps assume that the wire-server-deploy artifact has been downloaded on the `adminhost` (your physical machine) and extracted at `/home/ansible_user/wire-server-deploy`. All commands from here on will be issued from this directory on the `adminhost`. Make sure you SSH into the node before proceeding. ### Environment Setup - **[Making tooling available in your environment](docs_ubuntu_22.04.md#making-tooling-available-in-your-environment)** - - Source the `bin/offline-env.sh` shell script by running `source bin/offline-env.sh` to set up a `d` alias that runs commands inside a Docker container with all necessary tools for offline deployment. + - Source the `bin/offline-env.sh` shell script by running following command to set up a `d` alias that runs commands inside a Docker container with all necessary tools for offline deployment. + ```bash + source bin/offline-env.sh + ``` + - You can always use this alias `d` later to interact with the ansible playbooks, k8s cluster and the helm charts. + - The docker container mounts everything here from the `wire-server-deploy` directory, hence this acts an entry point for all the future interactions with ansible, k8s and helm charts. - **[Generating secrets](docs_ubuntu_22.04.md#generating-secrets)** - - Run `./bin/offline-secrets.sh` to generate fresh secrets for Minio and coturn services. This creates two secret files: `ansible/inventory/group_vars/all/secrets.yaml` and `values/wire-server/secrets.yaml`. + - Run `bin/offline-secrets.sh` to generate fresh secrets for Minio and coturn services. It uses the docker container images shipped inside the `wire-server-deploy` directory. + ```bash + ./bin/offline-secrets.sh + ``` + - This creates following secret files: + - `ansible/inventory/group_vars/all/secrets.yaml` + - `values/wire-server/secrets.yaml` + - `values/coturn/prod-secrets.example.yaml` ### Kubernetes & Data Services Deployment - **[Deploying Kubernetes and stateful services](docs_ubuntu_22.04.md#deploying-kubernetes-and-stateful-services)** - - Run `d ./bin/offline-cluster.sh` to deploy Kubernetes and stateful services (Cassandra, PostgreSQL, Elasticsearch, Minio, RabbitMQ). This script deploys all infrastructure needed for Wire backend operations. + ```bash + d ./bin/offline-cluster.sh + ``` + - Run the above command to deploy Kubernetes and stateful services (Cassandra, PostgreSQL, Elasticsearch, Minio, RabbitMQ). This script deploys all infrastructure needed for Wire backend operations. ### Helm Operations to install wire services and supporting helm charts @@ -164,8 +192,8 @@ d sh -c 'TARGET_SYSTEM="example.dev" CERT_MASTER_EMAIL="certmaster@example.dev" ``` **Charts deployed by the script:** -- External datastores and helpers: `cassandra-external`, `elasticsearch-external`, `minio-external`, `rabbitmq-external`,`postgresql-external`, `databases-ephemeral`, `reaper`, `fake-aws`, `demo-smtp`. -- Wire services: `wire-server`, `webapp`, `account-pages`, `team-settings`, `smallstep-accomp`. +- External datastores and helpers: `cassandra-external`, `elasticsearch-external`, `postgresql-external`, `minio-external`, `rabbitmq-external`, `databases-ephemeral`, `reaper`, `fake-aws`, `smtp`. +- Wire services: `wire-server`, `webapp`, `account-pages`, `team-settings`. - Ingress and certificates: `ingress-nginx-controller`, `cert-manager`, `nginx-ingress-services`. - Calling services: `sftd`, `coturn`. @@ -177,59 +205,111 @@ d sh -c 'TARGET_SYSTEM="example.dev" CERT_MASTER_EMAIL="certmaster@example.dev" ## Network Traffic Configuration -### Bring traffic from the physical machine to Wire services in the k8s cluster +### Bring traffic from the adminhost to Wire services in the k8s cluster -If you used the Ansible playbook earlier, nftables firewall rules are pre-configured to forward traffic. If you set up VMs manually with your own hypervisor, you must manually configure network traffic flow using nftables as descibed below. +Our Wire services are ready to receive traffic but we must enable network access from the `adminhost` network interface to the k8s pods running in the virtual network. We can achieve it by setting up [nftables](https://documentation.ubuntu.com/security/security-features/network/firewall/nftables/) rules on the `adminhost`. When using any other type of firewall tools, please ensure following network configuration is achieved. **Required Network Configuration:** -The physical machine (adminhost) must forward traffic from external clients to the Kubernetes cluster running Wire services. This involves: - -1. **HTTP/HTTPS Traffic (Ingress)** - Forward ports 80 and 443 to the nginx-ingress-controller running on a Kubernetes node - - Port 80 (HTTP) → Kubernetes node port 31772 - - Port 443 (HTTPS) → Kubernetes node port 31773 - -2. **Calling Services Traffic (Coturn/SFT)** - Forward media and TURN protocol traffic to Coturn/SFT - - Port 3478 (TCP/UDP) → Coturn control traffic - - Ports 32768-65535 (UDP) → Media relay traffic for WebRTC calling +The `adminhost` must forward traffic from external clients to the Kubernetes cluster running Wire services. This involves: + +1. **HTTP/HTTPS Traffic (Ingress)** – Forward external web traffic to Kubernetes ingress with load balancing across nodes + - Port 80 (TCP, from any external source to adminhost WAN IP) → DNAT to any Kubernetes node on port 31772 → HTTP ingress + - Port 443 (TCP, from any external source to adminhost WAN IP) → DNAT to any Kubernetes node on port 31773 → HTTPS ingress + +2. **Calling Services Traffic (Coturn/SFT)** – Forward TURN control and media traffic to the dedicated calling node + - Port 3478 (TCP/UDP, from any external source to adminhost WAN IP) → DNAT to calling node → TURN control traffic + - Ports 32768–65535 (UDP, from any external source to adminhost WAN IP) → DNAT to calling node → WebRTC media relay + +3. **Normal Access Rules (Host-Level Access)** – Restrict direct access to adminhost + - Port 22 (TCP, from allowed sources to adminhost) → allow → SSH access + - Traffic from loopback and VM bridge interfaces → allow → internal communication + - Any traffic within VM network → allowed → ensures inter-node communication + - All other inbound traffic to adminhost → drop → default deny policy + +4. **Masquerading (If [Internet access for VMs](#internet-access-for-vms) is required)** – Enable outbound connectivity for VMs + - Any traffic from VM subnet leaving via WAN interface → SNAT/masquerade → ensures return traffic from internet. + +5. **Conditional Rules (cert-manager / HTTP-01 in NAT setups)** – Temporary adjustments for certificate validation + - DNAT hairpin traffic (VM → public IP → VM) → may require SNAT/masquerade on VM bridge → ensures return path during HTTP-01 self-checks + - Asymmetric routing scenarios → may require relaxed reverse path filtering → prevents packet drops during validation + +```mermaid +flowchart TB + +%% External Clients +Client[External Client] +LetsEncrypt["(Optional)
Let's Encrypt"] +Internet["(If Required)
Internet Services
(AWS/FCM/APNS, Email Services etc)"] + +%% Admin Host +AdminHost["AdminHost
(Firewall)"] + +%% VM Network +subgraph VM_Network ["VM Network (virbr0)"] + K1[KubeNode1] + K2[KubeNode2] + K3["KubeNode3
(CALLING NODE)"] +end + +%% Ingress Traffic +Client -->|HTTPS → wire-records.example.com| AdminHost +AdminHost -->|"DNAT →31772/31773"| K1 +AdminHost -->|"DNAT →31772/31773"| K2 +AdminHost -->|"DNAT →31772/31773"| K3 + +%% Calling Traffic +Client -->|TCP/UDP Calling| AdminHost +AdminHost -->|DNAT → Calling Node| K3 + +%% Outbound Traffic (Masquerade) +K1 -.->|SNAT via AdminHost| Internet +K2 -.->|SNAT via AdminHost| Internet +K3 -.->|SNAT via AdminHost| Internet + +%% Cert-Manager Flow +K1 <-.->|HTTP-01 self-check| AdminHost +AdminHost-.->|Request TLS certificate| LetsEncrypt +``` **Implementation:** -Use the detailed nftables rules in [../ansible/files/wiab_server_nftables.conf.j2](../ansible/files/wiab_server_nftables.conf.j2) as the template. The nftable configuration template covers: -- Defining your network variables (Coturn IP, Kubernetes node IP, WAN interface) -- Creating NAT rules for HTTP/HTTPS ingress traffic -- Setting up TURN protocol forwarding for Coturn and traffic for SFTD +The nftables rules are detailed in [wiab_server_nftables.conf.j2](https://github.com/wireapp/wire-server-deploy/blob/master/ansible/files/wiab_server_nftables.conf.j2). Please ensure no other firewall services like `ufw` or `iptables` are configured on the node before continuing. -*Note: If you have already ran the playbook wiab-staging-provision.yml then it is already be configured for you. Confirm it by checking if the wire endpoint `https://webapp.TARGET_SYSTEM` is reachable from public internet or your private network (in case of private network), but not from the adminhost itself.* - -You can also apply these rules using the Ansible playbook against your adminhost, by following: +If you have already used the `wiab-staging-provision.yml` ansible playbook to create the VMs, then you can apply these rules using the same playbook (with the tag `nftables`) against your adminhost, by following: ```bash -ansible-playbook -i inventory.yml ansible/wiab-staging-nftables.yml +ansible-playbook -i ansible/inventory/demo/wiab-staging.yml ansible/wiab-staging-provision.yml --tags nftables +``` +Alternatively, if you have not used the `wiab-staging-provision.yml` ansible playbook to create the VMs but would like to configure nftables rules, you can invoke the ansible playbook [wiab-staging-nftables.yaml](https://github.com/wireapp/wire-server-deploy/blob/master/ansible/wiab-staging-nftables.yaml) against the physical node. The playbook is available in the directory `wire-server-deploy/ansible`. + +The inventory file `inventory.yml` should define the following variables: +```yaml +wiab-staging: + hosts: + deploy_node: + # this should be the adminhost + ansible_host: example.com + ansible_ssh_common_args: '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=60 -o ServerAliveCountMax=3 -o TCPKeepAlive=yes' + ansible_user: 'demo' + ansible_ssh_private_key_file: "~/.ssh/id_ed25519" + vars: + # Kubernetes node IPs + kubenode1_ip: 192.168.122.11 + kubenode2_ip: 192.168.122.12 + kubenode3_ip: 192.168.122.13 + # Calling services node(kubenode3) + calling_node_ip: 192.168.122.13 + wire_comment: "wiab-stag" + # it will disable internet access to VMs created on the private network + private_deployment: true + # the playbook will try to find the default interface i.e. INF_WAN from ansible_default_ipv4.interface ``` -You can run the above playbook from local system or where you have cloned/downloaded the [Wire server deploy ansible playbooks](#getting-the-ansible-playbooks). - -The inventory should define the following variables: - -```ini -[all:vars] -# Kubernetes node IPs -kubenode1_ip=192.168.122.11 -kubenode2_ip=192.168.122.12 -kubenode3_ip=192.168.122.13 - -# Calling services node (usually kubenode3) -calling_node_ip=192.168.122.13 - -# Host WAN interface name -inf_wan=eth0 - -# These are the same as wiab-staging.yml -# user and ssh key for adminhost -ansible_user='demo' -ansible_ssh_private_key_file='~/.ssh/id_ed25519' - +To implement the nftables rules, execute the following command: +```bash +# assuming the inventory.yml storead at wire-server-deploy and run command from the same directory +ansible-playbook -i inventory.yml ansible/wiab-staging-nftables.yaml ``` ### cert-manager behaviour in NAT / bridge environments @@ -238,6 +318,21 @@ When cert-manager performs HTTP-01 self-checks inside the cluster, traffic can h - Pod → Node → host public IP → DNAT → Node → Ingress +> **Note**: Using Let's encrypt with `cert-manager` requires internet access ([to at least `acme-v02.api.letsencrypt.org`](https://letsencrypt.org/docs/acme-protocol-updates/)) to issue TLS certs. If you have chosen to keep the network private i.e. `private_deployment=true` for the VMs when applying nftables rules aka no internet access to VMs, then we need to make a temporary exception for this. +> +> To add a nftables masquerading rule for all outgoing traffic run the following command on the `adminhost` or make a similar change in your firewall: +> +> ```bash +> # Host WAN interface name +> INF_WAN=enp41s0 +> sudo nft insert rule ip nat POSTROUTING position 0 \ +> oifname $INF_WAN \ +> counter masquerade \ +> comment "wire-masquerade-for-letsencrypt" +> ``` +> +> If you are using a different implementation than nftables then please ensure Internet access to VMs. + In NAT/bridge setups (for example, using `virbr0` on the host): - If nftables DNAT rules exist in `PREROUTING` without a matching SNAT on `virbr0 → virbr0`, return packets may bypass the host and break conntrack, causing HTTP-01 timeouts and certificate verification failures. @@ -246,15 +341,25 @@ In NAT/bridge setups (for example, using `virbr0` on the host): Before changing anything, first verify whether certificate issuance is actually failing: 1. Check whether certificates are successfully issued: - ```bash - d kubectl get certificates - ``` -2. If certificates are not in `Ready=True` state, inspect cert-manager logs for HTTP-01 self-check or timeout errors: - ```bash - d kubectl logs -n cert-manager-ns - ``` + ```bash + d kubectl get certificates + ``` +2. Check if k8s pods can access to its own domain: + ```bash + # Replace below. To find the aws-sns pod id, run the command: + # d kubectl get pods -l 'app=fake-aws-sns' + d kubectl exec -ti fake-aws-sns- -- sh -c 'curl --connect-timeout 10 -v webapp.' + ``` +3. If certificates are not in `Ready=True` state, inspect cert-manager logs for HTTP-01 self-check or timeout errors: + ```bash + # To find the , run the following command: + # d kubectl get pods -n cert-manager-ns -l 'app=cert-manager' + d kubectl logs -n cert-manager-ns + ``` + +If you observe HTTP-01 challenge timeouts or self-check failures in a NAT/bridge environment, hairpin SNAT and relaxed reverse-path filtering handling may be required. One possible approach is by making following changes to the adminhost: -If you observe HTTP-01 challenge timeouts or self-check failures in a NAT/bridge environment, hairpin SNAT and relaxed reverse-path filtering handling may be required. One possible approach is: +> **Note:** All `nft` and `sysctl` commands should run on the adminhost. - Relax reverse-path filtering to loose mode to allow asymmetric flows: ```bash @@ -295,6 +400,20 @@ If you observe HTTP-01 challenge timeouts or self-check failures in a NAT/bridge xargs -r -I {} sudo nft delete rule ip nat POSTROUTING handle {} ``` +> **Note**: If above you had made an exception to allow temporary internet access to VMs by adding a nftables rules, then this should be removed now. +> +> To remove the nftables masquerading rule for all outgoing traffic run the following command: +> +> ```bash +> # remove the masquerading rule +> sudo nft -a list chain ip nat POSTROUTING | \ +> grep wire-masquerade-for-letsencrypt | \ +> sed -E 's/.*handle ([0-9]+).*/\1/' | \ +> xargs -r -I {} sudo nft delete rule ip nat POSTROUTING handle {} +> ``` +> +> If you are using a different implementation than nftables then please ensure temporary Internet access to VMs has been removed. + For additional background on when hairpin NAT is required and how it relates to WIAB Dev and WIAB Staging, see [Hairpin networking for WIAB Dev and WIAB Staging](tls-certificates.md#hairpin-networking-for-wiab-dev-and-wiab-staging). ## Further Reading @@ -304,7 +423,6 @@ For additional background on when hairpin NAT is required and how it relates to - **[Deploying webapp](docs_ubuntu_22.04.md#deploying-webapp)**: Read more about webapp deployment and domain configuration. - **[Deploying team-settings](docs_ubuntu_22.04.md#deploying-team-settings)**: Read more about team settings services. - **[Deploying account-pages](docs_ubuntu_22.04.md#deploying-account-pages)**: Read more about account management services. -- **[Deploying smallstep-accomp](docs_ubuntu_22.04.md#deploying-smallstep-accomp)**: Read more about the ACME companion. - **[Enabling emails for wire](smtp.md)**: Read more about SMTP options for onboarding email delivery and relay setup. - **[Deploy ingress-nginx-controller](docs_ubuntu_22.04.md#deploy-ingress-nginx-controller)**: Read more about ingress configuration and traffic forwarding requirements. - **[Acquiring / Deploying SSL Certificates](docs_ubuntu_22.04.md#acquiring--deploying-ssl-certificates)**: Read more about TLS options (Bring Your Own or cert-manager) and certificate requirements.