diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index a82dc3e..05758ee 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -1,11 +1,6 @@ -name: Update Changelog +name: Update Changelog (disabled) on: - push: - branches: [main] - paths-ignore: - - 'CHANGELOG.md' - - '.github/**' workflow_dispatch: inputs: since_tag: diff --git a/.github/workflows/publish-oci.yml b/.github/workflows/publish-oci.yml index 20c8faf..2ddf08a 100644 --- a/.github/workflows/publish-oci.yml +++ b/.github/workflows/publish-oci.yml @@ -30,6 +30,15 @@ jobs: with: version: v3.17.0 + - name: Build chart dependencies + run: | + for chart in charts/*/; do + if grep -q '^dependencies:' "${chart}Chart.yaml" 2>/dev/null; then + echo "Building dependencies for $(basename "${chart}")..." + helm dependency build "${chart}" + fi + done + - name: Lint all charts run: | for chart in charts/*/; do @@ -68,6 +77,15 @@ jobs: echo "${GITHUB_TOKEN}" | helm registry login ${REGISTRY} -u "${GH_ACTOR}" --password-stdin echo "${GITHUB_TOKEN}" | cosign login ${REGISTRY} -u "${GH_ACTOR}" --password-stdin + - name: Build chart dependencies + run: | + for chart in charts/*/; do + if grep -q '^dependencies:' "${chart}Chart.yaml" 2>/dev/null; then + echo "Building dependencies for $(basename "${chart}")..." + helm dependency build "${chart}" + fi + done + - name: Package, push, sign, and attach SBOM env: RELEASE_TAG: ${{ github.event.release.tag_name }} diff --git a/.github/workflows/validate-charts.yml b/.github/workflows/validate-charts.yml index a05ea7c..f5c4cdc 100644 --- a/.github/workflows/validate-charts.yml +++ b/.github/workflows/validate-charts.yml @@ -33,6 +33,15 @@ jobs: with: version: v3.17.0 + - name: Build chart dependencies + run: | + for chart in charts/*/; do + if grep -q '^dependencies:' "${chart}Chart.yaml" 2>/dev/null; then + echo "Building dependencies for $(basename "${chart}")..." + helm dependency build "${chart}" + fi + done + - name: Lint all charts run: | exit_code=0 @@ -57,6 +66,15 @@ jobs: with: version: v3.17.0 + - name: Build chart dependencies + run: | + for chart in charts/*/; do + if grep -q '^dependencies:' "${chart}Chart.yaml" 2>/dev/null; then + echo "Building dependencies for $(basename "${chart}")..." + helm dependency build "${chart}" + fi + done + - name: Template render all charts run: | exit_code=0 @@ -90,6 +108,12 @@ jobs: --set users.metrics.password=test \ > /dev/null || exit_code=1 ;; + countly-migration) + helm template test-release "${chart}" \ + --set backingServices.mongodb.uri=mongodb://test \ + --set backingServices.clickhouse.url=http://test \ + > /dev/null || exit_code=1 + ;; *) helm template test-release "${chart}" > /dev/null || exit_code=1 ;; diff --git a/README.md b/README.md index a9f5a97..6a9dc6d 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Helm charts for deploying Countly analytics on Kubernetes. ## Architecture -Five charts, each in its own namespace: +Six charts, each in its own namespace: | Chart | Namespace | Purpose | |-------|-----------|---------| @@ -13,6 +13,7 @@ Five charts, each in its own namespace: | `countly-clickhouse` | clickhouse | ClickHouse via ClickHouse Operator | | `countly-kafka` | kafka | Kafka via Strimzi Operator | | `countly-observability` | observability | Prometheus, Grafana, Loki, Tempo, Pyroscope | +| `countly-migration` | countly-migration | MongoDB to ClickHouse batch migration (with bundled Redis) | ### Architecture Overview @@ -46,6 +47,11 @@ flowchart TB mongo["MongoDB\n:27017"] end + subgraph mig-ns["countly-migration"] + migsvc["Migration Service\n:8080"] + redis["Redis\n:6379"] + end + subgraph obs-ns["observability"] prom["Prometheus"] grafana["Grafana"] @@ -66,6 +72,10 @@ flowchart TB brokers --> connect --> chserver keeper -.-> chserver + migsvc -->|read batches| mongo + migsvc -->|insert rows| chserver + migsvc <-.->|hot state| redis + alloy -.-> prom & loki & tempo & pyroscope prom & loki & tempo & pyroscope --> grafana ``` @@ -201,6 +211,12 @@ helm install countly-observability ./charts/countly-observability -n observabili -f profiles/sizing/production/observability.yaml \ -f profiles/observability/full/observability.yaml \ -f environments/my-deployment/observability.yaml + +# Optional: MongoDB to ClickHouse batch migration (includes bundled Redis) +helm install countly-migration ./charts/countly-migration -n countly-migration --create-namespace \ + --wait --timeout 5m \ + -f environments/my-deployment/migration.yaml \ + -f environments/my-deployment/secrets-migration.yaml ``` ## Configuration Model diff --git a/charts/countly-argocd/Chart.yaml b/charts/countly-argocd/Chart.yaml new file mode 100644 index 0000000..9ccc535 --- /dev/null +++ b/charts/countly-argocd/Chart.yaml @@ -0,0 +1,20 @@ +apiVersion: v2 +name: countly-argocd +description: ArgoCD app-of-apps for deploying Countly to one or more clusters +type: application +version: 0.1.0 +appVersion: "1.0.0" +home: https://countly.com +icon: https://count.ly/images/logos/countly-logo.svg +sources: + - https://github.com/Countly/countly-server +keywords: + - argocd + - gitops + - countly + - multi-cluster +maintainers: + - name: Countly + url: https://countly.com +annotations: + artifacthub.io/license: AGPL-3.0 diff --git a/charts/countly-argocd/examples/applicationset.yaml b/charts/countly-argocd/examples/applicationset.yaml new file mode 100644 index 0000000..9070151 --- /dev/null +++ b/charts/countly-argocd/examples/applicationset.yaml @@ -0,0 +1,142 @@ +# Alternative to the app-of-apps chart for 100+ customers. +# +# Instead of running `helm install` per customer, this single ApplicationSet +# generates all Applications from a list of customers. Add a new customer +# by adding an entry to the list — no new Helm release needed. +# +# Prerequisites: +# 1. ArgoCD ApplicationSet controller installed +# 2. Target clusters registered with ArgoCD +# 3. Environment directories exist per customer in the helm repo +# 4. Custom health checks in argocd-cm (see chart NOTES.txt) +# +# Apply: kubectl apply -f applicationset.yaml -n argocd +# Add customer: add entry to generators[].list.elements, re-apply + +# One ApplicationSet per component, each with the correct sync-wave. +# ArgoCD processes waves within a parent sync — use an app-of-apps +# root Application that points to a directory containing these files. + +--- +# Wave 0: MongoDB +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: countly-mongodb + namespace: argocd +spec: + generators: + - list: + elements: + - customer: customer-a + server: https://cluster-a.example.com + sizing: production + security: hardened + - customer: customer-b + server: https://cluster-b.example.com + sizing: small + security: open + # Add more customers here... + template: + metadata: + name: "{{customer}}-mongodb" + annotations: + argocd.argoproj.io/sync-wave: "0" + spec: + project: "{{customer}}" + source: + repoURL: https://github.com/Countly/helm.git + targetRevision: main + path: charts/countly-mongodb + helm: + releaseName: countly-mongodb + valueFiles: + - "../../environments/{{customer}}/global.yaml" + - "../../profiles/sizing/{{sizing}}/mongodb.yaml" + - "../../profiles/security/{{security}}/mongodb.yaml" + - "../../environments/{{customer}}/mongodb.yaml" + parameters: + - name: argocd.enabled + value: "true" + destination: + server: "{{server}}" + namespace: mongodb + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m + +--- +# Wave 0: ClickHouse (same pattern as MongoDB) +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: countly-clickhouse + namespace: argocd +spec: + generators: + - list: + elements: + - customer: customer-a + server: https://cluster-a.example.com + sizing: production + security: hardened + - customer: customer-b + server: https://cluster-b.example.com + sizing: small + security: open + template: + metadata: + name: "{{customer}}-clickhouse" + annotations: + argocd.argoproj.io/sync-wave: "0" + spec: + project: "{{customer}}" + source: + repoURL: https://github.com/Countly/helm.git + targetRevision: main + path: charts/countly-clickhouse + helm: + releaseName: countly-clickhouse + valueFiles: + - "../../environments/{{customer}}/global.yaml" + - "../../profiles/sizing/{{sizing}}/clickhouse.yaml" + - "../../profiles/security/{{security}}/clickhouse.yaml" + - "../../environments/{{customer}}/clickhouse.yaml" + parameters: + - name: argocd.enabled + value: "true" + destination: + server: "{{server}}" + namespace: clickhouse + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m + +# Repeat the same pattern for: +# - countly-kafka (wave 5) +# - countly (wave 10) +# - countly-observability (wave 15) +# - countly-migrations (wave 10, optional) +# +# For a DRY approach, use a Matrix generator combining the customer list +# with a component list to generate all Applications from a single spec. diff --git a/charts/countly-argocd/examples/multi-cluster.yaml b/charts/countly-argocd/examples/multi-cluster.yaml new file mode 100644 index 0000000..14eeb64 --- /dev/null +++ b/charts/countly-argocd/examples/multi-cluster.yaml @@ -0,0 +1,49 @@ +# Example: Deploy Countly to two clusters from a single ArgoCD instance. +# +# Prerequisites: +# 1. ArgoCD installed on a management cluster +# 2. Target clusters registered with ArgoCD: +# argocd cluster add cluster-a-context +# argocd cluster add cluster-b-context +# 3. Environment directories exist: +# environments/customer-a/ (global.yaml, mongodb.yaml, etc.) +# environments/customer-b/ (global.yaml, mongodb.yaml, etc.) +# 4. Custom health checks configured in argocd-cm (see NOTES.txt) +# +# Deploy: +# helm install customer-a charts/countly-argocd -f examples/multi-cluster.yaml -n argocd +# helm install customer-b charts/countly-argocd -f examples/multi-cluster.yaml --set environment=customer-b -n argocd + +# --- Customer A: Production, large, with migrations --- +repoURL: "https://github.com/Countly/helm.git" +targetRevision: main +environment: customer-a +project: countly-customer-a + +destination: + server: "https://cluster-a.example.com" + +global: + sizing: production + security: hardened + tls: letsencrypt + observability: full + kafkaConnect: throughput + +mongodb: + enabled: true +clickhouse: + enabled: true +kafka: + enabled: true +countly: + enabled: true +observability: + enabled: true +migrations: + enabled: true # This customer needs data migration + +syncPolicy: + automated: true + selfHeal: true + prune: true diff --git a/charts/countly-argocd/templates/NOTES.txt b/charts/countly-argocd/templates/NOTES.txt new file mode 100644 index 0000000..447620a --- /dev/null +++ b/charts/countly-argocd/templates/NOTES.txt @@ -0,0 +1,71 @@ +=== Countly ArgoCD Deployment === + +Environment: {{ .Values.environment }} +Cluster: {{ .Values.destination.server }} +Project: {{ include "countly-argocd.projectName" . }} + +Applications deployed (sync wave order): + Wave 0: {{ if .Values.mongodb.enabled }}mongodb{{ end }} {{ if .Values.clickhouse.enabled }}clickhouse{{ end }} + Wave 5: {{ if .Values.kafka.enabled }}kafka{{ end }} + Wave 10: {{ if .Values.countly.enabled }}countly{{ end }} {{ if .Values.migrations.enabled }}migrations{{ end }} + Wave 15: {{ if .Values.observability.enabled }}observability{{ end }} + +--- Status --- + + # List all Countly applications + kubectl get applications -n argocd -l app.kubernetes.io/instance={{ .Release.Name }} + + # Sync all + argocd app sync -l app.kubernetes.io/instance={{ .Release.Name }} + +--- Multi-Cluster --- + + # Deploy to another cluster + helm install countly- charts/countly-argocd \ + --set environment= \ + --set destination.server=https:// \ + -n argocd + +--- Required: ArgoCD Custom Health Checks --- + + Add these to your argocd-cm ConfigMap for sync waves to + block on actual readiness: + + resource.customizations.health.kafka.strimzi.io_Kafka: | + hs = {} + if obj.status ~= nil and obj.status.conditions ~= nil then + for _, c in ipairs(obj.status.conditions) do + if c.type == "Ready" and c.status == "True" then + hs.status = "Healthy"; hs.message = c.message or "Ready"; return hs + end + if c.type == "NotReady" then + hs.status = "Progressing"; hs.message = c.message or "Not ready"; return hs + end + end + end + hs.status = "Progressing"; hs.message = "Waiting for status"; return hs + + # Same pattern for: KafkaConnect, KafkaNodePool, KafkaConnector + + resource.customizations.health.clickhouse.com_ClickHouseCluster: | + hs = {} + if obj.status ~= nil and obj.status.status ~= nil then + if obj.status.status == "Completed" then + hs.status = "Healthy"; hs.message = "Completed"; return hs + end + end + hs.status = "Progressing"; hs.message = "Provisioning"; return hs + + resource.customizations.health.mongodbcommunity.mongodb.com_MongoDBCommunity: | + hs = {} + if obj.status ~= nil and obj.status.phase ~= nil then + if obj.status.phase == "Running" then + hs.status = "Healthy"; hs.message = "Running"; return hs + end + end + hs.status = "Progressing"; hs.message = "Provisioning"; return hs + +--- Teardown --- + + helm uninstall {{ .Release.Name }} -n argocd + # Cascading finalizers will delete all child Applications diff --git a/charts/countly-argocd/templates/_helpers.tpl b/charts/countly-argocd/templates/_helpers.tpl new file mode 100644 index 0000000..9aee842 --- /dev/null +++ b/charts/countly-argocd/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "countly-argocd.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "countly-argocd.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "countly-argocd.labels" -}} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +app.kubernetes.io/name: {{ include "countly-argocd.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +ArgoCD project name — unique per release to prevent multi-tenant collisions. +*/}} +{{- define "countly-argocd.projectName" -}} +{{- .Values.project | default (include "countly-argocd.fullname" .) }} +{{- end -}} + +{{/* +Sync policy block — reused by all Application templates. +Includes retry policy for resilience at scale (100+ customers = 600+ Applications). +*/}} +{{- define "countly-argocd.syncPolicy" -}} +syncPolicy: + {{- if .Values.syncPolicy.automated }} + automated: + prune: {{ .Values.syncPolicy.prune }} + selfHeal: {{ .Values.syncPolicy.selfHeal }} + {{- end }} + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + retry: + limit: {{ .Values.syncPolicy.retry.limit }} + backoff: + duration: {{ .Values.syncPolicy.retry.backoff.duration }} + factor: {{ .Values.syncPolicy.retry.backoff.factor }} + maxDuration: {{ .Values.syncPolicy.retry.backoff.maxDuration }} +{{- end -}} diff --git a/charts/countly-argocd/templates/app-clickhouse.yaml b/charts/countly-argocd/templates/app-clickhouse.yaml new file mode 100644 index 0000000..b164050 --- /dev/null +++ b/charts/countly-argocd/templates/app-clickhouse.yaml @@ -0,0 +1,43 @@ +{{- if .Values.clickhouse.enabled }} +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: {{ include "countly-argocd.fullname" . }}-clickhouse + namespace: argocd + labels: + {{- include "countly-argocd.labels" . | nindent 4 }} + annotations: + argocd.argoproj.io/sync-wave: "0" + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: {{ include "countly-argocd.projectName" . }} + source: + repoURL: {{ .Values.repoURL }} + targetRevision: {{ .Values.targetRevision }} + path: charts/countly-clickhouse + helm: + releaseName: countly-clickhouse + valueFiles: + - ../../environments/{{ .Values.environment }}/global.yaml + - ../../profiles/sizing/{{ .Values.global.sizing }}/clickhouse.yaml + - ../../profiles/security/{{ .Values.global.security }}/clickhouse.yaml + - ../../environments/{{ .Values.environment }}/clickhouse.yaml + - ../../environments/{{ .Values.environment }}/secrets-clickhouse.yaml + parameters: + - name: argocd.enabled + value: "true" + destination: + server: {{ .Values.destination.server }} + namespace: {{ .Values.clickhouse.namespace }} + {{- include "countly-argocd.syncPolicy" . | nindent 2 }} + ignoreDifferences: + - group: clickhouse.com + kind: ClickHouseCluster + jsonPointers: + - /status + - group: clickhouse.com + kind: KeeperCluster + jsonPointers: + - /status +{{- end }} diff --git a/charts/countly-argocd/templates/app-countly.yaml b/charts/countly-argocd/templates/app-countly.yaml new file mode 100644 index 0000000..54c8592 --- /dev/null +++ b/charts/countly-argocd/templates/app-countly.yaml @@ -0,0 +1,41 @@ +{{- if .Values.countly.enabled }} +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: {{ include "countly-argocd.fullname" . }}-countly + namespace: argocd + labels: + {{- include "countly-argocd.labels" . | nindent 4 }} + annotations: + argocd.argoproj.io/sync-wave: "10" + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: {{ include "countly-argocd.projectName" . }} + source: + repoURL: {{ .Values.repoURL }} + targetRevision: {{ .Values.targetRevision }} + path: charts/countly + helm: + releaseName: countly + valueFiles: + - ../../environments/{{ .Values.environment }}/global.yaml + - ../../profiles/sizing/{{ .Values.global.sizing }}/countly.yaml + - ../../profiles/tls/{{ .Values.global.tls }}/countly.yaml + - ../../profiles/observability/{{ .Values.global.observability }}/countly.yaml + - ../../profiles/security/{{ .Values.global.security }}/countly.yaml + - ../../environments/{{ .Values.environment }}/countly.yaml + - ../../environments/{{ .Values.environment }}/secrets-countly.yaml + parameters: + - name: argocd.enabled + value: "true" + destination: + server: {{ .Values.destination.server }} + namespace: {{ .Values.countly.namespace }} + {{- include "countly-argocd.syncPolicy" . | nindent 2 }} + ignoreDifferences: + - group: networking.k8s.io + kind: Ingress + jsonPointers: + - /status +{{- end }} diff --git a/charts/countly-argocd/templates/app-kafka.yaml b/charts/countly-argocd/templates/app-kafka.yaml new file mode 100644 index 0000000..b373087 --- /dev/null +++ b/charts/countly-argocd/templates/app-kafka.yaml @@ -0,0 +1,53 @@ +{{- if .Values.kafka.enabled }} +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: {{ include "countly-argocd.fullname" . }}-kafka + namespace: argocd + labels: + {{- include "countly-argocd.labels" . | nindent 4 }} + annotations: + argocd.argoproj.io/sync-wave: "5" + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: {{ include "countly-argocd.projectName" . }} + source: + repoURL: {{ .Values.repoURL }} + targetRevision: {{ .Values.targetRevision }} + path: charts/countly-kafka + helm: + releaseName: countly-kafka + valueFiles: + - ../../environments/{{ .Values.environment }}/global.yaml + - ../../profiles/sizing/{{ .Values.global.sizing }}/kafka.yaml + - ../../profiles/kafka-connect/{{ .Values.global.kafkaConnect }}/kafka.yaml + - ../../profiles/observability/{{ .Values.global.observability }}/kafka.yaml + - ../../profiles/security/{{ .Values.global.security }}/kafka.yaml + - ../../environments/{{ .Values.environment }}/kafka.yaml + - ../../environments/{{ .Values.environment }}/secrets-kafka.yaml + parameters: + - name: argocd.enabled + value: "true" + destination: + server: {{ .Values.destination.server }} + namespace: {{ .Values.kafka.namespace }} + {{- include "countly-argocd.syncPolicy" . | nindent 2 }} + ignoreDifferences: + - group: kafka.strimzi.io + kind: Kafka + jsonPointers: + - /status + - group: kafka.strimzi.io + kind: KafkaConnect + jsonPointers: + - /status + - group: kafka.strimzi.io + kind: KafkaConnector + jsonPointers: + - /status + - group: kafka.strimzi.io + kind: KafkaNodePool + jsonPointers: + - /status +{{- end }} diff --git a/charts/countly-argocd/templates/app-migration.yaml b/charts/countly-argocd/templates/app-migration.yaml new file mode 100644 index 0000000..86ae26f --- /dev/null +++ b/charts/countly-argocd/templates/app-migration.yaml @@ -0,0 +1,32 @@ +{{- if .Values.migration.enabled }} +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: {{ include "countly-argocd.fullname" . }}-migration + namespace: argocd + labels: + {{- include "countly-argocd.labels" . | nindent 4 }} + annotations: + argocd.argoproj.io/sync-wave: "10" + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: {{ include "countly-argocd.projectName" . }} + source: + repoURL: {{ .Values.repoURL }} + targetRevision: {{ .Values.targetRevision }} + path: charts/countly-migration + helm: + releaseName: countly-migration + valueFiles: + - ../../environments/{{ .Values.environment }}/global.yaml + - ../../environments/{{ .Values.environment }}/migration.yaml + - ../../environments/{{ .Values.environment }}/secrets-migration.yaml + parameters: + - name: argocd.enabled + value: "true" + destination: + server: {{ .Values.destination.server }} + namespace: {{ .Values.migration.namespace }} + {{- include "countly-argocd.syncPolicy" . | nindent 2 }} +{{- end }} diff --git a/charts/countly-argocd/templates/app-migrations.yaml b/charts/countly-argocd/templates/app-migrations.yaml new file mode 100644 index 0000000..2d34cc4 --- /dev/null +++ b/charts/countly-argocd/templates/app-migrations.yaml @@ -0,0 +1,45 @@ +{{- if .Values.migrations.enabled }} +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: {{ include "countly-argocd.fullname" . }}-migrations + namespace: argocd + labels: + {{- include "countly-argocd.labels" . | nindent 4 }} + annotations: + argocd.argoproj.io/sync-wave: "10" + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: {{ include "countly-argocd.projectName" . }} + source: + repoURL: {{ .Values.repoURL }} + targetRevision: {{ .Values.targetRevision }} + path: charts/countly-migrations + helm: + releaseName: countly-migrations + valueFiles: + - ../../environments/{{ .Values.environment }}/global.yaml + - ../../environments/{{ .Values.environment }}/migrations.yaml + - ../../environments/{{ .Values.environment }}/secrets-migrations.yaml + parameters: + - name: argocd.enabled + value: "true" + destination: + server: {{ .Values.destination.server }} + namespace: {{ .Values.migrations.namespace }} + {{- include "countly-argocd.syncPolicy" . | nindent 2 }} + ignoreDifferences: + - group: kafka.strimzi.io + kind: Kafka + jsonPointers: + - /status + - group: kafka.strimzi.io + kind: KafkaConnect + jsonPointers: + - /status + - group: kafka.strimzi.io + kind: KafkaConnector + jsonPointers: + - /status +{{- end }} diff --git a/charts/countly-argocd/templates/app-mongodb.yaml b/charts/countly-argocd/templates/app-mongodb.yaml new file mode 100644 index 0000000..ce470a3 --- /dev/null +++ b/charts/countly-argocd/templates/app-mongodb.yaml @@ -0,0 +1,39 @@ +{{- if .Values.mongodb.enabled }} +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: {{ include "countly-argocd.fullname" . }}-mongodb + namespace: argocd + labels: + {{- include "countly-argocd.labels" . | nindent 4 }} + annotations: + argocd.argoproj.io/sync-wave: "0" + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: {{ include "countly-argocd.projectName" . }} + source: + repoURL: {{ .Values.repoURL }} + targetRevision: {{ .Values.targetRevision }} + path: charts/countly-mongodb + helm: + releaseName: countly-mongodb + valueFiles: + - ../../environments/{{ .Values.environment }}/global.yaml + - ../../profiles/sizing/{{ .Values.global.sizing }}/mongodb.yaml + - ../../profiles/security/{{ .Values.global.security }}/mongodb.yaml + - ../../environments/{{ .Values.environment }}/mongodb.yaml + - ../../environments/{{ .Values.environment }}/secrets-mongodb.yaml + parameters: + - name: argocd.enabled + value: "true" + destination: + server: {{ .Values.destination.server }} + namespace: {{ .Values.mongodb.namespace }} + {{- include "countly-argocd.syncPolicy" . | nindent 2 }} + ignoreDifferences: + - group: mongodbcommunity.mongodb.com + kind: MongoDBCommunity + jsonPointers: + - /status +{{- end }} diff --git a/charts/countly-argocd/templates/app-observability.yaml b/charts/countly-argocd/templates/app-observability.yaml new file mode 100644 index 0000000..27276d7 --- /dev/null +++ b/charts/countly-argocd/templates/app-observability.yaml @@ -0,0 +1,35 @@ +{{- if .Values.observability.enabled }} +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: {{ include "countly-argocd.fullname" . }}-observability + namespace: argocd + labels: + {{- include "countly-argocd.labels" . | nindent 4 }} + annotations: + argocd.argoproj.io/sync-wave: "15" + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: {{ include "countly-argocd.projectName" . }} + source: + repoURL: {{ .Values.repoURL }} + targetRevision: {{ .Values.targetRevision }} + path: charts/countly-observability + helm: + releaseName: countly-observability + valueFiles: + - ../../environments/{{ .Values.environment }}/global.yaml + - ../../profiles/sizing/{{ .Values.global.sizing }}/observability.yaml + - ../../profiles/observability/{{ .Values.global.observability }}/observability.yaml + - ../../profiles/security/{{ .Values.global.security }}/observability.yaml + - ../../environments/{{ .Values.environment }}/observability.yaml + - ../../environments/{{ .Values.environment }}/secrets-observability.yaml + parameters: + - name: argocd.enabled + value: "true" + destination: + server: {{ .Values.destination.server }} + namespace: {{ .Values.observability.namespace }} + {{- include "countly-argocd.syncPolicy" . | nindent 2 }} +{{- end }} diff --git a/charts/countly-argocd/templates/project.yaml b/charts/countly-argocd/templates/project.yaml new file mode 100644 index 0000000..5cbad53 --- /dev/null +++ b/charts/countly-argocd/templates/project.yaml @@ -0,0 +1,28 @@ +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: {{ include "countly-argocd.projectName" . }} + namespace: argocd + labels: + {{- include "countly-argocd.labels" . | nindent 4 }} +spec: + description: "Countly analytics platform ({{ .Values.environment }})" + sourceRepos: + - {{ .Values.repoURL | quote }} + destinations: + - namespace: "*" + server: {{ .Values.destination.server }} + clusterResourceWhitelist: + - group: storage.k8s.io + kind: StorageClass + - group: rbac.authorization.k8s.io + kind: ClusterRole + - group: rbac.authorization.k8s.io + kind: ClusterRoleBinding + - group: cert-manager.io + kind: ClusterIssuer + namespaceResourceWhitelist: + - group: "*" + kind: "*" + orphanedResources: + warn: true diff --git a/charts/countly-argocd/values.yaml b/charts/countly-argocd/values.yaml new file mode 100644 index 0000000..eaa6aca --- /dev/null +++ b/charts/countly-argocd/values.yaml @@ -0,0 +1,63 @@ +# -- Git repo containing the Helm charts +repoURL: "https://github.com/Countly/helm.git" +targetRevision: main + +# -- Environment name (maps to environments// directory) +environment: example-production + +# -- Target cluster +destination: + server: "https://kubernetes.default.svc" + +# -- ArgoCD project name (defaults to release name if empty) +# Each customer MUST have a unique project to avoid collisions. +project: "" + +# -- Profile selections (passed to child charts via valueFiles) +global: + sizing: production + security: hardened + tls: letsencrypt + observability: full + kafkaConnect: balanced + +# -- Component toggles +mongodb: + enabled: true + namespace: mongodb + +clickhouse: + enabled: true + namespace: clickhouse + +kafka: + enabled: true + namespace: kafka + +countly: + enabled: true + namespace: countly + +observability: + enabled: true + namespace: observability + +migrations: + enabled: false + namespace: kafka-migrations + +migration: + enabled: false + namespace: countly-migration + +# -- Sync policy for child Applications +syncPolicy: + automated: true + selfHeal: true + prune: true + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m diff --git a/charts/countly-clickhouse/templates/_helpers.tpl b/charts/countly-clickhouse/templates/_helpers.tpl index 73c06a9..9dcd898 100644 --- a/charts/countly-clickhouse/templates/_helpers.tpl +++ b/charts/countly-clickhouse/templates/_helpers.tpl @@ -116,3 +116,12 @@ Password secret name for ClickHouse default user {{ .Values.auth.defaultUserPassword.secretName }} {{- end -}} {{- end -}} + +{{/* +ArgoCD sync-wave annotation (only when argocd.enabled). +*/}} +{{- define "countly-clickhouse.syncWave" -}} +{{- if .root.Values.argocd.enabled }} +argocd.argoproj.io/sync-wave: {{ .wave | quote }} +{{- end }} +{{- end -}} diff --git a/charts/countly-clickhouse/templates/clickhousecluster.yaml b/charts/countly-clickhouse/templates/clickhousecluster.yaml index 037612e..befc0a8 100644 --- a/charts/countly-clickhouse/templates/clickhousecluster.yaml +++ b/charts/countly-clickhouse/templates/clickhousecluster.yaml @@ -4,6 +4,7 @@ metadata: name: {{ include "countly-clickhouse.fullname" . }} annotations: "helm.sh/resource-policy": keep + {{- include "countly-clickhouse.syncWave" (dict "wave" "5" "root" .) | nindent 4 }} labels: {{- include "countly-clickhouse.labels" . | nindent 4 }} spec: diff --git a/charts/countly-clickhouse/templates/keepercluster.yaml b/charts/countly-clickhouse/templates/keepercluster.yaml index 8b92a05..19771e1 100644 --- a/charts/countly-clickhouse/templates/keepercluster.yaml +++ b/charts/countly-clickhouse/templates/keepercluster.yaml @@ -4,6 +4,7 @@ metadata: name: {{ include "countly-clickhouse.fullname" . }}-keeper annotations: "helm.sh/resource-policy": keep + {{- include "countly-clickhouse.syncWave" (dict "wave" "3" "root" .) | nindent 4 }} labels: {{- include "countly-clickhouse.labels" . | nindent 4 }} spec: diff --git a/charts/countly-clickhouse/templates/namespace.yaml b/charts/countly-clickhouse/templates/namespace.yaml index a372fc8..fe13a22 100644 --- a/charts/countly-clickhouse/templates/namespace.yaml +++ b/charts/countly-clickhouse/templates/namespace.yaml @@ -5,4 +5,8 @@ metadata: name: {{ .Release.Namespace }} labels: {{- include "countly-clickhouse.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-clickhouse.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} + {{- end }} {{- end }} diff --git a/charts/countly-clickhouse/templates/networkpolicy.yaml b/charts/countly-clickhouse/templates/networkpolicy.yaml index 0de5fb8..7de16b9 100644 --- a/charts/countly-clickhouse/templates/networkpolicy.yaml +++ b/charts/countly-clickhouse/templates/networkpolicy.yaml @@ -5,6 +5,10 @@ metadata: name: {{ include "countly-clickhouse.fullname" . }}-default-deny labels: {{- include "countly-clickhouse.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-clickhouse.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} + {{- end }} spec: podSelector: {} policyTypes: diff --git a/charts/countly-clickhouse/templates/pdb-keeper.yaml b/charts/countly-clickhouse/templates/pdb-keeper.yaml index 1c83556..5c64a93 100644 --- a/charts/countly-clickhouse/templates/pdb-keeper.yaml +++ b/charts/countly-clickhouse/templates/pdb-keeper.yaml @@ -5,6 +5,10 @@ metadata: name: {{ include "countly-clickhouse.fullname" . }}-keeper-pdb labels: {{- include "countly-clickhouse.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-clickhouse.syncWave" (dict "wave" "3" "root" .) | nindent 4 }} + {{- end }} spec: maxUnavailable: {{ .Values.podDisruptionBudget.keeper.maxUnavailable | default 1 }} selector: diff --git a/charts/countly-clickhouse/templates/pdb-server.yaml b/charts/countly-clickhouse/templates/pdb-server.yaml index 20aef77..fa8899a 100644 --- a/charts/countly-clickhouse/templates/pdb-server.yaml +++ b/charts/countly-clickhouse/templates/pdb-server.yaml @@ -5,6 +5,10 @@ metadata: name: {{ include "countly-clickhouse.fullname" . }}-server-pdb labels: {{- include "countly-clickhouse.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-clickhouse.syncWave" (dict "wave" "5" "root" .) | nindent 4 }} + {{- end }} spec: maxUnavailable: {{ .Values.podDisruptionBudget.server.maxUnavailable | default 1 }} selector: diff --git a/charts/countly-clickhouse/templates/secret-default-password.yaml b/charts/countly-clickhouse/templates/secret-default-password.yaml index b5cd2cf..cbeaa37 100644 --- a/charts/countly-clickhouse/templates/secret-default-password.yaml +++ b/charts/countly-clickhouse/templates/secret-default-password.yaml @@ -9,6 +9,7 @@ metadata: {{- if .Values.secrets.keep }} helm.sh/resource-policy: keep {{- end }} + {{- include "countly-clickhouse.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} type: Opaque data: {{- $existing := lookup "v1" "Secret" .Release.Namespace .Values.auth.defaultUserPassword.secretName }} diff --git a/charts/countly-clickhouse/templates/service-metrics.yaml b/charts/countly-clickhouse/templates/service-metrics.yaml index b5f420e..f301696 100644 --- a/charts/countly-clickhouse/templates/service-metrics.yaml +++ b/charts/countly-clickhouse/templates/service-metrics.yaml @@ -7,6 +7,10 @@ metadata: {{- include "countly-clickhouse.labels" . | nindent 4 }} countly.io/component: clickhouse-server countly.io/metrics: "true" + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-clickhouse.syncWave" (dict "wave" "10" "root" .) | nindent 4 }} + {{- end }} spec: type: ClusterIP {{- if eq (.Values.serviceMonitor.serviceType | default "headless") "headless" }} @@ -30,6 +34,10 @@ metadata: {{- include "countly-clickhouse.labels" . | nindent 4 }} countly.io/component: clickhouse-keeper countly.io/metrics: "true" + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-clickhouse.syncWave" (dict "wave" "10" "root" .) | nindent 4 }} + {{- end }} spec: type: ClusterIP {{- if eq (.Values.serviceMonitor.serviceType | default "headless") "headless" }} diff --git a/charts/countly-clickhouse/templates/servicemonitor.yaml b/charts/countly-clickhouse/templates/servicemonitor.yaml index 8aace64..46b0e8f 100644 --- a/charts/countly-clickhouse/templates/servicemonitor.yaml +++ b/charts/countly-clickhouse/templates/servicemonitor.yaml @@ -5,6 +5,10 @@ metadata: name: {{ include "countly-clickhouse.fullname" . }}-server labels: {{- include "countly-clickhouse.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-clickhouse.syncWave" (dict "wave" "10" "root" .) | nindent 4 }} + {{- end }} spec: namespaceSelector: matchNames: @@ -24,6 +28,10 @@ metadata: name: {{ include "countly-clickhouse.fullname" . }}-keeper labels: {{- include "countly-clickhouse.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-clickhouse.syncWave" (dict "wave" "10" "root" .) | nindent 4 }} + {{- end }} spec: namespaceSelector: matchNames: diff --git a/charts/countly-clickhouse/values.yaml b/charts/countly-clickhouse/values.yaml index 3affe4c..2e77eb2 100644 --- a/charts/countly-clickhouse/values.yaml +++ b/charts/countly-clickhouse/values.yaml @@ -12,6 +12,9 @@ fullnameOverride: "" createNamespace: false +argocd: + enabled: false + clickhouseOperator: apiVersion: clickhouse.com/v1alpha1 diff --git a/charts/countly-kafka/templates/_helpers.tpl b/charts/countly-kafka/templates/_helpers.tpl index ee8a001..f51a958 100644 --- a/charts/countly-kafka/templates/_helpers.tpl +++ b/charts/countly-kafka/templates/_helpers.tpl @@ -77,6 +77,16 @@ countly-clickhouse-clickhouse-headless.{{ .Values.clickhouseNamespace | default {{- end -}} {{- end -}} +{{/* +ArgoCD sync-wave annotation (only when argocd.enabled). +Usage: {{- include "countly-kafka.syncWave" (dict "wave" "5" "root" .) | nindent 4 }} +*/}} +{{- define "countly-kafka.syncWave" -}} +{{- if .root.Values.argocd.enabled }} +argocd.argoproj.io/sync-wave: {{ .wave | quote }} +{{- end }} +{{- end -}} + {{/* ClickHouse Connect secret name */}} diff --git a/charts/countly-kafka/templates/configmap-connect-env.yaml b/charts/countly-kafka/templates/configmap-connect-env.yaml index d250887..636766a 100644 --- a/charts/countly-kafka/templates/configmap-connect-env.yaml +++ b/charts/countly-kafka/templates/configmap-connect-env.yaml @@ -5,6 +5,10 @@ metadata: name: {{ include "countly-kafka.fullname" . }}-connect-env labels: {{- include "countly-kafka.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-kafka.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} + {{- end }} data: CLICKHOUSE_HOST: {{ include "countly-kafka.clickhouseHost" . | quote }} CLICKHOUSE_PORT: {{ .Values.kafkaConnect.clickhouse.port | quote }} diff --git a/charts/countly-kafka/templates/configmap-metrics.yaml b/charts/countly-kafka/templates/configmap-metrics.yaml index f8613cd..f4b9767 100644 --- a/charts/countly-kafka/templates/configmap-metrics.yaml +++ b/charts/countly-kafka/templates/configmap-metrics.yaml @@ -5,6 +5,10 @@ metadata: name: {{ include "countly-kafka.fullname" . }}-metrics labels: {{- include "countly-kafka.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-kafka.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} + {{- end }} data: kafka-metrics-config.yml: | lowercaseOutputName: true diff --git a/charts/countly-kafka/templates/hpa-connect.yaml b/charts/countly-kafka/templates/hpa-connect.yaml index 5173819..fdb9333 100644 --- a/charts/countly-kafka/templates/hpa-connect.yaml +++ b/charts/countly-kafka/templates/hpa-connect.yaml @@ -5,6 +5,10 @@ metadata: name: {{ include "countly-kafka.connectName" . }}-hpa labels: {{- include "countly-kafka.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-kafka.syncWave" (dict "wave" "10" "root" .) | nindent 4 }} + {{- end }} spec: scaleTargetRef: apiVersion: {{ .Values.strimzi.apiVersion }} diff --git a/charts/countly-kafka/templates/kafka.yaml b/charts/countly-kafka/templates/kafka.yaml index 821a4c5..1b174f5 100644 --- a/charts/countly-kafka/templates/kafka.yaml +++ b/charts/countly-kafka/templates/kafka.yaml @@ -43,6 +43,7 @@ metadata: name: {{ include "countly-kafka.fullname" . }}-brokers annotations: "helm.sh/resource-policy": keep + {{- include "countly-kafka.syncWave" (dict "wave" "5" "root" .) | nindent 4 }} labels: strimzi.io/cluster: {{ include "countly-kafka.fullname" . }} {{- include "countly-kafka.labels" . | nindent 4 }} @@ -100,6 +101,7 @@ metadata: name: {{ include "countly-kafka.fullname" . }}-controllers annotations: "helm.sh/resource-policy": keep + {{- include "countly-kafka.syncWave" (dict "wave" "5" "root" .) | nindent 4 }} labels: strimzi.io/cluster: {{ include "countly-kafka.fullname" . }} {{- include "countly-kafka.labels" . | nindent 4 }} @@ -152,6 +154,7 @@ metadata: "helm.sh/resource-policy": keep strimzi.io/kraft: enabled strimzi.io/node-pools: enabled + {{- include "countly-kafka.syncWave" (dict "wave" "5" "root" .) | nindent 4 }} spec: kafka: version: {{ .Values.version | quote }} diff --git a/charts/countly-kafka/templates/kafkaconnect.yaml b/charts/countly-kafka/templates/kafkaconnect.yaml index e28384d..fc6994c 100644 --- a/charts/countly-kafka/templates/kafkaconnect.yaml +++ b/charts/countly-kafka/templates/kafkaconnect.yaml @@ -7,6 +7,7 @@ metadata: {{- include "countly-kafka.labels" . | nindent 4 }} annotations: strimzi.io/use-connector-resources: "true" + {{- include "countly-kafka.syncWave" (dict "wave" "10" "root" .) | nindent 4 }} spec: version: {{ .Values.version | quote }} replicas: {{ .Values.kafkaConnect.replicas }} diff --git a/charts/countly-kafka/templates/kafkaconnectors.yaml b/charts/countly-kafka/templates/kafkaconnectors.yaml index 21fb798..6316901 100644 --- a/charts/countly-kafka/templates/kafkaconnectors.yaml +++ b/charts/countly-kafka/templates/kafkaconnectors.yaml @@ -9,6 +9,10 @@ metadata: labels: strimzi.io/cluster: {{ include "countly-kafka.connectName" $ }} {{- include "countly-kafka.labels" $ | nindent 4 }} + {{- if $.Values.argocd.enabled }} + annotations: + {{- include "countly-kafka.syncWave" (dict "wave" "15" "root" $) | nindent 4 }} + {{- end }} spec: class: {{ $connector.class }} tasksMax: {{ $connector.tasksMax }} diff --git a/charts/countly-kafka/templates/namespace.yaml b/charts/countly-kafka/templates/namespace.yaml index 9d6596a..38754c4 100644 --- a/charts/countly-kafka/templates/namespace.yaml +++ b/charts/countly-kafka/templates/namespace.yaml @@ -5,4 +5,8 @@ metadata: name: {{ .Release.Namespace }} labels: {{- include "countly-kafka.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-kafka.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} + {{- end }} {{- end }} diff --git a/charts/countly-kafka/templates/networkpolicy.yaml b/charts/countly-kafka/templates/networkpolicy.yaml index 7168a09..94d329b 100644 --- a/charts/countly-kafka/templates/networkpolicy.yaml +++ b/charts/countly-kafka/templates/networkpolicy.yaml @@ -10,6 +10,10 @@ metadata: name: {{ include "countly-kafka.fullname" . }}-connect-api labels: {{- include "countly-kafka.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-kafka.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} + {{- end }} spec: podSelector: matchLabels: @@ -36,6 +40,10 @@ metadata: name: {{ include "countly-kafka.fullname" . }}-default-deny labels: {{- include "countly-kafka.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-kafka.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} + {{- end }} spec: podSelector: {} policyTypes: @@ -48,6 +56,10 @@ metadata: name: {{ include "countly-kafka.fullname" . }}-allow-kafka labels: {{- include "countly-kafka.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-kafka.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} + {{- end }} spec: podSelector: {} policyTypes: @@ -74,6 +86,10 @@ metadata: name: {{ include "countly-kafka.fullname" . }}-allow-monitoring labels: {{- include "countly-kafka.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-kafka.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} + {{- end }} spec: podSelector: {} policyTypes: diff --git a/charts/countly-kafka/templates/secret-clickhouse-connect.yaml b/charts/countly-kafka/templates/secret-clickhouse-connect.yaml index 6e4ac86..6039fca 100644 --- a/charts/countly-kafka/templates/secret-clickhouse-connect.yaml +++ b/charts/countly-kafka/templates/secret-clickhouse-connect.yaml @@ -9,6 +9,7 @@ metadata: {{- if .Values.secrets.keep }} helm.sh/resource-policy: keep {{- end }} + {{- include "countly-kafka.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} type: Opaque data: {{- $secretName := .Values.kafkaConnect.clickhouse.secretName }} diff --git a/charts/countly-kafka/values.yaml b/charts/countly-kafka/values.yaml index 8bfcd0d..974a6d6 100644 --- a/charts/countly-kafka/values.yaml +++ b/charts/countly-kafka/values.yaml @@ -12,6 +12,9 @@ fullnameOverride: "" createNamespace: false +argocd: + enabled: false + strimzi: apiVersion: kafka.strimzi.io/v1 diff --git a/charts/countly-migration/Chart.yaml b/charts/countly-migration/Chart.yaml new file mode 100644 index 0000000..3632797 --- /dev/null +++ b/charts/countly-migration/Chart.yaml @@ -0,0 +1,29 @@ +apiVersion: v2 +name: countly-migration +description: MongoDB to ClickHouse batch migration service for Countly drill events +type: application +version: 0.1.0 +appVersion: "1.0.0" +home: https://countly.com +icon: https://count.ly/images/logos/countly-logo.svg +sources: + - https://github.com/Countly/countly-server +keywords: + - migration + - clickhouse + - mongodb + - countly + - batch-migration +maintainers: + - name: Countly + url: https://countly.com +dependencies: + - name: redis + version: ">=25.0.0" + repository: https://charts.bitnami.com/bitnami + condition: redis.enabled +annotations: + artifacthub.io/license: AGPL-3.0 + artifacthub.io/links: | + - name: Documentation + url: https://github.com/Countly/helm diff --git a/charts/countly-migration/README.md b/charts/countly-migration/README.md new file mode 100644 index 0000000..a3f30c2 --- /dev/null +++ b/charts/countly-migration/README.md @@ -0,0 +1,352 @@ +# Countly Migration Helm Chart + +Deploys the MongoDB-to-ClickHouse batch migration service for Countly drill events. Reads `drill_events*` collections from MongoDB, transforms documents, and inserts them into the ClickHouse `drill_events` table. Includes a bundled Redis instance for migration state tracking. + +**Chart version:** 0.1.0 +**App version:** 1.0.0 + +--- + +## Architecture + +```mermaid +flowchart LR + subgraph source["Source"] + mongo["MongoDB\ncountly_drill.drill_events*"] + end + + subgraph migration["countly-migration namespace"] + svc["Migration Service\n:8080"] + redis["Redis\n:6379"] + end + + subgraph target["Target"] + ch["ClickHouse\ncountly_drill.drill_events"] + end + + mongo -->|read batches| svc + svc -->|insert rows| ch + svc <-->|hot state, bitmaps,\nerror buffers| redis + svc -->|run manifests| mongo +``` + +The migration service is a **singleton Deployment** with `Recreate` strategy. It processes collections sequentially in batches, with full crash recovery and resume support. State is stored in MongoDB (run manifests) and Redis (hot state, processed document bitmaps, error buffers). + +--- + +## Quick Start + +```bash +helm install countly-migration ./charts/countly-migration \ + -n countly-migration --create-namespace \ + --set backingServices.mongodb.mode=external \ + --set backingServices.mongodb.uri="mongodb://app:PASSWORD@mongodb-host:27017/admin?replicaSet=rs0&ssl=false" \ + --set backingServices.clickhouse.mode=external \ + --set backingServices.clickhouse.url="http://clickhouse-host:8123" \ + --set backingServices.clickhouse.password="PASSWORD" +``` + +Redis is deployed automatically as a subchart. The migration service auto-discovers all `drill_events*` collections and begins migrating. + +--- + +## Prerequisites + +- **MongoDB** — Source database with `drill_events*` collections in `countly_drill` database +- **ClickHouse** — Target with `drill_events` table in `countly_drill` database +- **Redis** — Bundled by default (Bitnami subchart), or provide an external URL + +If deploying alongside other Countly charts, MongoDB and ClickHouse are already available via their respective namespaces. + +--- + +## Configuration + +### Backing Services + +The chart connects to MongoDB, ClickHouse, and Redis. Each can be configured in **bundled** or **external** mode. + +#### MongoDB + +| Mode | Description | +|------|-------------| +| `external` (default) | Provide a full connection URI via `backingServices.mongodb.uri` | +| `bundled` | Auto-constructs URI from sibling `countly-mongodb` chart using in-cluster DNS | + +```yaml +# External mode +backingServices: + mongodb: + mode: external + uri: "mongodb://app:pass@host:27017/admin?replicaSet=rs0&ssl=false" + +# Bundled mode (uses countly-mongodb chart in same cluster) +backingServices: + mongodb: + mode: bundled + password: "app-user-password" + namespace: mongodb +``` + +#### ClickHouse + +| Mode | Description | +|------|-------------| +| `external` (default) | Provide a full HTTP URL via `backingServices.clickhouse.url` | +| `bundled` | Auto-constructs URL from sibling `countly-clickhouse` chart using in-cluster DNS | + +```yaml +# External mode +backingServices: + clickhouse: + mode: external + url: "http://clickhouse-host:8123" + password: "default-password" + +# Bundled mode +backingServices: + clickhouse: + mode: bundled + password: "default-password" + namespace: clickhouse +``` + +#### Redis + +Redis is **enabled by default** as a Bitnami subchart with AOF persistence. + +```yaml +# Default: bundled Redis (already enabled) +redis: + enabled: true + +# External Redis: disable subchart and provide URL +redis: + enabled: false +backingServices: + redis: + url: "redis://my-external-redis:6379" +``` + +### Redis Configuration + +The bundled Redis defaults: + +| Setting | Default | Description | +|---------|---------|-------------| +| `redis.architecture` | `standalone` | Single-node Redis | +| `redis.auth.enabled` | `false` | No password (internal cluster traffic) | +| `redis.master.persistence.enabled` | `true` | Persistent volume for data | +| `redis.master.persistence.size` | `8Gi` | PVC size | +| `redis.commonConfiguration` | AOF + RDB | `appendonly yes`, `appendfsync everysec`, RDB snapshots | +| `redis.master.resources.requests.cpu` | `500m` | CPU request | +| `redis.master.resources.requests.memory` | `2Gi` | Memory request | +| `redis.master.resources.limits.cpu` | `1` | CPU limit | +| `redis.master.resources.limits.memory` | `2Gi` | Memory limit | + +To disable persistence (dev/test only): + +```yaml +redis: + master: + persistence: + enabled: false +``` + +### Secrets + +Three modes for managing credentials: + +| Mode | Description | Use Case | +|------|-------------|----------| +| `values` (default) | Secret created from Helm values | Development, testing | +| `existingSecret` | Reference a pre-created Kubernetes Secret | Production with manual secret management | +| `externalSecret` | External Secrets Operator (AWS SM, Azure KV) | Production with vault integration | + +The Secret must contain these keys: `MONGO_URI`, `CLICKHOUSE_URL`, `CLICKHOUSE_PASSWORD`, `REDIS_URL`. + +```yaml +# Production: use pre-created secret +secrets: + mode: existingSecret + existingSecret: + name: countly-migration-secrets +``` + +### Migration Config + +Key environment variables (set via `config.*`): + +| Variable | Default | Description | +|----------|---------|-------------| +| `RERUN_MODE` | `resume` | `resume` (crash recovery), `new-run`, `clone-run` | +| `LOG_LEVEL` | `info` | `fatal`, `error`, `warn`, `info`, `debug`, `trace` | +| `MONGO_DB` | `countly_drill` | Source MongoDB database | +| `MONGO_COLLECTION_PREFIX` | `drill_events` | Collection name prefix to discover | +| `MONGO_BATCH_ROWS_TARGET` | `10000` | Documents per batch | +| `CLICKHOUSE_DB` | `countly_drill` | Target ClickHouse database | +| `CLICKHOUSE_TABLE` | `drill_events` | Target table | +| `CLICKHOUSE_USE_DEDUP_TOKEN` | `true` | Deduplication on insert | +| `BACKPRESSURE_ENABLED` | `true` | Monitor ClickHouse compaction pressure | +| `GC_ENABLED` | `true` | Automatic garbage collection | +| `GC_RSS_SOFT_LIMIT_MB` | `1536` | Trigger GC at this RSS | +| `GC_RSS_HARD_LIMIT_MB` | `2048` | Force exit at this RSS | + +### ArgoCD Integration + +Enable sync-wave annotations and external progress link: + +```yaml +argocd: + enabled: true + +externalLink: + enabled: true + url: "https://migration.example.internal/runs/current" +``` + +Sync-wave ordering: +- Wave 0: ServiceAccount, ConfigMap +- Wave 1: Secret +- Wave 10: Deployment, Service, Ingress, ServiceMonitor + +Namespace is created by the ArgoCD Application (`CreateNamespace=true`), not by the chart. + +--- + +## Endpoints + +| Method | Path | Purpose | +|--------|------|---------| +| GET | `/healthz` | Liveness probe — always returns 200 | +| GET | `/readyz` | Readiness probe — checks MongoDB, ClickHouse, Redis, ManifestStore, BatchRunner | +| GET | `/stats` | Comprehensive JSON stats (throughput, integrity, memory, backpressure) | +| GET | `/runs/current` | Current active run details | +| GET | `/runs` | Paginated list of all runs (`?status=active\|completed\|failed&limit=20`) | +| GET | `/runs/:id` | Single run details | +| GET | `/runs/:id/batches` | Batches for a run (`?status=done\|failed&limit=50`) | +| GET | `/runs/:id/failures` | Failure analysis (errors, mismatches, retries) | +| GET | `/runs/:id/timeline` | Historical timeline snapshots | +| GET | `/runs/:id/coverage` | Coverage percentage and batch counts | +| POST | `/control/pause` | Pause after current batch | +| POST | `/control/resume` | Resume processing | +| POST | `/control/stop-after-batch` | Graceful stop | +| POST | `/control/gc` | Trigger garbage collection (`{"mode":"now\|after-batch\|force"}`) | +| DELETE | `/runs/:id/cache` | Cleanup Redis cache for a completed run | + +--- + +## Verifying the Deployment + +### 1. Check pods are running + +```bash +kubectl get pods -n countly-migration +``` + +Expected: migration pod `1/1 Running`, redis-master pod `1/1 Running`. + +### 2. Check health + +```bash +kubectl exec -n countly-migration deploy/-countly-migration -- \ + node -e "fetch('http://localhost:8080/healthz').then(r=>r.text()).then(console.log)" +``` + +Expected: `{"status":"alive"}` + +### 3. Check readiness + +```bash +kubectl exec -n countly-migration deploy/-countly-migration -- \ + node -e "fetch('http://localhost:8080/readyz').then(r=>r.text()).then(console.log)" +``` + +Expected: `{"ready":true,"checks":{"mongo":true,"clickhouse":true,"redis":true,"manifestStore":true,"batchRunner":true}}` + +### 4. Check migration stats + +```bash +kubectl exec -n countly-migration deploy/-countly-migration -- \ + node -e "fetch('http://localhost:8080/stats').then(r=>r.json()).then(d=>console.log(JSON.stringify(d,null,2)))" +``` + +### 5. Check run status + +```bash +kubectl exec -n countly-migration deploy/-countly-migration -- \ + node -e "fetch('http://localhost:8080/runs?limit=5').then(r=>r.text()).then(console.log)" +``` + +### 6. Verify data in ClickHouse + +```bash +kubectl exec -n clickhouse -- \ + clickhouse-client --password \ + --query "SELECT count() FROM countly_drill.drill_events" +``` + +### 7. Port-forward for browser access + +```bash +kubectl port-forward -n countly-migration svc/-countly-migration 8080:8080 +# Then open: http://localhost:8080/stats +``` + +--- + +## Operations + +### Pause / Resume + +```bash +# Pause after current batch completes +kubectl exec -n countly-migration deploy/-countly-migration -- \ + node -e "fetch('http://localhost:8080/control/pause',{method:'POST'}).then(r=>r.text()).then(console.log)" + +# Resume +kubectl exec -n countly-migration deploy/-countly-migration -- \ + node -e "fetch('http://localhost:8080/control/resume',{method:'POST'}).then(r=>r.text()).then(console.log)" +``` + +### Check failures + +```bash +kubectl exec -n countly-migration deploy/-countly-migration -- \ + node -e "fetch('http://localhost:8080/runs/RUNID/failures').then(r=>r.text()).then(console.log)" +``` + +### Cleanup Redis cache after a completed run + +```bash +kubectl exec -n countly-migration deploy/-countly-migration -- \ + node -e "fetch('http://localhost:8080/runs/RUNID/cache',{method:'DELETE'}).then(r=>r.text()).then(console.log)" +``` + +### View logs + +```bash +kubectl logs -n countly-migration -l app.kubernetes.io/name=countly-migration -f +``` + +--- + +## Schema Guardrails + +The chart includes `values.schema.json` that enforces: + +- **`deployment.replicas`** must be `1` — this is a singleton workload +- **`deployment.strategy.type`** must be `Recreate` — prevents concurrent pods during rollout +- **`secrets.mode`** must be one of: `values`, `existingSecret`, `externalSecret` + +Attempting to set `replicas: 2` or `strategy.type: RollingUpdate` will fail at `helm install/upgrade/template` time. + +--- + +## Examples + +See the `examples/` directory: + +- **`values-production.yaml`** — Production setup with `existingSecret` mode +- **`values-development.yaml`** — Development setup with bundled backing services +- **`argocd-application.yaml`** — ArgoCD Application manifest with `CreateNamespace=true` diff --git a/charts/countly-migration/examples/argocd-application.yaml b/charts/countly-migration/examples/argocd-application.yaml new file mode 100644 index 0000000..6b36665 --- /dev/null +++ b/charts/countly-migration/examples/argocd-application.yaml @@ -0,0 +1,32 @@ +# Example ArgoCD Application for countly-migration. +# Namespace is created by ArgoCD (CreateNamespace=true), not by the chart. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: countly-migration + namespace: argocd +spec: + project: default + source: + repoURL: https://github.com/Countly/helm.git + targetRevision: main + path: charts/countly-migration + helm: + releaseName: countly-migration + valueFiles: + - ../../environments/prod/migration.yaml + parameters: + - name: argocd.enabled + value: "true" + destination: + server: https://kubernetes.default.svc + namespace: countly-migration + syncPolicy: + syncOptions: + - CreateNamespace=true + - ApplyOutOfSyncOnly=true + managedNamespaceMetadata: + labels: + app.kubernetes.io/part-of: countly + annotations: + owner: data-platform diff --git a/charts/countly-migration/examples/values-development.yaml b/charts/countly-migration/examples/values-development.yaml new file mode 100644 index 0000000..c9c402e --- /dev/null +++ b/charts/countly-migration/examples/values-development.yaml @@ -0,0 +1,42 @@ +# Development values example for countly-migration. +# Uses bundled mode — constructs credentials from sibling chart DNS. + +image: + repository: countly/migration + tag: "1.0.0" + +deployment: + replicas: 1 + strategy: + type: Recreate + +# Construct MongoDB URI from sibling countly-mongodb chart +backingServices: + mongodb: + mode: bundled + password: "devpassword" + namespace: mongodb + clickhouse: + mode: bundled + password: "devpassword" + namespace: clickhouse + redis: + url: "redis://redis:6379" + +secrets: + mode: values + +config: + SERVICE_NAME: countly-migration + LOG_LEVEL: "debug" + RERUN_MODE: "resume" + MONGO_DB: "countly_drill" + CLICKHOUSE_DB: "countly_drill" + +resources: + requests: + cpu: "250m" + memory: "512Mi" + limits: + cpu: "1" + memory: "2Gi" diff --git a/charts/countly-migration/examples/values-production.yaml b/charts/countly-migration/examples/values-production.yaml new file mode 100644 index 0000000..838ec41 --- /dev/null +++ b/charts/countly-migration/examples/values-production.yaml @@ -0,0 +1,61 @@ +# Production values example for countly-migration. +# Uses existingSecret mode — credentials are pre-created or managed externally. + +image: + repository: registry.example.com/countly/migration + tag: "1.0.0" + +argocd: + enabled: true + +deployment: + replicas: 1 + strategy: + type: Recreate + terminationGracePeriodSeconds: 90 + +service: + port: 8080 + +ingress: + enabled: true + className: nginx + annotations: + nginx.ingress.kubernetes.io/whitelist-source-range: "10.0.0.0/8,192.168.0.0/16" + argocd.argoproj.io/ignore-default-links: "true" + hosts: + - host: migration.example.internal + paths: + - path: / + pathType: Prefix + +externalLink: + enabled: true + url: "https://migration.example.internal/runs/current" + +# Reference pre-created secret containing MONGO_URI, CLICKHOUSE_URL, CLICKHOUSE_PASSWORD, REDIS_URL +secrets: + mode: existingSecret + keep: true + existingSecret: + name: countly-migration-secrets + +config: + SERVICE_NAME: countly-migration + SERVICE_PORT: "8080" + RERUN_MODE: "resume" + LOG_LEVEL: "info" + MONGO_DB: "countly_drill" + MONGO_COLLECTION_PREFIX: "drill_events" + CLICKHOUSE_DB: "countly_drill" + CLICKHOUSE_TABLE: "drill_events" + MANIFEST_DB: "countly_drill" + REDIS_KEY_PREFIX: "mig" + +resources: + requests: + cpu: "500m" + memory: "1Gi" + limits: + cpu: "2" + memory: "3Gi" diff --git a/charts/countly-migration/templates/NOTES.txt b/charts/countly-migration/templates/NOTES.txt new file mode 100644 index 0000000..64e5ef9 --- /dev/null +++ b/charts/countly-migration/templates/NOTES.txt @@ -0,0 +1,70 @@ +Countly Migration service has been deployed! + +=== Service Endpoints === + + Health: GET /healthz (liveness probe) + Ready: GET /readyz (readiness — checks mongo, clickhouse, redis) + Stats: GET /stats (throughput, memory, backpressure) + Runs: GET /runs/current (active run details) + Control: POST /control/{pause,resume,stop-after-batch} + +=== Access === +{{- if .Values.ingress.enabled }} +{{- range .Values.ingress.hosts }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ .host }} +{{- end }} +{{- else }} + + kubectl port-forward svc/{{ include "countly-migration.fullname" . }} {{ .Values.service.port }}:{{ .Values.service.port }} -n {{ .Release.Namespace }} + # Then open: http://localhost:{{ .Values.service.port }}/stats +{{- end }} +{{- if and .Values.externalLink.enabled .Values.externalLink.url }} + + ArgoCD progress link: {{ .Values.externalLink.url }} +{{- end }} + +=== Redis === +{{- if .Values.redis.enabled }} + + Bundled Redis (Bitnami subchart): + URL: redis://{{ include "countly-migration.fullname" . }}-redis-master:6379 + Persistence: {{ if .Values.redis.master.persistence.enabled }}enabled ({{ .Values.redis.master.persistence.size }}){{ else }}disabled{{ end }} + Auth: {{ if .Values.redis.auth.enabled }}enabled{{ else }}disabled{{ end }} +{{- else if .Values.backingServices.redis.url }} + + External Redis: {{ .Values.backingServices.redis.url }} +{{- else }} + + WARNING: No Redis configured. The migration service requires Redis for state tracking. + Either enable the bundled Redis (redis.enabled=true) or provide backingServices.redis.url. +{{- end }} + +=== Secrets === + + Mode: {{ .Values.secrets.mode }} +{{- if eq .Values.secrets.mode "existingSecret" }} + Secret: {{ .Values.secrets.existingSecret.name }} +{{- end }} +{{- if eq .Values.secrets.mode "externalSecret" }} + External Secrets Operator will provision the secret. +{{- end }} + +=== Verify Deployment === + + 1. Check pods: + kubectl get pods -n {{ .Release.Namespace }} + + 2. Check health: + kubectl exec -n {{ .Release.Namespace }} deploy/{{ include "countly-migration.fullname" . }} -- \ + node -e "fetch('http://localhost:{{ .Values.service.port }}/healthz').then(r=>r.text()).then(console.log)" + + 3. Check readiness (all backing services connected): + kubectl exec -n {{ .Release.Namespace }} deploy/{{ include "countly-migration.fullname" . }} -- \ + node -e "fetch('http://localhost:{{ .Values.service.port }}/readyz').then(r=>r.text()).then(console.log)" + + 4. Check migration progress: + kubectl exec -n {{ .Release.Namespace }} deploy/{{ include "countly-migration.fullname" . }} -- \ + node -e "fetch('http://localhost:{{ .Values.service.port }}/runs?limit=5').then(r=>r.text()).then(console.log)" + + 5. View logs: + kubectl logs -n {{ .Release.Namespace }} -l app.kubernetes.io/name=countly-migration -f diff --git a/charts/countly-migration/templates/_helpers.tpl b/charts/countly-migration/templates/_helpers.tpl new file mode 100644 index 0000000..47adeeb --- /dev/null +++ b/charts/countly-migration/templates/_helpers.tpl @@ -0,0 +1,140 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "countly-migration.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "countly-migration.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "countly-migration.labels" -}} +helm.sh/chart: {{ include "countly-migration.chart" . }} +{{ include "countly-migration.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Chart label +*/}} +{{- define "countly-migration.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "countly-migration.selectorLabels" -}} +app.kubernetes.io/name: {{ include "countly-migration.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Service account name +*/}} +{{- define "countly-migration.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "countly-migration.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +ArgoCD sync-wave annotation (only when argocd.enabled). +Usage: include "countly-migration.syncWave" (dict "wave" "0" "root" .) +*/}} +{{- define "countly-migration.syncWave" -}} +{{- if .root.Values.argocd.enabled }} +argocd.argoproj.io/sync-wave: {{ .wave | quote }} +{{- end }} +{{- end -}} + +{{/* +Secret name resolution across three modes. +*/}} +{{- define "countly-migration.secretName" -}} +{{- if eq (.Values.secrets.mode | default "values") "existingSecret" }} +{{- required "secrets.existingSecret.name is required when secrets.mode=existingSecret" .Values.secrets.existingSecret.name }} +{{- else }} +{{- include "countly-migration.fullname" . }} +{{- end }} +{{- end }} + +{{/* +MongoDB URI computation. +External mode: use provided URI directly. +Bundled mode: construct from sibling countly-mongodb chart DNS. +*/}} +{{- define "countly-migration.mongoUri" -}} +{{- $bs := .Values.backingServices.mongodb -}} +{{- if eq ($bs.mode | default "external") "external" -}} +{{- required "backingServices.mongodb.uri is required when mode=external" $bs.uri -}} +{{- else -}} +{{- $host := $bs.host | default (printf "%s-mongodb-svc.%s.svc.cluster.local" .Release.Name ($bs.namespace | default "mongodb")) -}} +{{- $port := $bs.port | default "27017" -}} +{{- $user := $bs.username | default "app" -}} +{{- $pass := required "backingServices.mongodb.password is required when mode=bundled" $bs.password -}} +{{- $db := $bs.database | default "admin" -}} +{{- $rs := $bs.replicaSet | default (printf "%s-mongodb" .Release.Name) -}} +mongodb://{{ $user }}:{{ $pass }}@{{ $host }}:{{ $port }}/{{ $db }}?replicaSet={{ $rs }}&ssl=false +{{- end -}} +{{- end -}} + +{{/* +ClickHouse URL computation. +External mode: use provided URL directly. +Bundled mode: construct from sibling countly-clickhouse chart DNS. +*/}} +{{- define "countly-migration.clickhouseUrl" -}} +{{- $bs := .Values.backingServices.clickhouse -}} +{{- if eq ($bs.mode | default "external") "external" -}} +{{- required "backingServices.clickhouse.url is required when mode=external" $bs.url -}} +{{- else -}} +{{- $host := $bs.host | default (printf "%s-clickhouse-clickhouse-headless.%s.svc" .Release.Name ($bs.namespace | default "clickhouse")) -}} +{{- $port := $bs.port | default "8123" -}} +{{- $tls := $bs.tls | default "false" -}} +{{- $scheme := ternary "https" "http" (eq (toString $tls) "true") -}} +{{- $scheme }}://{{ $host }}:{{ $port }} +{{- end -}} +{{- end -}} + +{{/* +Redis URL computation. +If backingServices.redis.url is set, use it. +If redis subchart is enabled, auto-wire to the subchart service. +*/}} +{{- define "countly-migration.redisUrl" -}} +{{- if .Values.backingServices.redis.url -}} +{{- .Values.backingServices.redis.url -}} +{{- else if .Values.redis.enabled -}} +redis://{{ include "countly-migration.fullname" . }}-redis-master:6379 +{{- end -}} +{{- end -}} + +{{/* +Image reference with tag defaulting to appVersion. +*/}} +{{- define "countly-migration.image" -}} +{{- $tag := .Values.image.tag | default .Chart.AppVersion -}} +{{ .Values.image.repository }}:{{ $tag }} +{{- end }} diff --git a/charts/countly-migration/templates/configmap.yaml b/charts/countly-migration/templates/configmap.yaml new file mode 100644 index 0000000..1566278 --- /dev/null +++ b/charts/countly-migration/templates/configmap.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "countly-migration.fullname" . }} + labels: + {{- include "countly-migration.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-migration.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} + {{- end }} +data: + {{- range $k, $v := .Values.config }} + {{ $k }}: {{ $v | quote }} + {{- end }} diff --git a/charts/countly-migration/templates/deployment.yaml b/charts/countly-migration/templates/deployment.yaml new file mode 100644 index 0000000..4718850 --- /dev/null +++ b/charts/countly-migration/templates/deployment.yaml @@ -0,0 +1,92 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "countly-migration.fullname" . }} + labels: + {{- include "countly-migration.labels" . | nindent 4 }} + {{- if or (and .Values.externalLink.enabled .Values.externalLink.url) .Values.argocd.enabled }} + annotations: + {{- if and .Values.externalLink.enabled .Values.externalLink.url }} + link.argocd.argoproj.io/external-link: {{ .Values.externalLink.url | quote }} + {{- end }} + {{- include "countly-migration.syncWave" (dict "wave" "10" "root" .) | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.deployment.replicas }} + strategy: + type: {{ .Values.deployment.strategy.type }} + selector: + matchLabels: + {{- include "countly-migration.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "countly-migration.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "countly-migration.serviceAccountName" . }} + terminationGracePeriodSeconds: {{ .Values.deployment.terminationGracePeriodSeconds }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.image.pullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ include "countly-migration.name" . }} + image: {{ include "countly-migration.image" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + envFrom: + - configMapRef: + name: {{ include "countly-migration.fullname" . }} + - secretRef: + name: {{ include "countly-migration.secretName" . }} + livenessProbe: + httpGet: + path: {{ .Values.probes.liveness.path }} + port: http + initialDelaySeconds: {{ .Values.probes.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.probes.liveness.periodSeconds }} + timeoutSeconds: {{ .Values.probes.liveness.timeoutSeconds }} + failureThreshold: {{ .Values.probes.liveness.failureThreshold }} + readinessProbe: + httpGet: + path: {{ .Values.probes.readiness.path }} + port: http + initialDelaySeconds: {{ .Values.probes.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.probes.readiness.periodSeconds }} + timeoutSeconds: {{ .Values.probes.readiness.timeoutSeconds }} + failureThreshold: {{ .Values.probes.readiness.failureThreshold }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/countly-migration/templates/external-secret.yaml b/charts/countly-migration/templates/external-secret.yaml new file mode 100644 index 0000000..aec8814 --- /dev/null +++ b/charts/countly-migration/templates/external-secret.yaml @@ -0,0 +1,33 @@ +{{- if eq (.Values.secrets.mode | default "values") "externalSecret" }} +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: {{ include "countly-migration.fullname" . }} + labels: + {{- include "countly-migration.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-migration.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} + {{- end }} +spec: + refreshInterval: {{ .Values.secrets.externalSecret.refreshInterval | default "1h" }} + secretStoreRef: + name: {{ required "secrets.externalSecret.secretStoreRef.name is required when secrets.mode=externalSecret" .Values.secrets.externalSecret.secretStoreRef.name }} + kind: {{ .Values.secrets.externalSecret.secretStoreRef.kind | default "ClusterSecretStore" }} + target: + name: {{ include "countly-migration.fullname" . }} + creationPolicy: Owner + data: + - secretKey: MONGO_URI + remoteRef: + key: {{ required "secrets.externalSecret.remoteRefs.mongoUri is required" .Values.secrets.externalSecret.remoteRefs.mongoUri }} + - secretKey: CLICKHOUSE_URL + remoteRef: + key: {{ required "secrets.externalSecret.remoteRefs.clickhouseUrl is required" .Values.secrets.externalSecret.remoteRefs.clickhouseUrl }} + - secretKey: CLICKHOUSE_PASSWORD + remoteRef: + key: {{ required "secrets.externalSecret.remoteRefs.clickhousePassword is required" .Values.secrets.externalSecret.remoteRefs.clickhousePassword }} + - secretKey: REDIS_URL + remoteRef: + key: {{ required "secrets.externalSecret.remoteRefs.redisUrl is required" .Values.secrets.externalSecret.remoteRefs.redisUrl }} +{{- end }} diff --git a/charts/countly-migration/templates/ingress.yaml b/charts/countly-migration/templates/ingress.yaml new file mode 100644 index 0000000..110acbe --- /dev/null +++ b/charts/countly-migration/templates/ingress.yaml @@ -0,0 +1,41 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "countly-migration.fullname" . }} + labels: + {{- include "countly-migration.labels" . | nindent 4 }} + {{- if or (and .Values.externalLink.enabled .Values.externalLink.url) .Values.argocd.enabled .Values.ingress.annotations }} + annotations: + {{- if and .Values.externalLink.enabled .Values.externalLink.url }} + link.argocd.argoproj.io/external-link: {{ .Values.externalLink.url | quote }} + {{- end }} + {{- include "countly-migration.syncWave" (dict "wave" "10" "root" .) | nindent 4 }} + {{- with .Values.ingress.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +spec: + {{- if .Values.ingress.className }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ include "countly-migration.fullname" $ }} + port: + number: {{ $.Values.service.port }} + {{- end }} + {{- end }} + {{- with .Values.ingress.tls }} + tls: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/countly-migration/templates/networkpolicy.yaml b/charts/countly-migration/templates/networkpolicy.yaml new file mode 100644 index 0000000..bc9da35 --- /dev/null +++ b/charts/countly-migration/templates/networkpolicy.yaml @@ -0,0 +1,27 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "countly-migration.fullname" . }} + labels: + {{- include "countly-migration.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-migration.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} + {{- end }} +spec: + podSelector: + matchLabels: + {{- include "countly-migration.selectorLabels" . | nindent 6 }} + policyTypes: + - Ingress + - Egress + {{- with .Values.networkPolicy.ingress }} + ingress: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.networkPolicy.egress }} + egress: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/countly-migration/templates/pdb.yaml b/charts/countly-migration/templates/pdb.yaml new file mode 100644 index 0000000..2196a5f --- /dev/null +++ b/charts/countly-migration/templates/pdb.yaml @@ -0,0 +1,13 @@ +{{- if .Values.pdb.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "countly-migration.fullname" . }} + labels: + {{- include "countly-migration.labels" . | nindent 4 }} +spec: + minAvailable: {{ .Values.pdb.minAvailable }} + selector: + matchLabels: + {{- include "countly-migration.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/countly-migration/templates/secret.yaml b/charts/countly-migration/templates/secret.yaml new file mode 100644 index 0000000..fea62e9 --- /dev/null +++ b/charts/countly-migration/templates/secret.yaml @@ -0,0 +1,36 @@ +{{- if eq (.Values.secrets.mode | default "values") "values" }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "countly-migration.fullname" . }} + labels: + {{- include "countly-migration.labels" . | nindent 4 }} + {{- if or .Values.secrets.keep .Values.argocd.enabled }} + annotations: + {{- if .Values.secrets.keep }} + helm.sh/resource-policy: keep + {{- end }} + {{- include "countly-migration.syncWave" (dict "wave" "1" "root" .) | nindent 4 }} + {{- end }} +type: Opaque +data: + {{- $secretName := include "countly-migration.fullname" . }} + {{- $existing := lookup "v1" "Secret" .Release.Namespace $secretName }} + MONGO_URI: {{ include "countly-migration.mongoUri" . | b64enc }} + CLICKHOUSE_URL: {{ include "countly-migration.clickhouseUrl" . | b64enc }} + {{- if .Values.backingServices.clickhouse.password }} + CLICKHOUSE_PASSWORD: {{ .Values.backingServices.clickhouse.password | b64enc }} + {{- else if and $existing (index $existing.data "CLICKHOUSE_PASSWORD") }} + CLICKHOUSE_PASSWORD: {{ index $existing.data "CLICKHOUSE_PASSWORD" }} + {{- else }} + CLICKHOUSE_PASSWORD: {{ "" | b64enc }} + {{- end }} + {{- $redisUrl := include "countly-migration.redisUrl" . }} + {{- if $redisUrl }} + REDIS_URL: {{ $redisUrl | b64enc }} + {{- else if and $existing (index $existing.data "REDIS_URL") }} + REDIS_URL: {{ index $existing.data "REDIS_URL" }} + {{- else }} + REDIS_URL: {{ "" | b64enc }} + {{- end }} +{{- end }} diff --git a/charts/countly-migration/templates/service.yaml b/charts/countly-migration/templates/service.yaml new file mode 100644 index 0000000..5e90d65 --- /dev/null +++ b/charts/countly-migration/templates/service.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "countly-migration.fullname" . }} + labels: + {{- include "countly-migration.labels" . | nindent 4 }} + {{- if or .Values.argocd.enabled .Values.service.annotations }} + annotations: + {{- include "countly-migration.syncWave" (dict "wave" "10" "root" .) | nindent 4 }} + {{- with .Values.service.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: {{ .Values.service.type }} + selector: + {{- include "countly-migration.selectorLabels" . | nindent 4 }} + ports: + - name: http + port: {{ .Values.service.port }} + targetPort: http + protocol: TCP diff --git a/charts/countly-migration/templates/serviceaccount.yaml b/charts/countly-migration/templates/serviceaccount.yaml new file mode 100644 index 0000000..529813e --- /dev/null +++ b/charts/countly-migration/templates/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "countly-migration.serviceAccountName" . }} + labels: + {{- include "countly-migration.labels" . | nindent 4 }} + {{- if or .Values.argocd.enabled .Values.serviceAccount.annotations }} + annotations: + {{- include "countly-migration.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +automountServiceAccountToken: false +{{- end }} diff --git a/charts/countly-migration/templates/servicemonitor.yaml b/charts/countly-migration/templates/servicemonitor.yaml new file mode 100644 index 0000000..7d480ce --- /dev/null +++ b/charts/countly-migration/templates/servicemonitor.yaml @@ -0,0 +1,21 @@ +{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "countly-migration.fullname" . }} + labels: + {{- include "countly-migration.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-migration.syncWave" (dict "wave" "10" "root" .) | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "countly-migration.selectorLabels" . | nindent 6 }} + endpoints: + - port: http + path: {{ .Values.serviceMonitor.path }} + interval: {{ .Values.serviceMonitor.interval }} + scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }} +{{- end }} diff --git a/charts/countly-migration/values.schema.json b/charts/countly-migration/values.schema.json new file mode 100644 index 0000000..696f1a3 --- /dev/null +++ b/charts/countly-migration/values.schema.json @@ -0,0 +1,204 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "title": "countly-migration values", + "type": "object", + "properties": { + "image": { + "type": "object", + "properties": { + "repository": { + "type": "string", + "minLength": 1, + "description": "Container image repository" + }, + "tag": { + "type": "string", + "description": "Image tag (defaults to appVersion)" + }, + "pullPolicy": { + "type": "string", + "enum": ["Always", "IfNotPresent", "Never"] + } + }, + "required": ["repository"] + }, + "deployment": { + "type": "object", + "properties": { + "replicas": { + "type": "integer", + "const": 1, + "description": "Must be 1 — this workload is a singleton" + }, + "strategy": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "Recreate", + "description": "Must be Recreate — concurrent pods are unsafe" + } + }, + "required": ["type"] + }, + "terminationGracePeriodSeconds": { + "type": "integer", + "minimum": 30 + } + }, + "required": ["replicas", "strategy"] + }, + "service": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["ClusterIP", "NodePort", "LoadBalancer"] + }, + "port": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + } + }, + "required": ["port"] + }, + "secrets": { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": ["values", "existingSecret", "externalSecret"], + "description": "Secret provisioning mode" + }, + "keep": { + "type": "boolean" + }, + "existingSecret": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + } + }, + "externalSecret": { + "type": "object", + "properties": { + "refreshInterval": { + "type": "string" + }, + "secretStoreRef": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "kind": { + "type": "string" + } + } + }, + "remoteRefs": { + "type": "object", + "properties": { + "mongoUri": { "type": "string" }, + "clickhouseUrl": { "type": "string" }, + "clickhousePassword": { "type": "string" }, + "redisUrl": { "type": "string" } + } + } + } + } + }, + "required": ["mode"] + }, + "backingServices": { + "type": "object", + "properties": { + "mongodb": { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": ["external", "bundled"] + } + } + }, + "clickhouse": { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": ["external", "bundled"] + } + } + }, + "redis": { + "type": "object", + "properties": { + "url": { + "type": "string" + } + } + } + } + }, + "config": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "Non-secret environment variables" + }, + "probes": { + "type": "object", + "properties": { + "liveness": { + "type": "object", + "properties": { + "path": { "type": "string" }, + "initialDelaySeconds": { "type": "integer", "minimum": 0 }, + "periodSeconds": { "type": "integer", "minimum": 1 }, + "timeoutSeconds": { "type": "integer", "minimum": 1 }, + "failureThreshold": { "type": "integer", "minimum": 1 } + } + }, + "readiness": { + "type": "object", + "properties": { + "path": { "type": "string" }, + "initialDelaySeconds": { "type": "integer", "minimum": 0 }, + "periodSeconds": { "type": "integer", "minimum": 1 }, + "timeoutSeconds": { "type": "integer", "minimum": 1 }, + "failureThreshold": { "type": "integer", "minimum": 1 } + } + } + } + }, + "resources": { + "type": "object", + "properties": { + "requests": { "type": "object" }, + "limits": { "type": "object" } + } + }, + "externalLink": { + "type": "object", + "properties": { + "enabled": { "type": "boolean" }, + "url": { "type": "string" } + } + }, + "serviceMonitor": { + "type": "object", + "properties": { + "enabled": { "type": "boolean" }, + "interval": { "type": "string" }, + "scrapeTimeout": { "type": "string" }, + "path": { "type": "string" } + } + } + }, + "required": ["image", "deployment", "service", "secrets"] +} diff --git a/charts/countly-migration/values.yaml b/charts/countly-migration/values.yaml new file mode 100644 index 0000000..51f12a9 --- /dev/null +++ b/charts/countly-migration/values.yaml @@ -0,0 +1,255 @@ +# -- Override the chart name used in resource names +nameOverride: "" +# -- Override the full resource name +fullnameOverride: "" + +# -- Container image configuration +image: + repository: countly/migration + # -- Defaults to Chart.appVersion when empty + tag: "" + pullPolicy: IfNotPresent + pullSecrets: [] + +# -- Service account configuration +serviceAccount: + create: true + name: "" + annotations: {} + +# -- ArgoCD integration +argocd: + enabled: false + +# -- Deployment configuration (singleton, Recreate only) +deployment: + replicas: 1 + strategy: + type: Recreate + terminationGracePeriodSeconds: 90 + +# -- Additional pod annotations +podAnnotations: {} +# -- Additional pod labels +podLabels: {} + +# -- Service configuration +service: + type: ClusterIP + port: 8080 + annotations: {} + +# -- Ingress configuration (optional) +ingress: + enabled: false + className: "" + annotations: {} + hosts: [] + # - host: migration.example.internal + # paths: + # - path: / + # pathType: Prefix + tls: [] + +# -- ArgoCD external link on the Deployment resource +externalLink: + enabled: false + # -- URL shown in ArgoCD UI (e.g. https://migration.example.internal/runs/current) + url: "" + +# -- Backing services (credential sources for MongoDB, ClickHouse, Redis) +backingServices: + mongodb: + # -- bundled: construct URI from sibling countly-mongodb chart; external: provide full URI + mode: external + # -- Full MongoDB connection string (external mode) + uri: "" + # -- MongoDB host (bundled mode; auto-constructed from release name if empty) + host: "" + port: "27017" + username: "app" + password: "" + database: "admin" + replicaSet: "" + # -- Namespace where countly-mongodb chart is deployed + namespace: mongodb + clickhouse: + # -- bundled: construct URL from sibling countly-clickhouse chart; external: provide full URL + mode: external + # -- Full ClickHouse HTTP URL (external mode) + url: "" + # -- ClickHouse host (bundled mode; auto-constructed from release name if empty) + host: "" + port: "8123" + tls: "false" + username: "default" + password: "" + # -- Namespace where countly-clickhouse chart is deployed + namespace: clickhouse + redis: + # -- Full Redis connection URL (e.g. redis://redis:6379) + url: "" + +# -- Secrets management +secrets: + # -- values: create Secret from values; existingSecret: reference pre-created; externalSecret: use ESO + mode: values + # -- Preserve secrets on helm uninstall/upgrade + keep: true + existingSecret: + # -- Name of pre-created Secret containing MONGO_URI, CLICKHOUSE_URL, CLICKHOUSE_PASSWORD, REDIS_URL + name: "" + externalSecret: + refreshInterval: "1h" + secretStoreRef: + name: "" + kind: ClusterSecretStore + remoteRefs: + mongoUri: "" + clickhouseUrl: "" + clickhousePassword: "" + redisUrl: "" + +# -- Application config (non-secret environment variables) +config: + SERVICE_NAME: countly-migration + SERVICE_PORT: "8080" + SERVICE_HOST: "0.0.0.0" + GRACEFUL_SHUTDOWN_TIMEOUT_MS: "60000" + RERUN_MODE: "resume" + LOG_LEVEL: "info" + + # MongoDB source + MONGO_DB: "countly_drill" + MONGO_COLLECTION_PREFIX: "drill_events" + MONGO_READ_PREFERENCE: "primary" + MONGO_READ_CONCERN: "majority" + MONGO_RETRY_READS: "true" + MONGO_APP_NAME: "countly-migration" + MONGO_BATCH_ROWS_TARGET: "10000" + MONGO_CURSOR_BATCH_SIZE: "2000" + MONGO_MAX_TIME_MS: "120000" + + # Transform + TRANSFORM_VERSION: "v1" + + # ClickHouse target + CLICKHOUSE_DB: "countly_drill" + CLICKHOUSE_TABLE: "drill_events" + CLICKHOUSE_USERNAME: "default" + CLICKHOUSE_QUERY_TIMEOUT_MS: "120000" + CLICKHOUSE_MAX_RETRIES: "8" + CLICKHOUSE_RETRY_BASE_DELAY_MS: "1000" + CLICKHOUSE_RETRY_MAX_DELAY_MS: "30000" + CLICKHOUSE_USE_DEDUP_TOKEN: "true" + + # Backpressure + BACKPRESSURE_ENABLED: "true" + BACKPRESSURE_PARTS_TO_THROW_INSERT: "300" + BACKPRESSURE_MAX_PARTS_IN_TOTAL: "500" + BACKPRESSURE_PARTITION_PCT_HIGH: "0.50" + BACKPRESSURE_PARTITION_PCT_LOW: "0.35" + BACKPRESSURE_TOTAL_PCT_HIGH: "0.50" + BACKPRESSURE_TOTAL_PCT_LOW: "0.40" + BACKPRESSURE_POLL_INTERVAL_MS: "15000" + BACKPRESSURE_MAX_PAUSE_EPISODE_MS: "180000" + + # State management + MANIFEST_DB: "countly_drill" + REDIS_KEY_PREFIX: "mig" + TIMELINE_SNAPSHOT_INTERVAL: "10" + + # Garbage collection + GC_ENABLED: "true" + GC_RSS_SOFT_LIMIT_MB: "1536" + GC_RSS_HARD_LIMIT_MB: "2048" + GC_HEAP_USED_RATIO: "0.70" + GC_EVERY_N_BATCHES: "10" + +# -- Health probes +probes: + liveness: + path: /healthz + initialDelaySeconds: 20 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 6 + readiness: + path: /readyz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + +# -- Resource requests and limits +resources: + requests: + cpu: "500m" + memory: "1Gi" + limits: + cpu: "2" + memory: "3Gi" + +# -- Pod-level security context +podSecurityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + +# -- Container-level security context +containerSecurityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + +# -- Node selector +nodeSelector: {} +# -- Tolerations +tolerations: [] +# -- Affinity rules +affinity: {} + +# -- Network policy (optional) +networkPolicy: + enabled: false + ingress: {} + egress: {} + +# -- Pod disruption budget (optional; disabled by default for singleton) +pdb: + enabled: false + minAvailable: 1 + +# -- Prometheus ServiceMonitor (optional) +serviceMonitor: + enabled: false + interval: "30s" + scrapeTimeout: "10s" + path: /stats + +# -- Bundled Redis subchart (Bitnami) +# Deploys Redis alongside the migration service. +# When enabled and backingServices.redis.url is empty, the chart auto-wires the URL. +# Set redis.enabled=false and provide backingServices.redis.url to use an external Redis. +redis: + enabled: true + architecture: standalone + auth: + enabled: false + master: + resources: + requests: + cpu: "500m" + memory: "2Gi" + limits: + cpu: "1" + memory: "2Gi" + persistence: + enabled: true + size: 8Gi + commonConfiguration: |- + appendonly yes + appendfsync everysec + save 900 1 + save 300 10 + save 60 10000 diff --git a/charts/countly-migrations/.helmignore b/charts/countly-migrations/.helmignore new file mode 100644 index 0000000..2dbde13 --- /dev/null +++ b/charts/countly-migrations/.helmignore @@ -0,0 +1,18 @@ +.DS_Store +.git +.gitignore +.bzr +.bzrignore +.hg +.hgignore +.svn +*.swp +*.bak +*.tmp +*.orig +*~ +.project +.idea +*.tmproj +.vscode +examples/ diff --git a/charts/countly-migrations/Chart.yaml b/charts/countly-migrations/Chart.yaml new file mode 100644 index 0000000..09411a4 --- /dev/null +++ b/charts/countly-migrations/Chart.yaml @@ -0,0 +1,26 @@ +apiVersion: v2 +name: countly-migrations +description: MongoDB to ClickHouse migration pipeline via Kafka (Strimzi) +type: application +version: 0.1.0 +appVersion: "0.47.0" +home: https://countly.com +icon: https://count.ly/images/logos/countly-logo.svg +sources: + - https://github.com/Countly/countly-server +keywords: + - migration + - kafka + - strimzi + - debezium + - clickhouse + - mongodb + - countly +maintainers: + - name: Countly + url: https://countly.com +annotations: + artifacthub.io/license: AGPL-3.0 + artifacthub.io/links: | + - name: Documentation + url: https://github.com/Countly/helm diff --git a/charts/countly-migrations/examples/argocd-application.yaml b/charts/countly-migrations/examples/argocd-application.yaml new file mode 100644 index 0000000..41e41bd --- /dev/null +++ b/charts/countly-migrations/examples/argocd-application.yaml @@ -0,0 +1,67 @@ +# Example ArgoCD Application for countly-migrations chart. +# This is NOT deployed by the chart — copy and customize for your setup. +# +# Prerequisites: +# - ArgoCD installed in the cluster +# - Strimzi CRDs installed (from production Kafka or separately) +# - Set argocd.enabled=true in your values to add sync-wave annotations +# - Custom health checks for Strimzi CRDs in argocd-cm (see below) +# +# Strimzi health checks (add to argocd-cm ConfigMap): +# resource.customizations.health.kafka.strimzi.io_Kafka: | +# hs = {} +# if obj.status ~= nil and obj.status.conditions ~= nil then +# for _, c in ipairs(obj.status.conditions) do +# if c.type == "Ready" and c.status == "True" then +# hs.status = "Healthy"; hs.message = "Ready"; return hs +# end +# end +# end +# hs.status = "Progressing"; hs.message = "Waiting"; return hs +# +# (Repeat for KafkaConnect, KafkaNodePool, KafkaConnector) + +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: countly-migrations + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: https://github.com/Countly/helm.git + targetRevision: main + path: charts/countly-migrations + helm: + releaseName: mig + valueFiles: + - ../../environments/production/migrations.yaml + parameters: + - name: argocd.enabled + value: "true" + destination: + server: https://kubernetes.default.svc + namespace: kafka-migrations + syncPolicy: + automated: + prune: true + selfHeal: false # false: don't fight manual pauses + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + ignoreDifferences: + - group: kafka.strimzi.io + kind: Kafka + jsonPointers: + - /status + - group: kafka.strimzi.io + kind: KafkaConnect + jsonPointers: + - /status + - group: kafka.strimzi.io + kind: KafkaConnector + jsonPointers: + - /status diff --git a/charts/countly-migrations/templates/NOTES.txt b/charts/countly-migrations/templates/NOTES.txt new file mode 100644 index 0000000..2eff5be --- /dev/null +++ b/charts/countly-migrations/templates/NOTES.txt @@ -0,0 +1,64 @@ +=== Countly Migration Pipeline === + +Namespace: {{ .Release.Namespace }} +Migration state: {{ .Values.migration.state }} + +Pipeline: + MongoDB -> Debezium ({{ include "countly-migrations.connectSrcName" . }}) + -> Kafka ({{ include "countly-migrations.fullname" . }}) + -> ClickHouse ({{ include "countly-migrations.connectSinkName" . }}) + -> MongoDB writeback + +--- Status --- + + kubectl -n {{ .Release.Namespace }} get kafkaconnectors + kubectl -n {{ .Release.Namespace }} get kafka,kafkaconnect + +--- Migration Control --- + + # Pause (preserves offsets): + helm upgrade {{ .Release.Name }} --reuse-values --set migration.state=paused + + # Resume: + helm upgrade {{ .Release.Name }} --reuse-values --set migration.state=running + + # Stop: + helm upgrade {{ .Release.Name }} --reuse-values --set migration.state=stopped + + # Hard stop (free compute): + helm upgrade {{ .Release.Name }} --reuse-values \ + --set connectSrc.replicas=0 --set connectSink.replicas=0 + +--- Scaling --- + + # Scale ClickHouse sink parallelism: + helm upgrade {{ .Release.Name }} --reuse-values \ + --set connectors.clickhouseSink.tasksMax=8 + + # Scale sink Connect resources: + helm upgrade {{ .Release.Name }} --reuse-values \ + --set connectSink.resources.requests.cpu=2 \ + --set connectSink.resources.limits.cpu=4 + +--- Teardown Checklist --- + + 1. Verify ClickHouse row count matches MongoDB: + kubectl exec -n clickhouse -- clickhouse-client \ + --query "SELECT count() FROM countly_drill.drill_events" + + 2. Verify DLQ is empty: + kubectl exec -n {{ .Release.Namespace }} {{ include "countly-migrations.fullname" . }}-brokers-0 \ + -c kafka -- bin/kafka-get-offsets.sh \ + --bootstrap-server localhost:9092 --topic drill-migration-dlq + + 3. Stop all connectors: + helm upgrade {{ .Release.Name }} --reuse-values --set migration.state=stopped + + 4. Enable PVC cleanup: + helm upgrade {{ .Release.Name }} --reuse-values --set migration.deleteClaim=true + + 5. Uninstall: + helm uninstall {{ .Release.Name }} -n {{ .Release.Namespace }} + + 6. (Optional) Delete namespace: + kubectl delete namespace {{ .Release.Namespace }} diff --git a/charts/countly-migrations/templates/_helpers.tpl b/charts/countly-migrations/templates/_helpers.tpl new file mode 100644 index 0000000..d6bf5bf --- /dev/null +++ b/charts/countly-migrations/templates/_helpers.tpl @@ -0,0 +1,129 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "countly-migrations.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "countly-migrations.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "countly-migrations.labels" -}} +helm.sh/chart: {{ include "countly-migrations.chart" . }} +{{ include "countly-migrations.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Chart label +*/}} +{{- define "countly-migrations.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "countly-migrations.selectorLabels" -}} +app.kubernetes.io/name: {{ include "countly-migrations.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Kafka bootstrap servers computation +*/}} +{{- define "countly-migrations.bootstrapServers" -}} +{{ include "countly-migrations.fullname" . }}-kafka-bootstrap:9092 +{{- end -}} + +{{/* +KafkaConnect source cluster name +*/}} +{{- define "countly-migrations.connectSrcName" -}} +{{- .Values.connectSrc.name | default (printf "%s-connect-src" (include "countly-migrations.fullname" .)) -}} +{{- end -}} + +{{/* +KafkaConnect sink cluster name +*/}} +{{- define "countly-migrations.connectSinkName" -}} +{{- .Values.connectSink.name | default (printf "%s-connect-sink" (include "countly-migrations.fullname" .)) -}} +{{- end -}} + +{{/* +ClickHouse secret name (supports existingSecret) +*/}} +{{- define "countly-migrations.clickhouseSecretName" -}} +{{- if .Values.clickhouse.existingSecret -}} +{{ .Values.clickhouse.existingSecret }} +{{- else -}} +{{ .Values.clickhouse.secretName }} +{{- end -}} +{{- end -}} + +{{/* +Storage class name for a component. +Falls back to storageClass.name, then global.storageClass. +Usage: {{ include "countly-migrations.storageClassName" (dict "component" .Values.brokers.persistence "root" .) }} +*/}} +{{- define "countly-migrations.storageClassName" -}} +{{- if .component.storageClass -}} +{{ .component.storageClass }} +{{- else if .root.Values.storageClass.name -}} +{{ .root.Values.storageClass.name }} +{{- else -}} +{{ .root.Values.global.storageClass }} +{{- end -}} +{{- end -}} + +{{/* +ArgoCD sync-wave annotation (only when argocd.enabled). +Usage: {{- include "countly-migrations.syncWave" (dict "wave" "5" "root" .) | nindent 4 }} +*/}} +{{- define "countly-migrations.syncWave" -}} +{{- if .root.Values.argocd.enabled }} +argocd.argoproj.io/sync-wave: {{ .wave | quote }} +{{- end }} +{{- end -}} + +{{/* +Resolve connector state: per-connector override > global migration.state. +Usage: {{ include "countly-migrations.connectorState" (dict "connector" .Values.connectors.source "global" .Values.migration) }} +*/}} +{{- define "countly-migrations.connectorState" -}} +{{- if .connector.state -}} +{{ .connector.state }} +{{- else -}} +{{ .global.state }} +{{- end -}} +{{- end -}} + +{{/* +Resolve deleteClaim: migration.deleteClaim overrides per-component setting. +*/}} +{{- define "countly-migrations.deleteClaim" -}} +{{- if .Values.migration.deleteClaim -}} +true +{{- else -}} +false +{{- end -}} +{{- end -}} diff --git a/charts/countly-migrations/templates/configmap-connect-env.yaml b/charts/countly-migrations/templates/configmap-connect-env.yaml new file mode 100644 index 0000000..dc81ad1 --- /dev/null +++ b/charts/countly-migrations/templates/configmap-connect-env.yaml @@ -0,0 +1,56 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "countly-migrations.fullname" . }}-connect-env + labels: + {{- include "countly-migrations.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-migrations.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} + {{- end }} +data: + # ClickHouse target + CLICKHOUSE_HOST: {{ .Values.clickhouse.host | quote }} + CLICKHOUSE_PORT: {{ .Values.clickhouse.port | quote }} + CLICKHOUSE_SSL: {{ .Values.clickhouse.ssl | quote }} + CLICKHOUSE_DB: {{ .Values.clickhouse.database | quote }} + + # Behavior + EXACTLY_ONCE: {{ .Values.behavior.exactlyOnce | quote }} + ERRORS_RETRY_TIMEOUT: {{ .Values.behavior.errorsRetryTimeout | quote }} + ERRORS_TOLERANCE: {{ .Values.behavior.errorsTolerance | quote }} + + # ClickHouse session / JSON handling + CLICKHOUSE_SETTINGS: {{ .Values.clickhouse.settings | quote }} + BYPASS_ROW_BINARY: {{ .Values.clickhouse.bypassRowBinary | quote }} + TABLE_REFRESH_INTERVAL: {{ .Values.clickhouse.tableRefreshInterval | quote }} + + # Converters + KEY_CONVERTER: {{ .Values.converters.key | quote }} + VALUE_CONVERTER: {{ .Values.converters.value | quote }} + VALUE_CONVERTER_SCHEMAS_ENABLE: {{ .Values.converters.valueSchemasEnable | quote }} + + # Sink consumer batching / throughput + KAFKA_CONSUMER_FETCH_MIN_BYTES: {{ .Values.consumerTuning.sink.fetchMinBytes | quote }} + KAFKA_CONSUMER_FETCH_MAX_WAIT_MS: {{ .Values.consumerTuning.sink.fetchMaxWaitMs | quote }} + KAFKA_CONSUMER_MAX_POLL_RECORDS: {{ .Values.consumerTuning.sink.maxPollRecords | quote }} + KAFKA_CONSUMER_MAX_PARTITION_FETCH_BYTES: {{ .Values.consumerTuning.sink.maxPartitionFetchBytes | quote }} + KAFKA_CONSUMER_FETCH_MAX_BYTES: {{ .Values.consumerTuning.sink.fetchMaxBytes | quote }} + + # Writeback consumer tuning + WB_CONSUMER_MAX_POLL_RECORDS: {{ .Values.consumerTuning.writeback.maxPollRecords | quote }} + WB_CONSUMER_FETCH_MAX_WAIT_MS: {{ .Values.consumerTuning.writeback.fetchMaxWaitMs | quote }} + + # Debezium snapshot tuning (source) + DBZ_SNAPSHOT_MODE: {{ .Values.debezium.snapshotMode | quote }} + DBZ_SNAPSHOT_MAX_THREADS: {{ .Values.debezium.snapshotMaxThreads | quote }} + DBZ_SNAPSHOT_FETCH_SIZE: {{ .Values.debezium.snapshotFetchSize | quote }} + DBZ_MAX_BATCH_SIZE: {{ .Values.debezium.maxBatchSize | quote }} + DBZ_MAX_QUEUE_SIZE: {{ .Values.debezium.maxQueueSize | quote }} + + # MongoDB connection (source) + MONGO_CONN_STR: {{ .Values.mongodb.connectionString | quote }} + MONGO_DB: {{ .Values.mongodb.database | quote }} + DBZ_FILTERS_MATCH_MODE: {{ .Values.mongodb.filtersMatchMode | quote }} + DRILL_EVENTS_NS_SELECTOR: {{ .Values.mongodb.namespaceSelector | quote }} + TOPIC_PREFIX: {{ .Values.mongodb.topicPrefix | quote }} diff --git a/charts/countly-migrations/templates/configmap-metrics.yaml b/charts/countly-migrations/templates/configmap-metrics.yaml new file mode 100644 index 0000000..0c279b9 --- /dev/null +++ b/charts/countly-migrations/templates/configmap-metrics.yaml @@ -0,0 +1,59 @@ +{{- if .Values.metrics.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "countly-migrations.fullname" . }}-metrics + labels: + {{- include "countly-migrations.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-migrations.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} + {{- end }} +data: + kafka-metrics-config.yml: | + lowercaseOutputName: true + rules: + - pattern: "kafka.server<>Count" + name: "kafka_brokertopic_$1_total" + labels: + topic: "$2" + type: COUNTER + + connect-metrics-config.yml: | + lowercaseOutputName: true + rules: + # Worker metrics + - pattern: "kafka.connect([^:]+):" + name: "kafka_connect_worker_$1" + - pattern: "kafka.connect([^:]+):" + name: "kafka_connect_worker_rebalance_$1" + # Connector-level metrics + - pattern: "kafka.connect<>(connector-class|connector-type|connector-version|status): (.+)" + name: "kafka_connect_connector_$2" + value: 1 + labels: + connector: "$1" + $2: "$3" + type: UNTYPED + # Task-level metrics + - pattern: "kafka.connect<>(.+): (.+)" + name: "kafka_connect_task_$3" + labels: + connector: "$1" + task: "$2" + # Source task metrics + - pattern: "kafka.connect<>(.+): (.+)" + name: "kafka_connect_source_task_$3" + labels: + connector: "$1" + task: "$2" + # Sink task metrics + - pattern: "kafka.connect<>(.+): (.+)" + name: "kafka_connect_sink_task_$3" + labels: + connector: "$1" + task: "$2" + # Fallback + - pattern: "kafka.connect<>([^:]+)" + name: "kafka_connect_$1_$2" +{{- end }} diff --git a/charts/countly-migrations/templates/connector-clickhouse-sink.yaml b/charts/countly-migrations/templates/connector-clickhouse-sink.yaml new file mode 100644 index 0000000..db93066 --- /dev/null +++ b/charts/countly-migrations/templates/connector-clickhouse-sink.yaml @@ -0,0 +1,53 @@ +{{- if .Values.connectors.clickhouseSink.enabled }} +apiVersion: {{ .Values.strimzi.apiVersion }} +kind: KafkaConnector +metadata: + name: ch-sink-drill-migration-mig + labels: + strimzi.io/cluster: {{ include "countly-migrations.connectSinkName" . }} + {{- include "countly-migrations.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-migrations.syncWave" (dict "wave" "15" "root" .) | nindent 4 }} + {{- end }} +spec: + class: com.clickhouse.kafka.connect.ClickHouseSinkConnector + tasksMax: {{ .Values.connectors.clickhouseSink.tasksMax }} + state: {{ include "countly-migrations.connectorState" (dict "connector" .Values.connectors.clickhouseSink "global" .Values.migration) }} + autoRestart: + enabled: true + config: + topics: {{ .Values.connectors.clickhouseSink.topics | quote }} + topic2TableMap: {{ .Values.connectors.clickhouseSink.topic2TableMap | quote }} + + hostname: "${env:CLICKHOUSE_HOST}" + port: "${env:CLICKHOUSE_PORT}" + ssl: "${env:CLICKHOUSE_SSL}" + database: "${env:CLICKHOUSE_DB}" + username: "${env:CLICKHOUSE_USER}" + password: "${env:CLICKHOUSE_PASSWORD}" + + exactlyOnce: "${env:EXACTLY_ONCE}" + errors.retry.timeout: "${env:ERRORS_RETRY_TIMEOUT}" + errors.tolerance: "${env:ERRORS_TOLERANCE}" + + clickhouseSettings: "${env:CLICKHOUSE_SETTINGS}" + bypassRowBinary: "${env:BYPASS_ROW_BINARY}" + tableRefreshInterval: "${env:TABLE_REFRESH_INTERVAL}" + + key.converter: "${env:KEY_CONVERTER}" + value.converter: "${env:VALUE_CONVERTER}" + value.converter.schemas.enable: "${env:VALUE_CONVERTER_SCHEMAS_ENABLE}" + + consumer.override.fetch.min.bytes: "${env:KAFKA_CONSUMER_FETCH_MIN_BYTES}" + consumer.override.fetch.max.wait.ms: "${env:KAFKA_CONSUMER_FETCH_MAX_WAIT_MS}" + consumer.override.max.poll.records: "${env:KAFKA_CONSUMER_MAX_POLL_RECORDS}" + consumer.override.max.partition.fetch.bytes: "${env:KAFKA_CONSUMER_MAX_PARTITION_FETCH_BYTES}" + consumer.override.fetch.max.bytes: "${env:KAFKA_CONSUMER_FETCH_MAX_BYTES}" + + {{- with .Values.connectors.clickhouseSink.dlq }} + errors.deadletterqueue.topic.name: {{ .topicName | quote }} + errors.deadletterqueue.topic.replication.factor: {{ .replicationFactor | quote }} + errors.deadletterqueue.context.headers.enable: {{ .headerEnabled | quote }} + {{- end }} +{{- end }} diff --git a/charts/countly-migrations/templates/connector-debezium-mongo.yaml b/charts/countly-migrations/templates/connector-debezium-mongo.yaml new file mode 100644 index 0000000..3b12387 --- /dev/null +++ b/charts/countly-migrations/templates/connector-debezium-mongo.yaml @@ -0,0 +1,51 @@ +{{- if .Values.connectors.source.enabled }} +apiVersion: {{ .Values.strimzi.apiVersion }} +kind: KafkaConnector +metadata: + name: mongo-snapshot-only-mig + labels: + strimzi.io/cluster: {{ include "countly-migrations.connectSrcName" . }} + {{- include "countly-migrations.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-migrations.syncWave" (dict "wave" "15" "root" .) | nindent 4 }} + {{- end }} +spec: + class: io.debezium.connector.mongodb.MongoDbConnector + tasksMax: {{ .Values.connectors.source.tasksMax }} + state: {{ include "countly-migrations.connectorState" (dict "connector" .Values.connectors.source "global" .Values.migration) }} + autoRestart: + enabled: true + config: + mongodb.connection.string: "${env:MONGO_CONN_STR}" + topic.prefix: "${env:TOPIC_PREFIX}" + filters.match.mode: "${env:DBZ_FILTERS_MATCH_MODE}" + database.include.list: "${env:MONGO_DB}" + collection.include.list: "${env:DRILL_EVENTS_NS_SELECTOR}" + + snapshot.mode: "${env:DBZ_SNAPSHOT_MODE}" + snapshot.max.threads: "${env:DBZ_SNAPSHOT_MAX_THREADS}" + snapshot.fetch.size: "${env:DBZ_SNAPSHOT_FETCH_SIZE}" + # Only snapshot documents not yet marked as migrated + snapshot.collection.filter.override.countly_drill.drill_events: {{ .Values.connectors.source.snapshotFilter | quote }} + + max.batch.size: "${env:DBZ_MAX_BATCH_SIZE}" + max.queue.size: "${env:DBZ_MAX_QUEUE_SIZE}" + tombstones.on.delete: "false" + + # SMT transformation chain + transforms: "normalize,route" + transforms.normalize.type: "dev.you.smt.ClyEventNormalize" + transforms.route.type: "org.apache.kafka.connect.transforms.RegexRouter" + transforms.route.regex: ".*" + transforms.route.replacement: "drill-events-migration" + + # Topic creation + topic.creation.enable: "true" + topic.creation.default.partitions: {{ .Values.connectors.source.topicPartitions | quote }} + topic.creation.default.replication.factor: {{ .Values.connectors.source.topicReplicationFactor | quote }} + + key.converter: "${env:KEY_CONVERTER}" + value.converter: "${env:VALUE_CONVERTER}" + value.converter.schemas.enable: "${env:VALUE_CONVERTER_SCHEMAS_ENABLE}" +{{- end }} diff --git a/charts/countly-migrations/templates/connector-mongo-writeback.yaml b/charts/countly-migrations/templates/connector-mongo-writeback.yaml new file mode 100644 index 0000000..8ce3185 --- /dev/null +++ b/charts/countly-migrations/templates/connector-mongo-writeback.yaml @@ -0,0 +1,61 @@ +{{- if .Values.connectors.mongoWriteback.enabled }} +apiVersion: {{ .Values.strimzi.apiVersion }} +kind: KafkaConnector +metadata: + name: mongo-writeback-mig + labels: + strimzi.io/cluster: {{ include "countly-migrations.connectSinkName" . }} + {{- include "countly-migrations.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-migrations.syncWave" (dict "wave" "15" "root" .) | nindent 4 }} + {{- end }} +spec: + class: com.mongodb.kafka.connect.MongoSinkConnector + tasksMax: {{ .Values.connectors.mongoWriteback.tasksMax }} + state: {{ include "countly-migrations.connectorState" (dict "connector" .Values.connectors.mongoWriteback "global" .Values.migration) }} + autoRestart: + enabled: true + config: + topics: "drill-events-migration" + + connection.uri: "${env:MONGO_CONN_STR}" + + # Route each update to the originating collection + namespace.mapper: "com.mongodb.kafka.connect.sink.namespace.mapping.FieldPathNamespaceMapper" + namespace.mapper.value.database.field: "_src_db" + namespace.mapper.value.collection.field: "_src_col" + + # UpdateOne by _id, $set remaining fields + writemodel.strategy: "com.mongodb.kafka.connect.sink.writemodel.strategy.UpdateOneDefaultWriteModelStrategy" + + # Unordered bulk writes for parallelism (safe: each op targets distinct _id) + bulk.write.ordered: {{ .Values.connectors.mongoWriteback.bulkWriteOrdered | quote }} + bulk.write.size: {{ .Values.connectors.mongoWriteback.bulkWriteSize | quote }} + + # PrepareMongoWriteback SMT: outputs { _id, migrated: true, _src_col, _src_db } + transforms: "prepareWriteback" + transforms.prepareWriteback.type: "dev.you.smt.PrepareMongoWriteback" + + key.converter: "${env:KEY_CONVERTER}" + value.converter: "${env:VALUE_CONVERTER}" + value.converter.schemas.enable: "${env:VALUE_CONVERTER_SCHEMAS_ENABLE}" + + # Writeback lags behind ClickHouse sink intentionally + consumer.override.fetch.min.bytes: "${env:KAFKA_CONSUMER_FETCH_MIN_BYTES}" + consumer.override.fetch.max.wait.ms: "${env:WB_CONSUMER_FETCH_MAX_WAIT_MS}" + consumer.override.max.poll.records: "${env:WB_CONSUMER_MAX_POLL_RECORDS}" + consumer.override.max.partition.fetch.bytes: "${env:KAFKA_CONSUMER_MAX_PARTITION_FETCH_BYTES}" + consumer.override.fetch.max.bytes: "${env:KAFKA_CONSUMER_FETCH_MAX_BYTES}" + + errors.log.enable: "true" + max.num.retries: {{ .Values.connectors.mongoWriteback.maxRetries | quote }} + errors.retry.timeout: "${env:ERRORS_RETRY_TIMEOUT}" + errors.tolerance: "${env:ERRORS_TOLERANCE}" + + {{- with .Values.connectors.mongoWriteback.dlq }} + errors.deadletterqueue.topic.name: {{ .topicName | quote }} + errors.deadletterqueue.topic.replication.factor: {{ .replicationFactor | quote }} + errors.deadletterqueue.context.headers.enable: {{ .headerEnabled | quote }} + {{- end }} +{{- end }} diff --git a/charts/countly-migrations/templates/kafka.yaml b/charts/countly-migrations/templates/kafka.yaml new file mode 100644 index 0000000..b8d898a --- /dev/null +++ b/charts/countly-migrations/templates/kafka.yaml @@ -0,0 +1,135 @@ +{{- /* ===== Broker config vs replica count validations ===== */}} +{{- $brokerCount := int .Values.brokers.replicas }} +{{- $rf := index .Values.brokers.config "default.replication.factor" | default 1 }} +{{- if gt (int $rf) $brokerCount }} +{{- fail (printf "default.replication.factor (%d) cannot exceed broker replicas (%d)" (int $rf) $brokerCount) }} +{{- end }} +{{- $minIsr := index .Values.brokers.config "min.insync.replicas" | default 1 }} +{{- if gt (int $minIsr) $brokerCount }} +{{- fail (printf "min.insync.replicas (%d) cannot exceed broker replicas (%d)" (int $minIsr) $brokerCount) }} +{{- end }} +{{- $offsetsRf := index .Values.brokers.config "offsets.topic.replication.factor" | default 1 }} +{{- if gt (int $offsetsRf) $brokerCount }} +{{- fail (printf "offsets.topic.replication.factor (%d) cannot exceed broker replicas (%d)" (int $offsetsRf) $brokerCount) }} +{{- end }} +{{- $txnRf := index .Values.brokers.config "transaction.state.log.replication.factor" | default 1 }} +{{- if gt (int $txnRf) $brokerCount }} +{{- fail (printf "transaction.state.log.replication.factor (%d) cannot exceed broker replicas (%d)" (int $txnRf) $brokerCount) }} +{{- end }} +{{- /* Validate connect workerConfig replication factors */}} +{{- $srcConfigRf := index .Values.connectSrc.workerConfig "config.storage.replication.factor" | default 1 }} +{{- if gt (int $srcConfigRf) $brokerCount }} +{{- fail (printf "connectSrc config.storage.replication.factor (%d) cannot exceed broker replicas (%d)" (int $srcConfigRf) $brokerCount) }} +{{- end }} +{{- $sinkConfigRf := index .Values.connectSink.workerConfig "config.storage.replication.factor" | default 1 }} +{{- if gt (int $sinkConfigRf) $brokerCount }} +{{- fail (printf "connectSink config.storage.replication.factor (%d) cannot exceed broker replicas (%d)" (int $sinkConfigRf) $brokerCount) }} +{{- end }} +--- +apiVersion: {{ .Values.strimzi.apiVersion }} +kind: KafkaNodePool +metadata: + name: {{ include "countly-migrations.fullname" . }}-brokers + annotations: + "helm.sh/resource-policy": keep + {{- include "countly-migrations.syncWave" (dict "wave" "5" "root" .) | nindent 4 }} + labels: + strimzi.io/cluster: {{ include "countly-migrations.fullname" . }} + {{- include "countly-migrations.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.brokers.replicas }} + roles: + - broker + storage: + type: persistent-claim + size: {{ .Values.brokers.persistence.size }} + {{- $brokerSC := include "countly-migrations.storageClassName" (dict "component" .Values.brokers.persistence "root" .) }} + {{- if $brokerSC }} + class: {{ $brokerSC | quote }} + {{- end }} + deleteClaim: {{ include "countly-migrations.deleteClaim" . }} + resources: + {{- toYaml .Values.brokers.resources | nindent 4 }} + jvmOptions: + -Xms: {{ .Values.brokers.jvmOptions.xms }} + -Xmx: {{ .Values.brokers.jvmOptions.xmx }} + template: + pod: + {{- with .Values.brokers.scheduling.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.brokers.scheduling.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +--- +apiVersion: {{ .Values.strimzi.apiVersion }} +kind: KafkaNodePool +metadata: + name: {{ include "countly-migrations.fullname" . }}-controllers + annotations: + "helm.sh/resource-policy": keep + {{- include "countly-migrations.syncWave" (dict "wave" "5" "root" .) | nindent 4 }} + labels: + strimzi.io/cluster: {{ include "countly-migrations.fullname" . }} + {{- include "countly-migrations.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.controllers.replicas }} + roles: + - controller + storage: + type: persistent-claim + size: {{ .Values.controllers.persistence.size }} + {{- $controllerSC := include "countly-migrations.storageClassName" (dict "component" .Values.controllers.persistence "root" .) }} + {{- if $controllerSC }} + class: {{ $controllerSC | quote }} + {{- end }} + deleteClaim: {{ include "countly-migrations.deleteClaim" . }} + resources: + {{- toYaml .Values.controllers.resources | nindent 4 }} + template: + pod: + {{- with .Values.controllers.scheduling.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.controllers.scheduling.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +--- +apiVersion: {{ .Values.strimzi.apiVersion }} +kind: Kafka +metadata: + name: {{ include "countly-migrations.fullname" . }} + labels: + {{- include "countly-migrations.labels" . | nindent 4 }} + annotations: + "helm.sh/resource-policy": keep + strimzi.io/kraft: enabled + strimzi.io/node-pools: enabled + {{- include "countly-migrations.syncWave" (dict "wave" "5" "root" .) | nindent 4 }} +spec: + kafka: + version: {{ .Values.version | quote }} + listeners: + {{- range .Values.listeners }} + - name: {{ .name }} + port: {{ .port }} + type: {{ .type }} + tls: {{ .tls }} + {{- end }} + config: + {{- range $key, $value := .Values.brokers.config }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- if .Values.metrics.enabled }} + metricsConfig: + type: jmxPrometheusExporter + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-metrics + key: kafka-metrics-config.yml + {{- end }} + entityOperator: {} diff --git a/charts/countly-migrations/templates/kafkaconnect-sink.yaml b/charts/countly-migrations/templates/kafkaconnect-sink.yaml new file mode 100644 index 0000000..02d5b67 --- /dev/null +++ b/charts/countly-migrations/templates/kafkaconnect-sink.yaml @@ -0,0 +1,166 @@ +apiVersion: {{ .Values.strimzi.apiVersion }} +kind: KafkaConnect +metadata: + name: {{ include "countly-migrations.connectSinkName" . }} + labels: + {{- include "countly-migrations.labels" . | nindent 4 }} + annotations: + strimzi.io/use-connector-resources: "true" + {{- include "countly-migrations.syncWave" (dict "wave" "10" "root" .) | nindent 4 }} +spec: + version: {{ .Values.version | quote }} + replicas: {{ .Values.connectSink.replicas }} + bootstrapServers: {{ include "countly-migrations.bootstrapServers" . }} + image: {{ .Values.connectSink.image }} + {{- if .Values.metrics.enabled }} + metricsConfig: + type: jmxPrometheusExporter + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-metrics + key: connect-metrics-config.yml + {{- end }} + resources: + {{- toYaml .Values.connectSink.resources | nindent 4 }} + jvmOptions: + -Xms: {{ .Values.connectSink.jvmOptions.xms }} + -Xmx: {{ .Values.connectSink.jvmOptions.xmx }} + javaSystemProperties: + - name: com.sun.management.jmxremote + value: "true" + - name: com.sun.management.jmxremote.port + value: "9999" + - name: com.sun.management.jmxremote.authenticate + value: "false" + - name: com.sun.management.jmxremote.ssl + value: "false" + config: + {{- range $key, $value := .Values.connectSink.workerConfig }} + {{ $key }}: {{ $value | quote }} + {{- end }} + template: + connectContainer: + env: + # ClickHouse credentials from Secret + - name: CLICKHOUSE_USER + valueFrom: + secretKeyRef: + name: {{ include "countly-migrations.clickhouseSecretName" . }} + key: username + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "countly-migrations.clickhouseSecretName" . }} + key: password + # ClickHouse connection from ConfigMap + - name: CLICKHOUSE_HOST + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: CLICKHOUSE_HOST + - name: CLICKHOUSE_PORT + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: CLICKHOUSE_PORT + - name: CLICKHOUSE_SSL + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: CLICKHOUSE_SSL + - name: CLICKHOUSE_DB + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: CLICKHOUSE_DB + # Behavior + - name: EXACTLY_ONCE + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: EXACTLY_ONCE + - name: ERRORS_RETRY_TIMEOUT + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: ERRORS_RETRY_TIMEOUT + - name: ERRORS_TOLERANCE + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: ERRORS_TOLERANCE + # ClickHouse settings + - name: CLICKHOUSE_SETTINGS + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: CLICKHOUSE_SETTINGS + - name: BYPASS_ROW_BINARY + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: BYPASS_ROW_BINARY + - name: TABLE_REFRESH_INTERVAL + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: TABLE_REFRESH_INTERVAL + # Converters + - name: KEY_CONVERTER + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: KEY_CONVERTER + - name: VALUE_CONVERTER + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: VALUE_CONVERTER + - name: VALUE_CONVERTER_SCHEMAS_ENABLE + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: VALUE_CONVERTER_SCHEMAS_ENABLE + # Sink consumer batching + - name: KAFKA_CONSUMER_FETCH_MIN_BYTES + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: KAFKA_CONSUMER_FETCH_MIN_BYTES + - name: KAFKA_CONSUMER_FETCH_MAX_WAIT_MS + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: KAFKA_CONSUMER_FETCH_MAX_WAIT_MS + - name: KAFKA_CONSUMER_MAX_POLL_RECORDS + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: KAFKA_CONSUMER_MAX_POLL_RECORDS + - name: KAFKA_CONSUMER_MAX_PARTITION_FETCH_BYTES + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: KAFKA_CONSUMER_MAX_PARTITION_FETCH_BYTES + - name: KAFKA_CONSUMER_FETCH_MAX_BYTES + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: KAFKA_CONSUMER_FETCH_MAX_BYTES + # Writeback consumer tuning + - name: WB_CONSUMER_MAX_POLL_RECORDS + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: WB_CONSUMER_MAX_POLL_RECORDS + - name: WB_CONSUMER_FETCH_MAX_WAIT_MS + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: WB_CONSUMER_FETCH_MAX_WAIT_MS + # MongoDB connection for writeback + - name: MONGO_CONN_STR + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: MONGO_CONN_STR diff --git a/charts/countly-migrations/templates/kafkaconnect-src.yaml b/charts/countly-migrations/templates/kafkaconnect-src.yaml new file mode 100644 index 0000000..1fae938 --- /dev/null +++ b/charts/countly-migrations/templates/kafkaconnect-src.yaml @@ -0,0 +1,108 @@ +apiVersion: {{ .Values.strimzi.apiVersion }} +kind: KafkaConnect +metadata: + name: {{ include "countly-migrations.connectSrcName" . }} + labels: + {{- include "countly-migrations.labels" . | nindent 4 }} + annotations: + strimzi.io/use-connector-resources: "true" + {{- include "countly-migrations.syncWave" (dict "wave" "10" "root" .) | nindent 4 }} +spec: + version: {{ .Values.version | quote }} + replicas: {{ .Values.connectSrc.replicas }} + bootstrapServers: {{ include "countly-migrations.bootstrapServers" . }} + image: {{ .Values.connectSrc.image }} + {{- if .Values.metrics.enabled }} + metricsConfig: + type: jmxPrometheusExporter + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-metrics + key: connect-metrics-config.yml + {{- end }} + resources: + {{- toYaml .Values.connectSrc.resources | nindent 4 }} + jvmOptions: + -Xms: {{ .Values.connectSrc.jvmOptions.xms }} + -Xmx: {{ .Values.connectSrc.jvmOptions.xmx }} + javaSystemProperties: + - name: com.sun.management.jmxremote + value: "true" + - name: com.sun.management.jmxremote.port + value: "9999" + - name: com.sun.management.jmxremote.authenticate + value: "false" + - name: com.sun.management.jmxremote.ssl + value: "false" + config: + {{- range $key, $value := .Values.connectSrc.workerConfig }} + {{ $key }}: {{ $value | quote }} + {{- end }} + template: + connectContainer: + env: + - name: DBZ_SNAPSHOT_MODE + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: DBZ_SNAPSHOT_MODE + - name: DBZ_SNAPSHOT_MAX_THREADS + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: DBZ_SNAPSHOT_MAX_THREADS + - name: DBZ_SNAPSHOT_FETCH_SIZE + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: DBZ_SNAPSHOT_FETCH_SIZE + - name: DBZ_MAX_BATCH_SIZE + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: DBZ_MAX_BATCH_SIZE + - name: DBZ_MAX_QUEUE_SIZE + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: DBZ_MAX_QUEUE_SIZE + - name: MONGO_CONN_STR + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: MONGO_CONN_STR + - name: MONGO_DB + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: MONGO_DB + - name: DBZ_FILTERS_MATCH_MODE + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: DBZ_FILTERS_MATCH_MODE + - name: DRILL_EVENTS_NS_SELECTOR + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: DRILL_EVENTS_NS_SELECTOR + - name: TOPIC_PREFIX + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: TOPIC_PREFIX + - name: KEY_CONVERTER + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: KEY_CONVERTER + - name: VALUE_CONVERTER + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: VALUE_CONVERTER + - name: VALUE_CONVERTER_SCHEMAS_ENABLE + valueFrom: + configMapKeyRef: + name: {{ include "countly-migrations.fullname" . }}-connect-env + key: VALUE_CONVERTER_SCHEMAS_ENABLE diff --git a/charts/countly-migrations/templates/namespace.yaml b/charts/countly-migrations/templates/namespace.yaml new file mode 100644 index 0000000..d778a10 --- /dev/null +++ b/charts/countly-migrations/templates/namespace.yaml @@ -0,0 +1,12 @@ +{{- if .Values.createNamespace }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Release.Namespace }} + labels: + {{- include "countly-migrations.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-migrations.syncWave" (dict "wave" "-5" "root" .) | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/countly-migrations/templates/operator/deployment.yaml b/charts/countly-migrations/templates/operator/deployment.yaml new file mode 100644 index 0000000..d8f6c6c --- /dev/null +++ b/charts/countly-migrations/templates/operator/deployment.yaml @@ -0,0 +1,84 @@ +{{- if .Values.operator.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: strimzi-cluster-operator + labels: + {{- include "countly-migrations.labels" . | nindent 4 }} + app: strimzi + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-migrations.syncWave" (dict "wave" "-1" "root" .) | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.operator.replicas }} + selector: + matchLabels: + name: strimzi-cluster-operator + strimzi.io/kind: cluster-operator + template: + metadata: + labels: + name: strimzi-cluster-operator + strimzi.io/kind: cluster-operator + {{- include "countly-migrations.selectorLabels" . | nindent 8 }} + spec: + serviceAccountName: strimzi-cluster-operator + containers: + - name: strimzi-cluster-operator + image: {{ .Values.operator.image }} + ports: + - containerPort: 8080 + name: http + resources: + {{- toYaml .Values.operator.resources | nindent 12 }} + env: + - name: STRIMZI_NAMESPACE + value: {{ .Release.Namespace | quote }} + - name: STRIMZI_RBAC_SCOPE + value: "NAMESPACED" + - name: STRIMZI_KAFKA_IMAGES + value: {{ printf "%s=%s" .Values.version .Values.operator.imageMaps.kafka | quote }} + - name: STRIMZI_KAFKA_CONNECT_IMAGES + value: {{ printf "%s=%s" .Values.version .Values.operator.imageMaps.kafkaConnect | quote }} + - name: STRIMZI_KAFKA_BRIDGE_IMAGES + value: {{ printf "%s=%s" .Values.version .Values.operator.imageMaps.kafkaBridge | quote }} + - name: STRIMZI_KAFKA_MIRROR_MAKER_2_IMAGES + value: {{ printf "%s=%s" .Values.version .Values.operator.imageMaps.kafkaMirrorMaker2 | quote }} + - name: STRIMZI_DEFAULT_KANIKO_EXECUTOR_IMAGE + value: {{ .Values.operator.kanikoImage | quote }} + - name: STRIMZI_DEFAULT_MAVEN_BUILDER_IMAGE + value: {{ .Values.operator.mavenBuilderImage | quote }} + - name: STRIMZI_FULL_RECONCILIATION_INTERVAL_MS + value: {{ .Values.operator.reconciliationIntervalMs | quote }} + - name: STRIMZI_OPERATION_TIMEOUT_MS + value: {{ .Values.operator.operationTimeoutMs | quote }} + - name: STRIMZI_LEADER_ELECTION_ENABLED + value: "true" + - name: STRIMZI_LEADER_ELECTION_LEASE_NAME + value: "strimzi-cluster-operator" + - name: STRIMZI_OPERATOR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: STRIMZI_LEADER_ELECTION_LEASE_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: STRIMZI_LEADER_ELECTION_IDENTITY + valueFrom: + fieldRef: + fieldPath: metadata.name + livenessProbe: + httpGet: + path: /healthy + port: http + initialDelaySeconds: 10 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /ready + port: http + initialDelaySeconds: 10 + periodSeconds: 30 +{{- end }} diff --git a/charts/countly-migrations/templates/operator/role-leader-election.yaml b/charts/countly-migrations/templates/operator/role-leader-election.yaml new file mode 100644 index 0000000..fe2ed7b --- /dev/null +++ b/charts/countly-migrations/templates/operator/role-leader-election.yaml @@ -0,0 +1,16 @@ +{{- if .Values.operator.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: strimzi-cluster-operator-leader-election + labels: + {{- include "countly-migrations.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-migrations.syncWave" (dict "wave" "-1" "root" .) | nindent 4 }} + {{- end }} +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +{{- end }} diff --git a/charts/countly-migrations/templates/operator/role-namespace.yaml b/charts/countly-migrations/templates/operator/role-namespace.yaml new file mode 100644 index 0000000..cf4ad59 --- /dev/null +++ b/charts/countly-migrations/templates/operator/role-namespace.yaml @@ -0,0 +1,43 @@ +{{- if .Values.operator.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: strimzi-cluster-operator-namespaced + labels: + {{- include "countly-migrations.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-migrations.syncWave" (dict "wave" "-1" "root" .) | nindent 4 }} + {{- end }} +rules: + - apiGroups: [""] + resources: ["pods", "pods/log", "services", "endpoints", "configmaps", "secrets", "events", "persistentvolumeclaims", "serviceaccounts"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["apps"] + resources: ["deployments", "statefulsets", "replicasets"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies", "ingresses"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["kafka.strimzi.io"] + resources: ["kafkas", "kafkaconnects", "kafkaconnectors", "kafkamirrormaker2s", "kafkabridges", "kafkarebalances", "kafkanodepools"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["kafka.strimzi.io"] + resources: ["*/status"] + verbs: ["get", "patch", "update"] + - apiGroups: ["kafka.strimzi.io"] + resources: ["*/finalizers"] + verbs: ["update"] + - apiGroups: ["core.strimzi.io"] + resources: ["strimzipodsets", "strimzipodsets/status"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +{{- end }} diff --git a/charts/countly-migrations/templates/operator/rolebinding-leader-election.yaml b/charts/countly-migrations/templates/operator/rolebinding-leader-election.yaml new file mode 100644 index 0000000..82f58d6 --- /dev/null +++ b/charts/countly-migrations/templates/operator/rolebinding-leader-election.yaml @@ -0,0 +1,20 @@ +{{- if .Values.operator.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: strimzi-cluster-operator-leader-election + labels: + {{- include "countly-migrations.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-migrations.syncWave" (dict "wave" "-1" "root" .) | nindent 4 }} + {{- end }} +subjects: + - kind: ServiceAccount + name: strimzi-cluster-operator + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: strimzi-cluster-operator-leader-election +{{- end }} diff --git a/charts/countly-migrations/templates/operator/rolebinding-namespace.yaml b/charts/countly-migrations/templates/operator/rolebinding-namespace.yaml new file mode 100644 index 0000000..fab1ae2 --- /dev/null +++ b/charts/countly-migrations/templates/operator/rolebinding-namespace.yaml @@ -0,0 +1,20 @@ +{{- if .Values.operator.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: strimzi-cluster-operator-namespaced + labels: + {{- include "countly-migrations.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-migrations.syncWave" (dict "wave" "-1" "root" .) | nindent 4 }} + {{- end }} +subjects: + - kind: ServiceAccount + name: strimzi-cluster-operator + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: strimzi-cluster-operator-namespaced +{{- end }} diff --git a/charts/countly-migrations/templates/operator/serviceaccount.yaml b/charts/countly-migrations/templates/operator/serviceaccount.yaml new file mode 100644 index 0000000..048f4c1 --- /dev/null +++ b/charts/countly-migrations/templates/operator/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.operator.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: strimzi-cluster-operator + labels: + {{- include "countly-migrations.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-migrations.syncWave" (dict "wave" "-1" "root" .) | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/countly-migrations/templates/progress/configmap-script.yaml b/charts/countly-migrations/templates/progress/configmap-script.yaml new file mode 100644 index 0000000..7bd34b2 --- /dev/null +++ b/charts/countly-migrations/templates/progress/configmap-script.yaml @@ -0,0 +1,82 @@ +{{- if .Values.progressMonitor.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "countly-migrations.fullname" . }}-progress-script + labels: + {{- include "countly-migrations.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-migrations.syncWave" (dict "wave" "20" "root" .) | nindent 4 }} + {{- end }} +data: + check-progress.sh: | + #!/bin/bash + set -e + + RED='\033[0;31m' + GREEN='\033[0;32m' + YELLOW='\033[1;33m' + CYAN='\033[0;36m' + BLUE='\033[0;34m' + NC='\033[0m' + + NAMESPACE_MIGRATION="{{ .Release.Namespace }}" + NAMESPACE_MONGODB="{{ .Values.progressMonitor.mongodb.namespace }}" + NAMESPACE_CLICKHOUSE="{{ .Values.progressMonitor.clickhouse.namespace }}" + MONGO_POD="{{ .Values.progressMonitor.mongodb.pod }}" + MONGO_CONTAINER="{{ .Values.progressMonitor.mongodb.container }}" + MONGO_USER="{{ .Values.progressMonitor.mongodb.user }}" + MONGO_DB="{{ .Values.progressMonitor.mongodb.database }}" + CLICKHOUSE_POD="{{ .Values.progressMonitor.clickhouse.pod }}" + CLICKHOUSE_CONTAINER="{{ .Values.progressMonitor.clickhouse.container }}" + CLICKHOUSE_DB="{{ .Values.progressMonitor.clickhouse.database }}" + CLICKHOUSE_TABLE="{{ .Values.progressMonitor.clickhouse.table }}" + BROKER_POD="{{ include "countly-migrations.fullname" . }}-brokers-0" + SOURCE_CONNECTOR="mongo-snapshot-only-mig" + SINK_CONNECTOR="ch-sink-drill-migration-mig" + + # Get MongoDB count + MONGO_COUNT=$(kubectl exec -n $NAMESPACE_MONGODB $MONGO_POD -c $MONGO_CONTAINER -- \ + mongosh --quiet --eval " + var d = db.getSiblingDB('$MONGO_DB'); + var total = 0; + d.getCollectionNames().filter(function(c){ return c.startsWith('drill_events'); }).forEach(function(c){ + total += d.getCollection(c).estimatedDocumentCount(); + }); + print(total); + " 2>&1 | grep -E '^[0-9]+$' | tail -1) || MONGO_COUNT=0 + + # Get ClickHouse count + CLICKHOUSE_COUNT=$(kubectl exec -n $NAMESPACE_CLICKHOUSE $CLICKHOUSE_POD -c $CLICKHOUSE_CONTAINER -- \ + clickhouse-client --query "SELECT count() FROM $CLICKHOUSE_DB.$CLICKHOUSE_TABLE FORMAT TSV" 2>/dev/null) || CLICKHOUSE_COUNT=0 + + # Get connector status + SOURCE_STATUS=$(kubectl get kafkaconnector $SOURCE_CONNECTOR -n $NAMESPACE_MIGRATION \ + -o jsonpath='{.status.connectorStatus.connector.state}' 2>/dev/null) || SOURCE_STATUS="UNKNOWN" + SINK_STATUS=$(kubectl get kafkaconnector $SINK_CONNECTOR -n $NAMESPACE_MIGRATION \ + -o jsonpath='{.status.connectorStatus.connector.state}' 2>/dev/null) || SINK_STATUS="UNKNOWN" + + echo "" + echo -e "${CYAN}======================================${NC}" + echo -e "${CYAN} MIGRATION PROGRESS${NC}" + echo -e "${CYAN}======================================${NC}" + echo "" + echo -e "${YELLOW}MongoDB:${NC} $(printf "%'d" $MONGO_COUNT 2>/dev/null || echo $MONGO_COUNT) documents" + echo -e "${YELLOW}ClickHouse:${NC} $(printf "%'d" $CLICKHOUSE_COUNT 2>/dev/null || echo $CLICKHOUSE_COUNT) documents" + echo "" + + if [ "$MONGO_COUNT" -gt 0 ] && [ "$CLICKHOUSE_COUNT" -ge 0 ] 2>/dev/null; then + REMAINING=$((MONGO_COUNT - CLICKHOUSE_COUNT)) + PROGRESS=$(echo "scale=2; ($CLICKHOUSE_COUNT * 100) / $MONGO_COUNT" | bc 2>/dev/null || echo "0") + echo -e "${YELLOW}Progress:${NC} ${GREEN}${PROGRESS}%${NC} ($(printf "%'d" $REMAINING 2>/dev/null || echo $REMAINING) remaining)" + fi + + echo "" + echo -e "${YELLOW}Connectors:${NC}" + echo -e " Source: $SOURCE_STATUS" + echo -e " Sink: $SINK_STATUS" + echo "" + echo -e "${BLUE}$(date '+%Y-%m-%d %H:%M:%S')${NC}" + echo -e "${CYAN}======================================${NC}" +{{- end }} diff --git a/charts/countly-migrations/templates/progress/cronjob.yaml b/charts/countly-migrations/templates/progress/cronjob.yaml new file mode 100644 index 0000000..9d41be9 --- /dev/null +++ b/charts/countly-migrations/templates/progress/cronjob.yaml @@ -0,0 +1,37 @@ +{{- if .Values.progressMonitor.enabled }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "countly-migrations.fullname" . }}-progress + labels: + {{- include "countly-migrations.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-migrations.syncWave" (dict "wave" "20" "root" .) | nindent 4 }} + {{- end }} +spec: + schedule: {{ .Values.progressMonitor.schedule | quote }} + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 0 + template: + spec: + serviceAccountName: {{ include "countly-migrations.fullname" . }}-progress + restartPolicy: Never + containers: + - name: progress + image: {{ .Values.progressMonitor.image }} + command: ["/bin/bash", "/scripts/check-progress.sh"] + volumeMounts: + - name: script + mountPath: /scripts + readOnly: true + volumes: + - name: script + configMap: + name: {{ include "countly-migrations.fullname" . }}-progress-script + defaultMode: 0755 +{{- end }} diff --git a/charts/countly-migrations/templates/progress/rbac.yaml b/charts/countly-migrations/templates/progress/rbac.yaml new file mode 100644 index 0000000..650599f --- /dev/null +++ b/charts/countly-migrations/templates/progress/rbac.yaml @@ -0,0 +1,38 @@ +{{- if .Values.progressMonitor.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "countly-migrations.fullname" . }}-progress + labels: + {{- include "countly-migrations.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-migrations.syncWave" (dict "wave" "20" "root" .) | nindent 4 }} + {{- end }} +rules: + - apiGroups: [""] + resources: ["pods", "pods/exec"] + verbs: ["get", "list", "create"] + - apiGroups: ["kafka.strimzi.io"] + resources: ["kafkaconnectors"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "countly-migrations.fullname" . }}-progress + labels: + {{- include "countly-migrations.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-migrations.syncWave" (dict "wave" "20" "root" .) | nindent 4 }} + {{- end }} +subjects: + - kind: ServiceAccount + name: {{ include "countly-migrations.fullname" . }}-progress + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "countly-migrations.fullname" . }}-progress +{{- end }} diff --git a/charts/countly-migrations/templates/progress/serviceaccount.yaml b/charts/countly-migrations/templates/progress/serviceaccount.yaml new file mode 100644 index 0000000..615e430 --- /dev/null +++ b/charts/countly-migrations/templates/progress/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.progressMonitor.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "countly-migrations.fullname" . }}-progress + labels: + {{- include "countly-migrations.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-migrations.syncWave" (dict "wave" "20" "root" .) | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/countly-migrations/templates/secret-clickhouse-auth.yaml b/charts/countly-migrations/templates/secret-clickhouse-auth.yaml new file mode 100644 index 0000000..8ab0b7d --- /dev/null +++ b/charts/countly-migrations/templates/secret-clickhouse-auth.yaml @@ -0,0 +1,19 @@ +{{- if not .Values.clickhouse.existingSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.clickhouse.secretName }} + labels: + {{- include "countly-migrations.labels" . | nindent 4 }} + {{- if or .Values.secrets.keep .Values.argocd.enabled }} + annotations: + {{- if .Values.secrets.keep }} + "helm.sh/resource-policy": keep + {{- end }} + {{- include "countly-migrations.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} + {{- end }} +type: Opaque +data: + username: {{ .Values.clickhouse.username | b64enc | quote }} + password: {{ .Values.clickhouse.password | b64enc | quote }} +{{- end }} diff --git a/charts/countly-migrations/templates/storageclass.yaml b/charts/countly-migrations/templates/storageclass.yaml new file mode 100644 index 0000000..a4d5efc --- /dev/null +++ b/charts/countly-migrations/templates/storageclass.yaml @@ -0,0 +1,20 @@ +{{- if .Values.storageClass.create }} +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.storageClass.name }} + labels: + {{- include "countly-migrations.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-migrations.syncWave" (dict "wave" "-3" "root" .) | nindent 4 }} + {{- end }} +provisioner: {{ .Values.storageClass.provisioner }} +{{- with .Values.storageClass.parameters }} +parameters: + {{- toYaml . | nindent 2 }} +{{- end }} +reclaimPolicy: {{ .Values.storageClass.reclaimPolicy }} +allowVolumeExpansion: {{ .Values.storageClass.allowVolumeExpansion }} +volumeBindingMode: {{ .Values.storageClass.volumeBindingMode }} +{{- end }} diff --git a/charts/countly-migrations/values.yaml b/charts/countly-migrations/values.yaml new file mode 100644 index 0000000..be9c7df --- /dev/null +++ b/charts/countly-migrations/values.yaml @@ -0,0 +1,291 @@ +global: + imageRegistry: "" + imagePullSecrets: [] + storageClass: "" + scheduling: + nodeSelector: {} + tolerations: [] + +nameOverride: "" +fullnameOverride: "" + +createNamespace: false + +strimzi: + apiVersion: kafka.strimzi.io/v1beta2 + +# ── MIGRATION CONTROL ────────────────────────────────── +# Master switch for all connectors. Per-connector overrides available below. +migration: + state: running # running | paused | stopped + deleteClaim: false # Set true before teardown to auto-delete PVCs + +# ── ARGOCD (optional) ───────────────────────────────── +# When enabled, adds sync-wave annotations for ordered deployment. +# When disabled (default), chart works with plain helm install. +argocd: + enabled: false + +# ── STRIMZI OPERATOR ────────────────────────────────── +# Deploys a namespace-scoped Strimzi operator. CRDs must be pre-installed. +operator: + enabled: true + image: quay.io/strimzi/operator:0.47.0 + replicas: 1 + resources: + requests: + cpu: 200m + memory: 384Mi + limits: + cpu: 500m + memory: 384Mi + reconciliationIntervalMs: "120000" + operationTimeoutMs: "300000" + # Strimzi image maps (operator resolves container images from these) + imageMaps: + kafka: "quay.io/strimzi/kafka:0.47.0-kafka-4.0.0" + kafkaConnect: "quay.io/strimzi/kafka:0.47.0-kafka-4.0.0" + kafkaBridge: "quay.io/strimzi/kafka:0.47.0-kafka-4.0.0" + kafkaMirrorMaker2: "quay.io/strimzi/kafka:0.47.0-kafka-4.0.0" + kanikoImage: "quay.io/strimzi/kaniko-executor:0.47.0" + mavenBuilderImage: "quay.io/strimzi/maven-builder:0.47.0" + +# ── STORAGE ─────────────────────────────────────────── +storageClass: + create: true + name: migrations-storage + provisioner: pd.csi.storage.gke.io # GKE default + parameters: + type: pd-ssd + reclaimPolicy: Retain + allowVolumeExpansion: true + volumeBindingMode: WaitForFirstConsumer + +# ── KAFKA ───────────────────────────────────────────── +version: "4.0.0" + +listeners: + - name: internal + port: 9092 + type: internal + tls: false + +brokers: + replicas: 1 + resources: + requests: + cpu: "500m" + memory: "2Gi" + limits: + cpu: "1" + memory: "4Gi" + jvmOptions: + xms: "1g" + xmx: "2g" + config: + default.replication.factor: 1 + min.insync.replicas: 1 + unclean.leader.election.enable: false + log.retention.hours: 48 + log.segment.bytes: "1073741824" + log.cleanup.policy: delete + compression.type: zstd + auto.create.topics.enable: true + offsets.topic.replication.factor: 1 + transaction.state.log.replication.factor: 1 + transaction.state.log.min.isr: 1 + persistence: + size: 50Gi + storageClass: "" + deleteClaim: false + scheduling: + nodeSelector: {} + tolerations: [] + +controllers: + replicas: 1 + resources: + requests: + cpu: "200m" + memory: "1Gi" + limits: + cpu: "500m" + memory: "2Gi" + persistence: + size: 10Gi + storageClass: "" + deleteClaim: false + scheduling: + nodeSelector: {} + tolerations: [] + +# ── CONNECT SOURCE (Debezium) ───────────────────────── +connectSrc: + name: connect-mig-src + image: "kanwarujjaval/connect-migrations:0.47.0-kafka-4.0.0-dbz-3.2.2-ch-1.3.3" + replicas: 1 + resources: + requests: + cpu: "500m" + memory: "2Gi" + limits: + cpu: "1" + memory: "3Gi" + jvmOptions: + xms: "1g" + xmx: "2g" + workerConfig: + group.id: connect-mig-src + config.storage.topic: mig_src_configs + offset.storage.topic: mig_src_offsets + status.storage.topic: mig_src_status + config.storage.replication.factor: 1 + offset.storage.replication.factor: 1 + status.storage.replication.factor: 1 + key.converter: org.apache.kafka.connect.storage.StringConverter + value.converter: org.apache.kafka.connect.json.JsonConverter + value.converter.schemas.enable: "false" + connector.client.config.override.policy: All + config.providers: env + config.providers.env.class: org.apache.kafka.common.config.provider.EnvVarConfigProvider + +# ── CONNECT SINK (ClickHouse + Writeback) ───────────── +connectSink: + name: connect-mig-sink + image: "kanwarujjaval/connect-migrations:0.47.0-kafka-4.0.0-dbz-3.2.2-ch-1.3.3" + replicas: 1 + resources: + requests: + cpu: "1" + memory: "4Gi" + limits: + cpu: "2" + memory: "8Gi" + jvmOptions: + xms: "2g" + xmx: "4g" + workerConfig: + group.id: connect-mig-sink + config.storage.topic: mig_sink_configs + offset.storage.topic: mig_sink_offsets + status.storage.topic: mig_sink_status + config.storage.replication.factor: 1 + offset.storage.replication.factor: 1 + status.storage.replication.factor: 1 + key.converter: org.apache.kafka.connect.storage.StringConverter + value.converter: org.apache.kafka.connect.json.JsonConverter + value.converter.schemas.enable: "false" + connector.client.config.override.policy: All + config.providers: env + config.providers.env.class: org.apache.kafka.common.config.provider.EnvVarConfigProvider + +# ── MONGODB SOURCE ──────────────────────────────────── +mongodb: + connectionString: "mongodb://host:27017/?replicaSet=rs0" + database: countly_drill + filtersMatchMode: regex # literal | regex + namespaceSelector: "countly_drill[.]drill_events.*" + topicPrefix: mig + +# ── CLICKHOUSE TARGET ───────────────────────────────── +clickhouse: + existingSecret: "" + secretName: clickhouse-auth + host: ch.clickhouse.svc.cluster.local + port: "8123" + ssl: "false" + database: countly_drill + username: default + password: "" + settings: "input_format_binary_read_json_as_string=1,allow_experimental_json_type=1,enable_json_type=1,async_insert=1,wait_for_async_insert=1,input_format_skip_unknown_fields=1" + bypassRowBinary: "false" + tableRefreshInterval: "300" + +# ── DEBEZIUM TUNING ────────────────────────────────── +debezium: + snapshotMode: initial + snapshotMaxThreads: 8 + snapshotFetchSize: 5000 + maxBatchSize: 2048 + maxQueueSize: 8192 + +# ── CONVERTERS ──────────────────────────────────────── +converters: + key: org.apache.kafka.connect.storage.StringConverter + value: org.apache.kafka.connect.json.JsonConverter + valueSchemasEnable: "false" + +# ── BEHAVIOR ────────────────────────────────────────── +behavior: + exactlyOnce: "false" + errorsRetryTimeout: "60" + errorsTolerance: all + +# ── CONSUMER TUNING ────────────────────────────────── +consumerTuning: + sink: + fetchMinBytes: "2097152" + fetchMaxWaitMs: "5000" + maxPollRecords: "50000" + maxPartitionFetchBytes: "5242880" + fetchMaxBytes: "134217728" + writeback: + maxPollRecords: "10000" + fetchMaxWaitMs: "500" + +# ── CONNECTORS ──────────────────────────────────────── +connectors: + source: + enabled: true + state: "" # Empty = inherit migration.state + tasksMax: 1 + topicPartitions: 4 + topicReplicationFactor: 1 + snapshotFilter: '{"migrated": {"$ne": true}}' + clickhouseSink: + enabled: true + state: "" + tasksMax: 4 + topics: drill-events-migration + topic2TableMap: "drill-events-migration=drill_events" + dlq: + topicName: drill-migration-dlq + replicationFactor: 1 + headerEnabled: true + mongoWriteback: + enabled: true + state: "" + tasksMax: 2 + bulkWriteOrdered: "false" + bulkWriteSize: 10000 + maxRetries: 3 + dlq: + topicName: writeback-dlq + replicationFactor: 1 + headerEnabled: true + +# ── METRICS ─────────────────────────────────────────── +metrics: + enabled: true + +# ── PROGRESS MONITOR ───────────────────────────────── +progressMonitor: + enabled: false + schedule: "*/5 * * * *" + image: bitnami/kubectl:1.30 + mongodb: + namespace: mongodb + pod: app-mongodb-0 + container: mongod + user: app + database: countly_drill + clickhouse: + namespace: clickhouse + pod: chi-ch-prod-main-0-0-0 + container: clickhouse + database: countly_drill + table: drill_events + +# ── SECRETS ─────────────────────────────────────────── +secrets: + keep: true diff --git a/charts/countly-mongodb/templates/_helpers.tpl b/charts/countly-mongodb/templates/_helpers.tpl index f9315e6..bf3482d 100644 --- a/charts/countly-mongodb/templates/_helpers.tpl +++ b/charts/countly-mongodb/templates/_helpers.tpl @@ -48,6 +48,15 @@ app.kubernetes.io/name: {{ include "countly-mongodb.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} +{{/* +ArgoCD sync-wave annotation (only when argocd.enabled). +*/}} +{{- define "countly-mongodb.syncWave" -}} +{{- if .root.Values.argocd.enabled }} +argocd.argoproj.io/sync-wave: {{ .wave | quote }} +{{- end }} +{{- end -}} + {{/* MongoDB connection string secret name (generated by operator) */}} diff --git a/charts/countly-mongodb/templates/deployment-exporter.yaml b/charts/countly-mongodb/templates/deployment-exporter.yaml index 0e29ad9..99b0d91 100644 --- a/charts/countly-mongodb/templates/deployment-exporter.yaml +++ b/charts/countly-mongodb/templates/deployment-exporter.yaml @@ -6,6 +6,10 @@ metadata: labels: {{- include "countly-mongodb.labels" . | nindent 4 }} app.kubernetes.io/component: exporter + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-mongodb.syncWave" (dict "wave" "10" "root" .) | nindent 4 }} + {{- end }} spec: replicas: 1 selector: diff --git a/charts/countly-mongodb/templates/mongodbcommunity.yaml b/charts/countly-mongodb/templates/mongodbcommunity.yaml index eed5f31..e3ba8d5 100644 --- a/charts/countly-mongodb/templates/mongodbcommunity.yaml +++ b/charts/countly-mongodb/templates/mongodbcommunity.yaml @@ -4,6 +4,7 @@ metadata: name: {{ include "countly-mongodb.fullname" . }} annotations: "helm.sh/resource-policy": keep + {{- include "countly-mongodb.syncWave" (dict "wave" "5" "root" .) | nindent 4 }} labels: {{- include "countly-mongodb.labels" . | nindent 4 }} spec: diff --git a/charts/countly-mongodb/templates/namespace.yaml b/charts/countly-mongodb/templates/namespace.yaml index c18077f..395dd89 100644 --- a/charts/countly-mongodb/templates/namespace.yaml +++ b/charts/countly-mongodb/templates/namespace.yaml @@ -5,4 +5,8 @@ metadata: name: {{ .Release.Namespace }} labels: {{- include "countly-mongodb.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-mongodb.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} + {{- end }} {{- end }} diff --git a/charts/countly-mongodb/templates/networkpolicy.yaml b/charts/countly-mongodb/templates/networkpolicy.yaml index 585fe2d..98e3dc1 100644 --- a/charts/countly-mongodb/templates/networkpolicy.yaml +++ b/charts/countly-mongodb/templates/networkpolicy.yaml @@ -5,6 +5,10 @@ metadata: name: {{ include "countly-mongodb.fullname" . }}-default-deny labels: {{- include "countly-mongodb.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-mongodb.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} + {{- end }} spec: podSelector: {} policyTypes: diff --git a/charts/countly-mongodb/templates/pdb.yaml b/charts/countly-mongodb/templates/pdb.yaml index b5a239d..75af5ae 100644 --- a/charts/countly-mongodb/templates/pdb.yaml +++ b/charts/countly-mongodb/templates/pdb.yaml @@ -5,6 +5,10 @@ metadata: name: {{ include "countly-mongodb.fullname" . }}-pdb labels: {{- include "countly-mongodb.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-mongodb.syncWave" (dict "wave" "5" "root" .) | nindent 4 }} + {{- end }} spec: {{- if .Values.podDisruptionBudget.minAvailable }} minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} diff --git a/charts/countly-mongodb/templates/secret-passwords.yaml b/charts/countly-mongodb/templates/secret-passwords.yaml index 94d09b7..1200229 100644 --- a/charts/countly-mongodb/templates/secret-passwords.yaml +++ b/charts/countly-mongodb/templates/secret-passwords.yaml @@ -8,6 +8,7 @@ metadata: {{- if .Values.secrets.keep }} helm.sh/resource-policy: keep {{- end }} + {{- include "countly-mongodb.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} type: Opaque data: {{ .Values.users.app.passwordSecretKey }}: |- @@ -31,6 +32,7 @@ metadata: {{- if .Values.secrets.keep }} helm.sh/resource-policy: keep {{- end }} + {{- include "countly-mongodb.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} type: Opaque data: {{ .Values.users.metrics.passwordSecretKey }}: |- diff --git a/charts/countly-mongodb/templates/service-metrics.yaml b/charts/countly-mongodb/templates/service-metrics.yaml index f0dc265..b8d2ebd 100644 --- a/charts/countly-mongodb/templates/service-metrics.yaml +++ b/charts/countly-mongodb/templates/service-metrics.yaml @@ -6,6 +6,10 @@ metadata: labels: {{- include "countly-mongodb.labels" . | nindent 4 }} countly.io/metrics: "true" + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly-mongodb.syncWave" (dict "wave" "10" "root" .) | nindent 4 }} + {{- end }} spec: selector: {{- include "countly-mongodb.selectorLabels" . | nindent 4 }} diff --git a/charts/countly-mongodb/values.yaml b/charts/countly-mongodb/values.yaml index 7a57a4a..ed96741 100644 --- a/charts/countly-mongodb/values.yaml +++ b/charts/countly-mongodb/values.yaml @@ -12,6 +12,9 @@ fullnameOverride: "" createNamespace: false +argocd: + enabled: false + mongodb: version: "8.2.5" members: 2 diff --git a/charts/countly-observability/templates/_helpers.tpl b/charts/countly-observability/templates/_helpers.tpl index 5e366a4..1cf2837 100644 --- a/charts/countly-observability/templates/_helpers.tpl +++ b/charts/countly-observability/templates/_helpers.tpl @@ -258,3 +258,12 @@ Args: dict "component" "storage" "default" "al {{- fail (printf "%s.storage.forcePathStyle requires storage.endpoint (used for S3-compatible endpoints like MinIO)" .component) -}} {{- end -}} {{- end }} + +{{/* +ArgoCD sync-wave annotation (only when argocd.enabled). +*/}} +{{- define "obs.syncWave" -}} +{{- if .root.Values.argocd.enabled }} +argocd.argoproj.io/sync-wave: {{ .wave | quote }} +{{- end }} +{{- end -}} diff --git a/charts/countly-observability/templates/ingress.yaml b/charts/countly-observability/templates/ingress.yaml index 1c7b14d..98b0aee 100644 --- a/charts/countly-observability/templates/ingress.yaml +++ b/charts/countly-observability/templates/ingress.yaml @@ -7,9 +7,12 @@ metadata: labels: {{- include "obs.labels" . | nindent 4 }} app.kubernetes.io/component: grafana - {{- with .Values.ingress.annotations }} + {{- if or .Values.argocd.enabled .Values.ingress.annotations }} annotations: + {{- include "obs.syncWave" (dict "wave" "5" "root" .) | nindent 4 }} + {{- with .Values.ingress.annotations }} {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} spec: ingressClassName: {{ .Values.ingress.className }} diff --git a/charts/countly-observability/templates/networkpolicy.yaml b/charts/countly-observability/templates/networkpolicy.yaml index b129833..13ae249 100644 --- a/charts/countly-observability/templates/networkpolicy.yaml +++ b/charts/countly-observability/templates/networkpolicy.yaml @@ -8,6 +8,10 @@ metadata: namespace: {{ .Release.Namespace }} labels: {{- include "obs.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "obs.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} + {{- end }} spec: podSelector: {} policyTypes: diff --git a/charts/countly-observability/templates/serviceaccount.yaml b/charts/countly-observability/templates/serviceaccount.yaml index 394e66a..c44ad0f 100644 --- a/charts/countly-observability/templates/serviceaccount.yaml +++ b/charts/countly-observability/templates/serviceaccount.yaml @@ -5,4 +5,8 @@ metadata: labels: {{- include "obs.labels" . | nindent 4 }} app.kubernetes.io/component: prometheus + {{- if .Values.argocd.enabled }} + annotations: + {{- include "obs.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} + {{- end }} automountServiceAccountToken: false diff --git a/charts/countly-observability/values.yaml b/charts/countly-observability/values.yaml index 0522a67..5929a36 100644 --- a/charts/countly-observability/values.yaml +++ b/charts/countly-observability/values.yaml @@ -1,3 +1,7 @@ +# -- ArgoCD integration (optional) +argocd: + enabled: false + # -- Deployment mode: "full" (all in-cluster), "hybrid" (no Grafana), "external" (no backends) mode: full diff --git a/charts/countly/templates/_countly-component.tpl b/charts/countly/templates/_countly-component.tpl index a35d032..f922930 100644 --- a/charts/countly/templates/_countly-component.tpl +++ b/charts/countly/templates/_countly-component.tpl @@ -22,6 +22,10 @@ metadata: labels: {{- include "countly.labels" $root | nindent 4 }} app.kubernetes.io/component: {{ $component }} + {{- if $root.Values.argocd.enabled }} + annotations: + {{- include "countly.syncWave" (dict "wave" "5" "root" $root) | nindent 4 }} + {{- end }} spec: {{- if not $values.hpa.enabled }} replicas: {{ $values.replicaCount }} diff --git a/charts/countly/templates/_helpers.tpl b/charts/countly/templates/_helpers.tpl index 9c10009..639492b 100644 --- a/charts/countly/templates/_helpers.tpl +++ b/charts/countly/templates/_helpers.tpl @@ -184,3 +184,12 @@ Called from NOTES.txt to surface errors during install. {{- end -}} {{- end -}} {{- end -}} + +{{/* +ArgoCD sync-wave annotation (only when argocd.enabled). +*/}} +{{- define "countly.syncWave" -}} +{{- if .root.Values.argocd.enabled }} +argocd.argoproj.io/sync-wave: {{ .wave | quote }} +{{- end }} +{{- end -}} diff --git a/charts/countly/templates/configmap-aggregator.yaml b/charts/countly/templates/configmap-aggregator.yaml index 0f48cc9..0c87e56 100644 --- a/charts/countly/templates/configmap-aggregator.yaml +++ b/charts/countly/templates/configmap-aggregator.yaml @@ -4,6 +4,10 @@ metadata: name: {{ include "countly.fullname" . }}-config-aggregator labels: {{- include "countly.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly.syncWave" (dict "wave" "2" "root" .) | nindent 4 }} + {{- end }} data: NODE_OPTIONS: {{ .Values.nodeOptions.aggregator | quote }} {{- range $key, $value := .Values.config.aggregator }} diff --git a/charts/countly/templates/configmap-api.yaml b/charts/countly/templates/configmap-api.yaml index f757e28..0998e7e 100644 --- a/charts/countly/templates/configmap-api.yaml +++ b/charts/countly/templates/configmap-api.yaml @@ -4,6 +4,10 @@ metadata: name: {{ include "countly.fullname" . }}-config-api labels: {{- include "countly.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly.syncWave" (dict "wave" "2" "root" .) | nindent 4 }} + {{- end }} data: NODE_OPTIONS: {{ .Values.nodeOptions.api | quote }} {{- range $key, $value := .Values.config.api }} diff --git a/charts/countly/templates/configmap-clickhouse.yaml b/charts/countly/templates/configmap-clickhouse.yaml index aec37f3..fc46c56 100644 --- a/charts/countly/templates/configmap-clickhouse.yaml +++ b/charts/countly/templates/configmap-clickhouse.yaml @@ -4,6 +4,10 @@ metadata: name: {{ include "countly.fullname" . }}-config-clickhouse labels: {{- include "countly.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly.syncWave" (dict "wave" "2" "root" .) | nindent 4 }} + {{- end }} data: {{- range $key, $value := .Values.config.clickhouse }} {{ $key }}: {{ $value | quote }} diff --git a/charts/countly/templates/configmap-common.yaml b/charts/countly/templates/configmap-common.yaml index 8b7818c..c11c169 100644 --- a/charts/countly/templates/configmap-common.yaml +++ b/charts/countly/templates/configmap-common.yaml @@ -4,6 +4,10 @@ metadata: name: {{ include "countly.fullname" . }}-config-common labels: {{- include "countly.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly.syncWave" (dict "wave" "2" "root" .) | nindent 4 }} + {{- end }} data: {{- range $key, $value := .Values.config.common }} {{ $key }}: {{ $value | quote }} diff --git a/charts/countly/templates/configmap-frontend.yaml b/charts/countly/templates/configmap-frontend.yaml index 9374bd1..3da6842 100644 --- a/charts/countly/templates/configmap-frontend.yaml +++ b/charts/countly/templates/configmap-frontend.yaml @@ -4,6 +4,10 @@ metadata: name: {{ include "countly.fullname" . }}-config-frontend labels: {{- include "countly.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly.syncWave" (dict "wave" "2" "root" .) | nindent 4 }} + {{- end }} data: NODE_OPTIONS: {{ .Values.nodeOptions.frontend | quote }} {{- range $key, $value := .Values.config.frontend }} diff --git a/charts/countly/templates/configmap-ingestor.yaml b/charts/countly/templates/configmap-ingestor.yaml index 10179f5..fab5046 100644 --- a/charts/countly/templates/configmap-ingestor.yaml +++ b/charts/countly/templates/configmap-ingestor.yaml @@ -4,6 +4,10 @@ metadata: name: {{ include "countly.fullname" . }}-config-ingestor labels: {{- include "countly.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly.syncWave" (dict "wave" "2" "root" .) | nindent 4 }} + {{- end }} data: NODE_OPTIONS: {{ .Values.nodeOptions.ingestor | quote }} {{- range $key, $value := .Values.config.ingestor }} diff --git a/charts/countly/templates/configmap-jobserver.yaml b/charts/countly/templates/configmap-jobserver.yaml index ee93bc3..d6bf84a 100644 --- a/charts/countly/templates/configmap-jobserver.yaml +++ b/charts/countly/templates/configmap-jobserver.yaml @@ -4,6 +4,10 @@ metadata: name: {{ include "countly.fullname" . }}-config-jobserver labels: {{- include "countly.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly.syncWave" (dict "wave" "2" "root" .) | nindent 4 }} + {{- end }} data: NODE_OPTIONS: {{ .Values.nodeOptions.jobserver | quote }} {{- range $key, $value := .Values.config.jobserver }} diff --git a/charts/countly/templates/configmap-kafka.yaml b/charts/countly/templates/configmap-kafka.yaml index 5f1c663..73e4fdb 100644 --- a/charts/countly/templates/configmap-kafka.yaml +++ b/charts/countly/templates/configmap-kafka.yaml @@ -4,6 +4,10 @@ metadata: name: {{ include "countly.fullname" . }}-config-kafka labels: {{- include "countly.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly.syncWave" (dict "wave" "2" "root" .) | nindent 4 }} + {{- end }} data: {{- range $key, $value := .Values.config.kafka }} {{ $key }}: {{ $value | quote }} diff --git a/charts/countly/templates/configmap-otel.yaml b/charts/countly/templates/configmap-otel.yaml index cdb42b3..abd3fe8 100644 --- a/charts/countly/templates/configmap-otel.yaml +++ b/charts/countly/templates/configmap-otel.yaml @@ -4,6 +4,10 @@ metadata: name: {{ include "countly.fullname" . }}-config-otel labels: {{- include "countly.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly.syncWave" (dict "wave" "2" "root" .) | nindent 4 }} + {{- end }} data: {{- range $key, $value := .Values.config.otel }} {{ $key }}: {{ $value | quote }} diff --git a/charts/countly/templates/external-secret-clickhouse.yaml b/charts/countly/templates/external-secret-clickhouse.yaml index 27adf2e..9b57d3e 100644 --- a/charts/countly/templates/external-secret-clickhouse.yaml +++ b/charts/countly/templates/external-secret-clickhouse.yaml @@ -6,6 +6,10 @@ metadata: name: {{ include "countly.fullname" . }}-clickhouse labels: {{- include "countly.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly.syncWave" (dict "wave" "1" "root" .) | nindent 4 }} + {{- end }} spec: refreshInterval: {{ .Values.secrets.externalSecret.refreshInterval | default "1h" }} secretStoreRef: diff --git a/charts/countly/templates/external-secret-common.yaml b/charts/countly/templates/external-secret-common.yaml index 8b9d4ed..3ebab1c 100644 --- a/charts/countly/templates/external-secret-common.yaml +++ b/charts/countly/templates/external-secret-common.yaml @@ -6,6 +6,10 @@ metadata: name: {{ include "countly.fullname" . }}-common labels: {{- include "countly.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly.syncWave" (dict "wave" "1" "root" .) | nindent 4 }} + {{- end }} spec: refreshInterval: {{ .Values.secrets.externalSecret.refreshInterval | default "1h" }} secretStoreRef: diff --git a/charts/countly/templates/external-secret-kafka.yaml b/charts/countly/templates/external-secret-kafka.yaml index 8bba188..dc4e145 100644 --- a/charts/countly/templates/external-secret-kafka.yaml +++ b/charts/countly/templates/external-secret-kafka.yaml @@ -6,6 +6,10 @@ metadata: name: {{ include "countly.fullname" . }}-kafka labels: {{- include "countly.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly.syncWave" (dict "wave" "1" "root" .) | nindent 4 }} + {{- end }} spec: refreshInterval: {{ .Values.secrets.externalSecret.refreshInterval | default "1h" }} secretStoreRef: diff --git a/charts/countly/templates/external-secret-mongodb.yaml b/charts/countly/templates/external-secret-mongodb.yaml index e1f5947..5f0603c 100644 --- a/charts/countly/templates/external-secret-mongodb.yaml +++ b/charts/countly/templates/external-secret-mongodb.yaml @@ -6,6 +6,10 @@ metadata: name: {{ include "countly.fullname" . }}-mongodb labels: {{- include "countly.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly.syncWave" (dict "wave" "1" "root" .) | nindent 4 }} + {{- end }} spec: refreshInterval: {{ .Values.secrets.externalSecret.refreshInterval | default "1h" }} secretStoreRef: diff --git a/charts/countly/templates/ingress.yaml b/charts/countly/templates/ingress.yaml index ac406c8..413d051 100644 --- a/charts/countly/templates/ingress.yaml +++ b/charts/countly/templates/ingress.yaml @@ -8,6 +8,7 @@ metadata: labels: {{- include "countly.labels" . | nindent 4 }} annotations: + {{- include "countly.syncWave" (dict "wave" "10" "root" .) | nindent 4 }} {{- with .Values.ingress.annotations }} {{- toYaml . | nindent 4 }} {{- end }} diff --git a/charts/countly/templates/namespace.yaml b/charts/countly/templates/namespace.yaml index 6a5e355..f584956 100644 --- a/charts/countly/templates/namespace.yaml +++ b/charts/countly/templates/namespace.yaml @@ -5,4 +5,8 @@ metadata: name: {{ .Release.Namespace }} labels: {{- include "countly.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} + {{- end }} {{- end }} diff --git a/charts/countly/templates/networkpolicy.yaml b/charts/countly/templates/networkpolicy.yaml index 776a5e3..86e74b4 100644 --- a/charts/countly/templates/networkpolicy.yaml +++ b/charts/countly/templates/networkpolicy.yaml @@ -5,6 +5,10 @@ metadata: name: {{ include "countly.fullname" . }}-default-deny labels: {{- include "countly.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} + {{- end }} spec: podSelector: {} policyTypes: diff --git a/charts/countly/templates/secret-clickhouse.yaml b/charts/countly/templates/secret-clickhouse.yaml index 1f6bdb4..65fa4c7 100644 --- a/charts/countly/templates/secret-clickhouse.yaml +++ b/charts/countly/templates/secret-clickhouse.yaml @@ -12,6 +12,7 @@ metadata: {{- if .Values.secrets.rotationId }} countly.io/rotation-id: {{ .Values.secrets.rotationId | quote }} {{- end }} + {{- include "countly.syncWave" (dict "wave" "1" "root" .) | nindent 4 }} type: Opaque data: COUNTLY_CONFIG__CLICKHOUSE_URL: {{ include "countly.clickhouse.url" . | b64enc }} diff --git a/charts/countly/templates/secret-common.yaml b/charts/countly/templates/secret-common.yaml index 2aad8ab..2308a71 100644 --- a/charts/countly/templates/secret-common.yaml +++ b/charts/countly/templates/secret-common.yaml @@ -12,6 +12,7 @@ metadata: {{- if .Values.secrets.rotationId }} countly.io/rotation-id: {{ .Values.secrets.rotationId | quote }} {{- end }} + {{- include "countly.syncWave" (dict "wave" "1" "root" .) | nindent 4 }} type: Opaque data: {{- $secretName := printf "%s-common" (include "countly.fullname" .) }} diff --git a/charts/countly/templates/secret-kafka.yaml b/charts/countly/templates/secret-kafka.yaml index 5d8cab8..18ed231 100644 --- a/charts/countly/templates/secret-kafka.yaml +++ b/charts/countly/templates/secret-kafka.yaml @@ -12,6 +12,7 @@ metadata: {{- if .Values.secrets.rotationId }} countly.io/rotation-id: {{ .Values.secrets.rotationId | quote }} {{- end }} + {{- include "countly.syncWave" (dict "wave" "1" "root" .) | nindent 4 }} type: Opaque data: COUNTLY_CONFIG__KAFKA_RDKAFKA_BROKERS: {{ include "countly.kafka.brokers" . | b64enc }} diff --git a/charts/countly/templates/secret-mongodb.yaml b/charts/countly/templates/secret-mongodb.yaml index 8cde015..6a49d23 100644 --- a/charts/countly/templates/secret-mongodb.yaml +++ b/charts/countly/templates/secret-mongodb.yaml @@ -12,6 +12,7 @@ metadata: {{- if .Values.secrets.rotationId }} countly.io/rotation-id: {{ .Values.secrets.rotationId | quote }} {{- end }} + {{- include "countly.syncWave" (dict "wave" "1" "root" .) | nindent 4 }} type: Opaque data: {{ .Values.secrets.mongodb.key }}: {{ include "countly.mongodb.connectionString" . | b64enc }} diff --git a/charts/countly/templates/serviceaccount.yaml b/charts/countly/templates/serviceaccount.yaml index 849898d..0d4e066 100644 --- a/charts/countly/templates/serviceaccount.yaml +++ b/charts/countly/templates/serviceaccount.yaml @@ -5,9 +5,12 @@ metadata: name: {{ include "countly.serviceAccountName" . }} labels: {{- include "countly.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} + {{- if or .Values.argocd.enabled .Values.serviceAccount.annotations }} annotations: + {{- include "countly.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} automountServiceAccountToken: false {{- end }} diff --git a/charts/countly/templates/tls-selfsigned.yaml b/charts/countly/templates/tls-selfsigned.yaml index 833f38f..93719b4 100644 --- a/charts/countly/templates/tls-selfsigned.yaml +++ b/charts/countly/templates/tls-selfsigned.yaml @@ -10,6 +10,10 @@ apiVersion: cert-manager.io/v1 kind: ClusterIssuer metadata: name: {{ $selfSignedIssuerName }} + {{- if $.Values.argocd.enabled }} + annotations: + {{- include "countly.syncWave" (dict "wave" "10" "root" $) | nindent 4 }} + {{- end }} labels: {{- include "countly.labels" . | nindent 4 }} spec: @@ -25,6 +29,10 @@ metadata: namespace: cert-manager labels: {{- include "countly.labels" . | nindent 4 }} + {{- if $.Values.argocd.enabled }} + annotations: + {{- include "countly.syncWave" (dict "wave" "11" "root" $) | nindent 4 }} + {{- end }} spec: isCA: true commonName: {{ $fullname }}-ca @@ -42,6 +50,10 @@ metadata: name: {{ $issuerName }} labels: {{- include "countly.labels" . | nindent 4 }} + {{- if $.Values.argocd.enabled }} + annotations: + {{- include "countly.syncWave" (dict "wave" "12" "root" $) | nindent 4 }} + {{- end }} spec: ca: secretName: {{ $caSecretName }} diff --git a/charts/countly/values.yaml b/charts/countly/values.yaml index 9dc197d..28d06b0 100644 --- a/charts/countly/values.yaml +++ b/charts/countly/values.yaml @@ -12,6 +12,9 @@ fullnameOverride: "" createNamespace: false +argocd: + enabled: false + serviceAccount: create: true name: "" diff --git a/helmfile.yaml.gotmpl b/helmfile.yaml.gotmpl index a28654a..8e33433 100644 --- a/helmfile.yaml.gotmpl +++ b/helmfile.yaml.gotmpl @@ -12,6 +12,9 @@ environments: example-production: values: - environments/example-production/global.yaml + test-local-full: + values: + - environments/test-local-full/global.yaml --- repositories: [] @@ -92,3 +95,17 @@ releases: - environments/{{ .Environment.Name }}/secrets-observability.yaml needs: - countly/countly + + # Optional: MongoDB to ClickHouse migration pipeline + # Set migration.enabled: true in your environment's global.yaml to deploy + - name: countly-migrations + installed: {{ .Values | get "migration.enabled" false }} + chart: ./charts/countly-migrations + namespace: kafka-migrations + values: + - environments/{{ .Environment.Name }}/global.yaml + - environments/{{ .Environment.Name }}/migrations.yaml + - environments/{{ .Environment.Name }}/secrets-migrations.yaml + needs: + - mongodb/countly-mongodb + - clickhouse/countly-clickhouse