diff --git a/chart/files/pod-template-file.kubernetes-helm-yaml b/chart/files/pod-template-file.kubernetes-helm-yaml index 25c2de6f81318..e885f3aeb91d0 100644 --- a/chart/files/pod-template-file.kubernetes-helm-yaml +++ b/chart/files/pod-template-file.kubernetes-helm-yaml @@ -257,6 +257,9 @@ spec: {{- if and .Values.dags.gitSync.enabled (or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey) }} {{- include "git_sync_ssh_key_volume" . | nindent 2 }} {{- end }} + {{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) .Values.dags.gitSync.credentialsSecret .Values.dags.gitSync.usePasswordFile }} + {{- include "git_sync_credentials_volume" . | nindent 2 }} + {{- end }} - configMap: name: {{ include "airflow_config" . }} name: config diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index 42848a9e22e14..3030cf7c78e0e 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -204,6 +204,14 @@ If release name contains chart name it will be used as a full name. defaultMode: 288 {{- end }} +{{/* Git credentials volume */}} +{{- define "git_sync_credentials_volume" }} +- name: git-sync-credentials + secret: + secretName: {{ .Values.dags.gitSync.credentialsSecret | quote }} + defaultMode: 288 +{{- end }} + {{/* Git sync container */}} {{- define "git_sync_container" }} - name: {{ .Values.dags.gitSync.containerName }}{{ if .is_init }}-init{{ end }} @@ -247,6 +255,12 @@ If release name contains chart name it will be used as a full name. secretKeyRef: name: {{ .Values.dags.gitSync.credentialsSecret | quote }} key: GITSYNC_USERNAME + {{- if .Values.dags.gitSync.usePasswordFile }} + - name: GIT_SYNC_PASSWORD_FILE + value: "/etc/git-secret/credentials/GIT_SYNC_PASSWORD" + - name: GITSYNC_PASSWORD_FILE + value: "/etc/git-secret/credentials/GITSYNC_PASSWORD" + {{- else }} - name: GIT_SYNC_PASSWORD valueFrom: secretKeyRef: @@ -258,6 +272,7 @@ If release name contains chart name it will be used as a full name. name: {{ .Values.dags.gitSync.credentialsSecret | quote }} key: GITSYNC_PASSWORD {{- end }} + {{- end }} - name: GIT_SYNC_REV value: {{ .Values.dags.gitSync.rev | quote }} - name: GITSYNC_REF @@ -351,6 +366,11 @@ If release name contains chart name it will be used as a full name. subPath: known_hosts {{- end }} {{- end }} + {{- if and .Values.dags.gitSync.credentialsSecret .Values.dags.gitSync.usePasswordFile }} + - name: git-sync-credentials + mountPath: /etc/git-secret/credentials + readOnly: true + {{- end }} {{- if .Values.dags.gitSync.extraVolumeMounts }} {{- tpl (toYaml .Values.dags.gitSync.extraVolumeMounts) . | nindent 2 }} {{- end }} diff --git a/chart/templates/dag-processor/dag-processor-deployment.yaml b/chart/templates/dag-processor/dag-processor-deployment.yaml index c5045e6ecefea..f02e6f44af1a1 100644 --- a/chart/templates/dag-processor/dag-processor-deployment.yaml +++ b/chart/templates/dag-processor/dag-processor-deployment.yaml @@ -270,6 +270,9 @@ spec: {{- if and .Values.dags.gitSync.enabled (or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey) }} {{- include "git_sync_ssh_key_volume" . | indent 8 }} {{- end }} + {{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.credentialsSecret .Values.dags.gitSync.usePasswordFile }} + {{- include "git_sync_credentials_volume" . | indent 8 }} + {{- end }} {{- if .Values.volumes }} {{- toYaml .Values.volumes | nindent 8 }} {{- end }} diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml index 3514180c874c5..b7a5ad422bca3 100644 --- a/chart/templates/scheduler/scheduler-deployment.yaml +++ b/chart/templates/scheduler/scheduler-deployment.yaml @@ -344,6 +344,9 @@ spec: {{- if or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey}} {{- include "git_sync_ssh_key_volume" . | indent 8 }} {{- end }} + {{- if and $localOrDagProcessorDisabled .Values.dags.gitSync.enabled .Values.dags.gitSync.credentialsSecret .Values.dags.gitSync.usePasswordFile }} + {{- include "git_sync_credentials_volume" . | indent 8 }} + {{- end }} {{- end }} {{- end }} {{- if .Values.volumes }} diff --git a/chart/templates/triggerer/triggerer-deployment.yaml b/chart/templates/triggerer/triggerer-deployment.yaml index 41a2f0d3d5501..dc59a8110ee3f 100644 --- a/chart/templates/triggerer/triggerer-deployment.yaml +++ b/chart/templates/triggerer/triggerer-deployment.yaml @@ -300,6 +300,9 @@ spec: {{- if or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey}} {{- include "git_sync_ssh_key_volume" . | nindent 8 }} {{- end }} + {{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) .Values.dags.gitSync.credentialsSecret .Values.dags.gitSync.usePasswordFile }} + {{- include "git_sync_credentials_volume" . | nindent 8 }} + {{- end }} {{- end }} {{- if .Values.volumes }} {{- toYaml .Values.volumes | nindent 8 }} diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index c810581bf7485..737e3f81a252f 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -482,6 +482,9 @@ spec: {{- if or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey}} {{- include "git_sync_ssh_key_volume" . | indent 8 }} {{- end }} + {{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) .Values.dags.gitSync.credentialsSecret .Values.dags.gitSync.usePasswordFile }} + {{- include "git_sync_credentials_volume" . | indent 8 }} + {{- end }} {{- end }} {{- if .Values.logs.persistence.enabled }} - name: logs diff --git a/chart/values.schema.json b/chart/values.schema.json index 0891db13bd5f7..2cdab3d044dfe 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -10801,13 +10801,18 @@ } }, "credentialsSecret": { - "description": "Name of a Secret containing the repo `GIT_SYNC_USERNAME` and `GIT_SYNC_PASSWORD`.", + "description": "Name of a Secret containing git credentials (`GIT_SYNC_USERNAME`/`GIT_SYNC_PASSWORD` and optionally `GITSYNC_USERNAME`/`GITSYNC_PASSWORD`).", "type": [ "string", "null" ], "default": null }, + "usePasswordFile": { + "description": "When true and `credentialsSecret` is set, mount the credentials secret and pass password keys via `*_PASSWORD_FILE` env vars.", + "type": "boolean", + "default": false + }, "sshKey": { "description": "SSH private key", "type": [ diff --git a/chart/values.yaml b/chart/values.yaml index 15fa5aeefdb4b..cc21d2cfeb784 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -3558,10 +3558,17 @@ dags: # # For git-sync v4 # GITSYNC_USERNAME: # GITSYNC_PASSWORD: - # and specify the name of the secret below + # and specify the name of the secret below. # # credentialsSecret: git-credentials # + # If set to true, credentialsSecret will also be mounted into git-sync at + # /etc/git-secret/credentials and password keys will be passed via + # GIT_SYNC_PASSWORD_FILE/GITSYNC_PASSWORD_FILE. + # + # usePasswordFile: true + usePasswordFile: false + # # # If you are using an ssh clone url, you can load # the ssh private key to a k8s secret like the one below diff --git a/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py b/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py index a1f0f03aa1dbe..23575bf9f57eb 100644 --- a/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py +++ b/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py @@ -255,6 +255,50 @@ def test_validate_if_ssh_known_hosts_are_added(self): } in jmespath.search("spec.initContainers[0].volumeMounts", docs[0]) def test_should_set_username_and_pass_env_variables(self): + docs = render_chart( + values={ + "dags": { + "gitSync": { + "enabled": True, + "credentialsSecret": "user-pass-secret", + "usePasswordFile": True, + "sshKeySecret": None, + } + } + }, + show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, + ) + + assert { + "name": "GIT_SYNC_USERNAME", + "valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key": "GIT_SYNC_USERNAME"}}, + } in jmespath.search("spec.initContainers[0].env", docs[0]) + assert { + "name": "GIT_SYNC_PASSWORD_FILE", + "value": "/etc/git-secret/credentials/GIT_SYNC_PASSWORD", + } in jmespath.search("spec.initContainers[0].env", docs[0]) + + # Testing git-sync v4 + assert { + "name": "GITSYNC_USERNAME", + "valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key": "GITSYNC_USERNAME"}}, + } in jmespath.search("spec.initContainers[0].env", docs[0]) + assert { + "name": "GITSYNC_PASSWORD_FILE", + "value": "/etc/git-secret/credentials/GITSYNC_PASSWORD", + } in jmespath.search("spec.initContainers[0].env", docs[0]) + assert { + "mountPath": "/etc/git-secret/credentials", + "name": "git-sync-credentials", + "readOnly": True, + } in jmespath.search("spec.initContainers[0].volumeMounts", docs[0]) + assert { + "name": "git-sync-credentials", + "secret": {"defaultMode": 288, "secretName": "user-pass-secret"}, + } in jmespath.search("spec.volumes", docs[0]) + + def test_should_set_username_and_pass_env_variables_by_default(self): docs = render_chart( values={ "dags": { @@ -278,7 +322,6 @@ def test_should_set_username_and_pass_env_variables(self): "valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key": "GIT_SYNC_PASSWORD"}}, } in jmespath.search("spec.initContainers[0].env", docs[0]) - # Testing git-sync v4 assert { "name": "GITSYNC_USERNAME", "valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key": "GITSYNC_USERNAME"}}, @@ -287,6 +330,7 @@ def test_should_set_username_and_pass_env_variables(self): "name": "GITSYNC_PASSWORD", "valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key": "GITSYNC_PASSWORD"}}, } in jmespath.search("spec.initContainers[0].env", docs[0]) + assert "git-sync-credentials" not in jmespath.search("spec.volumes[].name", docs[0]) def test_should_set_the_dags_volume_claim_correctly_when_using_an_existing_claim(self): docs = render_chart( diff --git a/helm-tests/tests/helm_tests/airflow_core/test_dag_processor.py b/helm-tests/tests/helm_tests/airflow_core/test_dag_processor.py index 75879ebd9883f..7261e6e454fbf 100644 --- a/helm-tests/tests/helm_tests/airflow_core/test_dag_processor.py +++ b/helm-tests/tests/helm_tests/airflow_core/test_dag_processor.py @@ -802,6 +802,40 @@ def test_validate_if_ssh_params_are_added_with_git_ssh_key(self): "secret": {"secretName": "release-name-ssh-secret", "defaultMode": 288}, } in jmespath.search("spec.template.spec.volumes", docs[0]) + def test_should_set_password_file_env_variables_when_credentials_secret_is_configured(self): + docs = render_chart( + values={ + "dagProcessor": {"enabled": True}, + "dags": { + "gitSync": { + "enabled": True, + "credentialsSecret": "user-pass-secret", + "usePasswordFile": True, + }, + "persistence": {"enabled": False}, + }, + }, + show_only=["templates/dag-processor/dag-processor-deployment.yaml"], + ) + + assert { + "name": "GIT_SYNC_PASSWORD_FILE", + "value": "/etc/git-secret/credentials/GIT_SYNC_PASSWORD", + } in jmespath.search("spec.template.spec.containers[1].env", docs[0]) + assert { + "name": "GITSYNC_PASSWORD_FILE", + "value": "/etc/git-secret/credentials/GITSYNC_PASSWORD", + } in jmespath.search("spec.template.spec.containers[1].env", docs[0]) + assert { + "mountPath": "/etc/git-secret/credentials", + "name": "git-sync-credentials", + "readOnly": True, + } in jmespath.search("spec.template.spec.containers[1].volumeMounts", docs[0]) + assert { + "name": "git-sync-credentials", + "secret": {"defaultMode": 288, "secretName": "user-pass-secret"}, + } in jmespath.search("spec.template.spec.volumes", docs[0]) + class TestDagProcessorLogGroomer(LogGroomerTestBase): """DAG processor log groomer.""" diff --git a/helm-tests/tests/helm_tests/other/test_git_sync_scheduler.py b/helm-tests/tests/helm_tests/other/test_git_sync_scheduler.py index 71b5f69abd0f6..a28b13a2e20c2 100644 --- a/helm-tests/tests/helm_tests/other/test_git_sync_scheduler.py +++ b/helm-tests/tests/helm_tests/other/test_git_sync_scheduler.py @@ -284,7 +284,7 @@ def test_validate_sshkeysecret_not_added_when_persistence_is_enabled(self): ) assert "git-sync-ssh-key" not in jmespath.search("spec.template.spec.volumes[].name", docs[0]) - def test_should_set_username_and_pass_env_variables(self): + def test_should_set_username_and_password_file_env_variables(self): docs = render_chart( values={ "airflowVersion": "2.11.0", @@ -292,6 +292,7 @@ def test_should_set_username_and_pass_env_variables(self): "gitSync": { "enabled": True, "credentialsSecret": "user-pass-secret", + "usePasswordFile": True, "sshKeySecret": None, } }, @@ -304,8 +305,8 @@ def test_should_set_username_and_pass_env_variables(self): "valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key": "GIT_SYNC_USERNAME"}}, } in jmespath.search("spec.template.spec.containers[1].env", docs[0]) assert { - "name": "GIT_SYNC_PASSWORD", - "valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key": "GIT_SYNC_PASSWORD"}}, + "name": "GIT_SYNC_PASSWORD_FILE", + "value": "/etc/git-secret/credentials/GIT_SYNC_PASSWORD", } in jmespath.search("spec.template.spec.containers[1].env", docs[0]) # Testing git-sync v4 @@ -314,9 +315,18 @@ def test_should_set_username_and_pass_env_variables(self): "valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key": "GITSYNC_USERNAME"}}, } in jmespath.search("spec.template.spec.containers[1].env", docs[0]) assert { - "name": "GITSYNC_PASSWORD", - "valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key": "GITSYNC_PASSWORD"}}, + "name": "GITSYNC_PASSWORD_FILE", + "value": "/etc/git-secret/credentials/GITSYNC_PASSWORD", } in jmespath.search("spec.template.spec.containers[1].env", docs[0]) + assert { + "mountPath": "/etc/git-secret/credentials", + "name": "git-sync-credentials", + "readOnly": True, + } in jmespath.search("spec.template.spec.containers[1].volumeMounts", docs[0]) + assert { + "name": "git-sync-credentials", + "secret": {"defaultMode": 288, "secretName": "user-pass-secret"}, + } in jmespath.search("spec.template.spec.volumes", docs[0]) def test_should_set_the_volume_claim_correctly_when_using_an_existing_claim(self): docs = render_chart( diff --git a/helm-tests/tests/helm_tests/other/test_git_sync_triggerer.py b/helm-tests/tests/helm_tests/other/test_git_sync_triggerer.py index 15a2a3b49b38c..aa224dbb4334f 100644 --- a/helm-tests/tests/helm_tests/other/test_git_sync_triggerer.py +++ b/helm-tests/tests/helm_tests/other/test_git_sync_triggerer.py @@ -77,6 +77,38 @@ def test_validate_if_ssh_params_are_added_with_git_ssh_key(self): "secret": {"secretName": "release-name-ssh-secret", "defaultMode": 288}, } in jmespath.search("spec.template.spec.volumes", docs[0]) + def test_should_set_password_file_env_variables_when_credentials_secret_is_configured(self): + docs = render_chart( + values={ + "dags": { + "gitSync": { + "enabled": True, + "credentialsSecret": "user-pass-secret", + "usePasswordFile": True, + } + } + }, + show_only=["templates/triggerer/triggerer-deployment.yaml"], + ) + + assert { + "name": "GIT_SYNC_PASSWORD_FILE", + "value": "/etc/git-secret/credentials/GIT_SYNC_PASSWORD", + } in jmespath.search("spec.template.spec.containers[1].env", docs[0]) + assert { + "name": "GITSYNC_PASSWORD_FILE", + "value": "/etc/git-secret/credentials/GITSYNC_PASSWORD", + } in jmespath.search("spec.template.spec.containers[1].env", docs[0]) + assert { + "mountPath": "/etc/git-secret/credentials", + "name": "git-sync-credentials", + "readOnly": True, + } in jmespath.search("spec.template.spec.containers[1].volumeMounts", docs[0]) + assert { + "name": "git-sync-credentials", + "secret": {"defaultMode": 288, "secretName": "user-pass-secret"}, + } in jmespath.search("spec.template.spec.volumes", docs[0]) + def test_liveness_probe_configuration(self): livenessProbe = { "failureThreshold": 10, diff --git a/helm-tests/tests/helm_tests/other/test_git_sync_worker.py b/helm-tests/tests/helm_tests/other/test_git_sync_worker.py index 03fdfc1217c7c..134ee1967d796 100644 --- a/helm-tests/tests/helm_tests/other/test_git_sync_worker.py +++ b/helm-tests/tests/helm_tests/other/test_git_sync_worker.py @@ -168,6 +168,38 @@ def test_validate_if_ssh_params_are_added_with_git_ssh_key(self): "secret": {"secretName": "release-name-ssh-secret", "defaultMode": 288}, } in jmespath.search("spec.template.spec.volumes", docs[0]) + def test_should_set_password_file_env_variables_when_credentials_secret_is_configured(self): + docs = render_chart( + values={ + "dags": { + "gitSync": { + "enabled": True, + "credentialsSecret": "user-pass-secret", + "usePasswordFile": True, + } + } + }, + show_only=["templates/workers/worker-deployment.yaml"], + ) + + assert { + "name": "GIT_SYNC_PASSWORD_FILE", + "value": "/etc/git-secret/credentials/GIT_SYNC_PASSWORD", + } in jmespath.search("spec.template.spec.containers[1].env", docs[0]) + assert { + "name": "GITSYNC_PASSWORD_FILE", + "value": "/etc/git-secret/credentials/GITSYNC_PASSWORD", + } in jmespath.search("spec.template.spec.containers[1].env", docs[0]) + assert { + "mountPath": "/etc/git-secret/credentials", + "name": "git-sync-credentials", + "readOnly": True, + } in jmespath.search("spec.template.spec.containers[1].volumeMounts", docs[0]) + assert { + "name": "git-sync-credentials", + "secret": {"defaultMode": 288, "secretName": "user-pass-secret"}, + } in jmespath.search("spec.template.spec.volumes", docs[0]) + def test_container_lifecycle_hooks(self): docs = render_chart( values={