feat: Added support for external-secrets, modified secret generation during apply step to use this new process (#203)

* feat: Added support for external-secrets, modified secret generation during apply step to use this new process

* fix: Renamed secret templates from yml to json

* fix: Add tf refresh before teardown, tear down application secrets too

Author: bmonkman

templates/Makefile

@@ -80,20 +80,25 @@ teardown-secrets:
 	aws secretsmanager list-secrets --region $(AWS_DEFAULT_REGION) --query "SecretList[?Tags[?Key=='cf-keypair' && Value=='$(PROJECT)']].[Name] | [0][0]" | xargs aws secretsmanager delete-secret --region $(AWS_DEFAULT_REGION) --secret-id || echo "Secret already removed"
 	aws secretsmanager list-secrets --region $(AWS_DEFAULT_REGION) --query "SecretList[?Tags[?Key=='wg' && Value=='$(PROJECT)-$(ENVIRONMENT)']].[Name] | [0][0]" | xargs aws secretsmanager delete-secret --region $(AWS_DEFAULT_REGION) --secret-id || echo "Secret already removed"
 	aws secretsmanager list-secrets --region $(AWS_DEFAULT_REGION) --query "SecretList[?Tags[?Key=='oathkeeper-jwks' && Value=='$(PROJECT)-$(ENVIRONMENT)']].[Name] | [0][0]" | xargs aws secretsmanager delete-secret --region $(AWS_DEFAULT_REGION) --secret-id || echo "Secret already removed"
+	aws secretsmanager list-secrets --region $(AWS_DEFAULT_REGION) --query "SecretList[?Tags[?Key=='application-secret' && Value=='$(PROJECT)-$(ENVIRONMENT)-$(PROJECT)']].[Name] | [0][0]" | xargs aws secretsmanager delete-secret --region $(AWS_DEFAULT_REGION) --secret-id || echo "Secret already removed"
+	aws secretsmanager list-secrets --region $(AWS_DEFAULT_REGION) --query "SecretList[?Tags[?Key=='application-secret' && Value=='$(PROJECT)-$(ENVIRONMENT)-user-auth']].[Name] | [0][0]" | xargs aws secretsmanager delete-secret --region $(AWS_DEFAULT_REGION) --secret-id || echo "Secret already removed"
 	aws iam list-role-policies --role-name $(PROJECT)-eks-cluster-creator --query "PolicyNames"   | jq -r ".[]" | xargs -n1 aws iam delete-role-policy --role-name $(PROJECT)-eks-cluster-creator --policy-name
 	aws iam list-attached-role-policies --role-name $(PROJECT)-eks-cluster-creator --query "AttachedPolicies[].PolicyArn" | jq -r ".[]" | xargs -n1 aws iam detach-role-policy --role-name $(PROJECT)-eks-cluster-creator --policy-arn
 	aws iam delete-role --role-name $(PROJECT)-eks-cluster-creator
 
 teardown-env:
 	cd terraform/environments/$(ENVIRONMENT) && \
+	terraform refresh && \
 	terraform destroy
 
 teardown-shared-env:
 	cd terraform/environments/shared && \
+	terraform refresh && \
 	terraform destroy
 
 teardown-k8s-utils:
 	cd kubernetes/terraform/environments/$(ENVIRONMENT) && \
+	terraform refresh && \
 	terraform destroy
 
 .PHONY: apply apply-remote-state apply-secrets apply-env apply-k8s-utils teardown-k8s-utils teardown-env teardown-shared-env teardown-secrets teardown-remote-state teardown-shared-remote-state

templates/kubernetes/terraform/environments/prod/main.tf

@@ -89,7 +89,7 @@ module "kubernetes" {
       jwks_secret_name              = "${local.project}-${local.environment}-oathkeeper-jwks-${local.random_seed}"
       # This domain or address must be verified by the mail provider (Sendgrid, SES, etc.)
       user_auth_mail_from_address   = "noreply@${local.domain_name}"
-      cookie_sigining_secret_key    = "${local.project}-${local.environment}-${local.random_seed}"
+      cookie_signing_secret_key    = "${local.project}-${local.environment}-${local.random_seed}"
     }
     ## User auth: Kratos requires database and a secret (as: `user_auth[0].name`)
     ## Oathkeeper requires a private key (as `user_auth[0].jwks_secret_name`)

templates/kubernetes/terraform/environments/stage/main.tf

@@ -88,7 +88,7 @@ module "kubernetes" {
       jwks_secret_name              = "${local.project}-${local.environment}-oathkeeper-jwks-${local.random_seed}"
       # This domain or address must be verified by the mail provider (Sendgrid, SES, etc.)
       user_auth_mail_from_address   = "noreply@${local.domain_name}"
-      cookie_sigining_secret_key    = "${local.project}-${local.environment}-${local.random_seed}"
+      cookie_signing_secret_key    = "${local.project}-${local.environment}-${local.random_seed}"
     }
     ## User auth: Kratos requires database and a secret (as: `user_auth[0].name`)
     ## Oathkeeper requires a private key (as `user_auth[0].jwks_secret_name`)

templates/kubernetes/terraform/modules/kubernetes/external_secrets.tf

@@ -0,0 +1,67 @@
+locals {
+  external_secrets_namespace = "kube-system"
+
+  // Set / expose poller interval?
+  external_secrets_helm_values = {
+    nameOverride : "external-secrets"
+    serviceMonitor : {
+      enabled : (var.metrics_type == "prometheus")
+      namespace : "metrics"
+    }
+    serviceAccount : {
+      name : "external-secrets"
+      annotations : {
+        "eks.amazonaws.com/role-arn" : module.iam_assumable_role_external_secrets.this_iam_role_arn
+      }
+    }
+    securityContext : {
+      fsGroup : 65534
+    }
+    env : {
+      AWS_REGION : var.region
+      LOG_LEVEL : "warn" # use "info" to see all polling events
+      # Each request to Secrets Manager has a small cost ($0.05 per 10,000 API calls) so a longer interval will reduce the number of calls but it will take longer to get updated secret values
+      POLLER_INTERVAL_MILLISECONDS : 15000
+    }
+  }
+}
+
+resource "helm_release" "external_secrets" {
+  name       = "external-secrets"
+  repository = "https://external-secrets.github.io/kubernetes-external-secrets/"
+  chart      = "kubernetes-external-secrets"
+  version    = "7.2.1"
+  namespace  = local.external_secrets_namespace
+  values     = [jsonencode(local.external_secrets_helm_values)]
+}
+
+# Create a role using oidc to map service accounts
+module "iam_assumable_role_external_secrets" {
+  source                        = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc"
+  version                       = "~> v3.15.0"
+  create_role                   = true
+  role_name                     = "${var.project}-k8s-${var.environment}-external-secrets"
+  provider_url                  = replace(data.aws_eks_cluster.cluster.identity.0.oidc.0.issuer, "https://", "")
+  role_policy_arns              = [aws_iam_policy.external_secrets.arn]
+  oidc_fully_qualified_subjects = ["system:serviceaccount:${local.external_secrets_namespace}:external-secrets"]
+}
+
+resource "aws_iam_policy" "external_secrets" {
+  name_prefix = "kubernetes-external-secrets"
+  description = "Kubernetes External Secrets Policy"
+  policy      = data.aws_iam_policy_document.external_secrets_policy_doc.json
+}
+
+data "aws_iam_policy_document" "external_secrets_policy_doc" {
+  statement {
+    effect    = "Allow"
+    resources = ["arn:aws:secretsmanager:${var.region}:*:secret:${var.project}/kubernetes/${var.environment}/*"]
+
+    actions = [
+      "secretsmanager:GetResourcePolicy",
+      "secretsmanager:GetSecretValue",
+      "secretsmanager:DescribeSecret",
+      "secretsmanager:ListSecretVersionIds",
+    ]
+  }
+}

templates/kubernetes/terraform/modules/kubernetes/user_auth.tf

@@ -1,7 +1,23 @@
+locals {
+  # This secret is created by the /scripts/create-db-user.sh script and contains environment variables that will be pulled into a k8s secret automatically by external-secrets
+  secrets_manager_secret_name = "${var.project}/kubernetes/${var.environment}/user-auth"
+}
+
+
+## Get generated JWKS content from secret
+data "aws_secretsmanager_secret" "jwks_content" {
+  count = length(var.user_auth)
+  name  = var.user_auth[count.index].jwks_secret_name
+}
+data "aws_secretsmanager_secret_version" "jwks_content" {
+  count     = length(data.aws_secretsmanager_secret.jwks_content)
+  secret_id = data.aws_secretsmanager_secret.jwks_content[count.index].id
+}
+
 module "user_auth" {
   count   = length(var.user_auth)
   source  = "commitdev/zero/aws//modules/user_auth"
-  version = "0.1.21"
+  version = "0.3.6"
 
   name                        = var.user_auth[count.index].name
   auth_namespace              = var.user_auth[count.index].auth_namespace
@@ -11,7 +27,10 @@ module "user_auth" {
   backend_service_domain      = var.user_auth[count.index].backend_service_domain
   user_auth_mail_from_address = var.user_auth[count.index].user_auth_mail_from_address
   whitelisted_return_urls     = var.user_auth[count.index].whitelisted_return_urls
-  jwks_secret_name            = var.user_auth[count.index].jwks_secret_name
-  cookie_sigining_secret_key  = var.user_auth[count.index].cookie_sigining_secret_key
-  k8s_local_exec_context      = local.k8s_exec_context
+  jwks_content                = data.aws_secretsmanager_secret_version.jwks_content[count.index].secret_string
+  cookie_signing_secret_key   = var.user_auth[count.index].cookie_signing_secret_key
+  kubectl_extra_args          = local.k8s_exec_context
+  external_secret_name        = local.secrets_manager_secret_name
+
+  depends_on = [helm_release.external_secrets]
 }

templates/kubernetes/terraform/modules/kubernetes/variables.tf

@@ -144,7 +144,7 @@ variable "user_auth" {
     jwks_secret_name            = string
     user_auth_mail_from_address = string
     whitelisted_return_urls     = list(string)
-    cookie_sigining_secret_key  = string
+    cookie_signing_secret_key   = string
   }))
 }
 

templates/scripts/create-db-user.sh

@@ -9,26 +9,25 @@ usage () {
 # check parameters
 # REGION        - AWS region to use
 # SEED          - Random seed that is part of the name of the AWS secret containing the db master password
-# PROJECT_NAME  - Name of the project and the k8s namespace containing the database service
+# PROJECT_NAME  - Name of the project
 # ENVIRONMENT   - stage or prod
-# NAMESPACE     - The target k8s namespace to create and create a secret in
+# NAMESPACE     - The target k8s namespace to create a secret in
 # DATABASE_TYPE - The type of database - mysql, postgres
 # DATABASE_NAME - The name of the database(s) to create in the database server
 # USER_NAME     - The name of the user to create and grant access to the database specified above
 # USER_PASSWORD - The password of the user to create and grant access to the database specified above (optional)
-# SECRET_NAME   - The secret of the database user to get password
+# SECRET_NAME   - The suffix name of the secret created in AWS Secret Manager that will contain the created credentials
 # CREATE_SECRET - A template file to render to create a secret (optional)
-# CREATE_DB_POD - A template file to render to create a db pod for troubleshooting (optional)
 ([[ -z "${REGION}" ]]        || \
  [[ -z "${SEED}" ]]          || \
  [[ -z "${PROJECT_NAME}" ]]  || \
  [[ -z "${ENVIRONMENT}" ]]   || \
  [[ -z "${NAMESPACE}" ]]     || \
+ [[ -z "${SECRET_NAME}" ]]   || \
  [[ -z "${DATABASE_TYPE}" ]] || \
  [[ -z "${DATABASE_NAME}" ]] || \
- [[ -z "${SECRET_NAME}" ]]  || \
  [[ -z "${USER_NAME}" ]] )  && \
-echo "Some environment variables (REGION, SEED, PROJECT_NAME, ENVIRONMENT, NAMESPACE, DATABASE_TYPE, DATABASE_NAME, USER_NAME) are not set properly." && usage
+echo "Some environment variables (REGION, SEED, PROJECT_NAME, ENVIRONMENT, NAMESPACE, SECRET_NAME, DATABASE_TYPE, DATABASE_NAME, USER_NAME) are not set properly." && usage
 
 # docker image with postgres + mysql clients
 DOCKER_IMAGE_TAG=commitdev/zero-k8s-utilities:0.0.3
@@ -46,6 +45,7 @@ MASTER_RDS_PASSWORD=$(aws secretsmanager get-secret-value --region=${REGION} --s
 ## get application user/pass
 DB_APP_USERNAME=$(echo "${USER_NAME}" | tr -dc 'A-Za-z0-9')
 DB_APP_PASSWORD=${USER_PASSWORD:-$(LC_ALL=C tr -dc 'A-Za-z0-9' < /dev/urandom | base64 | head -c 24)}
+JOB_ID=$(LC_ALL=C tr -dc 'a-z0-9' < /dev/urandom | head -c 8)
 
 # get correct dsn string for db type
 if [[ "${DB_TYPE}" == "postgres" ]]; then
@@ -54,28 +54,24 @@ elif [[ "${DB_TYPE}" == "mysql" ]]; then
   DB_ENDPOINT_FOR_DSN="tcp(${DB_ENDPOINT})"
 fi
 
-# fill in env-vars to db user creation manifest
-JOB_ID=$(LC_ALL=C tr -dc 'a-z0-9' < /dev/urandom | head -c 8)
-eval "echo \"$(cat ./db-ops/job-create-db-${DATABASE_TYPE}.yml.tpl)\"" > ./k8s-job-create-db.yml
-[[ -z "${CREATE_SECRET}" ]] || eval "echo \"$(cat ./db-ops/${CREATE_SECRET})\"" >> ./k8s-job-create-db.yml
 # the manifest creates these things
 # 1. Namespaces: db-ops, $NAMESPACE
 # 2. Secret in db-ops: db-create-users (with master password, and a .sql file
 # 3. Job in db-ops: db-create-users (runs the .sql file against the RDS given master_password from env)
-# 4. Secret in $NAMESPACE namespace with DB_USERNAME / DB_PASSWORD
 
-# execution
-kubectl apply -f ./k8s-job-create-db.yml
-rm -f ./k8s-job-create-db.yml
+# Run the job in the kubernetes cluster that will create the database user
+eval "echo \"$(cat ./db-ops/job-create-db-${DATABASE_TYPE}.yml.tpl)\"" | kubectl apply -f -
+
+# Create a secret in AWS Secrets Manager. The contents of this secret will be automatically pulled into a kubernetes secret by external-secrets
+[[ -z "${CREATE_SECRET}" ]] || aws secretsmanager create-secret --name "${PROJECT_NAME}/kubernetes/${ENVIRONMENT}/${SECRET_NAME}" --description "Application secrets" --tags "[{\"Key\":\"application-secret\",\"Value\":\"${PROJECT}-${ENVIRONMENT}-${SECRET_NAME}\"}]" --secret-string "$(eval "echo \"$(cat ./db-ops/${CREATE_SECRET})\"")"
 
-# clean up
-## Deleting the entire db-ops namespace, leaving ONLY application-namespace's secret behind
+## Delete the entire db-ops namespace
 kubectl -n db-ops wait --for=condition=complete --timeout=10s job db-create-users-$NAMESPACE-${JOB_ID}
 if [ $? -eq 0 ]
 then
     kubectl delete namespace db-ops
 else
     echo "Failed to create application database user, please see 'kubectl logs -n db-ops -l job-name=db-create-users-$NAMESPACE-${JOB_ID}'"
-    kubectl delete secret -n db-ops ${SECRET_NAME}
+    kubectl delete secret -n db-ops db-create-users
 fi
 

templates/scripts/db-ops/secret-application.json.tpl

@@ -0,0 +1,4 @@
+{
+  \"DATABASE_USERNAME\":\"$DB_APP_USERNAME\",
+  \"DATABASE_PASSWORD\":\"$DB_APP_PASSWORD\"
+}

templates/scripts/db-ops/secret-application.yml.tpl

@@ -0,0 +1,6 @@
+{
+  \"dsn\":\"$DB_TYPE://$DB_APP_USERNAME:$DB_APP_PASSWORD@$DB_ENDPOINT_FOR_DSN/$DB_NAME\",
+  \"secretsCookie\":\"cookie-secret-$PROJECT_NAME-$SEED\",
+  \"secretsDefault\":\"default-secret-$PROJECT_NAME-$SEED\",
+  \"smtpConnectionURI\":\"$SMTP_URI\"
+}