From 9f28d486f751fab8a2f16c6dbe977b3d040db2e0 Mon Sep 17 00:00:00 2001 From: Mark Boyd Date: Fri, 30 Jan 2026 12:48:48 -0500 Subject: [PATCH 1/4] add script to find non self-setting pipelines --- concourse/get-pipelines-without-self-set.sh | 64 +++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100755 concourse/get-pipelines-without-self-set.sh diff --git a/concourse/get-pipelines-without-self-set.sh b/concourse/get-pipelines-without-self-set.sh new file mode 100755 index 00000000..13f76147 --- /dev/null +++ b/concourse/get-pipelines-without-self-set.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +function usage { + echo -e " + ./$( basename "$0" ) [pipeline-name] [--help, -h] + + Get all git resources in Concourse pipelines that don\'t have commit signing + configured + + Optional environment variable \$CI_URL matching your Concourse URL. + example: CI_URL=https://ci.fr.cloud.gov ./$( basename "$0" ) + + Optional argument for specific pipeline to check + example: ./$( basename "$0" ) pipeline-name + + \$CI_URL, Defaults to https://ci.fr.cloud.gov + " + exit +} + +while getopts ":h" opt; do + case ${opt} in + h ) + usage + exit 0 + ;; + * ) + usage + exit 0 + ;; + esac +done + + +CI_URL="${CI_URL:-"https://ci.fr.cloud.gov"}" +FLY_TARGET=$(fly targets | grep "${CI_URL}" | head -n 1 | awk '{print $1}') + +if ! fly --target "${FLY_TARGET}" workers > /dev/null; then + echo "Not logged in to concourse" + exit 1 +fi + +function get_number_of_self_set_tasks { + fly -t "${FLY_TARGET}" get-pipeline --pipeline "$1" --json \ + | jq '[.jobs[].plan[] | has("set_pipeline")] | map(select(.)) | length' +} + +function report_pipelines_without_self_set { + length=$(get_number_of_self_set_tasks "$1") + if [[ $length != "1" ]]; then + printf 'pipeline: %s\n' "$1" + fi +} + +if [ -z "$1" ]; then + fly --target "${FLY_TARGET}" pipelines | tail -n +1 | while read -r line; do + pipeline_name=$(echo "$line" | awk '{print $2}') + + report_pipelines_without_self_set "$pipeline_name" + done +else + report_pipelines_without_self_set "$1" +fi + From 45c4dc3be1a8d65a170b99591d7bf108acb9cee2 Mon Sep 17 00:00:00 2001 From: Mark Boyd Date: Mon, 9 Feb 2026 17:39:23 -0500 Subject: [PATCH 2/4] add script for preparing bosh manifest variables for import to credhub --- bosh/credhub-format.sh | 58 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100755 bosh/credhub-format.sh diff --git a/bosh/credhub-format.sh b/bosh/credhub-format.sh new file mode 100755 index 00000000..a98b91f6 --- /dev/null +++ b/bosh/credhub-format.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# set -eo pipefail + +# yq operates on yaml documents using a jq-like syntax. +# For jq program syntax, see: https://stedolan.github.io/jq/manual/ + +if [ "$#" -ne 2 ]; then + echo " +Usage: ./credhub-format.sh +Example: ./credhub-format.sh vars.yml deployment > output.yml + +./credhub-format.sh credentials.yml deploy-something > output.yml + +Read from a credential file and reformat the contents to a JSON format that CredHub +can import, excluding any keys that do not appear in the manifests + +Run this script from the root of the repository so it can find the pipeline configuration +file. The output file is written to stdout. Additional output, such as the keys that were +excluded, is written to stderr." >&2 + exit 1 +fi + +MANIFEST_PATH=${MANIFEST_PATH:-manifest.yml} + +if [ ! -f "$MANIFEST_PATH" ]; then + echo "$MANIFEST_PATH not found. Is the script being run from the root directory of the repository?" >&2 + exit 1 +fi + +vars_file=$1 +deployment_name=$2 + +echo "The following values do not appear in $MANIFEST_PATH and will not be exported: +" >&2 +# without setting -S, strings longer than the default of 255 will not be fully interpolated by xargs. + +yq 'keys | .[]' < "$vars_file" | xargs -I % -S 512 bash -c \ +"if ! grep -q % $MANIFEST_PATH; then + echo % +fi" >&2 + +# reformat the credential file to the Credhub format, excluding entries +# that don't appear in pipeline.yml. +yq --output-format json < "$vars_file" | \ +jq --arg deploymentname "$deployment_name" \ + --rawfile varsfile $vars_file ' + to_entries | + map(select(.key | inside($varsfile))) | + { + credentials: [ .[] | + { + name: ("/bosh/"+$deploymentname+"/"+.key), + type: (if .value | type == "object" then (if .value | has("certificate") then "certificate" else "json" end) else "value" end), + value: .value + } + ] + } +' From 0dc4299c3e40ae3ae244d45aa518ac7bae2f3258 Mon Sep 17 00:00:00 2001 From: Mark Boyd Date: Mon, 9 Feb 2026 17:40:41 -0500 Subject: [PATCH 3/4] update script usage text --- bosh/credhub-format.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bosh/credhub-format.sh b/bosh/credhub-format.sh index a98b91f6..63e12cf9 100755 --- a/bosh/credhub-format.sh +++ b/bosh/credhub-format.sh @@ -9,12 +9,10 @@ if [ "$#" -ne 2 ]; then Usage: ./credhub-format.sh Example: ./credhub-format.sh vars.yml deployment > output.yml -./credhub-format.sh credentials.yml deploy-something > output.yml - -Read from a credential file and reformat the contents to a JSON format that CredHub +Read from a BOSH variables file and reformat the contents to a JSON format that CredHub can import, excluding any keys that do not appear in the manifests -Run this script from the root of the repository so it can find the pipeline configuration +Run this script from the root of the repository so it can find the manifest file. The output file is written to stdout. Additional output, such as the keys that were excluded, is written to stderr." >&2 exit 1 From 1d9de5fd9573cd6a5009003de1ce069a38ea8867 Mon Sep 17 00:00:00 2001 From: Mark Boyd Date: Tue, 10 Feb 2026 11:36:19 -0500 Subject: [PATCH 4/4] improve logic for determining type of bosh variables --- bosh/credhub-format.sh | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/bosh/credhub-format.sh b/bosh/credhub-format.sh index 63e12cf9..e4b8e646 100755 --- a/bosh/credhub-format.sh +++ b/bosh/credhub-format.sh @@ -1,13 +1,18 @@ #!/bin/bash -# set -eo pipefail +set -eo pipefail # yq operates on yaml documents using a jq-like syntax. # For jq program syntax, see: https://stedolan.github.io/jq/manual/ if [ "$#" -ne 2 ]; then echo " -Usage: ./credhub-format.sh -Example: ./credhub-format.sh vars.yml deployment > output.yml +Usage: ./credhub-format.sh +Example: ./credhub-format.sh vars.yml /bosh/deployment > output.json + +You can optionally override the location for your manifest, which +defaults to manifest.yml: + +MANIFEST_PATH=deploy/manifest.yml ./credhub-format.sh credentials.yml deploy-something > output.yml Read from a BOSH variables file and reformat the contents to a JSON format that CredHub can import, excluding any keys that do not appear in the manifests @@ -25,30 +30,33 @@ if [ ! -f "$MANIFEST_PATH" ]; then exit 1 fi -vars_file=$1 -deployment_name=$2 +VARS_FILE=$1 +CREDHUB_VAR_PREFIX=$2 echo "The following values do not appear in $MANIFEST_PATH and will not be exported: " >&2 # without setting -S, strings longer than the default of 255 will not be fully interpolated by xargs. -yq 'keys | .[]' < "$vars_file" | xargs -I % -S 512 bash -c \ +yq 'keys | .[]' < "$VARS_FILE" | xargs -I % -S 512 bash -c \ "if ! grep -q % $MANIFEST_PATH; then echo % fi" >&2 +MANIFEST_VARS=$(cat "$MANIFEST_PATH" | yq '.variables' --output-format json | jq 'map({(.name): .}) | add') + # reformat the credential file to the Credhub format, excluding entries -# that don't appear in pipeline.yml. -yq --output-format json < "$vars_file" | \ -jq --arg deploymentname "$deployment_name" \ - --rawfile varsfile $vars_file ' +# that don't appear in the manifest +yq --output-format json < "$VARS_FILE" | \ +jq --arg credhub_var_prefix "$CREDHUB_VAR_PREFIX" \ + --argjson manifest_vars "$MANIFEST_VARS" \ + --rawfile varsfile "$VARS_FILE" ' to_entries | map(select(.key | inside($varsfile))) | { credentials: [ .[] | { - name: ("/bosh/"+$deploymentname+"/"+.key), - type: (if .value | type == "object" then (if .value | has("certificate") then "certificate" else "json" end) else "value" end), + name: ($credhub_var_prefix+"/"+.key), + type: ($manifest_vars[.key].type // if .value | type == "object" then "json" else "value" end), value: .value } ]