feat(go): add support for Go subprojects

.mise/tasks/stencil/post/go-sync

@@ -11,6 +11,9 @@ DEVBASE_LIB_DIR="$DEVBASE_ROOT_DIR/shell/lib"
 # shellcheck source=../../../../shell/lib/bootstrap.sh
 source "$DEVBASE_LIB_DIR"/bootstrap.sh
 
+# shellcheck source=../../../../shell/lib/go.sh
+source "$DEVBASE_LIB_DIR"/go.sh
+
 # shellcheck source=../../../../shell/lib/logging.sh
 source "$DEVBASE_LIB_DIR"/logging.sh
 
@@ -39,16 +42,11 @@ if [[ -z $gomodGoVersion ]]; then
   exit 0
 fi
 
-git ls-files '**/go.mod' | while read -r gomod; do
+for godir in $(go_mod_dirs); do
+  gomod="$godir/go.mod"
   if managed_by_stencil "$gomod"; then
     continue
   fi
-  for ignored in ${IGNORED_GO_MOD_DIRS:-}; do
-    if [[ "$(dirname "$gomod")" == "$ignored" ]]; then
-      warn "Ignoring $gomod as per IGNORED_GO_MOD_DIRS"
-      continue 2
-    fi
-  done
   info "Syncing Go version/toolchain in $gomod to $gomodGoVersion/$goVersion"
 
   sed_replace '^go .*$' "go $gomodGoVersion" "$appDir/$gomod"

root/Makefile

@@ -115,12 +115,12 @@ build:: pre-build gobuild
 lint:: pre-test pre-lint
 	@# Note that this requires the ensure_asdf.sh invocation at the top of
 	@# this file.
-	$(BASE_TEST_ENV) ./scripts/shell-wrapper.sh linters.sh
+	$(BASE_TEST_ENV) mise exec -- ./scripts/shell-wrapper.sh linters.sh
 
 ## test:            run unit tests
 .PHONY: test
 test:: pre-test lint
-	$(BASE_TEST_ENV) ./scripts/shell-wrapper.sh test.sh
+	$(BASE_TEST_ENV) mise exec -- ./scripts/shell-wrapper.sh test.sh
 
 ## test-e2e:        run only e2e test (use inside a dev pod)
 .PHONY: test-e2e

shell/lib/go.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+#
+# Go-related utility functions.
+
+# Retrieve list of directories containing go.mod files in the repository.
+# Use the IGNORED_GO_MOD_DIRS environment variable (space-separated
+# directories) to skip specific directories.
+go_mod_dirs() {
+  git ls-files --cached --others --modified --exclude-standard go.mod '**/go.mod' | xargs dirname | while read -r gomodDir; do
+    for ignored in ${IGNORED_GO_MOD_DIRS:-}; do
+      if [[ $gomodDir == "$ignored" ]]; then
+        continue 2
+      fi
+    done
+    echo "$gomodDir"
+  done | sort | uniq | xargs echo
+}

shell/lib/go_test.bats

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+bats_load_library "bats-support/load.bash"
+bats_load_library "bats-assert/load.bash"
+
+load go.sh
+load test_helper.sh
+
+setup() {
+  # This points us to use a temp file for a git repo to operate on, as
+  # opposed to the real one.
+  REPOPATH=$(mktempdir devbase-lib-go-XXXXXX)
+
+  git init --initial-branch=main "$REPOPATH"
+  cd "$REPOPATH" || exit 1
+  git config user.name "Test User"
+  git config user.email "testuser@example.com"
+}
+
+teardown() {
+  rm -rf "$REPOPATH"
+}
+
+@test "go_mod_dirs finds all go.mod files in the repo" {
+  mkdir -p moduleA moduleB/submodule
+  touch moduleA/go.mod
+  touch moduleB/submodule/go.mod
+  touch go.mod
+  git add .
+
+  run go_mod_dirs
+  assert_success
+  assert_output ". moduleA moduleB/submodule"
+}
+
+@test "go_mod_dirs excludes go.mod files in IGNORED_GO_MOD_DIRS" {
+  mkdir -p moduleA moduleB/submodule moduleC
+  touch moduleA/go.mod
+  touch moduleB/submodule/go.mod
+  touch moduleC/go.mod
+  touch go.mod
+  git add .
+
+  IGNORED_GO_MOD_DIRS="moduleB/submodule moduleC" run go_mod_dirs
+  assert_success
+  assert_output ". moduleA" # output is sorted
+}

shell/linters/go.sh

@@ -1,6 +1,9 @@
 #!/usr/bin/env bash
 # Linters for Golang
 
+# shellcheck source=../lib/go.sh
+source "$DIR/lib/go.sh"
+
 # Why: Used by the script that calls us
 # shellcheck disable=SC2034
 extensions=("go")
@@ -24,11 +27,18 @@ gofumpt() {
 }
 
 linter() {
-  run_command "go mod tidy" go mod tidy -diff || return 1
-  # gofmt/goimports/gofumpt checking is done by golangci-lint
-  run_command "golangci-lint" \
-    "$DIR/golangci-lint.sh" --build-tags "or_e2e,or_test" --timeout 10m run ./... || return 1
-  run_command "lintroller" lintroller || return 1
+  for godir in $(go_mod_dirs); do
+    pushd "$godir" >/dev/null || return 1
+    if [[ $godir != "." ]]; then
+      info "Linting module in $godir"
+    fi
+    run_command "go mod tidy" go mod tidy -diff || return 1
+    # gofmt/goimports/gofumpt checking is done by golangci-lint
+    run_command "golangci-lint" \
+      "$DIR/golangci-lint.sh" --build-tags "or_e2e,or_test" --timeout 10m run ./... || return 1
+    run_command "lintroller" lintroller || return 1
+    popd >/dev/null || return 1
+  done
 }
 
 formatter() {
@@ -37,13 +47,20 @@ formatter() {
   if [[ -f "$(get_repo_directory)/go.work" ]]; then
     run_command "go work use" go work use || return 1
   fi
-  run_command "go mod tidy" go mod tidy || return 1
-  if [[ -z $goFormatter || $goFormatter == "null" || $goFormatter == "gofmt" ]]; then
-    run_command goimports goimports || return 1
-    run_command gofmt gofmt || return 1
-  elif [[ $goFormatter == gofumpt ]]; then
-    run_command gofumpt gofumpt || return 1
-  else
-    fatal "Unknown Go formatter: $goFormatter"
-  fi
+  for godir in $(go_mod_dirs); do
+    pushd "$godir" >/dev/null || return 1
+    if [[ $godir != "." ]]; then
+      info "Formatting module in $godir"
+    fi
+    run_command "go mod tidy" go mod tidy || return 1
+    if [[ -z $goFormatter || $goFormatter == "null" || $goFormatter == "gofmt" ]]; then
+      run_command goimports goimports || return 1
+      run_command gofmt gofmt || return 1
+    elif [[ $goFormatter == gofumpt ]]; then
+      run_command gofumpt gofumpt || return 1
+    else
+      fatal "Unknown Go formatter: $goFormatter"
+    fi
+    popd >/dev/null || return 1
+  done
 }

shell/test.sh

@@ -11,6 +11,9 @@ source "$DIR/lib/bootstrap.sh"
 # shellcheck source=./lib/github.sh
 source "$DIR/lib/github.sh"
 
+# shellcheck source=./lib/go.sh
+source "$DIR/lib/go.sh"
+
 # shellcheck source=./lib/logging.sh
 source "$DIR/lib/logging.sh"
 
@@ -102,36 +105,89 @@ SHUFFLE="${SHUFFLE:-enabled}"
 # is "standard-verbose".
 TEST_OUTPUT_FORMAT="${TEST_OUTPUT_FORMAT:-}"
 
+# repoDir is the base directory of the repository.
+repoDir=$(get_repo_directory)
+
 # Generates the Go toolchain string to be used for E2E tests.
 # Go 1.25 and later have an issue with code coverage, so we append
 # "+auto" so that the `covdata` tool is available.
 # See: https://github.com/golang/go/issues/75031
 e2e_go_toolchain() {
-  local repoDir toolchain
-  repoDir="$(get_repo_directory)"
-  toolchain="$(grep ^toolchain "$repoDir/go.mod" | awk '{print $2}')"
+  local goDir="$1"
+  local toolchain
+  toolchain="$(grep ^toolchain "$goDir/go.mod" | awk '{print $2}')"
   if [[ -z $toolchain ]]; then
     toolchain="go$(grep ^golang "$repoDir/.tool-versions" | awk '{print $2}')"
   fi
   echo "$toolchain+auto"
 }
 
+go_ldflags() {
+  echo "-X github.com/getoutreach/go-outreach/v2/pkg/app.Version=testing -X github.com/getoutreach/gobox/pkg/app.Version=testing"
+}
+
+# run_go_tests <projectDir> [args...]
+#
+# Runs Go tests with gotestsum for the given project directory,
+# with optional additional arguments.
+run_go_tests() {
+  local projectDir="$1"
+  shift
+  pushd "$projectDir" >/dev/null || fatal "Failed to change directory to $projectDir"
+  info "Running go test (${TEST_TAGS[*]}) in $projectDir"
+  local exitCode=0
+
+  local junitFile
+  if [[ $projectDir == "." ]]; then
+    junitFile="$repoDir/bin/unit-tests.xml"
+  else
+    # Replace path separators with dashes for the junit file name
+    local sanitizedDir
+    sanitizedDir="$(echo "$projectDir" | tr '/' '-')"
+    junitFile="$repoDir/bin/unit-tests-${sanitizedDir}.xml"
+  fi
+
+  (
+    if [[ ${TEST_TAGS[*]} =~ "or_e2e" ]]; then
+      # Workaround from https://github.com/golang/go/issues/75031#issuecomment-3195256688
+      local toolchain
+      toolchain="$(e2e_go_toolchain "$projectDir")"
+      go env -w GOTOOLCHAIN="$toolchain"
+      info_sub "Running E2E tests with Go toolchain $toolchain"
+    fi
+    mise_exec_tool gotestsum --junitfile "$junitFile" --format "$format" -- \
+      "${BENCH_FLAGS[@]}" "${COVER_FLAGS[@]}" "${TEST_FLAGS[@]}" \
+      -ldflags "$(go_ldflags)" -tags="$test_tags_string" "$@" "${TEST_PACKAGES[@]}"
+  ) || exitCode=$?
+
+  if in_ci_environment; then
+    # Move this to a temporary directory so that we can control
+    # what gets uploaded via the store_test_results call
+    mv "$junitFile" /tmp/test-results/
+  fi
+
+  if [[ $exitCode -ne 0 ]]; then
+    error "Tests failed in $projectDir with exit code $exitCode"
+    return $exitCode
+  fi
+  popd >/dev/null || fatal "Failed to change directory back from $projectDir"
+}
+
 if in_ci_environment; then
   GOFLAGS+=(-mod=readonly)
   WITH_COVERAGE="true"
 
-  # Ensure that all processes recieve the value of GOFLAGS.
+  # Ensure that all processes receive the value of GOFLAGS.
   export GOFLAGS
+  # Coverage results directory
+  mkdir -p /tmp/test-results
 fi
 
 # If GO_TEST_TIMEOUT is set, we pass it to `go test` as a timeout.
 if [[ -n $GO_TEST_TIMEOUT ]]; then
   TEST_FLAGS+=(-timeout "$GO_TEST_TIMEOUT")
 fi
 
-# REPODIR is the base directory of the repository.
-REPODIR=$(get_repo_directory)
-
 # Catches test dependencies by shuffling tests if the installed Go version supports it
 currentver="$(go version | awk '{ print $3 }' | sed 's|go||')"
 requiredver="1.17.0"
@@ -160,8 +216,6 @@ if [[ -e $testInclude ]]; then
 fi
 
 if [[ "$(git ls-files '*_test.go' | wc -l | tr -d ' ')" -gt 0 ]]; then
-  info "Running go test (${TEST_TAGS[*]})"
-
   format="dots-v2"
   if in_ci_environment; then
     # When in CI, always use the pkgname format because it's easier to
@@ -201,36 +255,15 @@ if [[ "$(git ls-files '*_test.go' | wc -l | tr -d ' ')" -gt 0 ]]; then
     # complex linker flags very well right now (v1.7.3).
     go test -c -o "${TESTBIN}" \
       "${BENCH_FLAGS[@]}" "${COVER_FLAGS[@]}" "${TEST_FLAGS[@]}" \
-      -ldflags "-X github.com/getoutreach/go-outreach/v2/pkg/app.Version=testing -X github.com/getoutreach/gobox/pkg/app.Version=testing" \
-      -tags="$test_tags_string" "$PACKAGE_TO_DEBUG"
+      -ldflags "$(go_ldflags)" -tags="$test_tags_string" "$PACKAGE_TO_DEBUG"
 
     # We pass along command line args to the executable so you can specify
     # `-test.run <regex>`, `-test.bench <regex>`, etc. if desired.  Try `-help`
     # for more information.
     exec "$DIR/dlv.sh" exec "${TESTBIN}" -- "$@"
   else
-    exitCode=0
-
-    (
-      if [[ ${TEST_TAGS[*]} =~ "or_e2e" ]]; then
-        # Workaround from https://github.com/golang/go/issues/75031#issuecomment-3195256688
-        toolchain="$(e2e_go_toolchain)"
-        go env -w GOTOOLCHAIN="$toolchain"
-        info_sub "Running E2E tests with Go toolchain $toolchain"
-      fi
-      mise_exec_tool gotestsum --junitfile "$REPODIR/bin/unit-tests.xml" --format "$format" -- \
-        "${BENCH_FLAGS[@]}" "${COVER_FLAGS[@]}" "${TEST_FLAGS[@]}" \
-        -ldflags "-X github.com/getoutreach/go-outreach/v2/pkg/app.Version=testing -X github.com/getoutreach/gobox/pkg/app.Version=testing" \
-        -tags="$test_tags_string" "$@" "${TEST_PACKAGES[@]}"
-    ) || exitCode=$?
-
-    if in_ci_environment; then
-      # Move this to a temporary directory so that we can control
-      # what gets uploaded via the store_test_results call
-      mkdir -p /tmp/test-results
-      mv "$REPODIR/bin/unit-tests.xml" /tmp/test-results/
-    fi
-
-    exit $exitCode
+    for godir in $(go_mod_dirs); do
+      run_go_tests "$godir" "$@" || fatal "Tests failed in $godir"
+    done
   fi
 fi