diff --git a/.mise/tasks/stencil/post/go-sync b/.mise/tasks/stencil/post/go-sync index 5b501fea..8baafdd4 100755 --- a/.mise/tasks/stencil/post/go-sync +++ b/.mise/tasks/stencil/post/go-sync @@ -11,6 +11,9 @@ DEVBASE_LIB_DIR="$DEVBASE_ROOT_DIR/shell/lib" # shellcheck source=../../../../shell/lib/bootstrap.sh source "$DEVBASE_LIB_DIR"/bootstrap.sh +# shellcheck source=../../../../shell/lib/go.sh +source "$DEVBASE_LIB_DIR"/go.sh + # shellcheck source=../../../../shell/lib/logging.sh source "$DEVBASE_LIB_DIR"/logging.sh @@ -39,16 +42,11 @@ if [[ -z $gomodGoVersion ]]; then exit 0 fi -git ls-files '**/go.mod' | while read -r gomod; do +for godir in $(go_mod_dirs); do + gomod="$godir/go.mod" if managed_by_stencil "$gomod"; then continue fi - for ignored in ${IGNORED_GO_MOD_DIRS:-}; do - if [[ "$(dirname "$gomod")" == "$ignored" ]]; then - warn "Ignoring $gomod as per IGNORED_GO_MOD_DIRS" - continue 2 - fi - done info "Syncing Go version/toolchain in $gomod to $gomodGoVersion/$goVersion" sed_replace '^go .*$' "go $gomodGoVersion" "$appDir/$gomod" diff --git a/root/Makefile b/root/Makefile index d9bc26c1..42ea3753 100644 --- a/root/Makefile +++ b/root/Makefile @@ -115,12 +115,12 @@ build:: pre-build gobuild lint:: pre-test pre-lint @# Note that this requires the ensure_asdf.sh invocation at the top of @# this file. - $(BASE_TEST_ENV) ./scripts/shell-wrapper.sh linters.sh + $(BASE_TEST_ENV) mise exec -- ./scripts/shell-wrapper.sh linters.sh ## test: run unit tests .PHONY: test test:: pre-test lint - $(BASE_TEST_ENV) ./scripts/shell-wrapper.sh test.sh + $(BASE_TEST_ENV) mise exec -- ./scripts/shell-wrapper.sh test.sh ## test-e2e: run only e2e test (use inside a dev pod) .PHONY: test-e2e diff --git a/shell/lib/go.sh b/shell/lib/go.sh new file mode 100644 index 00000000..1d8362b6 --- /dev/null +++ b/shell/lib/go.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# +# Go-related utility functions. + +# Retrieve list of directories containing go.mod files in the repository. +# Use the IGNORED_GO_MOD_DIRS environment variable (space-separated +# directories) to skip specific directories. +go_mod_dirs() { + git ls-files --cached --others --modified --exclude-standard go.mod '**/go.mod' | xargs dirname | while read -r gomodDir; do + for ignored in ${IGNORED_GO_MOD_DIRS:-}; do + if [[ $gomodDir == "$ignored" ]]; then + continue 2 + fi + done + echo "$gomodDir" + done | sort | uniq | xargs echo +} diff --git a/shell/lib/go_test.bats b/shell/lib/go_test.bats new file mode 100644 index 00000000..674b25fc --- /dev/null +++ b/shell/lib/go_test.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +bats_load_library "bats-support/load.bash" +bats_load_library "bats-assert/load.bash" + +load go.sh +load test_helper.sh + +setup() { + # This points us to use a temp file for a git repo to operate on, as + # opposed to the real one. + REPOPATH=$(mktempdir devbase-lib-go-XXXXXX) + + git init --initial-branch=main "$REPOPATH" + cd "$REPOPATH" || exit 1 + git config user.name "Test User" + git config user.email "testuser@example.com" +} + +teardown() { + rm -rf "$REPOPATH" +} + +@test "go_mod_dirs finds all go.mod files in the repo" { + mkdir -p moduleA moduleB/submodule + touch moduleA/go.mod + touch moduleB/submodule/go.mod + touch go.mod + git add . + + run go_mod_dirs + assert_success + assert_output ". moduleA moduleB/submodule" +} + +@test "go_mod_dirs excludes go.mod files in IGNORED_GO_MOD_DIRS" { + mkdir -p moduleA moduleB/submodule moduleC + touch moduleA/go.mod + touch moduleB/submodule/go.mod + touch moduleC/go.mod + touch go.mod + git add . + + IGNORED_GO_MOD_DIRS="moduleB/submodule moduleC" run go_mod_dirs + assert_success + assert_output ". moduleA" # output is sorted +} diff --git a/shell/linters/go.sh b/shell/linters/go.sh index c11956d2..505151b7 100644 --- a/shell/linters/go.sh +++ b/shell/linters/go.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash # Linters for Golang +# shellcheck source=../lib/go.sh +source "$DIR/lib/go.sh" + # Why: Used by the script that calls us # shellcheck disable=SC2034 extensions=("go") @@ -24,11 +27,18 @@ gofumpt() { } linter() { - run_command "go mod tidy" go mod tidy -diff || return 1 - # gofmt/goimports/gofumpt checking is done by golangci-lint - run_command "golangci-lint" \ - "$DIR/golangci-lint.sh" --build-tags "or_e2e,or_test" --timeout 10m run ./... || return 1 - run_command "lintroller" lintroller || return 1 + for godir in $(go_mod_dirs); do + pushd "$godir" >/dev/null || return 1 + if [[ $godir != "." ]]; then + info "Linting module in $godir" + fi + run_command "go mod tidy" go mod tidy -diff || return 1 + # gofmt/goimports/gofumpt checking is done by golangci-lint + run_command "golangci-lint" \ + "$DIR/golangci-lint.sh" --build-tags "or_e2e,or_test" --timeout 10m run ./... || return 1 + run_command "lintroller" lintroller || return 1 + popd >/dev/null || return 1 + done } formatter() { @@ -37,13 +47,20 @@ formatter() { if [[ -f "$(get_repo_directory)/go.work" ]]; then run_command "go work use" go work use || return 1 fi - run_command "go mod tidy" go mod tidy || return 1 - if [[ -z $goFormatter || $goFormatter == "null" || $goFormatter == "gofmt" ]]; then - run_command goimports goimports || return 1 - run_command gofmt gofmt || return 1 - elif [[ $goFormatter == gofumpt ]]; then - run_command gofumpt gofumpt || return 1 - else - fatal "Unknown Go formatter: $goFormatter" - fi + for godir in $(go_mod_dirs); do + pushd "$godir" >/dev/null || return 1 + if [[ $godir != "." ]]; then + info "Formatting module in $godir" + fi + run_command "go mod tidy" go mod tidy || return 1 + if [[ -z $goFormatter || $goFormatter == "null" || $goFormatter == "gofmt" ]]; then + run_command goimports goimports || return 1 + run_command gofmt gofmt || return 1 + elif [[ $goFormatter == gofumpt ]]; then + run_command gofumpt gofumpt || return 1 + else + fatal "Unknown Go formatter: $goFormatter" + fi + popd >/dev/null || return 1 + done } diff --git a/shell/test.sh b/shell/test.sh index 79d1d646..5560bd3b 100755 --- a/shell/test.sh +++ b/shell/test.sh @@ -11,6 +11,9 @@ source "$DIR/lib/bootstrap.sh" # shellcheck source=./lib/github.sh source "$DIR/lib/github.sh" +# shellcheck source=./lib/go.sh +source "$DIR/lib/go.sh" + # shellcheck source=./lib/logging.sh source "$DIR/lib/logging.sh" @@ -102,26 +105,82 @@ SHUFFLE="${SHUFFLE:-enabled}" # is "standard-verbose". TEST_OUTPUT_FORMAT="${TEST_OUTPUT_FORMAT:-}" +# repoDir is the base directory of the repository. +repoDir=$(get_repo_directory) + # Generates the Go toolchain string to be used for E2E tests. # Go 1.25 and later have an issue with code coverage, so we append # "+auto" so that the `covdata` tool is available. # See: https://github.com/golang/go/issues/75031 e2e_go_toolchain() { - local repoDir toolchain - repoDir="$(get_repo_directory)" - toolchain="$(grep ^toolchain "$repoDir/go.mod" | awk '{print $2}')" + local goDir="$1" + local toolchain + toolchain="$(grep ^toolchain "$goDir/go.mod" | awk '{print $2}')" if [[ -z $toolchain ]]; then toolchain="go$(grep ^golang "$repoDir/.tool-versions" | awk '{print $2}')" fi echo "$toolchain+auto" } +go_ldflags() { + echo "-X github.com/getoutreach/go-outreach/v2/pkg/app.Version=testing -X github.com/getoutreach/gobox/pkg/app.Version=testing" +} + +# run_go_tests [args...] +# +# Runs Go tests with gotestsum for the given project directory, +# with optional additional arguments. +run_go_tests() { + local projectDir="$1" + shift + pushd "$projectDir" >/dev/null || fatal "Failed to change directory to $projectDir" + info "Running go test (${TEST_TAGS[*]}) in $projectDir" + local exitCode=0 + + local junitFile + if [[ $projectDir == "." ]]; then + junitFile="$repoDir/bin/unit-tests.xml" + else + # Replace path separators with dashes for the junit file name + local sanitizedDir + sanitizedDir="$(echo "$projectDir" | tr '/' '-')" + junitFile="$repoDir/bin/unit-tests-${sanitizedDir}.xml" + fi + + ( + if [[ ${TEST_TAGS[*]} =~ "or_e2e" ]]; then + # Workaround from https://github.com/golang/go/issues/75031#issuecomment-3195256688 + local toolchain + toolchain="$(e2e_go_toolchain "$projectDir")" + go env -w GOTOOLCHAIN="$toolchain" + info_sub "Running E2E tests with Go toolchain $toolchain" + fi + mise_exec_tool gotestsum --junitfile "$junitFile" --format "$format" -- \ + "${BENCH_FLAGS[@]}" "${COVER_FLAGS[@]}" "${TEST_FLAGS[@]}" \ + -ldflags "$(go_ldflags)" -tags="$test_tags_string" "$@" "${TEST_PACKAGES[@]}" + ) || exitCode=$? + + if in_ci_environment; then + # Move this to a temporary directory so that we can control + # what gets uploaded via the store_test_results call + mv "$junitFile" /tmp/test-results/ + fi + + if [[ $exitCode -ne 0 ]]; then + error "Tests failed in $projectDir with exit code $exitCode" + return $exitCode + fi + popd >/dev/null || fatal "Failed to change directory back from $projectDir" +} + if in_ci_environment; then GOFLAGS+=(-mod=readonly) WITH_COVERAGE="true" - # Ensure that all processes recieve the value of GOFLAGS. + # Ensure that all processes receive the value of GOFLAGS. export GOFLAGS + # Coverage results directory + mkdir -p /tmp/test-results fi # If GO_TEST_TIMEOUT is set, we pass it to `go test` as a timeout. @@ -129,9 +188,6 @@ if [[ -n $GO_TEST_TIMEOUT ]]; then TEST_FLAGS+=(-timeout "$GO_TEST_TIMEOUT") fi -# REPODIR is the base directory of the repository. -REPODIR=$(get_repo_directory) - # Catches test dependencies by shuffling tests if the installed Go version supports it currentver="$(go version | awk '{ print $3 }' | sed 's|go||')" requiredver="1.17.0" @@ -160,8 +216,6 @@ if [[ -e $testInclude ]]; then fi if [[ "$(git ls-files '*_test.go' | wc -l | tr -d ' ')" -gt 0 ]]; then - info "Running go test (${TEST_TAGS[*]})" - format="dots-v2" if in_ci_environment; then # When in CI, always use the pkgname format because it's easier to @@ -201,36 +255,15 @@ if [[ "$(git ls-files '*_test.go' | wc -l | tr -d ' ')" -gt 0 ]]; then # complex linker flags very well right now (v1.7.3). go test -c -o "${TESTBIN}" \ "${BENCH_FLAGS[@]}" "${COVER_FLAGS[@]}" "${TEST_FLAGS[@]}" \ - -ldflags "-X github.com/getoutreach/go-outreach/v2/pkg/app.Version=testing -X github.com/getoutreach/gobox/pkg/app.Version=testing" \ - -tags="$test_tags_string" "$PACKAGE_TO_DEBUG" + -ldflags "$(go_ldflags)" -tags="$test_tags_string" "$PACKAGE_TO_DEBUG" # We pass along command line args to the executable so you can specify # `-test.run `, `-test.bench `, etc. if desired. Try `-help` # for more information. exec "$DIR/dlv.sh" exec "${TESTBIN}" -- "$@" else - exitCode=0 - - ( - if [[ ${TEST_TAGS[*]} =~ "or_e2e" ]]; then - # Workaround from https://github.com/golang/go/issues/75031#issuecomment-3195256688 - toolchain="$(e2e_go_toolchain)" - go env -w GOTOOLCHAIN="$toolchain" - info_sub "Running E2E tests with Go toolchain $toolchain" - fi - mise_exec_tool gotestsum --junitfile "$REPODIR/bin/unit-tests.xml" --format "$format" -- \ - "${BENCH_FLAGS[@]}" "${COVER_FLAGS[@]}" "${TEST_FLAGS[@]}" \ - -ldflags "-X github.com/getoutreach/go-outreach/v2/pkg/app.Version=testing -X github.com/getoutreach/gobox/pkg/app.Version=testing" \ - -tags="$test_tags_string" "$@" "${TEST_PACKAGES[@]}" - ) || exitCode=$? - - if in_ci_environment; then - # Move this to a temporary directory so that we can control - # what gets uploaded via the store_test_results call - mkdir -p /tmp/test-results - mv "$REPODIR/bin/unit-tests.xml" /tmp/test-results/ - fi - - exit $exitCode + for godir in $(go_mod_dirs); do + run_go_tests "$godir" "$@" || fatal "Tests failed in $godir" + done fi fi