diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2a310a374c6b..db954df7ca3e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -53,6 +53,85 @@ jobs: id: date run: echo "::set-output name=week_of_year::$(date +'%W' --utc)" - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: 8 + distribution: temurin + - name: "Set jvm system property environment variable for surefire plugin (unit tests)" + # Maven surefire plugin (unit tests) allows us to specify JVM to run the tests. + # https://maven.apache.org/surefire/maven-surefire-plugin/test-mojo.html#jvm + run: echo "SUREFIRE_JVM_OPT=-Djvm=${JAVA_HOME}/bin/java" >> $GITHUB_ENV + shell: bash + - uses: actions/setup-java@v4 + with: + java-version: 11 + distribution: temurin + cache: maven + - uses: actions/cache@v4 + id: mvn-cache + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-unified-${{ steps.date.outputs.week_of_year }} + - run: .kokoro/build.sh + shell: bash + env: + JOB_TYPE: test + JOB_NAME: units-8-runtime-${{matrix.java}} + # detect which libraries have changed + changes: + runs-on: ubuntu-latest + permissions: + pull-requests: read + outputs: + packages: ${{ steps.filter.outputs.changes }} + steps: + - uses: dorny/paths-filter@v4 + id: filter + with: + filters: | + java-bigquery: java-bigquery/** + java-bigquerystorage: java-bigquerystorage/** + java-datastore: java-datastore/** + java-logging: java-logging/** + java-logging-logback: java-logging-logback/** + java-spanner: java-spanner/** + sdk-platform-java: sdk-platform-java/** + split-units: + runs-on: ubuntu-latest + needs: changes + strategy: + fail-fast: false + matrix: + package: ${{ fromJSON(needs.changes.outputs.packages) }} + java: [11, 17, 21, 25] + steps: + - name: Get current week within the year + id: date + run: echo "::set-output name=week_of_year::$(date +'%W' --utc)" + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: ${{matrix.java}} + - run: .kokoro/build.sh + env: + BUILD_SUBDIR: ${{matrix.package}} + JOB_TYPE: test + JOB_NAME: units-${{matrix.package}}-${{matrix.java}} + split-units-8: + runs-on: ubuntu-latest + name: "split-units" + needs: changes + strategy: + fail-fast: false + matrix: + package: ${{ fromJSON(needs.changes.outputs.packages) }} + java: [8] + steps: + - name: Get current week within the year + id: date + run: echo "::set-output name=week_of_year::$(date +'%W' --utc)" + - uses: actions/checkout@v4 - uses: actions/setup-java@v4 with: java-version: 11 @@ -66,15 +145,16 @@ jobs: key: ${{ runner.os }}-maven-unified-${{ steps.date.outputs.week_of_year }} - name: Install all modules using Java 11 shell: bash - run: | - mvn -V -B -ntp clean install -DskipTests -Dfmt.skip -Dclirr.skip -Denforcer.skip -Dcheckstyle.skip -Dflatten.skip -Danimal-sniffer.skip -Djacoco.skip - working-directory: sdk-platform-java + run: .kokoro/build.sh + env: + BUILD_SUBDIR: ${{matrix.package}} + JOB_TYPE: install - uses: actions/setup-java@v4 with: - java-version: 8 + java-version: ${{matrix.java}} distribution: temurin - run: java -version - - name: Run tests in Java 8 with the source compiled in Java 11 + - name: Run tests in Java ${{matrix.java}} with the source compiled in Java 11 run: | mvn test \ -B -ntp \ @@ -86,11 +166,13 @@ jobs: -Dflatten.skip=true \ -Danimal.sniffer.skip=true \ -Dmaven.wagon.http.retryHandler.count=5 \ + --also-make \ -T 1C env: + BUILD_SUBDIR: ${{matrix.package}} JOB_TYPE: test JOB_NAME: units-8-runtime-${{matrix.java}} - working-directory: sdk-platform-java + working-directory: ${{matrix.package}} windows: runs-on: windows-latest steps: diff --git a/.github/workflows/java-bigquery-ci.yaml b/.github/workflows/java-bigquery-ci.yaml deleted file mode 100644 index 79c8afd7bec8..000000000000 --- a/.github/workflows/java-bigquery-ci.yaml +++ /dev/null @@ -1,143 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Github action job to test core java library features on -# downstream client libraries before they are released. -on: - push: - branches: - - main - pull_request: -name: java-bigquery ci -env: - BUILD_SUBDIR: java-bigquery -jobs: - filter: - runs-on: ubuntu-latest - outputs: - library: ${{ steps.filter.outputs.library }} - steps: - - uses: actions/checkout@v4 - - uses: dorny/paths-filter@v3 - id: filter - with: - filters: | - library: - - 'java-bigquery/**' - units: - needs: filter - if: ${{ needs.filter.outputs.library == 'true' }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - java: [11, 17, 21, 25] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: ${{matrix.java}} - - run: java -version - - run: .kokoro/build.sh - env: - JOB_TYPE: test - units-java8: - needs: filter - if: ${{ needs.filter.outputs.library == 'true' }} - # Building using Java 17 and run the tests with Java 8 runtime - name: "units (8)" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - java-version: 8 - distribution: temurin - - name: "Set jvm system property environment variable for surefire plugin (unit tests)" - # Maven surefire plugin (unit tests) allows us to specify JVM to run the tests. - # https://maven.apache.org/surefire/maven-surefire-plugin/test-mojo.html#jvm - run: echo "SUREFIRE_JVM_OPT=-Djvm=${JAVA_HOME}/bin/java -P !java17" >> $GITHUB_ENV - shell: bash - - uses: actions/setup-java@v4 - with: - java-version: 17 - distribution: temurin - - run: .kokoro/build.sh - env: - JOB_TYPE: test - windows: - needs: filter - if: ${{ needs.filter.outputs.library == 'true' }} - runs-on: windows-latest - steps: - - name: Support longpaths - run: git config --system core.longpaths true - - name: Support longpaths - run: git config --system core.longpaths true - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: 8 - - run: java -version - - run: .kokoro/build.sh - env: - JOB_TYPE: test - dependencies: - needs: filter - if: ${{ needs.filter.outputs.library == 'true' }} - runs-on: ubuntu-latest - strategy: - matrix: - java: [17] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: ${{matrix.java}} - - run: java -version - - run: .kokoro/dependencies.sh - javadoc: - needs: filter - if: ${{ needs.filter.outputs.library == 'true' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: 17 - - run: java -version - - run: .kokoro/build.sh - env: - JOB_TYPE: javadoc - lint: - needs: filter - if: ${{ needs.filter.outputs.library == 'true' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: 17 - - run: java -version - - run: .kokoro/build.sh - env: - JOB_TYPE: lint - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} diff --git a/.github/workflows/sdk-platform-java-ci.yaml b/.github/workflows/sdk-platform-java-ci.yaml index c0e9904ea4f7..a88d45304d8e 100644 --- a/.github/workflows/sdk-platform-java-ci.yaml +++ b/.github/workflows/sdk-platform-java-ci.yaml @@ -430,8 +430,11 @@ jobs: - name: Checkout sdk-platform-java @ PR merge commit uses: actions/checkout@v3 - name: Install sdk-platform-java @ PR merge commit - run: mvn install -B -ntp -T 1C -DskipTests - working-directory: sdk-platform-java + shell: bash + run: .kokoro/build.sh + env: + JOB_TYPE: install + BUILD_SUBDIR: sdk-platform-java # Showcase golden test ensures that src changes are already reflected in the PR. - name: Clirr check working-directory: sdk-platform-java/java-showcase diff --git a/.kokoro/build.sh b/.kokoro/build.sh index c20f1633a357..b8524fbb94d8 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -43,7 +43,9 @@ case ${JOB_TYPE} in install_modules "${BUILD_SUBDIR}" echo "Running in subdir: ${BUILD_SUBDIR}" pushd "${BUILD_SUBDIR}" + EXTRA_PROFILE_OPTS=() else + EXTRA_PROFILE_OPTS=("-PbulkTests") install_modules "sdk-platform-java" fi echo "SUREFIRE_JVM_OPT: ${SUREFIRE_JVM_OPT}" @@ -59,7 +61,7 @@ case ${JOB_TYPE} in -Danimal.sniffer.skip=true \ -Dmaven.wagon.http.retryHandler.count=5 \ --also-make \ - ${SUREFIRE_JVM_OPT} + ${SUREFIRE_JVM_OPT} "${EXTRA_PROFILE_OPTS[@]}" RETURN_CODE=$? if [[ -n "${BUILD_SUBDIR}" ]] @@ -69,6 +71,28 @@ case ${JOB_TYPE} in fi echo "Finished running unit tests" ;; + install) + if [[ -n "${BUILD_SUBDIR}" ]] + then + echo "Compiling and building all modules for ${BUILD_SUBDIR}" + install_modules "${BUILD_SUBDIR}" + else + install_modules "sdk-platform-java" + mvn install \ + -B -ntp \ + -Dorg.slf4j.simpleLogger.showDateTime=true \ + -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \ + -Dclirr.skip=true \ + -Denforcer.skip=true \ + -Dcheckstyle.skip=true \ + -Dflatten.skip=true \ + -Danimal.sniffer.skip=true \ + -Dmaven.wagon.http.retryHandler.count=5 \ + -DskipTests=true \ + --also-make \ + -T 1C + fi + ;; integration) generate_modified_modules_list if [[ "$(release_please_snapshot_pull_request)" == "true" ]]; then diff --git a/java-bigquery/pom.xml b/java-bigquery/pom.xml index caf716fd21d9..e3818fef269d 100644 --- a/java-bigquery/pom.xml +++ b/java-bigquery/pom.xml @@ -162,5 +162,12 @@ benchmark + + + bulkTests + + true + + diff --git a/java-bigquerystorage/pom.xml b/java-bigquerystorage/pom.xml index 68ffc12a7357..94c4abca62b3 100644 --- a/java-bigquerystorage/pom.xml +++ b/java-bigquerystorage/pom.xml @@ -202,4 +202,14 @@ google-cloud-bigquerystorage-bom - + + + + bulkTests + + true + + + + + diff --git a/java-datastore/pom.xml b/java-datastore/pom.xml index 763eab057d29..ef7709bc84a4 100644 --- a/java-datastore/pom.xml +++ b/java-datastore/pom.xml @@ -232,4 +232,13 @@ google-cloud-datastore-utils - + + + + bulkTests + + true + + + + diff --git a/java-logging-logback/pom.xml b/java-logging-logback/pom.xml index 78fcf1f08ad8..b2849e45d121 100644 --- a/java-logging-logback/pom.xml +++ b/java-logging-logback/pom.xml @@ -168,5 +168,12 @@ !LoggingAppenderTest + + + bulkTests + + true + + diff --git a/java-logging/pom.xml b/java-logging/pom.xml index 576aa35928de..057b72b96da1 100644 --- a/java-logging/pom.xml +++ b/java-logging/pom.xml @@ -101,4 +101,13 @@ google-cloud-logging-bom - + + + + bulkTests + + true + + + + diff --git a/java-spanner/pom.xml b/java-spanner/pom.xml index 9e0293d05e8f..bd8ec86fb359 100644 --- a/java-spanner/pom.xml +++ b/java-spanner/pom.xml @@ -153,4 +153,14 @@ + + + + bulkTests + + true + + + + diff --git a/sdk-platform-java/gapic-generator-java-pom-parent/pom.xml b/sdk-platform-java/gapic-generator-java-pom-parent/pom.xml index 8d0be132a5b5..57b697858660 100644 --- a/sdk-platform-java/gapic-generator-java-pom-parent/pom.xml +++ b/sdk-platform-java/gapic-generator-java-pom-parent/pom.xml @@ -202,6 +202,13 @@ + + + bulkTests + + true + +