diff --git a/.github/workflows/sdk-platform-java-downstream_protobuf_compatibility_check_nightly.yaml b/.github/workflows/sdk-platform-java-downstream_protobuf_compatibility_check_nightly.yaml index 858c02de00f3..9d6e40d734d6 100644 --- a/.github/workflows/sdk-platform-java-downstream_protobuf_compatibility_check_nightly.yaml +++ b/.github/workflows/sdk-platform-java-downstream_protobuf_compatibility_check_nightly.yaml @@ -12,44 +12,25 @@ on: schedule: - cron: '0 1 * * *' # Nightly at 1am +# This job intends to test the compatibility of the Protobuf version against repos not in the +# monorepo. As repos are migrated to the monorepo, this job will be obsolete and turned off. name: sdk-platform-java Downstream Protobuf Compatibility Check Nightly -env: - BUILD_SUBDIR: sdk-platform-java jobs: - filter: - runs-on: ubuntu-latest - outputs: - library: ${{ steps.filter.outputs.library }} - steps: - - uses: actions/checkout@v4 - - uses: dorny/paths-filter@v3 - id: filter - with: - filters: | - library: - - 'sdk-platform-java/**' downstream-protobuf-test: - needs: filter # This job runs if any of the three conditions match: # 1. PR is raised from Release-Please (PR comes from branch: release-please--branches-main) # 2. Job is invoked by the nightly job (scheduled event) # 3. Job is manually invoked via Github UI (workflow_dispatch event) - if: needs.filter.outputs.library == 'true' && (github.head_ref == 'release-please--branches--main' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') + if: github.head_ref == 'release-please--branches--main' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: repo: - - google-cloud-java - java-bigtable - - java-bigquery - java-firestore - java-pubsub - java-pubsublite - - java-spanner-jdbc - - java-spanner - - java-storage - - java-storage-nio # Default Protobuf-Java versions to use are specified here. Without this, the nightly workflow won't know # which values to use and would resolve to ''. protobuf-version: ${{ fromJSON(format('[{0}]', inputs.protobuf_runtime_versions || '"4.33.5"')) }} @@ -70,7 +51,7 @@ jobs: - name: Print Protobuf-Java testing version run: echo "Testing with Protobuf-Java v${{ matrix.protobuf-version }}" - name: Perform downstream source compatibility testing - run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version }}" ./.kokoro/nightly/downstream-protobuf-source-compatibility.sh + run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version }}" ./sdk-platform-java/.kokoro/nightly/downstream-protobuf-source-compatibility.sh - name: Perform downstream binary compatibility testing - run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version }}" ./.kokoro/nightly/downstream-protobuf-binary-compatibility.sh + run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version }}" ./sdk-platform-java/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh diff --git a/sdk-platform-java/.kokoro/nightly/common.sh b/sdk-platform-java/.kokoro/nightly/common.sh new file mode 100644 index 000000000000..589ee00bfbf7 --- /dev/null +++ b/sdk-platform-java/.kokoro/nightly/common.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# For google-cloud-java, only test specific handwritten libraries included in the monorepo. This is to +# help speed up the execution as building the entire repo is an expensive operation. Specify the nested +# `google-cloud-*` path (except for grafeas as it doesn't have one) because maven -pl will only build the +# specified folder (i.e. parent folder) and ignore all the related sub-modules inside +google_cloud_java_handwritten_maven_args="java-grafeas,java-vertexai/google-cloud-vertexai,java-resourcemanager/google-cloud-resourcemanager,java-translate/google-cloud-translate" + +# Checks that the protobuf compatibility scripts provide non-empty input +function validate_protobuf_compatibility_script_inputs { + # Comma-delimited list of repos to test + if [ -z "${REPOS_UNDER_TEST}" ]; then + echo "REPOS_UNDER_TEST Env Var must be set. This script expects a" + echo "comma-delimited list: i.e REPOS_UNDER_TEST=\"java-bigtable,java-bigquery\"" + exit 1 + fi + + # A single version of Protobuf-Java runtime to test + if [ -z "${PROTOBUF_RUNTIME_VERSION}" ]; then + echo "PROTOBUF_RUNTIME_VERSION Env Var must be set. This script expects a single " + echo "Protobuf-Java runtime version i.e. PROTOBUF_RUNTIME_VERSION=\"4.28.3\"" + exit 1 + fi +} \ No newline at end of file diff --git a/sdk-platform-java/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/sdk-platform-java/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh new file mode 100755 index 000000000000..5877eb614c73 --- /dev/null +++ b/sdk-platform-java/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -0,0 +1,105 @@ +#!/bin/bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +scriptDir=$(realpath "$(dirname "${BASH_SOURCE[0]}")") +source "${scriptDir}/common.sh" + +validate_protobuf_compatibility_script_inputs + +# Declare a map of downstream handwritten libraries and the relevant artifacts to test. The map stores a +# K/V pairing of (Key: repo name, Value: comma separate list of Group ID:Artifact ID pairings). Note: The +# value list doesn't hold the version and this needs to be parsed from the repo's versions.txt file +declare -A repo_linkage_checker_arguments +repo_linkage_checker_arguments["google-cloud-java"]="io.grafeas:grafeas,com.google.cloud:google-cloud-vertexai,com.google.cloud:google-cloud-resourcemanager,com.google.cloud:google-cloud-translate,com.google.api.grpc:grpc-google-cloud-vertexai-v1,com.google.api.grpc:grpc-google-cloud-vertexai-v1beta1,com.google.api.grpc:grpc-google-cloud-resourcemanager-v3,com.google.api.grpc:grpc-google-cloud-translate-v3,com.google.api.grpc:grpc-google-cloud-translate-v3beta1" +repo_linkage_checker_arguments["java-bigtable"]="com.google.cloud:google-cloud-bigtable,com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2,com.google.api.grpc:grpc-google-cloud-bigtable-v2" +repo_linkage_checker_arguments["java-bigquery"]="com.google.cloud:google-cloud-bigquery" +repo_linkage_checker_arguments["java-bigquerystorage"]="com.google.cloud:google-cloud-bigquerystorage,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1alpha" +repo_linkage_checker_arguments["java-datastore"]="com.google.cloud:google-cloud-datastore,com.google.cloud.datastre:datastore-v1-proto-client,com.google.api.grpc:grpc-google-cloud-datastore-admin-v1" +repo_linkage_checker_arguments["java-firestore"]="com.google.cloud:google-cloud-firestore,com.google.cloud:google-cloud-firestore-admin,com.google.api.grpc:grpc-google-cloud-firestore-admin-v1,com.google.api.grpc:grpc-google-cloud-firestore-v1" +repo_linkage_checker_arguments["java-logging"]="com.google.cloud:google-cloud-logging,com.google.api.grpc:grpc-google-cloud-logging-v2" +repo_linkage_checker_arguments["java-logging-logback"]="com.google.cloud:google-cloud-logging-logback" +repo_linkage_checker_arguments["java-pubsub"]="com.google.cloud:google-cloud-pubsub,com.google.api.grpc:grpc-google-cloud-pubsub-v1" +repo_linkage_checker_arguments["java-pubsublite"]="com.google.cloud:google-cloud-pubsublite,com.google.api.grpc:grpc-google-cloud-pubsublite-v1" +repo_linkage_checker_arguments["java-spanner-jdbc"]="com.google.cloud:google-cloud-spanner-jdbc" +repo_linkage_checker_arguments["java-spanner"]="com.google.cloud:google-cloud-spanner,com.google.cloud:google-cloud-spanner-executor,com.google.api.grpc:grpc-google-cloud-spanner-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1,com.google.api.grpc:grpc-google-cloud-spanner-executor-v1" +repo_linkage_checker_arguments["java-storage"]="com.google.cloud:google-cloud-storage,com.google.api.grpc:gapic-google-cloud-storage-v2,com.google.api.grpc:grpc-google-cloud-storage-v2,com.google.cloud:google-cloud-storage-control,com.google.api.grpc:grpc-google-cloud-storage-control-v2" +repo_linkage_checker_arguments["java-storage-nio"]="com.google.cloud:google-cloud-nio" + +# This function requires access to the versions.txt to retrieve the versions for the artifacts +# It will try to match the artifact_id in the versions.txt file and attach it to form the GAV +# The GAV list is required by Linkage Checker as program arguments +function build_program_arguments() { + artifact_list="${repo_linkage_checker_arguments[$1]}" + + for artifact in ${artifact_list//,/ }; do # Split on comma + artifact_id=$(echo "${artifact}" | cut -d ':' -f2) + + # The grep query tries to match `{artifact_id}:{released_version}:{current_version}`. + # The artifact_id must be exact otherwise multiple entries may match + version=$(cat "versions.txt" | grep -E "^${artifact_id}:.*:.*$" | cut -d ':' -f3) + repo_gav_coordinate="${artifact}:${version}" + + # The first entry added is not separated with a comma. Avoids generating `,{ARTIFACT_LIST}` + if [ -z "${linkage_checker_arguments}" ]; then + linkage_checker_arguments="${repo_gav_coordinate}" + else + linkage_checker_arguments="${linkage_checker_arguments},${repo_gav_coordinate}" + fi + done +} + +# TODO(https://github.com/GoogleCloudPlatform/cloud-opensource-java/issues/2395): Java 17+ support for Linkage Checker +# cloud-opensource-java contains the Linkage Checker tool +git clone https://github.com/GoogleCloudPlatform/cloud-opensource-java.git +pushd cloud-opensource-java +mvn -B -ntp clean compile -T 1C +# Linkage Checker tool resides in the /dependencies subfolder +pushd dependencies + +# REPOS_UNDER_TEST Env Var accepts a comma separated list of googleapis repos to test. For Github CI, +# this will be a single repo as Github will build a matrix of repos with each repo being tested in parallel. +# For local invocation, you can pass a list of repos to test multiple repos together. +for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma + # Perform testing on main (with latest changes). Shallow copy as history is not important + git clone "https://github.com/googleapis/${repo}.git" --depth=1 + pushd "${repo}" + + if [ "${repo}" == "google-cloud-java" ]; then + # The `-am` command also builds anything these libraries depend on (i.e. proto-* and grpc-* sub modules) + mvn clean install -B -V -ntp -T 1C -DskipTests -Dclirr.skip -Denforcer.skip -Dmaven.javadoc.skip \ + -pl "${google_cloud_java_handwritten_maven_args}" -am + else + # Install all repo modules to ~/.m2 (there can be multiple relevant artifacts to test i.e. core, admin, control) + mvn clean install -B -V -ntp -T 1C -DskipTests -Dclirr.skip -Denforcer.skip -Dmaven.javadoc.skip + fi + + + linkage_checker_arguments="" + build_program_arguments "${repo}" + + # Linkage Checker /dependencies + popd + + echo "Artifact List: ${linkage_checker_arguments}" + # The `-s` argument filters the linkage check problems that stem from the artifact + program_args="-r --artifacts ${linkage_checker_arguments},com.google.protobuf:protobuf-java:${PROTOBUF_RUNTIME_VERSION},com.google.protobuf:protobuf-java-util:${PROTOBUF_RUNTIME_VERSION} -s ${linkage_checker_arguments}" + echo "Running Linkage Checker on the repo's handwritten modules" + echo "Linkage Checker Program Arguments: ${program_args}" + mvn -B -ntp exec:java -Dexec.args="${program_args}" -P exec-linkage-checker +done +popd +popd diff --git a/sdk-platform-java/.kokoro/nightly/downstream-protobuf-source-compatibility.sh b/sdk-platform-java/.kokoro/nightly/downstream-protobuf-source-compatibility.sh new file mode 100755 index 000000000000..aaa65b82087e --- /dev/null +++ b/sdk-platform-java/.kokoro/nightly/downstream-protobuf-source-compatibility.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +scriptDir=$(realpath "$(dirname "${BASH_SOURCE[0]}")") +source "${scriptDir}/common.sh" + +validate_protobuf_compatibility_script_inputs + +# REPOS_UNDER_TEST Env Var accepts a comma separated list of googleapis repos to test. For Github CI, +# this will be a single repo as Github will build a matrix of repos with each repo being tested in parallel. +# For local invocation, you can pass a list of repos to test multiple repos together. +for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma + # Perform source-compatibility testing on main (latest changes) + git clone "https://github.com/googleapis/$repo.git" --depth=1 + pushd "$repo" + + # Compile with Java 11 and run the tests with Java 8 JVM + mvn compile -T 1C + + # JAVA8_HOME is set by the GH Actions CI + if [ -n "${JAVA8_HOME}" ]; then + surefire_opt="-Djvm=${JAVA8_HOME}/bin/java" + else + # Provide a default value for local executions that don't configure JAVA8_HOME + surefire_opt="-Djvm=${JAVA_HOME}/bin/java" + fi + + # Compile the Handwritten Library with the Protobuf-Java version to test source compatibility + # Run unit tests to help check for any behavior differences (dependant on coverage) + if [ "${repo}" == "google-cloud-java" ]; then + # The `-am` command also builds anything these libraries depend on (i.e. proto-* and grpc-* sub modules) + mvn test -B -V -ntp \ + -Dclirr.skip \ + -Denforcer.skip \ + -Dmaven.javadoc.skip \ + -Denforcer.skip \ + -Dprotobuf.version=${PROTOBUF_RUNTIME_VERSION} \ + -pl "${google_cloud_java_handwritten_maven_args}" -am \ + "${surefire_opt}" \ + -T 1C + else + mvn test -B -V -ntp \ + -Dclirr.skip \ + -Denforcer.skip \ + -Dmaven.javadoc.skip \ + -Denforcer.skip \ + -Dprotobuf.version=${PROTOBUF_RUNTIME_VERSION} \ + "${surefire_opt}" \ + -T 1C + fi + + popd +done \ No newline at end of file