diff --git a/java-sdk/.gitattributes b/java-sdk/.gitattributes index 428710f2ee4ea..6cd69336377c2 100644 --- a/java-sdk/.gitattributes +++ b/java-sdk/.gitattributes @@ -10,6 +10,11 @@ # Binary files should be left untouched *.jar binary -# Keep the Gradle wrapper jar out of source releases. +# Keep the Gradle wrapper jar and scripts out of source releases. # See: https://issues.apache.org/jira/browse/LEGAL-570 -gradle/wrapper/gradle-wrapper.jar export-ignore +# This intentionally does not exclude the entire gradle/wrapper directory so +# gradle-wrapper.properties is kept. This file carries the pinned Gradle version +# and distribution checksum for a verifier to use when regenerating the wrapper. +/gradlew export-ignore +/gradlew.bat export-ignore +/gradle/wrapper/gradle-wrapper.jar export-ignore diff --git a/java-sdk/README.md b/java-sdk/README.md index cd8874a95526a..c9d54f942fed4 100644 --- a/java-sdk/README.md +++ b/java-sdk/README.md @@ -117,22 +117,23 @@ Every release therefore requires a PMC vote before it is published; only Edit `gradle.properties` and set the version for this release: ```properties -projectVersion=1.0.0 +projectVersion= ``` Commit the change and push it to the release branch. Use a Maven-compatible version string, as defined by the -[Maven version order specification](https://maven.apache.org/pom.html#Version_Order_Specification). -For example, version 1 beta 1 is `1.0.0-beta1`, and the eventual -general-availability release is `1.0.0`. +[Maven version order specification]. For example, version 1 beta 1 is +`1.0.0-beta1`, and the eventual version 1 release is `1.0.0`. + +[Maven version order specification]: https://maven.apache.org/pom.html#Version_Order_Specification *NOTE:* Editing `gradle.properties` as above is the standard procedure. You can alternatively override the version for a single command without editing the -file, by passing `-PprojectVersion=1.0.0-beta1` to any Gradle invocation. This +file, by passing `-PprojectVersion=` to any Gradle invocation. This is handy for one-off or pre-release builds. Either way, `main` should stay on a -`-SNAPSHOT` version between releases: a snapshot sorts after `1.0.0-beta1` and -before the `1.0.0` GA, so no extra bump is needed after a beta. +`-SNAPSHOT` version between releases: a snapshot sorts after `` and +before the GA, so no extra bump is needed after a beta. ### Tag the release candidate @@ -142,8 +143,8 @@ suffix). Push the tag before sending the vote so reviewers can check out the exact source being voted on. ```bash -git tag -s java-sdk/1.0.0-beta1-rc1 -m "Java SDK 1.0.0-beta1 RC 1" -git push upstream java-sdk/1.0.0-beta1-rc1 +git tag -s java-sdk/-rc -m "Java SDK RC " +git push upstream java-sdk/-rc ``` ### Verify the POM locally @@ -246,22 +247,23 @@ signature and checksum in one step: ```bash # Signing uses your local gpg keyring, so have your key/passphrase ready. -./gradlew sourceRelease -PgitRef=java-sdk/1.0.0-beta1-rc1 +./gradlew sourceRelease -PgitRef=java-sdk/-rc ``` This writes three files to `build/distributions/`: ``` -apache-airflow-java-sdk-1.0.0-beta1-src.tar.gz -apache-airflow-java-sdk-1.0.0-beta1-src.tar.gz.asc -apache-airflow-java-sdk-1.0.0-beta1-src.tar.gz.sha512 +apache-airflow-java-sdk--src.tar.gz +apache-airflow-java-sdk--src.tar.gz.asc +apache-airflow-java-sdk--src.tar.gz.sha512 ``` -*NOTE:* The source archive deliberately omits `gradle/wrapper/gradle-wrapper.jar` -since ASF source releases must not contain compiled code (see [LEGAL-570]). To -build an *extracted* source package, use a locally installed Gradle instead: -either run `gradle build`, or use `gradle wrapper` once to regenerate the jar -and then `./gradlew build`. +**NOTE:** The source archive omits the Gradle wrapper scripts (`gradlew`, +`gradlew.bat`) and `gradle/wrapper/gradle-wrapper.jar` since ASF source releases +must not contain compiled code (see [LEGAL-570]), and the scripts are not useful +without the jar. The `gradle/wrapper/gradle-wrapper.properties` file is kept so +the archive pins the Gradle version and distribution checksum for verification +when the wrapper is regenerated. [LEGAL-570]: https://issues.apache.org/jira/browse/LEGAL-570 @@ -275,23 +277,164 @@ project's location: svn checkout https://dist.apache.org/repos/dist/dev/airflow cd -mkdir -p java-sdk/1.0.0-beta1-rc1 -cp /java-sdk/build/distributions/apache-airflow-java-sdk-1.0.0-beta1-src.tar.gz* \ - java-sdk/1.0.0-beta1-rc1/ +mkdir -p java-sdk/-rc +cp /java-sdk/build/distributions/apache-airflow-java-sdk--src.tar.gz* \ + java-sdk/-rc/ -svn add --parents java-sdk/1.0.0-beta1-rc1 -svn commit -m "Add Apache Airflow Java SDK 1.0.0-beta1-rc1 source release candidate" +svn add --parents java-sdk/-rc +svn commit -m "Add Apache Airflow Java SDK -rc source release candidate" ``` The commit publishes them under -`https://dist.apache.org/repos/dist/dev/airflow/java-sdk/1.0.0-beta1-rc1/`, which +`https://dist.apache.org/repos/dist/dev/airflow/java-sdk/-rc/`, which is the source-package URL you link in the vote. ### Call the vote Send a `[VOTE]` email to `dev@airflow.apache.org` linking the git tag and commit, the source package in `dist/dev`, the closed Nexus staging repository, -and the `KEYS` file. +and the `KEYS` file. See "Vote email template" below for the exact fields to +fill in, and "Verifying a release" for what to ask reviewers to check. + +### Vote email template + +```text +Subject: [VOTE] Release Apache Airflow Java SDK based on -rc + +Hi, + +I would like to call a vote to release Apache Airflow Java SDK , based +on release candidate -rc. + +Changes since : + +The release candidate contains the following Maven artifacts, all under group id org.apache.airflow: + +- Main SDK API + - airflow-sdk + - airflow-sdk-processor +- Logger helpers + - airflow-sdk-jpl + - airflow-sdk-jul + - airflow-sdk-log4j2 + - airflow-sdk-slf4j +- Gradle plugin and the marker artifact + - airflow-sdk-gradle-plugin + - org.apache.airflow.sdk.gradle.plugin +- BOM + - airflow-sdk-bom + +Git information: + +- Tag: java-sdk/-rc +- Commit: +- https://github.com/apache/airflow/releases/tag/java-sdk%2F-rc + +Source release, signatures and checksums: +https://dist.apache.org/repos/dist/dev/airflow/java-sdk/-rc/ + +Convenience binaries (staged in the ASF Nexus repository): +https://repository.apache.org/content/repositories/orgapacheairflow-/ + +KEYS file (public keys used to sign the release): +https://downloads.apache.org/airflow/KEYS + +Please review and vote. The vote will remain open for at least 72 hours, until , or until the necessary number of binding votes is reached. + +[ ] +1 Release this package as Apache Airflow Java SDK +[ ] +0 No opinion +[ ] -1 Do not release, because ... + +Only votes from Airflow PMC members are binding, but everyone is welcome and encouraged to test the release and vote. + +The verification process can be found in the main repository: +https://github.com/apache/airflow/tree/java-sdk/-rc/java-sdk#verifying-a-release + +For more details on ASF release verification, see: https://www.apache.org/info/verification.html + +Best, + +``` + +Pre-send checklist: + +* Every staged artifact above resolves in the Nexus staging repository. Cross + check against the BOM, not just this list from memory. +* `Changes since rc` and the vote deadline are filled in. +* Run `grep '<' email.txt` on the rendered email and confirm **no output** — + any match means a template placeholder (``, ``, ``, + etc.) was left unfilled. + +### Verifying a release + +Anyone on `dev@airflow.apache.org` can (and should) independently verify a +candidate before voting. Below is the checklist a reviewer — or the release +manager, before sending the vote — should run against the source package in +`dist/dev`. + +1. **Checksum.** Confirm the published SHA-512 matches the downloaded tarball: + + ```bash + sha512sum -c apache-airflow-java-sdk--src.tar.gz.sha512 + ``` + +2. **Signature.** Import the `KEYS` file and verify the GPG signature: + + ```bash + curl -O https://downloads.apache.org/airflow/KEYS + gpg --import KEYS + gpg --verify apache-airflow-java-sdk--src.tar.gz.asc \ + apache-airflow-java-sdk--src.tar.gz + ``` + +3. **Diff against the git tag.** Extract the tarball and compare it with a + clean checkout of the tag it claims to be built from. They should be + identical except `gradlew`, `gradlew.bat`, and `gradle-wrapper.jar`. The + extracted top-level directory should be `apache-airflow-java-sdk-` + without the `-src` suffix that only appears in the tarball's own filename: + + ```bash + tar xzf apache-airflow-java-sdk--src.tar.gz + git clone --branch java-sdk/-rc \ + https://github.com/apache/airflow.git tag-checkout + diff -rq apache-airflow-java-sdk-/ tag-checkout/java-sdk/ \ + | grep -vE ': (gradlew|gradlew\.bat|gradle-wrapper\.jar)$' + ``` + + Any remaining diff output is unexpected and should block the vote. + +4. **No binary files.** ASF source releases must not contain compiled code. + Scan for anything that isn't text: + + ```bash + find apache-airflow-java-sdk-/ -type f \ + -exec sh -c 'file "$1" | grep -qv text && echo "$1"' _ {} \; + ``` + + This should print nothing. + +5. **Build from source.** Regenerate the Gradle wrapper from a locally installed + Gradle (see the *Upload the source package* section above): + + ```bash + cd apache-airflow-java-sdk- + gradle wrapper \ + --gradle-version \ + --gradle-distribution-sha256-sum + ./gradlew build + ``` + + Use the values for `` and `` from + `distributionUrl` and `distributionSha256Sum` in the bundled + `gradle/wrapper/gradle-wrapper.properties`. + +6. **Staged-binary smoke test.** Resolve the staged Nexus artifacts from a + throwaway project to confirm they're actually consumable, following the + same pattern as the "Dry-run against a local repository" step: point a + `repositories {}` block at the staging repository URL, declare a dependency + on `org.apache.airflow:airflow-sdk-bom:`, and confirm the + transitive artifacts (including `airflow-sdk-jpl`) resolve and the example + bundle builds against them. ### After a successful vote @@ -309,16 +452,16 @@ Reply with a `[RESULT][VOTE]` tally, then: 2. **Move** the source package from `dist/dev` to `dist/release`: ```bash - svn mv https://dist.apache.org/repos/dist/dev/airflow/java-sdk/1.0.0-beta1-rc1 \ - https://dist.apache.org/repos/dist/release/airflow/java-sdk/1.0.0-beta1 \ - -m "Release Apache Airflow Java SDK 1.0.0-beta1" + svn mv https://dist.apache.org/repos/dist/dev/airflow/java-sdk/-rc \ + https://dist.apache.org/repos/dist/release/airflow/java-sdk/ \ + -m "Release Apache Airflow Java SDK " ``` 3. **Tag** the final version on the same commit that was voted: ```bash - git tag -s java-sdk/1.0.0-beta1 -m "Apache Airflow Java SDK 1.0.0-beta1" - git push upstream java-sdk/1.0.0-beta1 + git tag -s java-sdk/ -m "Apache Airflow Java SDK " + git push upstream java-sdk/ ``` Keep the RC tag for traceability. @@ -330,7 +473,7 @@ Reply with a `[RESULT][VOTE]` tally, then: Close the vote, **drop** the staging repository in Nexus, remove the `dist/dev` candidate, fix the issue, and cut the next RC (`...-rc2`). The released version -stays the same (e.g. `1.0.0-beta1`); only the RC counter in the tag increments. +stays the same (e.g. ``); only the RC counter in the tag increments. ## Contributing diff --git a/java-sdk/bom/build.gradle.kts b/java-sdk/bom/build.gradle.kts index 1656dc3ca9729..8babd7c8876a9 100644 --- a/java-sdk/bom/build.gradle.kts +++ b/java-sdk/bom/build.gradle.kts @@ -17,7 +17,10 @@ * under the License. */ +import org.gradle.api.publish.PublishingExtension + plugins { + base `java-platform` id("airflow-publish") } @@ -28,6 +31,8 @@ val airflowSupervisorSchemaVersion: String by project dependencies { constraints { api("org.apache.airflow:airflow-sdk:$projectVersion") + api("org.apache.airflow:airflow-sdk-gradle-plugin:$projectVersion") + api("org.apache.airflow:airflow-sdk-jpl:$projectVersion") api("org.apache.airflow:airflow-sdk-jul:$projectVersion") api("org.apache.airflow:airflow-sdk-log4j2:$projectVersion") api("org.apache.airflow:airflow-sdk-processor:${projectVersion}") @@ -48,3 +53,45 @@ publishing { } } } + +// What the BOM currently constrains (group org.apache.airflow). +val bomArtifacts = + configurations["api"].dependencyConstraints + .filter { it.group == "org.apache.airflow" } + .map { it.name } + .toSet() + +val publishedArtifacts = + rootProject.subprojects + .filter { it.path != project.path } + .flatMap { sub -> + evaluationDependsOn(sub.path) + sub.extensions.findByType(PublishingExtension::class.java) + ?.publications + ?.withType(MavenPublication::class.java) + ?.filter { it.groupId == "org.apache.airflow" } + ?.map { it.artifactId } + ?: emptyList() + } + .toSet() + +val verifyBomCoverage by tasks.registering { + group = "verification" + description = "Fail if airflow-sdk-bom does not constrain the same set of published Java SDK artifacts." + doLast { + val missing = (publishedArtifacts - bomArtifacts).sorted() + val stale = (bomArtifacts - publishedArtifacts).sorted() + if (missing.isNotEmpty() || stale.isNotEmpty()) { + throw GradleException( + buildString { + appendLine("airflow-sdk-bom is out of sync with the published Java SDK artifacts:") + if (missing.isNotEmpty()) appendLine(" published but missing from the BOM: $missing") + if (stale.isNotEmpty()) appendLine(" listed in the BOM but not published: $stale") + append("Update constraints in bom/build.gradle.kts to match.") + }, + ) + } + } +} + +tasks.named("check") { dependsOn(verifyBomCoverage) } diff --git a/java-sdk/buildSrc/src/main/kotlin/airflow-jvm-conventions.gradle.kts b/java-sdk/buildSrc/src/main/kotlin/airflow-jvm-conventions.gradle.kts index 5a5c6b873aa9b..75fa4282f9367 100644 --- a/java-sdk/buildSrc/src/main/kotlin/airflow-jvm-conventions.gradle.kts +++ b/java-sdk/buildSrc/src/main/kotlin/airflow-jvm-conventions.gradle.kts @@ -53,3 +53,17 @@ configure { endWithNewline() } } + +tasks.withType().configureEach { + metaInf { + from(rootProject.layout.projectDirectory.file("LICENSE")) + from(rootProject.layout.projectDirectory.file("NOTICE")) + } +} + +tasks.withType().configureEach { + isPreserveFileTimestamps = false + isReproducibleFileOrder = true + dirPermissions { unix("755") } + filePermissions { unix("644") } +}