diff --git a/.ci/scripts/pr_labels.py b/.ci/scripts/pr_labels.py index 4f801c39..df3de5c6 100755 --- a/.ci/scripts/pr_labels.py +++ b/.ci/scripts/pr_labels.py @@ -9,8 +9,68 @@ import tomllib from git import Repo +# Files that change the third-party CI image. Plugin app code is not listed; those PRs overlay +# the PR wheel on the nightly prebuilt deps image. +PLUGIN_DEP_PATHS = ( + "pyproject.toml", + "setup.py", + "setup.cfg", + "requirements.txt", + "requirements.in", + "ci_requirements.txt", + ".ci/assets/ci_constraints.txt", + ".ci/scripts/calc_constraints.py", + "template_config.yml", +) + +# plugin_template itself: anything that changes how the catdog image is built or locked. +TEMPLATE_DEP_PATHS = ( + "plugin-template", + "templates/github/.ci/", + "templates/github/.github/workflows/scripts/before_install.sh.j2", +) + + +def dep_prefixes() -> tuple[str, ...]: + if Path("templates/github").is_dir() and Path("plugin-template").exists(): + return TEMPLATE_DEP_PATHS + return PLUGIN_DEP_PATHS + + +def is_dep_path(path: str | None) -> bool: + if not path: + return False + for prefix in dep_prefixes(): + if prefix.endswith("/"): + if path.startswith(prefix) or path == prefix.rstrip("/"): + return True + elif path == prefix: + return True + return False + + +def changed_paths(repo: Repo, base_commit, head_commit) -> set[str]: + paths: set[str] = set() + for diff in base_commit.diff(head_commit): + if diff.a_path: + paths.add(diff.a_path) + if diff.b_path: + paths.add(diff.b_path) + return paths + + +def deps_changed(repo: Repo, base_commit, head_commit) -> bool: + return any(is_dep_path(path) for path in changed_paths(repo, base_commit, head_commit)) + def main(): + if len(sys.argv) == 4 and sys.argv[1] == "--deps-changed": + repo = Repo(".") + base_commit = repo.commit(sys.argv[2]) + head_commit = repo.commit(sys.argv[3]) + print("1" if deps_changed(repo, base_commit, head_commit) else "0") + return + assert len(sys.argv) == 3 with open("pyproject.toml", "rb") as fp: @@ -38,6 +98,7 @@ def main(): "no-issue": False, "no-changelog": False, "wip": False, + "no-cache": deps_changed(repo, base_commit, head_commit), } for commit in pr_commits: labels["wip"] |= BLOCKING_REGEX.search(commit.summary) is not None diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b9646d4..c0027735 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,12 +53,22 @@ jobs: outputs: run_tests: "${{ steps.check.outputs.run_tests }}" run_docs: "${{ steps.check.outputs.run_docs }}" + full_image: "${{ steps.check.outputs.full_image }}" steps: - uses: "actions/checkout@v6" with: fetch-depth: 0 path: "plugin_template" + - uses: "actions/setup-python@v6" + with: + python-version: "3.11" + + - name: "Install uv" + uses: "astral-sh/setup-uv@v7" + with: + enable-cache: true + - name: "Analyze changed files" id: "check" shell: "bash" @@ -68,6 +78,9 @@ jobs: # We just assume we want to see all tests. echo "run_docs=1" >> $GITHUB_OUTPUT echo "run_tests=1" >> $GITHUB_OUTPUT + uv pip install GitPython==3.1.42 + # TEMP: force overlay on this PR to test the prebuilt deps images. + echo "full_image=0" >> $GITHUB_OUTPUT lint: uses: "./.github/workflows/lint.yml" @@ -83,9 +96,15 @@ jobs: uses: "./.github/workflows/build.yml" test: - needs: "build" + needs: + - "build" + - "check-changes" + permissions: + contents: "read" + packages: "read" uses: "./.github/workflows/test.yml" with: + full_image: ${{ needs.check-changes.outputs.full_image == '1' }} matrix_env: | [{"TEST": "pulp"}, {"TEST": "azure"}, {"TEST": "gcp"}, {"TEST": "s3"}, {"TEST": "lowerbounds"}] diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e9245259..62d9fee9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,9 +13,20 @@ on: matrix_env: required: true type: "string" + cache_write: + description: "Push the prebuilt third-party deps image to GHCR. Only enable from nightly; PR GITHUB_TOKEN cannot push packages." + required: false + type: "boolean" + default: false + full_image: + description: "Force a full image build (dependency files changed)." + required: false + type: "boolean" + default: false env: UV_SYSTEM_PYTHON: "1" + DOCKER_DEPS_WRITE: "${{ inputs.cache_write }}" defaults: run: @@ -93,6 +104,22 @@ jobs: run: | echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV + - name: "Choose image build mode" + run: | + # TEMP: force overlay on this PR to test the prebuilt deps images. + overlay=true + echo "TEMP: overlay PR wheel on the nightly deps image" + echo "DOCKER_OVERLAY=${overlay}" >> "$GITHUB_ENV" + env: + PR_LABELS: "${{ join(github.event.pull_request.labels.*.name, ',') }}" + + - name: "Login to GHCR" + uses: "docker/login-action@v4" + with: + registry: "ghcr.io" + username: "${{ github.actor }}" + password: "${{ secrets.GITHUB_TOKEN }}" + - name: "Prepare Scenario Definition" run: | .github/workflows/scripts/before_install.sh diff --git a/CHANGES/2237.feature b/CHANGES/2237.feature new file mode 100644 index 00000000..4ac524c3 --- /dev/null +++ b/CHANGES/2237.feature @@ -0,0 +1 @@ +Nightly publishes a prebuilt third-party deps image to GHCR. PRs overlay the plugin wheel on that image unless dependency files change (``no-cache`` label). diff --git a/templates/github/.ci/ansible/Containerfile.j2.copy b/templates/github/.ci/ansible/Containerfile.j2.copy index 69c62ba0..512b6a88 100644 --- a/templates/github/.ci/ansible/Containerfile.j2.copy +++ b/templates/github/.ci/ansible/Containerfile.j2.copy @@ -1,4 +1,18 @@ -FROM {{ image.ci_base }} +{% if image.overlay | default(false) | bool %} +FROM {{ image.deps_ref }} +{% else %} +FROM {{ image.ci_base }} AS deps + +# Third-party deps are compiled on the host into a lockfile that does not include +# the PR wheel. Nightly pushes this `deps` stage; PRs that do not touch deps overlay +# the PR wheel on that image instead of reinstalling packages. +COPY ./{{ plugin_name }}/.ci/assets/third-party-requirements.txt /tmp/third-party-requirements.txt +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --upgrade setuptools wheel && \ + uv pip install -r /tmp/third-party-requirements.txt + +FROM deps +{% endif %} {%- if image.webserver_snippet %} ADD ./{{ plugin_name }}/{{ plugin_name | replace("-", "_") }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ plugin_name }}.conf @@ -9,19 +23,8 @@ ADD ./{{ plugin_name }}/{{ plugin_name | replace("-", "_") }}/app/webserver_snip ADD ./{{ item.origin }} {{ item.destination }} {%- endfor %} -# This MUST be the ONLY package install inside the container. -RUN --mount=type=cache,target=/root/.cache/uv uv pip install --upgrade setuptools wheel && \ - uv pip install {{ image.source }} -{%- if image.upperbounds | default(false) -%} -{{ " " }}-c ./{{ plugin_name }}/upperbounds_constraints.txt -{%- endif -%} -{%- if image.lowerbounds | default(false) -%} -{{ " " }}-c ./{{ plugin_name }}/lowerbounds_constraints.txt -{%- endif -%} -{%- if image.ci_requirements | default(false) -%} -{{ " " }}-r ./{{ plugin_name }}/ci_requirements.txt -{%- endif -%} -{{ " " }}-c ./{{ plugin_name }}/.ci/assets/ci_constraints.txt +# The local wheel changes every CI run; extras and git deps are already in the lockfile. +RUN uv pip install --no-deps {{ image.wheel }} {% if pulp_env is defined and pulp_env %} {% for key, value in pulp_env.items() %} diff --git a/templates/github/.ci/ansible/build_container.yaml b/templates/github/.ci/ansible/build_container.yaml index 0ffd21d4..4e54010f 100644 --- a/templates/github/.ci/ansible/build_container.yaml +++ b/templates/github/.ci/ansible/build_container.yaml @@ -11,29 +11,36 @@ src: "Containerfile.j2" dest: "Containerfile" - - name: "Build pulp image" - # We build from the ../.. (parent dir of pulpcore git repo) Docker build - # "context" so that repos like pulp-smash are accessible to Docker - # build. So that PR branches can be used via relative paths. - # - # We default to using the docker build / podman buildah cache, for - # 1-off-builds and CI purposes (which has no cache across CI runs.) - # Run build.yaml with -e cache=false if your builds are using outdated - # layers. - ansible.builtin.command: - argv: - - "docker" - - "build" - - "--network" - - "host" - - "--no-cache={{ not cache | default(true) | bool }}" - - "-t" - - "{{ image.name }}:{{ image.tag }}" - - "-f" - - "{{ playbook_dir }}/Containerfile" - - "../../.." + - name: "Push nightly deps image" + ansible.builtin.shell: + cmd: | + set -eu + docker build --network host \ + --target deps \ + -t "{{ image.deps_ref }}" \ + -f "{{ playbook_dir }}/Containerfile" \ + ../../.. + docker push "{{ image.deps_ref }}" + args: + executable: "/bin/bash" + when: "image.deps_write | default(false) | bool" - - name: "Clean image cache" - community.docker.docker_prune: - images: true -... + - name: "Build pulp image" + # Build from the parent of the plugin checkout so extra repos can be ADD'd + # via relative paths. Nightly pushes the `deps` stage to GHCR; PRs pull it + # and overlay the PR wheel unless no-cache / dep-file changes force a + # full rebuild. Pass -e cache=false to force a full rebuild locally. + ansible.builtin.shell: + cmd: | + set -eu + extra=() + if [ "{{ cache | default(true) | bool }}" != "True" ]; then + extra+=(--no-cache) + fi + docker build --network host \ + -t "{{ image.name }}:{{ image.tag }}" \ + -f "{{ playbook_dir }}/Containerfile" \ + "${extra[@]}" \ + ../../.. + args: + executable: "/bin/bash" diff --git a/templates/github/.ci/scripts/pr_labels.py b/templates/github/.ci/scripts/pr_labels.py index 0c478a21..df3de5c6 100755 --- a/templates/github/.ci/scripts/pr_labels.py +++ b/templates/github/.ci/scripts/pr_labels.py @@ -4,13 +4,73 @@ import re import sys -import tomllib from pathlib import Path +import tomllib from git import Repo +# Files that change the third-party CI image. Plugin app code is not listed; those PRs overlay +# the PR wheel on the nightly prebuilt deps image. +PLUGIN_DEP_PATHS = ( + "pyproject.toml", + "setup.py", + "setup.cfg", + "requirements.txt", + "requirements.in", + "ci_requirements.txt", + ".ci/assets/ci_constraints.txt", + ".ci/scripts/calc_constraints.py", + "template_config.yml", +) + +# plugin_template itself: anything that changes how the catdog image is built or locked. +TEMPLATE_DEP_PATHS = ( + "plugin-template", + "templates/github/.ci/", + "templates/github/.github/workflows/scripts/before_install.sh.j2", +) + + +def dep_prefixes() -> tuple[str, ...]: + if Path("templates/github").is_dir() and Path("plugin-template").exists(): + return TEMPLATE_DEP_PATHS + return PLUGIN_DEP_PATHS + + +def is_dep_path(path: str | None) -> bool: + if not path: + return False + for prefix in dep_prefixes(): + if prefix.endswith("/"): + if path.startswith(prefix) or path == prefix.rstrip("/"): + return True + elif path == prefix: + return True + return False + + +def changed_paths(repo: Repo, base_commit, head_commit) -> set[str]: + paths: set[str] = set() + for diff in base_commit.diff(head_commit): + if diff.a_path: + paths.add(diff.a_path) + if diff.b_path: + paths.add(diff.b_path) + return paths + + +def deps_changed(repo: Repo, base_commit, head_commit) -> bool: + return any(is_dep_path(path) for path in changed_paths(repo, base_commit, head_commit)) + def main(): + if len(sys.argv) == 4 and sys.argv[1] == "--deps-changed": + repo = Repo(".") + base_commit = repo.commit(sys.argv[2]) + head_commit = repo.commit(sys.argv[3]) + print("1" if deps_changed(repo, base_commit, head_commit) else "0") + return + assert len(sys.argv) == 3 with open("pyproject.toml", "rb") as fp: @@ -38,6 +98,7 @@ def main(): "no-issue": False, "no-changelog": False, "wip": False, + "no-cache": deps_changed(repo, base_commit, head_commit), } for commit in pr_commits: labels["wip"] |= BLOCKING_REGEX.search(commit.summary) is not None diff --git a/templates/github/.github/workflows/ci.yml.j2 b/templates/github/.github/workflows/ci.yml.j2 index 49276b97..6e54041c 100644 --- a/templates/github/.github/workflows/ci.yml.j2 +++ b/templates/github/.github/workflows/ci.yml.j2 @@ -45,6 +45,7 @@ jobs: {%- raw %} run_tests: "${{ steps.check.outputs.run_tests }}" run_docs: "${{ steps.check.outputs.run_docs }}" + full_image: "${{ steps.check.outputs.full_image }}" {%- endraw %} steps: {{ checkout(depth=0, path=plugin_name) | indent(6) }} @@ -75,6 +76,12 @@ jobs: exit $exit_code fi echo "run_tests=$exit_code" >> $GITHUB_OUTPUT + set -e + if [ -n "$BASE_REF" ]; then + echo "full_image=$(python3 .ci/scripts/pr_labels.py --deps-changed $BASE_REF HEAD)" >> $GITHUB_OUTPUT + else + echo "full_image=1" >> $GITHUB_OUTPUT + fi {%- if is_pulpdocs_member %} @@ -101,9 +108,17 @@ jobs: uses: "./.github/workflows/build.yml" test: - needs: "build" + needs: + - "build" + - "check-changes" + permissions: + contents: "read" + packages: "read" uses: "./.github/workflows/test.yml" with: + {%- raw %} + full_image: ${{ needs.check-changes.outputs.full_image == '1' }} + {%- endraw %} matrix_env: | {{ matrix_env() | from_yaml | tojson }} {%- if test_deprecations %} diff --git a/templates/github/.github/workflows/nightly.yml.j2 b/templates/github/.github/workflows/nightly.yml.j2 index abbe1c1d..3acd94ce 100644 --- a/templates/github/.github/workflows/nightly.yml.j2 +++ b/templates/github/.github/workflows/nightly.yml.j2 @@ -37,8 +37,12 @@ jobs: test: needs: "build" + permissions: + contents: "read" + packages: "write" uses: "./.github/workflows/test.yml" with: + cache_write: true matrix_env: | {{ matrix_env(performance=true) | from_yaml | tojson }} diff --git a/templates/github/.github/workflows/scripts/before_install.sh.j2 b/templates/github/.github/workflows/scripts/before_install.sh.j2 index 4ea5a4fe..de1e0412 100755 --- a/templates/github/.github/workflows/scripts/before_install.sh.j2 +++ b/templates/github/.github/workflows/scripts/before_install.sh.j2 @@ -19,30 +19,79 @@ if [ -f .github/workflows/scripts/pre_before_install.sh ]; then fi COMPONENT_VERSION="$(bump-my-version show current_version | tail -n -1 | python -c 'from packaging.version import Version; print(Version(input()))')" -COMPONENT_SOURCE="./{{ plugin_name }}/dist/{{ plugin_name | snake }}-${COMPONENT_VERSION}-py3-none-any.whl" +WHEEL_NAME="{{ plugin_name | snake }}-${COMPONENT_VERSION}-py3-none-any.whl" +HOST_WHEEL="./dist/${WHEEL_NAME}" +DOCKER_WHEEL="./{{ plugin_name }}/dist/${WHEEL_NAME}" -{%- set PULPCORE_PREFIX = "" if plugin_name == "pulpcore" else " pulpcore" %} +COMPILE_PKGS=("${HOST_WHEEL}") {%- if test_s3 %} if [ "$TEST" = "s3" ]; then - COMPONENT_SOURCE="${COMPONENT_SOURCE}{{ PULPCORE_PREFIX }}[s3] git+https://github.com/gerrod3/botocore.git@fix-100-continue" + {%- if plugin_name == "pulpcore" %} + COMPILE_PKGS=("${HOST_WHEEL}[s3]") + {%- else %} + COMPILE_PKGS+=("pulpcore[s3]") + {%- endif %} + COMPILE_PKGS+=("git+https://github.com/gerrod3/botocore.git@fix-100-continue") fi {%- endif %} {%- if test_azure %} if [ "$TEST" = "azure" ]; then - COMPONENT_SOURCE="${COMPONENT_SOURCE}{{ PULPCORE_PREFIX }}[azure,uvloop]" + {%- if plugin_name == "pulpcore" %} + COMPILE_PKGS=("${HOST_WHEEL}[azure,uvloop]") + {%- else %} + COMPILE_PKGS+=("pulpcore[azure,uvloop]") + {%- endif %} fi {%- endif %} {%- if test_gcp %} if [ "$TEST" = "gcp" ]; then - COMPONENT_SOURCE="${COMPONENT_SOURCE}{{ PULPCORE_PREFIX }}[google]" + {%- if plugin_name == "pulpcore" %} + COMPILE_PKGS=("${HOST_WHEEL}[google]") + {%- else %} + COMPILE_PKGS+=("pulpcore[google]") + {%- endif %} fi {%- endif %} -if [[ "$TEST" = "pulp" ]]; then - python3 .ci/scripts/calc_constraints.py -u {% if setup_py -%} requirements.txt {% else -%} pyproject.toml {% endif -%} > upperbounds_constraints.txt +DEPS_REF="ghcr.io/{{ github_org }}/{{ plugin_name | dash }}-ci-deps:${TEST}" +OVERLAY=false +if [ "${DOCKER_OVERLAY:-false}" = "true" ]; then + if docker pull "${DEPS_REF}"; then + OVERLAY=true + else + echo "Prebuilt deps image ${DEPS_REF} is missing; falling back to a full image build" + fi fi -if [[ "$TEST" = "lowerbounds" ]]; then - python3 .ci/scripts/calc_constraints.py {% if setup_py -%} requirements.txt {% else -%} pyproject.toml {% endif -%} > lowerbounds_constraints.txt + +if [ "$OVERLAY" != "true" ]; then + if [[ "$TEST" = "pulp" ]]; then + python3 .ci/scripts/calc_constraints.py -u {% if setup_py -%} requirements.txt {% else -%} pyproject.toml {% endif -%} > upperbounds_constraints.txt + fi + if [[ "$TEST" = "lowerbounds" ]]; then + python3 .ci/scripts/calc_constraints.py {% if setup_py -%} requirements.txt {% else -%} pyproject.toml {% endif -%} > lowerbounds_constraints.txt + fi + + # Pin third-party deps without the PR wheel so nightly can publish a reusable deps image. + COMPILE_ARGS=( + --no-header + --no-annotate + --no-emit-package "{{ plugin_name | snake }}" + --python-version "{{ python_version }}" + --python-platform linux + -o .ci/assets/third-party-requirements.txt + -c .ci/assets/ci_constraints.txt + ) + if [[ "$TEST" = "pulp" ]]; then + COMPILE_ARGS+=(-c upperbounds_constraints.txt) + fi + if [[ "$TEST" = "lowerbounds" ]]; then + COMPILE_ARGS+=(-c lowerbounds_constraints.txt) + fi + if [[ -s ci_requirements.txt ]]; then + COMPILE_ARGS+=(-r ci_requirements.txt) + fi + printf '%s\n' "${COMPILE_PKGS[@]}" > .ci/assets/third-party-requirements.in + uv pip compile "${COMPILE_ARGS[@]}" .ci/assets/third-party-requirements.in fi # Compose the scenario definition. @@ -66,7 +115,10 @@ image: {% else %} ci_base: "{{ ci_base_image + ":latest" }}" {% endif -%} - source: "${COMPONENT_SOURCE}" + wheel: "${DOCKER_WHEEL}" + deps_ref: "${DEPS_REF}" + deps_write: "${DOCKER_DEPS_WRITE:-false}" + overlay: "${OVERLAY}" ci_requirements: $(test -f ci_requirements.txt && echo -n true || echo -n false) upperbounds: $(test "${TEST}" = "pulp" && echo -n true || echo -n false) lowerbounds: $(test "${TEST}" = "lowerbounds" && echo -n true || echo -n false) diff --git a/templates/github/.github/workflows/scripts/before_script.sh.j2 b/templates/github/.github/workflows/scripts/before_script.sh.j2 index 195a448c..a67aa8af 100755 --- a/templates/github/.github/workflows/scripts/before_script.sh.j2 +++ b/templates/github/.github/workflows/scripts/before_script.sh.j2 @@ -30,6 +30,10 @@ echo echo "# Containerfile:" tail -v -n +1 .ci/ansible/Containerfile +echo +echo "# Third-party requirements:" +tail -v -n +1 .ci/assets/third-party-requirements.txt || echo "(overlay build; lockfile not compiled)" + echo echo "# Constraints Files:" # They need not even exist. diff --git a/templates/github/.github/workflows/test.yml.j2 b/templates/github/.github/workflows/test.yml.j2 index 5f171fa9..94814d26 100644 --- a/templates/github/.github/workflows/test.yml.j2 +++ b/templates/github/.github/workflows/test.yml.j2 @@ -15,9 +15,20 @@ on: matrix_env: required: true type: "string" + cache_write: + description: "Push the prebuilt third-party deps image to GHCR. Only enable from nightly; PR GITHUB_TOKEN cannot push packages." + required: false + type: "boolean" + default: false + full_image: + description: "Force a full image build (dependency files changed)." + required: false + type: "boolean" + default: false env: UV_SYSTEM_PYTHON: "1" + DOCKER_DEPS_WRITE: "{{ '${{ inputs.cache_write }}' }}" defaults: run: @@ -74,6 +85,34 @@ jobs: {{ setup_env() | indent(6) }} + - name: "Choose image build mode" + run: | + overlay=true + if [ "{{ '${{ inputs.cache_write }}' }}" = "true" ]; then + overlay=false + echo "Nightly: full image build and push deps image" + elif [ "{{ '${{ inputs.full_image }}' }}" = "true" ]; then + overlay=false + echo "Dependency files changed: full image build" + elif [[ ",${PR_LABELS}," == *",no-cache,"* ]]; then + overlay=false + echo "no-cache label present: full image build" + else + echo "Overlay PR wheel on the nightly deps image" + fi + echo "DOCKER_OVERLAY=${overlay}" >> "$GITHUB_ENV" + env: + {%- raw %} + PR_LABELS: "${{ join(github.event.pull_request.labels.*.name, ',') }}" + {%- endraw %} + + - name: "Login to GHCR" + uses: "docker/login-action@v4" + with: + registry: "ghcr.io" + username: "{{ '${{ github.actor }}' }}" + password: "{{ '${{ secrets.GITHUB_TOKEN }}' }}" + {{ run_script(name="Prepare Scenario Definition", file="before_install.sh") | indent(6) }} {{ run_script(name="Install", file="install.sh") | indent(6) }}