Cache CI container dependency layers in GHCR.

.ci/scripts/pr_labels.py

@@ -9,8 +9,68 @@
 import tomllib
 from git import Repo
 
+# Files that change the third-party CI image. Plugin app code is not listed; those PRs overlay
+# the PR wheel on the nightly prebuilt deps image.
+PLUGIN_DEP_PATHS = (
+    "pyproject.toml",
+    "setup.py",
+    "setup.cfg",
+    "requirements.txt",
+    "requirements.in",
+    "ci_requirements.txt",
+    ".ci/assets/ci_constraints.txt",
+    ".ci/scripts/calc_constraints.py",
+    "template_config.yml",
+)
+
+# plugin_template itself: anything that changes how the catdog image is built or locked.
+TEMPLATE_DEP_PATHS = (
+    "plugin-template",
+    "templates/github/.ci/",
+    "templates/github/.github/workflows/scripts/before_install.sh.j2",
+)
+
+
+def dep_prefixes() -> tuple[str, ...]:
+    if Path("templates/github").is_dir() and Path("plugin-template").exists():
+        return TEMPLATE_DEP_PATHS
+    return PLUGIN_DEP_PATHS
+
+
+def is_dep_path(path: str | None) -> bool:
+    if not path:
+        return False
+    for prefix in dep_prefixes():
+        if prefix.endswith("/"):
+            if path.startswith(prefix) or path == prefix.rstrip("/"):
+                return True
+        elif path == prefix:
+            return True
+    return False
+
+
+def changed_paths(repo: Repo, base_commit, head_commit) -> set[str]:
+    paths: set[str] = set()
+    for diff in base_commit.diff(head_commit):
+        if diff.a_path:
+            paths.add(diff.a_path)
+        if diff.b_path:
+            paths.add(diff.b_path)
+    return paths
+
+
+def deps_changed(repo: Repo, base_commit, head_commit) -> bool:
+    return any(is_dep_path(path) for path in changed_paths(repo, base_commit, head_commit))
+
 
 def main():
+    if len(sys.argv) == 4 and sys.argv[1] == "--deps-changed":
+        repo = Repo(".")
+        base_commit = repo.commit(sys.argv[2])
+        head_commit = repo.commit(sys.argv[3])
+        print("1" if deps_changed(repo, base_commit, head_commit) else "0")
+        return
+
     assert len(sys.argv) == 3
 
     with open("pyproject.toml", "rb") as fp:
@@ -38,6 +98,7 @@ def main():
         "no-issue": False,
         "no-changelog": False,
         "wip": False,
+        "no-cache": deps_changed(repo, base_commit, head_commit),
     }
     for commit in pr_commits:
         labels["wip"] |= BLOCKING_REGEX.search(commit.summary) is not None

.github/workflows/ci.yml

@@ -53,12 +53,22 @@ jobs:
     outputs:
       run_tests: "${{ steps.check.outputs.run_tests }}"
       run_docs: "${{ steps.check.outputs.run_docs }}"
+      full_image: "${{ steps.check.outputs.full_image }}"
     steps:
       - uses: "actions/checkout@v6"
         with:
           fetch-depth: 0
           path: "plugin_template"
 
+      - uses: "actions/setup-python@v6"
+        with:
+          python-version: "3.11"
+
+      - name: "Install uv"
+        uses: "astral-sh/setup-uv@v7"
+        with:
+          enable-cache: true
+
       - name: "Analyze changed files"
         id: "check"
         shell: "bash"
@@ -68,6 +78,9 @@ jobs:
           # We just assume we want to see all tests.
           echo "run_docs=1" >> $GITHUB_OUTPUT
           echo "run_tests=1" >> $GITHUB_OUTPUT
+          uv pip install GitPython==3.1.42
+          # TEMP: force overlay on this PR to test the prebuilt deps images.
+          echo "full_image=0" >> $GITHUB_OUTPUT
 
   lint:
     uses: "./.github/workflows/lint.yml"
@@ -83,9 +96,15 @@ jobs:
     uses: "./.github/workflows/build.yml"
 
   test:
-    needs: "build"
+    needs:
+      - "build"
+      - "check-changes"
+    permissions:
+      contents: "read"
+      packages: "read"
     uses: "./.github/workflows/test.yml"
     with:
+      full_image: ${{ needs.check-changes.outputs.full_image == '1' }}
       matrix_env: |
         [{"TEST": "pulp"}, {"TEST": "azure"}, {"TEST": "gcp"}, {"TEST": "s3"}, {"TEST": "lowerbounds"}]
 

.github/workflows/test.yml

@@ -13,9 +13,20 @@ on:
       matrix_env:
         required: true
         type: "string"
+      cache_write:
+        description: "Push the prebuilt third-party deps image to GHCR. Only enable from nightly; PR GITHUB_TOKEN cannot push packages."
+        required: false
+        type: "boolean"
+        default: false
+      full_image:
+        description: "Force a full image build (dependency files changed)."
+        required: false
+        type: "boolean"
+        default: false
 
 env:
   UV_SYSTEM_PYTHON: "1"
+  DOCKER_DEPS_WRITE: "${{ inputs.cache_write }}"
 
 defaults:
   run:
@@ -93,6 +104,22 @@ jobs:
         run: |
           echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV
 
+      - name: "Choose image build mode"
+        run: |
+          # TEMP: force overlay on this PR to test the prebuilt deps images.
+          overlay=true
+          echo "TEMP: overlay PR wheel on the nightly deps image"
+          echo "DOCKER_OVERLAY=${overlay}" >> "$GITHUB_ENV"
+        env:
+          PR_LABELS: "${{ join(github.event.pull_request.labels.*.name, ',') }}"
+
+      - name: "Login to GHCR"
+        uses: "docker/login-action@v4"
+        with:
+          registry: "ghcr.io"
+          username: "${{ github.actor }}"
+          password: "${{ secrets.GITHUB_TOKEN }}"
+
       - name: "Prepare Scenario Definition"
         run: |
           .github/workflows/scripts/before_install.sh

CHANGES/2237.feature

@@ -0,0 +1 @@
+Nightly publishes a prebuilt third-party deps image to GHCR. PRs overlay the plugin wheel on that image unless dependency files change (``no-cache`` label).

templates/github/.ci/ansible/Containerfile.j2.copy

@@ -1,4 +1,18 @@
-FROM {{ image.ci_base }}
+{% if image.overlay | default(false) | bool %}
+FROM {{ image.deps_ref }}
+{% else %}
+FROM {{ image.ci_base }} AS deps
+
+# Third-party deps are compiled on the host into a lockfile that does not include
+# the PR wheel. Nightly pushes this `deps` stage; PRs that do not touch deps overlay
+# the PR wheel on that image instead of reinstalling packages.
+COPY ./{{ plugin_name }}/.ci/assets/third-party-requirements.txt /tmp/third-party-requirements.txt
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install --upgrade setuptools wheel && \
+    uv pip install -r /tmp/third-party-requirements.txt
+
+FROM deps
+{% endif %}
 {%- if image.webserver_snippet %}
 
 ADD ./{{ plugin_name }}/{{ plugin_name | replace("-", "_") }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ plugin_name }}.conf
@@ -9,19 +23,8 @@ ADD ./{{ plugin_name }}/{{ plugin_name | replace("-", "_") }}/app/webserver_snip
 ADD ./{{ item.origin }} {{ item.destination }}
 {%- endfor %}
 
-# This MUST be the ONLY package install inside the container.
-RUN --mount=type=cache,target=/root/.cache/uv uv pip install --upgrade setuptools wheel && \
-  uv pip install {{ image.source }}
-{%- if image.upperbounds | default(false) -%}
-{{ " " }}-c ./{{ plugin_name }}/upperbounds_constraints.txt
-{%- endif -%}
-{%- if image.lowerbounds | default(false) -%}
-{{ " " }}-c ./{{ plugin_name }}/lowerbounds_constraints.txt
-{%- endif -%}
-{%- if image.ci_requirements | default(false) -%}
-{{ " " }}-r ./{{ plugin_name }}/ci_requirements.txt
-{%- endif -%}
-{{ " " }}-c ./{{ plugin_name }}/.ci/assets/ci_constraints.txt
+# The local wheel changes every CI run; extras and git deps are already in the lockfile.
+RUN uv pip install --no-deps {{ image.wheel }}
 
 {% if pulp_env is defined and pulp_env %}
 {% for key, value in pulp_env.items() %}

templates/github/.ci/ansible/build_container.yaml

@@ -11,29 +11,36 @@
         src: "Containerfile.j2"
         dest: "Containerfile"
 
-    - name: "Build pulp image"
-      # We build from the ../.. (parent dir of pulpcore git repo) Docker build
-      # "context" so that repos like pulp-smash are accessible to Docker
-      # build. So that PR branches can be used via relative paths.
-      #
-      # We default to using the docker build / podman buildah cache, for
-      # 1-off-builds and CI purposes (which has no cache across CI runs.)
-      # Run build.yaml with -e cache=false if your builds are using outdated
-      # layers.
-      ansible.builtin.command:
-        argv:
-          - "docker"
-          - "build"
-          - "--network"
-          - "host"
-          - "--no-cache={{ not cache | default(true) | bool }}"
-          - "-t"
-          - "{{ image.name }}:{{ image.tag }}"
-          - "-f"
-          - "{{ playbook_dir }}/Containerfile"
-          - "../../.."
+    - name: "Push nightly deps image"
+      ansible.builtin.shell:
+        cmd: |
+          set -eu
+          docker build --network host \
+            --target deps \
+            -t "{{ image.deps_ref }}" \
+            -f "{{ playbook_dir }}/Containerfile" \
+            ../../..
+          docker push "{{ image.deps_ref }}"
+      args:
+        executable: "/bin/bash"
+      when: "image.deps_write | default(false) | bool"
 
-    - name: "Clean image cache"
-      community.docker.docker_prune:
-        images: true
-...
+    - name: "Build pulp image"
+      # Build from the parent of the plugin checkout so extra repos can be ADD'd
+      # via relative paths. Nightly pushes the `deps` stage to GHCR; PRs pull it
+      # and overlay the PR wheel unless no-cache / dep-file changes force a
+      # full rebuild. Pass -e cache=false to force a full rebuild locally.
+      ansible.builtin.shell:
+        cmd: |
+          set -eu
+          extra=()
+          if [ "{{ cache | default(true) | bool }}" != "True" ]; then
+            extra+=(--no-cache)
+          fi
+          docker build --network host \
+            -t "{{ image.name }}:{{ image.tag }}" \
+            -f "{{ playbook_dir }}/Containerfile" \
+            "${extra[@]}" \
+            ../../..
+      args:
+        executable: "/bin/bash"

templates/github/.ci/scripts/pr_labels.py

@@ -4,13 +4,73 @@
 
 import re
 import sys
-import tomllib
 from pathlib import Path
 
+import tomllib
 from git import Repo
 
+# Files that change the third-party CI image. Plugin app code is not listed; those PRs overlay
+# the PR wheel on the nightly prebuilt deps image.
+PLUGIN_DEP_PATHS = (
+    "pyproject.toml",
+    "setup.py",
+    "setup.cfg",
+    "requirements.txt",
+    "requirements.in",
+    "ci_requirements.txt",
+    ".ci/assets/ci_constraints.txt",
+    ".ci/scripts/calc_constraints.py",
+    "template_config.yml",
+)
+
+# plugin_template itself: anything that changes how the catdog image is built or locked.
+TEMPLATE_DEP_PATHS = (
+    "plugin-template",
+    "templates/github/.ci/",
+    "templates/github/.github/workflows/scripts/before_install.sh.j2",
+)
+
+
+def dep_prefixes() -> tuple[str, ...]:
+    if Path("templates/github").is_dir() and Path("plugin-template").exists():
+        return TEMPLATE_DEP_PATHS
+    return PLUGIN_DEP_PATHS
+
+
+def is_dep_path(path: str | None) -> bool:
+    if not path:
+        return False
+    for prefix in dep_prefixes():
+        if prefix.endswith("/"):
+            if path.startswith(prefix) or path == prefix.rstrip("/"):
+                return True
+        elif path == prefix:
+            return True
+    return False
+
+
+def changed_paths(repo: Repo, base_commit, head_commit) -> set[str]:
+    paths: set[str] = set()
+    for diff in base_commit.diff(head_commit):
+        if diff.a_path:
+            paths.add(diff.a_path)
+        if diff.b_path:
+            paths.add(diff.b_path)
+    return paths
+
+
+def deps_changed(repo: Repo, base_commit, head_commit) -> bool:
+    return any(is_dep_path(path) for path in changed_paths(repo, base_commit, head_commit))
+
 
 def main():
+    if len(sys.argv) == 4 and sys.argv[1] == "--deps-changed":
+        repo = Repo(".")
+        base_commit = repo.commit(sys.argv[2])
+        head_commit = repo.commit(sys.argv[3])
+        print("1" if deps_changed(repo, base_commit, head_commit) else "0")
+        return
+
     assert len(sys.argv) == 3
 
     with open("pyproject.toml", "rb") as fp:
@@ -38,6 +98,7 @@ def main():
         "no-issue": False,
         "no-changelog": False,
         "wip": False,
+        "no-cache": deps_changed(repo, base_commit, head_commit),
     }
     for commit in pr_commits:
         labels["wip"] |= BLOCKING_REGEX.search(commit.summary) is not None

templates/github/.github/workflows/ci.yml.j2

@@ -45,6 +45,7 @@ jobs:
       {%- raw %}
       run_tests: "${{ steps.check.outputs.run_tests }}"
       run_docs: "${{ steps.check.outputs.run_docs }}"
+      full_image: "${{ steps.check.outputs.full_image }}"
       {%- endraw %}
     steps:
       {{ checkout(depth=0, path=plugin_name) | indent(6) }}
@@ -75,6 +76,12 @@ jobs:
             exit $exit_code
           fi
           echo "run_tests=$exit_code" >> $GITHUB_OUTPUT
+          set -e
+          if [ -n "$BASE_REF" ]; then
+            echo "full_image=$(python3 .ci/scripts/pr_labels.py --deps-changed $BASE_REF HEAD)" >> $GITHUB_OUTPUT
+          else
+            echo "full_image=1" >> $GITHUB_OUTPUT
+          fi
 
   {%- if is_pulpdocs_member %}
 
@@ -101,9 +108,17 @@ jobs:
     uses: "./.github/workflows/build.yml"
 
   test:
-    needs: "build"
+    needs:
+      - "build"
+      - "check-changes"
+    permissions:
+      contents: "read"
+      packages: "read"
     uses: "./.github/workflows/test.yml"
     with:
+      {%- raw %}
+      full_image: ${{ needs.check-changes.outputs.full_image == '1' }}
+      {%- endraw %}
       matrix_env: |
         {{ matrix_env() | from_yaml | tojson }}
   {%- if test_deprecations %}