Add a separate action for removing old wheels

.github/workflows/ci.yml

@@ -96,13 +96,27 @@ jobs:
       name: remove-old-wheels
 
     steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+
+      # Dogfood the remove-wheels action on PRs here. If we keep zero uploads,
+      # it marks every version of the package for removal.
+      - name: Remove test package uploads via the action
+        uses: ./remove-wheels
+        with:
+          n_latest_uploads_to_keep: 0
+          anaconda_nightly_upload_organization: scientific-python-nightly-wheels
+          anaconda_nightly_token: ${{ secrets.ANACONDA_TOKEN }}
+          package_name: test-package
+
+      # The action removes per version, so delete the (now empty) package
+      # registration as well
       - name: Set up pixi
         uses: prefix-dev/setup-pixi@a09b6247153796b190642a2b53fac4241043cf6f  # v0.10.0
         with:
-          run-install: false
+          locked: true
 
-      - name: Remove test package upload
-        shell: pixi exec --spec anaconda-client -- bash -e {0}
+      - name: Remove the empty test package
+        shell: pixi run --manifest-path pixi.toml bash -e {0}
         run: |
             anaconda --token ${{ secrets.ANACONDA_TOKEN }} remove \
               --force \

.github/workflows/keep-alive.yml

@@ -18,15 +18,15 @@ jobs:
       group: ${{ github.workflow }}-${{ github.ref }}
 
     steps:
-    - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+    - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
 
     - name: Create a minimal payload
       run: |
         mkdir _pass
         touch _pass/pass.txt
 
     - name: Keep workflow alive
-      uses: peaceiris/actions-gh-pages@84c30a85c19949d7eee79c4ff27748b70285e453 #v4.1.0
+      uses: peaceiris/actions-gh-pages@84c30a85c19949d7eee79c4ff27748b70285e453 # v4.1.0
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
         publish_branch: keep-alive

.github/workflows/remove-wheels.yml

@@ -3,109 +3,67 @@ name: Remove old wheels
 on:
   # Run daily at 1:23 UTC
   schedule:
-  - cron:  '23 1 * * *'
+    - cron: '23 1 * * *'
   workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
-# Needed for micromamba pickup
-defaults:
-  run:
-    shell: bash -l {0}
-
 env:
   N_LATEST_UPLOADS: 5
   ANACONDA_USER: "scientific-python-nightly-wheels"
 
 jobs:
-  remove:
-
+  get-pkgs:
     runs-on: ubuntu-latest
     if: github.repository_owner == 'scientific-python'
     # Set required workflow secrets in the environment for additional security
     # https://github.com/scientific-python/upload-nightly-action/settings/environments
     environment:
       name: remove-old-wheels
 
+    outputs:
+      pkgnames: ${{ steps.set-outputs.outputs.pkgnames }}
+
     steps:
-      - name: Install micromamba and anaconda-client
-        uses: mamba-org/setup-micromamba@d7c9bd84e824b79d2af72a2d4196c7f4300d3476  # v3.0.0
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+      - name: Set up pixi
+        uses: prefix-dev/setup-pixi@a09b6247153796b190642a2b53fac4241043cf6f # v0.10.0
         with:
-          environment-name: remove-wheels
-          create-args: >-
-            anaconda-client==1.14.0
-            curl
-            jq
-
-      - name: Show environment
-        run: env
-
-      - name: Show CLI API info
-        run: |
-          anaconda show --help
-          echo ""
-          anaconda remove --help
+          locked: true
 
-      - name: Query package index for packages
+      - name: Generate list of package names
+        id: set-outputs
+        shell: pixi run --manifest-path pixi.toml bash -e {0}
         run: |
-          curl https://raw.githubusercontent.com/scientific-python/upload-nightly-action/main/packages-ignore-from-cleanup.txt --output packages-ignore-from-cleanup.txt
-          anaconda show "${ANACONDA_USER}" &> >(grep "${ANACONDA_USER}/") | \
+          pkgs_json=$(anaconda show "${ANACONDA_USER}" 2>&1 | grep "${ANACONDA_USER}/" | \
               awk '{print $1}' | \
               sed 's|.*/||g' | \
-              grep -vf packages-ignore-from-cleanup.txt > package-names.txt
-
-      - name: Remove old uploads to save space
-        run: |
-          # Remove all _but_ the last ${N_LATEST_UPLOADS} package versions and
-          # remove all package versions older than 30 days.
-
-          if [ -s package-names.txt ]; then
-              threshold_date="$(date +%F -d '30 days ago')"
-
-              # Remember can't quote subshell as need to split on (space seperated) token
-              for package_name in $(cat package-names.txt); do
-
-                  echo -e "\n# package: ${package_name}"
+              grep -vf packages-ignore-from-cleanup.txt | \
+              python -c 'import sys, json; print(json.dumps([line.strip() for line in sys.stdin if line.strip()]))')
+          echo "pkgnames=${pkgs_json}" >> "$GITHUB_OUTPUT"
 
-                  curl --silent https://api.anaconda.org/package/"${ANACONDA_USER}/${package_name}" | \
-                      jq -r '.releases[].version' > package-versions.txt
-                  head --lines "-${N_LATEST_UPLOADS}" package-versions.txt > remove-package-versions.txt
-
-                  for package_version in $(cat package-versions.txt); do
-                    # c.f. https://github.com/Anaconda-Platform/anaconda-client/issues/682#issuecomment-1677283067
-                    upload_date=$(curl --silent https://api.anaconda.org/release/"${ANACONDA_USER}/${package_name}/${package_version}" | \
-                        jq -r '.distributions[].upload_time' | \
-                        sort | \
-                        tail --lines 1 | \
-                        awk '{print $1}')
-
-                    # check upload_date is YYYY-MM-DD formatted
-                    # c.f. https://github.com/scientific-python/upload-nightly-action/issues/73
-                    if [[ "${upload_date}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then
-                        if [[ "${upload_date}" < "${threshold_date}" ]]; then
-                            echo "# ${ANACONDA_USER}/${package_name}/${package_version} last uploaded on ${upload_date}"
-                            echo "${package_version}" >> remove-package-versions.txt
-                        fi
-                    else
-                        echo "# ERROR: ${ANACONDA_USER}/${package_name}/${package_version} upload date ${upload_date} is not YYYY-MM-DD."
-                    fi
-
-                  done
+  remove:
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'scientific-python'
+    # Set required workflow secrets in the environment for additional security
+    # https://github.com/scientific-python/upload-nightly-action/settings/environments
+    environment:
+      name: remove-old-wheels
 
-                  if [ -s remove-package-versions.txt ]; then
-                      # Guard against duplicate entries from packages over
-                      # count and time thresholds
-                      sort --output remove-package-versions.txt --unique remove-package-versions.txt
+    needs: [get-pkgs]
 
-                      for package_version in $(cat remove-package-versions.txt); do
-                          echo "# Removing ${ANACONDA_USER}/${package_name}/${package_version}"
-                          anaconda --token ${{ secrets.ANACONDA_TOKEN }} remove \
-                            --force \
-                            "${ANACONDA_USER}/${package_name}/${package_version}"
-                      done
-                  fi
+    strategy:
+      matrix:
+        pkgname: ${{fromJSON(needs.get-pkgs.outputs.pkgnames)}}
 
-              done
-          fi
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+      - name: Remove old wheels
+        uses: ./remove-wheels
+        with:
+          n_latest_uploads_to_keep: ${{ env.N_LATEST_UPLOADS }}
+          anaconda_nightly_upload_organization: ${{ env.ANACONDA_USER }}
+          anaconda_nightly_token: ${{ secrets.ANACONDA_TOKEN }}
+          package_name: ${{ matrix.pkgname }}

README.md

@@ -18,10 +18,47 @@ jobs:
         anaconda_nightly_upload_token: ${{secrets.UPLOAD_TOKEN}}
 ```
 
-Note that we recommend pinning the action against a specific SHA
+> [!IMPORTANT]
+> Note that we recommend pinning the action against a specific SHA
 (rather than a tag), to guard against the unlikely event of upstream
 being compromised.
 
+## Removing old nightly builds
+
+Note that ``scientific-python-nightly-wheels``, specifically, already removes
+old artifacts daily. The `remove-wheels` action is therefore intended for use with other channels.
+
+This repository also ships with an action to ease removals of older nightly wheels from a channel.
+Please note that the default configuration below will remove all but the `n_latest_uploads_to_keep`
+latest uploads from the channel. This is useful to avoid hosting outdated development
+versions, as well as to clean up space.
+
+If you do not wish to have this automated cleanup, please [open an issue](https://github.com/scientific-python/upload-nightly-action/)
+to be added to the list of packages exempt from it. The current ones are named in
+[`packages-ignore-from-cleanup.txt`](packages-ignore-from-cleanup.txt).
+
+Please refer to the [artifact cleanup policy][] for more information.
+
+To use this functionality, add the following snippet to your workflow:
+
+```yml
+jobs:
+  steps:
+    ...
+    - name: Remove old wheels
+      uses: scientific-python/upload-nightly-action/remove-wheels@main # pin to a SHA in practice
+      with:
+        n_latest_uploads_to_keep: 5
+        anaconda_nightly_upload_organization: "your-organization"
+        anaconda_nightly_token: ${{secrets.ANACONDA_TOKEN}}
+        package_name: "your-package"
+```
+
+Please note that the `anaconda_nightly_token` secret must have the necessary permissions to
+remove artifacts from the channel. A token scoped to a particular package will delete only
+the artifacts for that package. If you need to delete artifacts for multiple packages, run
+the action once per package (such as in a matrix).
+
 ## Updating the action
 
 You can [use Dependabot to keep the GitHub Action up to date][],
@@ -112,3 +149,4 @@ dependencies:
 [PyPI]: https://pypi.org/
 [scientific-python nightly channel]: https://anaconda.org/scientific-python-nightly-wheels
 [SPEC4 — Using and Creating Nightly Wheels]: https://scientific-python.org/specs/spec-0004/
+[artifact cleanup policy]: #artifact-cleanup-policy-at-the-scientific-python-nightly-wheels-channel