Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 54 additions & 31 deletions .github/workflows/release-dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ jobs:
python_version: ${{ steps.v.outputs.python }}
cargo_version: ${{ steps.v.outputs.cargo }}
deb_version: ${{ steps.v.outputs.deb }}
rpm_version: ${{ steps.v.outputs.rpm_version }}
rpm_release: ${{ steps.v.outputs.rpm_release }}
steps:
- uses: actions/checkout@v6
with:
Expand All @@ -48,6 +50,8 @@ jobs:
echo "python=$(uv run python tasks/scripts/release.py get-version --python)" >> "$GITHUB_OUTPUT"
echo "cargo=$(uv run python tasks/scripts/release.py get-version --cargo)" >> "$GITHUB_OUTPUT"
echo "deb=$(uv run python tasks/scripts/release.py get-version --deb)" >> "$GITHUB_OUTPUT"
echo "rpm_version=$(uv run python tasks/scripts/release.py get-version --rpm-version)" >> "$GITHUB_OUTPUT"
echo "rpm_release=$(uv run python tasks/scripts/release.py get-version --rpm-release)" >> "$GITHUB_OUTPUT"

build-gateway:
needs: [compute-versions]
Expand Down Expand Up @@ -651,6 +655,9 @@ jobs:
uses: ./.github/workflows/rpm-package.yml
with:
checkout-ref: ${{ github.sha }}
rpm-version: ${{ needs.compute-versions.outputs.rpm_version }}
rpm-release: ${{ needs.compute-versions.outputs.rpm_release }}
cargo-version: ${{ needs.compute-versions.outputs.cargo_version }}
secrets: inherit

# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -720,6 +727,31 @@ jobs:
path: release/
merge-multiple: true

- name: Normalize dev package filenames
run: |
set -euo pipefail
shopt -s nullglob

move_one() {
local dest="$1"
shift
local matches=("$@")
if [ "${#matches[@]}" -ne 1 ]; then
echo "expected exactly one source for ${dest}, found ${#matches[@]}: ${matches[*]-}" >&2
exit 1
fi
mv "${matches[0]}" "release/${dest}"
}

move_one openshell-dev-amd64.deb release/openshell_*_amd64.deb
move_one openshell-dev-arm64.deb release/openshell_*_arm64.deb
move_one openshell-dev-x86_64.rpm release/openshell-[0-9]*.x86_64.rpm
move_one openshell-dev-aarch64.rpm release/openshell-[0-9]*.aarch64.rpm
move_one openshell-gateway-dev-x86_64.rpm release/openshell-gateway-[0-9]*.x86_64.rpm
move_one openshell-gateway-dev-aarch64.rpm release/openshell-gateway-[0-9]*.aarch64.rpm

ls -la release/

- name: Capture wheel filenames
id: wheel_filenames
run: |
Expand All @@ -736,7 +768,7 @@ jobs:
openshell-x86_64-unknown-linux-musl.tar.gz \
openshell-aarch64-unknown-linux-musl.tar.gz \
openshell-aarch64-apple-darwin.tar.gz \
openshell_*.deb \
*.deb \
openshell-*.rpm \
*.whl > openshell-checksums-sha256.txt
cat openshell-checksums-sha256.txt
Expand All @@ -758,20 +790,12 @@ jobs:
release/openshell-driver-vm-aarch64-unknown-linux-gnu.tar.gz
release/openshell-driver-vm-aarch64-apple-darwin.tar.gz

- name: Prune stale wheel, deb, and rpm assets from dev release
- name: Prune managed assets from dev release
uses: actions/github-script@v7
env:
WHEEL_VERSION: ${{ needs.compute-versions.outputs.python_version }}
with:
script: |
const wheelVersion = process.env.WHEEL_VERSION;
const currentPrefix = `openshell-${wheelVersion}-`;
const [owner, repo] = process.env.GITHUB_REPOSITORY.split('/');

core.info(`=== Wheel pruning diagnostics ===`);
core.info(`WHEEL_VERSION: ${wheelVersion}`);
core.info(`CURRENT_PREFIX: ${currentPrefix}`);

// Fetch the dev release
let release;
try {
Expand All @@ -790,31 +814,29 @@ jobs:
core.info(` ${String(a.id).padStart(12)} ${a.name}`);
}

// Delete stale wheels, debs, rpms, and removed VM checksum assets.
let kept = 0, deleted = 0, debDeleted = 0, rpmDeleted = 0, removedVmChecksums = 0;
const managed = (name) => (
name.startsWith('openshell') &&
(
name.endsWith('.tar.gz') ||
name.endsWith('.txt') ||
name.endsWith('.whl') ||
name.endsWith('.deb') ||
name.endsWith('.rpm')
)
);

let deleted = 0, skipped = 0;
for (const asset of assets) {
if (asset.name.endsWith('.deb')) {
core.info(`Deleting stale deb package: ${asset.name} (id=${asset.id})`);
await github.rest.repos.deleteReleaseAsset({ owner, repo, asset_id: asset.id });
debDeleted++;
} else if (asset.name.endsWith('.rpm')) {
core.info(`Deleting stale rpm package: ${asset.name} (id=${asset.id})`);
await github.rest.repos.deleteReleaseAsset({ owner, repo, asset_id: asset.id });
rpmDeleted++;
} else if (asset.name === 'openshell-driver-vm-checksums-sha256.txt') {
core.info(`Deleting removed VM checksum asset: ${asset.name} (id=${asset.id})`);
await github.rest.repos.deleteReleaseAsset({ owner, repo, asset_id: asset.id });
removedVmChecksums++;
} else if (asset.name.endsWith('.whl') && asset.name.startsWith(currentPrefix)) {
core.info(`Keeping current wheel: ${asset.name}`);
kept++;
} else if (asset.name.endsWith('.whl')) {
core.info(`Deleting stale wheel: ${asset.name} (id=${asset.id})`);
if (managed(asset.name)) {
core.info(`Deleting managed dev asset: ${asset.name} (id=${asset.id})`);
await github.rest.repos.deleteReleaseAsset({ owner, repo, asset_id: asset.id });
deleted++;
} else {
core.info(`Skipping unmanaged asset: ${asset.name}`);
skipped++;
}
}
core.info(`Summary: kept_wheels=${kept}, deleted_wheels=${deleted}, deleted_debs=${debDeleted}, deleted_rpms=${rpmDeleted}, deleted_removed_vm_checksums=${removedVmChecksums}`);
core.info(`Summary: deleted=${deleted}, skipped=${skipped}`);

- name: Move dev tag
run: |
Expand Down Expand Up @@ -845,7 +867,8 @@ jobs:
release/openshell-x86_64-unknown-linux-musl.tar.gz
release/openshell-aarch64-unknown-linux-musl.tar.gz
release/openshell-aarch64-apple-darwin.tar.gz
release/openshell_*.deb
release/openshell-dev-amd64.deb
release/openshell-dev-arm64.deb
release/openshell-*.rpm
release/openshell-gateway-x86_64-unknown-linux-gnu.tar.gz
release/openshell-gateway-aarch64-unknown-linux-gnu.tar.gz
Expand Down
7 changes: 7 additions & 0 deletions .github/workflows/release-tag.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ jobs:
python_version: ${{ steps.v.outputs.python }}
cargo_version: ${{ steps.v.outputs.cargo }}
deb_version: ${{ steps.v.outputs.deb }}
rpm_version: ${{ steps.v.outputs.rpm_version }}
rpm_release: ${{ steps.v.outputs.rpm_release }}
# Semver without 'v' prefix (e.g. 0.6.0), used for image tags and release body
semver: ${{ steps.v.outputs.semver }}
steps:
Expand All @@ -62,6 +64,8 @@ jobs:
echo "python=$(uv run python tasks/scripts/release.py get-version --python)" >> "$GITHUB_OUTPUT"
echo "cargo=$(uv run python tasks/scripts/release.py get-version --cargo)" >> "$GITHUB_OUTPUT"
echo "deb=$(uv run python tasks/scripts/release.py get-version --deb)" >> "$GITHUB_OUTPUT"
echo "rpm_version=$(uv run python tasks/scripts/release.py get-version --rpm-version)" >> "$GITHUB_OUTPUT"
echo "rpm_release=$(uv run python tasks/scripts/release.py get-version --rpm-release)" >> "$GITHUB_OUTPUT"
echo "semver=${RELEASE_TAG#v}" >> "$GITHUB_OUTPUT"

build-gateway:
Expand Down Expand Up @@ -678,6 +682,9 @@ jobs:
uses: ./.github/workflows/rpm-package.yml
with:
checkout-ref: ${{ inputs.tag || github.ref }}
rpm-version: ${{ needs.compute-versions.outputs.rpm_version }}
rpm-release: ${{ needs.compute-versions.outputs.rpm_release }}
cargo-version: ${{ needs.compute-versions.outputs.cargo_version }}
secrets: inherit

# ---------------------------------------------------------------------------
Expand Down
30 changes: 22 additions & 8 deletions .github/workflows/rpm-package.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,18 @@ on:
checkout-ref:
required: true
type: string
rpm-version:
required: false
type: string
default: ""
rpm-release:
required: false
type: string
default: ""
cargo-version:
required: false
type: string
default: ""

permissions:
contents: read
Expand All @@ -21,6 +33,7 @@ jobs:
build-rpm-linux:
name: Build RPM Package (Linux ${{ matrix.arch }})
strategy:
fail-fast: false
matrix:
include:
- arch: x86_64
Expand Down Expand Up @@ -53,22 +66,23 @@ jobs:
run: git fetch --tags --force

- name: Build RPMs via Packit
env:
OPENSHELL_RPM_VERSION: ${{ inputs['rpm-version'] }}
OPENSHELL_RPM_RELEASE: ${{ inputs['rpm-release'] }}
OPENSHELL_CARGO_VERSION: ${{ inputs['cargo-version'] }}
run: packit build locally

- name: Collect RPM artifacts
run: |
set -euo pipefail
mkdir -p artifacts
for rpm_dir in "$HOME/rpmbuild/RPMS" "$PWD/${{ matrix.arch }}"; do
if [ -d "$rpm_dir" ]; then
find "$rpm_dir" -type f -name '*.rpm' -exec cp {} artifacts/ \;
fi
done
if ! compgen -G 'artifacts/*.rpm' > /dev/null; then
echo "::error::No RPM artifacts found"
find "$PWD" -maxdepth 3 -type f -name '*.rpm' -print
mapfile -t rpms < <(find "$GITHUB_WORKSPACE" -maxdepth 3 -type f -name '*.rpm' ! -name '*.src.rpm' | sort)
if [ "${#rpms[@]}" -eq 0 ]; then
echo "::error::No RPM artifacts found under $GITHUB_WORKSPACE"
find "$GITHUB_WORKSPACE" -maxdepth 3 -type f | sort
exit 1
fi
cp "${rpms[@]}" artifacts/
echo "=== Built RPMs ==="
ls -lah artifacts/

Expand Down
15 changes: 10 additions & 5 deletions .packit.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,11 @@ srpm_build_deps:

actions:
get-current-version:
# Derive version from the latest SemVer upstream tag on the current branch.
# Avoid operational tags such as vm-dev; Packit normalizes that to m.dev,
# which is not a valid Cargo package version.
- 'bash -c "git describe --tags --match ''v[0-9]*.[0-9]*.[0-9]*'' --abbrev=0 HEAD | sed ''s/^v//''"'
# Release workflows pass OPENSHELL_RPM_VERSION so every artifact shares one
# precomputed identity. Local Packit runs fall back to the latest SemVer
# upstream tag, avoiding operational tags such as vm-dev; Packit normalizes
# those to invalid Cargo package versions such as m.dev.
- 'bash -c "if [ -n \"${OPENSHELL_RPM_VERSION:-}\" ]; then echo \"${OPENSHELL_RPM_VERSION}\"; else git describe --tags --match ''v[0-9]*.[0-9]*.[0-9]*'' --abbrev=0 HEAD | sed ''s/^v//''; fi"'

create-archive:
# Step 1: Create source tarball from git working tree.
Expand All @@ -41,7 +42,11 @@ actions:
# Update Version
- 'bash -c "sed -i -r \"s/^Version:(\\s*)\\S+/Version:\\1${PACKIT_RPMSPEC_VERSION}/\" openshell.spec"'
# Update Release
- 'bash -c "sed -i -r \"s/^Release:(\\s*)\\S+/Release:\\1${PACKIT_RPMSPEC_RELEASE}%{?dist}/\" openshell.spec"'
- 'bash -c "RELEASE=${OPENSHELL_RPM_RELEASE:-${PACKIT_RPMSPEC_RELEASE}} && sed -i -r \"s/^Release:(\\s*)\\S+/Release:\\1${RELEASE}%{?dist}/\" openshell.spec"'
# Keep embedded binary metadata aligned with the release workflow. Python
# dist-info stays at the RPM Version; dev build identity is carried by
# Release so Fedora's Python RPM post-processing can normalize metadata.
- 'bash -c "if [ -n \"${OPENSHELL_CARGO_VERSION:-}\" ]; then sed -i -r \"s/^%global openshell_cargo_version .*/%global openshell_cargo_version ${OPENSHELL_CARGO_VERSION}/\" openshell.spec; fi"'

jobs:
# Build on every pull request targeting main for CI validation
Expand Down
16 changes: 16 additions & 0 deletions architecture/build-containers.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,22 @@ OpenShell also publishes Python wheels for `linux/amd64`, `linux/arm64`, and mac
- Release workflows mirror the CLI layout: a Linux matrix job for amd64/arm64, a separate macOS job, and release jobs that download the per-platform wheel artifacts directly before publishing.
- Release CPU jobs run on `linux-amd64-cpu8` and `linux-arm64-cpu8`; the macOS wheel is still cross-compiled in Docker from the amd64 Linux runner.

## Development Release Assets

The rolling `dev` release is installer-facing but still publishes the full
artifact set: CLI tarballs, standalone gateway and sandbox tarballs, Python
wheels, Debian packages, RPM packages, and checksums. Every artifact is built
from the version computed once in `release-dev.yml`.

Package-manager artifacts use stable dev aliases on the GitHub release
(`openshell-dev-*.deb`, `openshell-dev-*.rpm`, and
`openshell-gateway-dev-*.rpm`) so the rolling release stays readable. Python
wheels keep their versioned filenames because wheel metadata requires it.

The dev release workflow prunes workflow-owned `openshell*` assets before
uploading the fresh set. `openshell-driver-vm` artifacts are intentionally not
published on the main `dev` release; VM driver binaries live on `vm-dev`.

## Sandbox Images

Sandbox images are not built in this repository. They are maintained in the [openshell-community](https://github.com/nvidia/openshell-community) repository and pulled from `ghcr.io/nvidia/openshell-community/sandboxes/` at runtime.
Expand Down
5 changes: 3 additions & 2 deletions deploy/docker/Dockerfile.cli-macos
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,12 @@
# --build-arg OPENSHELL_CARGO_VERSION=0.6.0 \
# --output type=local,dest=out/ .

ARG OSXCROSS_IMAGE=crazymax/osxcross:latest
ARG OSXCROSS_IMAGE=ghcr.io/crazy-max/osxcross:latest
ARG PYTHON_IMAGE=public.ecr.aws/docker/library/python:3.12-slim

FROM ${OSXCROSS_IMAGE} AS osxcross

FROM python:3.12-slim AS builder
FROM ${PYTHON_IMAGE} AS builder

ARG CARGO_TARGET_CACHE_SCOPE=default

Expand Down
5 changes: 3 additions & 2 deletions deploy/docker/Dockerfile.driver-vm-macos
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,12 @@
# --build-context vm-runtime-compressed=/path/to/compressed-dir \
# --output type=local,dest=out/ .

ARG OSXCROSS_IMAGE=crazymax/osxcross:latest
ARG OSXCROSS_IMAGE=ghcr.io/crazy-max/osxcross:latest
ARG PYTHON_IMAGE=public.ecr.aws/docker/library/python:3.12-slim

FROM ${OSXCROSS_IMAGE} AS osxcross

FROM python:3.12-slim AS builder
FROM ${PYTHON_IMAGE} AS builder

ARG CARGO_TARGET_CACHE_SCOPE=default

Expand Down
5 changes: 3 additions & 2 deletions deploy/docker/Dockerfile.gateway-macos
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,12 @@
# Cross-compile the standalone openshell-gateway binary for macOS aarch64
# (Apple Silicon) using the osxcross toolchain.

ARG OSXCROSS_IMAGE=crazymax/osxcross:latest
ARG OSXCROSS_IMAGE=ghcr.io/crazy-max/osxcross:latest
ARG PYTHON_IMAGE=public.ecr.aws/docker/library/python:3.12-slim

FROM ${OSXCROSS_IMAGE} AS osxcross

FROM python:3.12-slim AS builder
FROM ${PYTHON_IMAGE} AS builder

ARG CARGO_TARGET_CACHE_SCOPE=default

Expand Down
5 changes: 3 additions & 2 deletions deploy/docker/Dockerfile.python-wheels-macos
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,13 @@
# SPDX-License-Identifier: Apache-2.0


ARG OSXCROSS_IMAGE=crazymax/osxcross:latest
ARG OSXCROSS_IMAGE=ghcr.io/crazy-max/osxcross:latest
ARG PYTHON_IMAGE=public.ecr.aws/docker/library/python
ARG PYTHON_VERSION=3.12

FROM ${OSXCROSS_IMAGE} AS osxcross

FROM python:${PYTHON_VERSION}-slim AS builder
FROM ${PYTHON_IMAGE}:${PYTHON_VERSION}-slim AS builder

ARG TARGETARCH
ARG CARGO_TARGET_CACHE_SCOPE=default
Expand Down
Loading
Loading