From ebc5398c58c1dfb8e6d1445d36c83257ecc4d554 Mon Sep 17 00:00:00 2001 From: Eliran Wolff Date: Sun, 2 Aug 2026 11:55:10 +0300 Subject: [PATCH 1/7] feat(atecontroller): mount NVIDIA toolkit into GPU worker pods Signed-off-by: Eliran Wolff --- .../internal/controllers/workerpool_apply.go | 92 +++++++++- .../controllers/workerpool_apply_test.go | 166 ++++++++++++++++++ 2 files changed, 257 insertions(+), 1 deletion(-) diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply.go b/cmd/atecontroller/internal/controllers/workerpool_apply.go index 48e1fef87..0128ab3d4 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply.go @@ -15,6 +15,8 @@ package controllers import ( + "os" + corev1 "k8s.io/api/core/v1" appsv1ac "k8s.io/client-go/applyconfigurations/apps/v1" corev1ac "k8s.io/client-go/applyconfigurations/core/v1" @@ -145,6 +147,7 @@ func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettin applyWorkerPoolPodTemplate(podSpecAC, containerAC, wp.Spec.Template) maybeApplyMicroVMPodShape(podSpecAC, containerAC, wp.Spec.SandboxClass) + maybeApplyGPUPodShape(podSpecAC, containerAC, wp.Spec.Template, wp.Spec.SandboxClass) podSpecAC.WithContainers(containerAC) podSpecAC.WithTerminationGracePeriodSeconds(workerTerminationGracePeriodSeconds) @@ -221,7 +224,8 @@ func fieldRefEnv(name, fieldPath string) *corev1ac.EnvVarApplyConfiguration { // veth and nftables rules (NET_ADMIN/NET_RAW), and the OCI rootfs is unpacked // and device nodes created as root over image-owned trees // (DAC_OVERRIDE/FOWNER/CHOWN/MKNOD). This replaces the former privileged worker; -// the default seccomp and AppArmor profiles are sufficient (no Unconfined). +// seccomp stays at the runtime default, but AppArmor must be Unconfined (see +// ateomSecurityContext) since runsc's own mounts trip the default profile. var ateomGvisorCapabilities = []corev1.Capability{ "NET_ADMIN", "SYS_ADMIN", "SYS_CHROOT", "SYS_PTRACE", "SETUID", "SETGID", "SETPCAP", "DAC_OVERRIDE", @@ -301,6 +305,92 @@ func maybeApplyMicroVMPodShape( WithEffect(corev1.TaintEffectNoSchedule)) } +// nvidiaToolkitContainerPath is where the host toolkit is mounted inside the +// worker; ateom-gvisor's toolkitDir must match this. It sits outside +// /usr/local/nvidia because the GPU device plugin mounts that tree into the +// container read-only, and a mount cannot create its own mountpoint there, so +// mounting under it only works when the toolkit happens to live inside the +// directory the plugin mounts. +const nvidiaToolkitContainerPath = "/opt/nvidia-toolkit" + +// defaultNvidiaToolkitHostPath is where gpu-operator installs the toolkit +// (toolkit.installDir defaults to /usr/local/nvidia). It is deliberately not the +// container path above: the two are independent, since where the node keeps the +// toolkit says nothing about where we can mount it. +const defaultNvidiaToolkitHostPath = "/usr/local/nvidia/toolkit" + +// nvidiaDriverRootEnv names the directory the GPU device plugin mounts the driver +// into a pod at. ateom derives the driver library and binary paths from it, both of +// which nvidia-ctk needs to generate a CDI spec. Only set it when the cluster's +// device plugin does not use the /usr/local/nvidia convention; the controller +// forwards its own value onto GPU worker pods. +const nvidiaDriverRootEnv = "ATE_NVIDIA_DRIVER_ROOT" + +// nvidiaToolkitHostPath is where the NVIDIA container toolkit lives on the node. +// It is platform-specific: gpu-operator and EKS install it at +// /usr/local/nvidia/toolkit, while GKE keeps NVIDIA assets under +// /home/kubernetes/bin/nvidia, so it is overridable via the +// ATE_NVIDIA_TOOLKIT_HOST_PATH env var on the controller. We mount it read-only +// so nvidia-ctk / nvidia-cdi-hook match whatever toolkit/driver the cluster runs. +var nvidiaToolkitHostPath = envOrDefault("ATE_NVIDIA_TOOLKIT_HOST_PATH", defaultNvidiaToolkitHostPath) + +func envOrDefault(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} + +// maybeApplyGPUPodShape shapes a gVisor worker pod that requests a GPU so ateom +// can inject the GPU into actors via CDI. It mounts the host NVIDIA toolkit +// (version-matched to the node) read-only, for the glibc-based ateom image to run +// directly. The pod keeps the same security posture as any other gVisor worker. +// No-op for non-GPU pools and non-gVisor classes; an empty class defaults to +// gVisor (WorkerPoolSpec kubebuilder default). +func maybeApplyGPUPodShape( + podSpecAC *corev1ac.PodSpecApplyConfiguration, + containerAC *corev1ac.ContainerApplyConfiguration, + tmpl *atev1alpha1.WorkerPoolPodTemplate, + sandboxClass atev1alpha1.SandboxClass, +) { + if sandboxClass != atev1alpha1.SandboxClassGvisor && sandboxClass != "" { + return + } + if !templateRequestsGPU(tmpl) { + return + } + // Mount the host NVIDIA toolkit (version-matched to the node) read-only. + containerAC.WithVolumeMounts(corev1ac.VolumeMount(). + WithName("nvidia-toolkit"). + WithMountPath(nvidiaToolkitContainerPath). + WithReadOnly(true)) + podSpecAC.WithVolumes(corev1ac.Volume(). + WithName("nvidia-toolkit"). + WithHostPath(corev1ac.HostPathVolumeSource(). + WithPath(nvidiaToolkitHostPath). + WithType(corev1.HostPathDirectory))) + // Only propagated when set, so a default deployment adds no env to worker pods. + if root := os.Getenv(nvidiaDriverRootEnv); root != "" { + containerAC.WithEnv(corev1ac.EnvVar().WithName(nvidiaDriverRootEnv).WithValue(root)) + } +} + +// templateRequestsGPU reports whether the pool template requests one or more +// nvidia.com/gpu devices (limits or requests). +func templateRequestsGPU(tmpl *atev1alpha1.WorkerPoolPodTemplate) bool { + if tmpl == nil || tmpl.Resources == nil { + return false + } + const gpu = corev1.ResourceName("nvidia.com/gpu") + if q, ok := tmpl.Resources.Limits[gpu]; ok && !q.IsZero() { + return true + } + if q, ok := tmpl.Resources.Requests[gpu]; ok && !q.IsZero() { + return true + } + return false +} + func applyWorkerPoolPodTemplate( podSpecAC *corev1ac.PodSpecApplyConfiguration, containerAC *corev1ac.ContainerApplyConfiguration, diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go index f208a23f4..89b7ac273 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go @@ -509,6 +509,172 @@ func envByName(env []corev1ac.EnvVarApplyConfiguration) map[string]envInfo { return m } +func TestGPUPoolMountsToolkit(t *testing.T) { + gpu := resource.MustParse("1") + wp := &atev1alpha1.WorkerPool{ + ObjectMeta: metav1.ObjectMeta{Name: "wp", Namespace: "ns"}, + Spec: atev1alpha1.WorkerPoolSpec{ + AteomImage: "img", + Template: &atev1alpha1.WorkerPoolPodTemplate{ + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{"nvidia.com/gpu": gpu}, + }, + }, + }, + } + dep := buildDeploymentApplyConfig(wp, ateomOTelSettings{}) + pod := dep.Spec.Template.Spec + + var found bool + for _, v := range pod.Volumes { + if v.Name != nil && *v.Name == "nvidia-toolkit" { + found = true + if v.HostPath == nil || *v.HostPath.Path != defaultNvidiaToolkitHostPath { + t.Fatalf("nvidia-toolkit volume has wrong hostPath: %+v", v.HostPath) + } + } + } + if !found { + t.Fatal("expected nvidia-toolkit host mount on a GPU pool") + } + + var mounted bool + for _, c := range pod.Containers { + for _, m := range c.VolumeMounts { + if m.Name != nil && *m.Name == "nvidia-toolkit" && *m.MountPath == nvidiaToolkitContainerPath { + mounted = true + } + } + } + if !mounted { + t.Fatal("expected nvidia-toolkit mount on the ateom container") + } + + // A GPU pool keeps the same posture as any other unprivileged gVisor worker: no + // user namespace and no unmasked /proc, which the skipped update-ldcache hook + // would otherwise force. + if pod.HostUsers != nil { + t.Error("did not expect hostUsers to be set on a GPU pool") + } + for _, c := range pod.Containers { + if c.SecurityContext != nil && c.SecurityContext.ProcMount != nil { + t.Errorf("did not expect procMount to be set, got %v", *c.SecurityContext.ProcMount) + } + } +} + +// TestGPUPoolDriverRootEnv covers the override reaching the worker: ateom derives the +// driver library and binary paths from it, and nvidia-ctk cannot generate a CDI spec +// without them. Unset, no env is added at all. +func TestGPUPoolDriverRootEnv(t *testing.T) { + gpu := resource.MustParse("1") + newGPUPool := func() *atev1alpha1.WorkerPool { + return &atev1alpha1.WorkerPool{ + ObjectMeta: metav1.ObjectMeta{Name: "wp", Namespace: "ns"}, + Spec: atev1alpha1.WorkerPoolSpec{ + AteomImage: "img", + Template: &atev1alpha1.WorkerPoolPodTemplate{ + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{"nvidia.com/gpu": gpu}, + }, + }, + }, + } + } + driverRootEnv := func(wp *atev1alpha1.WorkerPool) (string, bool) { + for _, c := range buildDeploymentApplyConfig(wp, ateomOTelSettings{}).Spec.Template.Spec.Containers { + for _, e := range c.Env { + if e.Name != nil && *e.Name == nvidiaDriverRootEnv { + return *e.Value, true + } + } + } + return "", false + } + + if v, ok := driverRootEnv(newGPUPool()); ok { + t.Errorf("unset: expected no %s on the worker, got %q", nvidiaDriverRootEnv, v) + } + + t.Setenv(nvidiaDriverRootEnv, "/opt/nvidia") + v, ok := driverRootEnv(newGPUPool()) + if !ok || v != "/opt/nvidia" { + t.Errorf("set: want %s=/opt/nvidia on the worker, got %q (present=%v)", nvidiaDriverRootEnv, v, ok) + } +} + +func TestNonGPUPoolHasNoToolkit(t *testing.T) { + wp := &atev1alpha1.WorkerPool{ + ObjectMeta: metav1.ObjectMeta{Name: "wp", Namespace: "ns"}, + Spec: atev1alpha1.WorkerPoolSpec{AteomImage: "img"}, + } + dep := buildDeploymentApplyConfig(wp, ateomOTelSettings{}) + pod := dep.Spec.Template.Spec + for _, v := range pod.Volumes { + if v.Name != nil && *v.Name == "nvidia-toolkit" { + t.Fatal("non-GPU pool must not mount the toolkit") + } + } + // Non-GPU workers keep the tighter base posture: no user namespace, no + // unmasked /proc. + if pod.HostUsers != nil { + t.Error("non-GPU pool must not set hostUsers") + } + for _, c := range pod.Containers { + if c.SecurityContext != nil && c.SecurityContext.ProcMount != nil { + t.Error("non-GPU pool must not set procMount") + } + } +} + +// TestGPUMicroVMPoolHasNoGPUPodShape asserts none of the GPU pod shaping is applied +// to a non-gVisor pool: no toolkit volume, no toolkit mount, no driver-root env. +// +// A WorkerPool like this is rejected at apply time by the CEL rule on +// WorkerPoolSpec, so it should never reach the controller. This covers the case +// where one already exists — the rule was added after the fact, or the object was +// written by a path that skipped CRD validation. The controller does not strip the +// resource request itself, so such a pod still schedules onto a GPU node and holds a +// device no actor can use; that gap is why the combination is rejected at the API +// rather than only here. +func TestGPUMicroVMPoolHasNoGPUPodShape(t *testing.T) { + // Set so the driver-root assertion below is not vacuous: a gVisor GPU pool would + // carry this env, a micro-VM one must not. + t.Setenv(nvidiaDriverRootEnv, "/opt/nvidia") + gpu := resource.MustParse("1") + wp := &atev1alpha1.WorkerPool{ + ObjectMeta: metav1.ObjectMeta{Name: "wp", Namespace: "ns"}, + Spec: atev1alpha1.WorkerPoolSpec{ + AteomImage: "img", + SandboxClass: atev1alpha1.SandboxClassMicroVM, + Template: &atev1alpha1.WorkerPoolPodTemplate{ + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{"nvidia.com/gpu": gpu}, + }, + }, + }, + } + pod := buildDeploymentApplyConfig(wp, ateomOTelSettings{}).Spec.Template.Spec + + for _, v := range pod.Volumes { + if v.Name != nil && *v.Name == "nvidia-toolkit" { + t.Error("micro-VM pool must not mount the NVIDIA toolkit even when it requests a GPU") + } + } + for _, c := range pod.Containers { + for _, m := range c.VolumeMounts { + if m.Name != nil && *m.Name == "nvidia-toolkit" { + t.Error("micro-VM pool must not get the toolkit volume mount") + } + } + for _, e := range c.Env { + if e.Name != nil && *e.Name == nvidiaDriverRootEnv { + t.Errorf("micro-VM pool must not get %s", nvidiaDriverRootEnv) + } + } + } +} + func testWorkerPoolApplyConfig(tmpl *atev1alpha1.WorkerPoolPodTemplate) *atev1alpha1.WorkerPool { return &atev1alpha1.WorkerPool{ ObjectMeta: metav1.ObjectMeta{Name: "pool", Namespace: "default", UID: "uid"}, From a09de36e30d4d2e199f559d6f0d779723c78d26b Mon Sep 17 00:00:00 2001 From: Eliran Wolff Date: Sun, 2 Aug 2026 11:55:10 +0300 Subject: [PATCH 2/7] feat(ateom-gvisor): pass GPUs through to gVisor actors via CDI Generate a per-pod CDI spec with nvidia-ctk, parse it in-tree, and inject the device nodes (major/minor stat-resolved) of the "all" CDI device, the driver- library mounts, and env into each actor's OCI spec. Run the CDI createContainer hooks from the mounted toolkit except update-ldcache, whose ldconfig needs a private /proc mount; stage the SONAME symlinks it would create from each library's ELF DT_SONAME instead. That keeps the GPU worker on the same unprivileged posture as any other gVisor worker (no user namespace, no procMount:Unmasked, cgroup delegation intact). Enable runsc --nvproxy at sandbox creation. Detect GPUs by device-node glob so any assigned index works (multi-GPU). The distroless ateom cannot exec nvidia-ctk, so GPU pools must run a glibc ateom build (WorkerPool.spec.ateomImage). Signed-off-by: Eliran Wolff --- cmd/ateom-gvisor/gpu.go | 516 +++++++++++++++++++++++++++++++++ cmd/ateom-gvisor/gpu_test.go | 374 ++++++++++++++++++++++++ cmd/ateom-gvisor/main.go | 12 +- cmd/ateom-gvisor/runsc.go | 40 ++- cmd/ateom-gvisor/runsc_test.go | 45 +++ 5 files changed, 976 insertions(+), 11 deletions(-) create mode 100644 cmd/ateom-gvisor/gpu.go create mode 100644 cmd/ateom-gvisor/gpu_test.go create mode 100644 cmd/ateom-gvisor/runsc_test.go diff --git a/cmd/ateom-gvisor/gpu.go b/cmd/ateom-gvisor/gpu.go new file mode 100644 index 000000000..25eef9d1d --- /dev/null +++ b/cmd/ateom-gvisor/gpu.go @@ -0,0 +1,516 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "cmp" + "context" + "debug/elf" + "encoding/json" + "fmt" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + + specs "github.com/opencontainers/runtime-spec/specs-go" + "golang.org/x/sys/unix" + + "github.com/agent-substrate/substrate/internal/ateompath" +) + +// toolkitDir is where the host's NVIDIA container toolkit (nvidia-ctk, +// nvidia-cdi-hook) is mounted into the worker pod — read-only from the node, so +// the binaries match whatever toolkit/driver the cluster installed. A var (not +// const) so tests can point it at a fixture directory. +var toolkitDir = "/opt/nvidia-toolkit" + +// gpuDeviceGlob matches the per-GPU device nodes. The device plugin can assign any +// indices (a worker sharing a multi-GPU node may get /dev/nvidia2,3 with no +// /dev/nvidia0), so detection must not assume index 0. The [0-9] excludes the control +// nodes (/dev/nvidiactl, /dev/nvidia-uvm). A var so tests can point it at a fixture. +var gpuDeviceGlob = "/dev/nvidia[0-9]*" + +// nvidiaDriverRoot is where the GPU device plugin mounts the driver into the pod. +// GKE and gpu-operator both use /usr/local/nvidia, but that is a convention rather +// than a contract, so it is overridable via ATE_NVIDIA_DRIVER_ROOT (propagated onto +// GPU worker pods by the controller). +var nvidiaDriverRoot = cmp.Or(os.Getenv("ATE_NVIDIA_DRIVER_ROOT"), "/usr/local/nvidia") + +// Both directories are load-bearing for CDI generation, not just for completeness: +// without the library path nvidia-ctk cannot load libnvidia-ml.so.1 to enumerate the +// GPUs and generation fails outright, and without the bin path (which it discovers +// via PATH) the generated spec carries libraries but no nvidia-smi. +var ( + driverLibDir = filepath.Join(nvidiaDriverRoot, "lib64") + driverBinDir = filepath.Join(nvidiaDriverRoot, "bin") +) + +const cdiOutputDir = "/run/ate-cdi" + +// enabledCDIHooks is the set of CDI createContainer hooks the actor runs. It is an +// allowlist because the toolkit is mounted from the host, so its version is the +// cluster's choice: a newer one can emit hooks that have never been reviewed +// against this worker's unprivileged posture, and those must not run by default. +// update-ldcache is absent deliberately — its ldconfig needs a private /proc +// mount, which the pod's masked /proc rejects, so the SONAME symlinks it would +// create are staged directly into the rootfs instead. +var enabledCDIHooks = map[string]bool{ + "create-symlinks": true, + "enable-cuda-compat": true, +} + +// toolkitBinary resolves a toolkit command to an executable path, preferring the +// unwrapped ".real" binary the NVIDIA toolkit ships: the plain name is often a +// /bin/sh wrapper. The ateom image is glibc-based (debian), so the glibc-dynamic +// toolkit binaries run directly — no ld-linux loader shim is needed. +func toolkitBinary(name string) string { + if real := filepath.Join(toolkitDir, name+".real"); fileExists(real) { + return real + } + return filepath.Join(toolkitDir, name) +} + +func fileExists(p string) bool { _, err := os.Stat(p); return err == nil } + +// dropEnvVar returns env with every "KEY=..." entry for the given key removed. +func dropEnvVar(env []string, key string) []string { + prefix := key + "=" + out := make([]string, 0, len(env)) + for _, e := range env { + if !strings.HasPrefix(e, prefix) { + out = append(out, e) + } + } + return out +} + +// gpuPresent reports whether any GPU is assigned to this worker pod, matching any +// device index (not just /dev/nvidia0 — the device plugin can assign 2,3 etc.). +func gpuPresent() bool { + matches, _ := filepath.Glob(gpuDeviceGlob) + return len(matches) > 0 +} + +var ( + generateMu sync.Mutex + cdiGenerated bool +) + +// generateCDISpec runs nvidia-ctk (from the host toolkit mounted into the pod) to +// produce a CDI spec scoped to this pod's assigned GPU. The glibc-based ateom image +// runs the glibc-dynamic toolkit binary directly. Runs under reapLock like every +// other subprocess in this process (a child reaper is running). +func generateCDISpec(ctx context.Context, outDir string) error { + if err := os.MkdirAll(outDir, 0o755); err != nil { + return fmt.Errorf("creating CDI output dir %s: %w", outDir, err) + } + reapLock.RLock() + defer reapLock.RUnlock() + // No --nvidia-cdi-hook-path: we discard the CDI hooks (staging SONAME symlinks + // ourselves), so the hook paths nvidia-ctk writes into the spec are never used. + cmd := exec.CommandContext(ctx, toolkitBinary("nvidia-ctk"), + "cdi", "generate", + "--format=json", + "--library-search-path="+driverLibDir, + "--output="+filepath.Join(outDir, "nvidia.json"), + ) + // nvidia-ctk finds the driver binaries (nvidia-smi, ...) via PATH. + cmd.Env = append(os.Environ(), "PATH="+driverBinDir+":"+os.Getenv("PATH")) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("nvidia-ctk cdi generate failed: %w: %s", err, out) + } + return nil +} + +// ensureCDISpec generates the per-pod CDI spec once, on the first actor. A failure is +// not memoized: a transient error (e.g. the toolkit mount not yet ready) is retried on +// the next actor rather than bricking GPU for the pod's lifetime. +func ensureCDISpec(ctx context.Context) error { + generateMu.Lock() + defer generateMu.Unlock() + if cdiGenerated { + return nil + } + if err := generateCDISpec(ctx, cdiOutputDir); err != nil { + return err + } + cdiGenerated = true + return nil +} + +// maybeInjectGPU is a no-op unless the worker pod has a GPU. When it does, it +// generates the per-pod CDI spec once and injects the GPU into the actor +// container's OCI bundle before runsc create. +func maybeInjectGPU(ctx context.Context, actorUID, containerName string) error { + if !gpuPresent() { + return nil + } + slog.InfoContext(ctx, "Injecting GPU into actor container", slog.String("container", containerName)) + if err := ensureCDISpec(ctx); err != nil { + return err + } + bundleDir := ateompath.OCIBundlePath(actorUID, containerName) + if err := injectGPUIntoBundle(ctx, bundleDir, cdiOutputDir); err != nil { + return fmt.Errorf("injecting GPU into %q bundle: %w", containerName, err) + } + return nil +} + +// cdiSpec is the minimal shape of the JSON CDI spec (nvidia-ctk --format=json) that +// we consume: the device nodes, driver-library mounts, and env. +type cdiSpec struct { + Devices []struct { + Name string `json:"name"` + ContainerEdits cdiEdits `json:"containerEdits"` + } `json:"devices"` + ContainerEdits cdiEdits `json:"containerEdits"` +} + +// cdiAllDevice is the CDI device that carries every GPU assigned to the pod. +// nvidia-ctk also emits per-index ("0") and per-UUID devices that repeat the same +// nodes, so we apply only this one (plus the spec-level edits) to avoid injecting +// each device node several times. +const cdiAllDevice = "all" + +type cdiEdits struct { + Env []string `json:"env,omitempty"` + DeviceNodes []cdiDev `json:"deviceNodes,omitempty"` + Mounts []cdiMount `json:"mounts,omitempty"` + Hooks []cdiHook `json:"hooks,omitempty"` +} + +type cdiDev struct { + Path string `json:"path"` + Type string `json:"type,omitempty"` + Major int64 `json:"major,omitempty"` + Minor int64 `json:"minor,omitempty"` +} + +type cdiMount struct { + HostPath string `json:"hostPath"` + ContainerPath string `json:"containerPath"` + Type string `json:"type,omitempty"` + Options []string `json:"options,omitempty"` +} + +type cdiHook struct { + HookName string `json:"hookName"` + Path string `json:"path"` + Args []string `json:"args,omitempty"` + Env []string `json:"env,omitempty"` +} + +// resolveDevNumbers fills a device node's major/minor from the host when the CDI spec +// omitted them. nvidia-ctk emits deviceNodes carrying only a path (CDI delegates +// number resolution to the OCI runtime, which stats the host); we merge into runsc's +// spec ourselves, so we stat here too, otherwise the actor gets bogus 0,0 char devices +// and NVML can't reach the driver. An nvidia device major is never 0. +func resolveDevNumbers(path string, major, minor int64) (int64, int64, error) { + if major != 0 { + return major, minor, nil + } + var st unix.Stat_t + if err := unix.Stat(path, &st); err != nil { + return 0, 0, fmt.Errorf("stat device %s: %w", path, err) + } + rdev := uint64(st.Rdev) + return int64(unix.Major(rdev)), int64(unix.Minor(rdev)), nil +} + +// injectGPUIntoBundle merges the CDI spec generated in cdiSpecDir into the actor's OCI +// config.json in bundleDir: device nodes (major/minor resolved from the host), the +// driver-library mounts, and env. It does NOT run the CDI hooks; instead it stages the +// SONAME symlinks (libcuda.so.1 -> libcuda.so.580.x) into the actor rootfs, which is +// what lets the GPU worker keep the plain unprivileged posture (no user namespace, no +// unmasked /proc). The CDI spec is plain JSON, so no CDI library is needed. +func injectGPUIntoBundle(ctx context.Context, bundleDir, cdiSpecDir string) error { + cdiData, err := os.ReadFile(filepath.Join(cdiSpecDir, "nvidia.json")) + if err != nil { + return fmt.Errorf("reading CDI spec: %w", err) + } + var cdi cdiSpec + if err := json.Unmarshal(cdiData, &cdi); err != nil { + return fmt.Errorf("parsing CDI spec: %w", err) + } + // Spec-level edits (driver libs, common device nodes, env) plus only the "all" + // device's edits — applying every device would inject each GPU node several times + // (nvidia-ctk repeats nodes across its per-index, per-UUID, and "all" devices). + edits := cdi.ContainerEdits + var foundAll bool + for _, d := range cdi.Devices { + if d.Name != cdiAllDevice { + continue + } + foundAll = true + edits.Env = append(edits.Env, d.ContainerEdits.Env...) + edits.DeviceNodes = append(edits.DeviceNodes, d.ContainerEdits.DeviceNodes...) + edits.Mounts = append(edits.Mounts, d.ContainerEdits.Mounts...) + edits.Hooks = append(edits.Hooks, d.ContainerEdits.Hooks...) + } + if !foundAll { + return fmt.Errorf("CDI spec in %s has no %q device", cdiSpecDir, cdiAllDevice) + } + if len(edits.DeviceNodes) == 0 { + return fmt.Errorf("CDI spec in %s resolved no devices", cdiSpecDir) + } + + cfgPath := filepath.Join(bundleDir, "config.json") + specData, err := os.ReadFile(cfgPath) + if err != nil { + return fmt.Errorf("reading %s: %w", cfgPath, err) + } + var spec specs.Spec + if err := json.Unmarshal(specData, &spec); err != nil { + return fmt.Errorf("parsing OCI spec: %w", err) + } + + // The edits below append to the bundle's config.json, so injecting twice would + // double every device, mount, env entry and hook. atelet re-unpacks the bundle + // before each Run and Restore, so this normally runs once per bundle — but that + // invariant lives in another component, so enforce it here rather than rely on it. + if hasGPUDevice(spec.Linux) { + slog.InfoContext(ctx, "Bundle already has GPU devices; skipping injection", + slog.String("bundle", bundleDir)) + return nil + } + + if spec.Linux == nil { + spec.Linux = &specs.Linux{} + } + if spec.Linux.Resources == nil { + spec.Linux.Resources = &specs.LinuxResources{} + } + for _, dn := range edits.DeviceNodes { + major, minor, err := resolveDevNumbers(dn.Path, dn.Major, dn.Minor) + if err != nil { + return err + } + devType := dn.Type + if devType == "" { + devType = "c" // nvidia-ctk omits type for char devices; runsc needs it. + } + spec.Linux.Devices = append(spec.Linux.Devices, specs.LinuxDevice{ + Path: dn.Path, Type: devType, Major: major, Minor: minor, + }) + spec.Linux.Resources.Devices = append(spec.Linux.Resources.Devices, specs.LinuxDeviceCgroup{ + Allow: true, Type: devType, Major: &major, Minor: &minor, Access: "rwm", + }) + } + + for _, m := range edits.Mounts { + mType := m.Type + if mType == "" { + mType = "bind" // CDI omits type for its bind mounts; runsc's gofer needs it. + } + spec.Mounts = append(spec.Mounts, specs.Mount{ + Source: m.HostPath, Destination: m.ContainerPath, Type: mType, Options: m.Options, + }) + } + + if spec.Process != nil { + spec.Process.Env = append(spec.Process.Env, edits.Env...) + // runsc's nvproxy runs nvidia-container-cli when it sees NVIDIA_VISIBLE_DEVICES + // (independent of --nvproxy); we set up the GPU via CDI, so strip it. + spec.Process.Env = dropEnvVar(spec.Process.Env, "NVIDIA_VISIBLE_DEVICES") + spec.Process.Env = prependLibraryPath(spec.Process.Env, []string{driverLibDir}) + } + + // Run the allowlisted CDI createContainer hooks (see enabledCDIHooks) from the + // mounted host toolkit. Anything else the toolkit emits is skipped and logged, + // so a toolkit upgrade that adds a hook is visible rather than silent. + if spec.Hooks == nil { + spec.Hooks = &specs.Hooks{} + } + for _, h := range edits.Hooks { + if h.HookName != "createContainer" || len(h.Args) < 2 { + continue + } + if !enabledCDIHooks[h.Args[1]] { + slog.InfoContext(ctx, "Skipping CDI hook outside the allowlist", + slog.String("hook", h.Args[1])) + continue + } + binary := toolkitBinary("nvidia-cdi-hook") + spec.Hooks.CreateContainer = append(spec.Hooks.CreateContainer, specs.Hook{ + Path: binary, + Args: append([]string{binary}, h.Args[1:]...), + Env: h.Env, + }) + } + + // Create the driver SONAME symlinks in the container's rootfs (spec.Root.Path, + // relative to the bundle). + rootfs := "rootfs" + if spec.Root != nil && spec.Root.Path != "" { + rootfs = spec.Root.Path + } + if !filepath.IsAbs(rootfs) { + rootfs = filepath.Join(bundleDir, rootfs) + } + if err := stageSonameSymlinks(ctx, rootfs, spec.Mounts); err != nil { + return fmt.Errorf("staging SONAME symlinks: %w", err) + } + + out, err := json.Marshal(&spec) + if err != nil { + return fmt.Errorf("serializing OCI spec: %w", err) + } + return os.WriteFile(cfgPath, out, 0o644) +} + +// hasGPUDevice reports whether the OCI spec already carries injected NVIDIA device +// nodes, which is how an already-injected bundle is recognized. +func hasGPUDevice(l *specs.Linux) bool { + if l == nil { + return false + } + for _, d := range l.Devices { + if strings.HasPrefix(d.Path, "/dev/nvidia") { + return true + } + } + return false +} + +// prependLibraryPath puts the driver library directories at the front of +// LD_LIBRARY_PATH, keeping whatever the image or ActorTemplate already set. The +// actor needs this to find libcuda.so.1: the update-ldcache hook that would +// normally add the directory to the loader cache is not run (see enabledCDIHooks), +// so an image that does not set LD_LIBRARY_PATH itself gets a CUDA runtime that +// reports zero devices even though the GPU is fully injected. Directories already +// on the path are left alone, so an NVIDIA base image keeps its own ordering. +// +// This is weaker than the ldcache the hook would have written — LD_LIBRARY_PATH is +// inherited by child processes and takes precedence over an executable's own +// DT_RUNPATH — so it carries only driverLibDir rather than every directory the CDI +// mounts touch, which would also sweep in the driver's X server modules. +func prependLibraryPath(env, dirs []string) []string { + if len(dirs) == 0 { + return env + } + existing := "" + for _, e := range env { + if v, ok := strings.CutPrefix(e, "LD_LIBRARY_PATH="); ok { + existing = v // OCI semantics: a later entry wins. + } + } + have := map[string]bool{} + for _, d := range strings.Split(existing, ":") { + have[d] = true + } + var add []string + for _, d := range dirs { + if !have[d] { + add = append(add, d) + } + } + if len(add) == 0 { + return env + } + val := strings.Join(add, ":") + if existing != "" { + val += ":" + existing + } + return append(dropEnvVar(env, "LD_LIBRARY_PATH"), "LD_LIBRARY_PATH="+val) +} + +// stageSonameSymlinks writes each driver library's SONAME symlink (e.g. +// libcuda.so.1 -> libcuda.so.580.65.06) into rootfs, so programs that link against +// the SONAME resolve it. For each CDI library mount it reads the library's ELF +// DT_SONAME and, when that differs from the mounted filename, writes a relative +// symlink alongside the mount destination; the real library file arrives at runtime +// via the CDI bind-mount into the same directory. +// +// A library whose SONAME cannot be read is skipped rather than failing the actor, so +// one odd file cannot cost us the whole GPU. That is logged: the missing symlink +// surfaces much later, as a loader error in the workload, with nothing pointing back +// here. +// Every path component under rootfs comes from the actor image, so the writes go +// through os.Root: an image that ships a driver-mount's parent directory as a +// symlink out of the rootfs would otherwise have the kernel resolve it in ateom's +// mount namespace, where the shared image cache and other actors' bundles are +// mounted. Same treatment as createExtraDirs in internal/imagecache. +func stageSonameSymlinks(ctx context.Context, rootfs string, mounts []specs.Mount) error { + root, err := os.OpenRoot(rootfs) + if err != nil { + return fmt.Errorf("while opening rootfs %q: %w", rootfs, err) + } + defer root.Close() + + for _, m := range mounts { + base := filepath.Base(m.Destination) + // Only shared libraries (…/lib…/foo.so.). + if !strings.Contains(base, ".so.") || m.Source == "" { + continue + } + soname, err := elfSonameFn(m.Source) + if err != nil { + slog.WarnContext(ctx, "Skipping SONAME symlink for driver library", + slog.String("library", m.Source), slog.Any("err", err)) + continue + } + if soname == "" || soname == base { + continue + } + // The SONAME is read out of the library and joined into a path, so require a + // bare filename rather than trusting the file's contents. + if soname != filepath.Base(soname) || !filepath.IsLocal(soname) { + slog.WarnContext(ctx, "Skipping driver library whose SONAME is not a filename", + slog.String("library", m.Source), slog.String("soname", soname)) + continue + } + dir := strings.TrimPrefix(filepath.Dir(m.Destination), "/") + if dir != "" && !filepath.IsLocal(dir) { + return fmt.Errorf("driver mount %q escapes the rootfs", m.Destination) + } + if err := root.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("mkdir %s: %w", dir, err) + } + link := filepath.Join(dir, soname) + _ = root.Remove(link) + if err := root.Symlink(base, link); err != nil { + return fmt.Errorf("symlink %s -> %s: %w", link, base, err) + } + } + return nil +} + +// elfSonameFn is elfSoname, as a var so tests can supply a SONAME without needing a +// real ELF file on disk. +var elfSonameFn = elfSoname + +// elfSoname returns a shared library's DT_SONAME. It returns "" with a nil error for +// a library that simply carries no DT_SONAME entry, and an error when the file could +// not be opened or parsed as ELF. +func elfSoname(path string) (string, error) { + f, err := elf.Open(path) + if err != nil { + return "", fmt.Errorf("opening ELF %s: %w", path, err) + } + defer f.Close() + names, err := f.DynString(elf.DT_SONAME) + if err != nil { + return "", fmt.Errorf("reading DT_SONAME from %s: %w", path, err) + } + if len(names) == 0 { + return "", nil + } + return names[0], nil +} diff --git a/cmd/ateom-gvisor/gpu_test.go b/cmd/ateom-gvisor/gpu_test.go new file mode 100644 index 000000000..a5d6753bf --- /dev/null +++ b/cmd/ateom-gvisor/gpu_test.go @@ -0,0 +1,374 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + specs "github.com/opencontainers/runtime-spec/specs-go" +) + +func TestMaybeInjectGPU_NoGPUIsNoop(t *testing.T) { + dir := t.TempDir() + old := gpuDeviceGlob + gpuDeviceGlob = filepath.Join(dir, "nvidia[0-9]*") // matches nothing + defer func() { gpuDeviceGlob = old }() + if err := maybeInjectGPU(context.Background(), "actor_uid", "c1"); err != nil { + t.Fatalf("expected no-op nil when no GPU is present, got %v", err) + } +} + +// TestGPUPresent checks detection matches any GPU index, not just nvidia0. +func TestGPUPresent(t *testing.T) { + dir := t.TempDir() + old := gpuDeviceGlob + gpuDeviceGlob = filepath.Join(dir, "nvidia[0-9]*") + defer func() { gpuDeviceGlob = old }() + + if gpuPresent() { + t.Fatal("expected absent before creation") + } + // A worker sharing a multi-GPU node can be assigned nvidia2, not nvidia0. + if err := os.WriteFile(filepath.Join(dir, "nvidia2"), nil, 0o644); err != nil { + t.Fatal(err) + } + if !gpuPresent() { + t.Fatal("expected present after creating nvidia2") + } +} + +func TestGenerateCDISpec_InvokesCtk(t *testing.T) { + dir := t.TempDir() + oldT := toolkitDir + toolkitDir = dir + defer func() { toolkitDir = oldT }() + + // Fake nvidia-ctk (run directly by the glibc ateom) that writes a minimal JSON + // spec to the --output= path. + const script = `#!/bin/sh +out="" +for a in "$@"; do + case "$a" in --output=*) out="${a#--output=}" ;; esac +done +printf '{"cdiVersion":"0.6.0","kind":"nvidia.com/gpu","devices":[{"name":"all","containerEdits":{"deviceNodes":[{"path":"/dev/nvidia0","type":"c","major":195,"minor":0}]}}]}' > "$out" +` + os.WriteFile(filepath.Join(dir, "nvidia-ctk"), []byte(script), 0o755) + + out := filepath.Join(dir, "cdi") + if err := generateCDISpec(context.Background(), out); err != nil { + t.Fatalf("generate: %v", err) + } + data, err := os.ReadFile(filepath.Join(out, "nvidia.json")) + if err != nil || !strings.Contains(string(data), "nvidia.com/gpu") { + t.Fatalf("spec not written correctly: %q err=%v", data, err) + } +} + +func TestGenerateCDISpec_NonZeroFails(t *testing.T) { + dir := t.TempDir() + oldT := toolkitDir + toolkitDir = dir + defer func() { toolkitDir = oldT }() + os.WriteFile(filepath.Join(dir, "nvidia-ctk"), []byte("#!/bin/sh\nexit 3\n"), 0o755) + if err := generateCDISpec(context.Background(), filepath.Join(dir, "cdi")); err == nil { + t.Fatal("expected error on non-zero exit") + } +} + +func TestInjectGPUIntoBundle(t *testing.T) { + dir := t.TempDir() + + // Minimal CDI spec: one device node with major/minor OMITTED (as nvidia-ctk + // emits) plus an env var. injectGPUIntoBundle must resolve major/minor from the + // host — /dev/null is a char device present everywhere as 1,3 — so we assert on it. + // The hooks cover all three cases: create-symlinks is allowlisted and kept, + // update-ldcache is excluded, and an unrecognized hook (as a newer host toolkit + // could emit) is dropped rather than run unreviewed. + specJSON := `{ + "cdiVersion": "0.6.0", + "kind": "nvidia.com/gpu", + "devices": [ + { + "name": "all", + "containerEdits": { + "deviceNodes": [{"path": "/dev/null"}], + "env": ["NVIDIA_TEST=1"], + "hooks": [ + {"hookName": "createContainer", "path": "/x/nvidia-cdi-hook", + "args": ["nvidia-cdi-hook", "create-symlinks", "--link", "a::b"]}, + {"hookName": "createContainer", "path": "/x/nvidia-cdi-hook", + "args": ["nvidia-cdi-hook", "update-ldcache", "--folder", "/usr/local/nvidia/lib64"]}, + {"hookName": "createContainer", "path": "/x/nvidia-cdi-hook", + "args": ["nvidia-cdi-hook", "some-future-hook", "--flag", "v"]} + ] + } + } + ] +}` + specDir := filepath.Join(dir, "cdi") + os.MkdirAll(specDir, 0o755) + os.WriteFile(filepath.Join(specDir, "nvidia.json"), []byte(specJSON), 0o644) + + bundle := filepath.Join(dir, "bundle") + // The rootfs is already mounted by SetupBundleRootfs before injection runs. + os.MkdirAll(filepath.Join(bundle, "rootfs"), 0o755) + // The CUDA base image sets NVIDIA_VISIBLE_DEVICES; injection must strip it. + base := &specs.Spec{Version: "1.0.0", Process: &specs.Process{ + Args: []string{"true"}, + Env: []string{"NVIDIA_VISIBLE_DEVICES=all"}, + }} + data, _ := json.Marshal(base) + os.WriteFile(filepath.Join(bundle, "config.json"), data, 0o644) + + if err := injectGPUIntoBundle(context.Background(), bundle, specDir); err != nil { + t.Fatalf("inject: %v", err) + } + + out, _ := os.ReadFile(filepath.Join(bundle, "config.json")) + var got specs.Spec + json.Unmarshal(out, &got) + + var dev *specs.LinuxDevice + for i := range got.Linux.Devices { + if got.Linux.Devices[i].Path == "/dev/null" { + dev = &got.Linux.Devices[i] + } + } + if dev == nil { + t.Fatalf("expected /dev/null device injected, spec=%s", out) + } + // The library resolved major/minor and type from the host (1,3,"c"). + if dev.Type != "c" || dev.Major != 1 || dev.Minor != 3 { + t.Fatalf("device not resolved from host: type=%q major=%d minor=%d", dev.Type, dev.Major, dev.Minor) + } + var hasEnv, hasVisibleDevices bool + for _, e := range got.Process.Env { + if e == "NVIDIA_TEST=1" { + hasEnv = true + } + if strings.HasPrefix(e, "NVIDIA_VISIBLE_DEVICES=") { + hasVisibleDevices = true + } + } + if !hasEnv { + t.Fatalf("expected NVIDIA_TEST env injected, spec=%s", out) + } + // NVIDIA_VISIBLE_DEVICES must be stripped so runsc's nvproxy does not invoke + // nvidia-container-cli (we set up the GPU via CDI instead). + if hasVisibleDevices { + t.Fatalf("expected NVIDIA_VISIBLE_DEVICES stripped, spec=%s", out) + } + // Only allowlisted hooks run. update-ldcache is excluded because its ldconfig + // needs a private /proc mount (its SONAME symlinks are staged directly instead), + // and an unrecognized hook is dropped rather than run unreviewed. + var kept []string + if got.Hooks != nil { + for _, h := range got.Hooks.CreateContainer { + if len(h.Args) > 1 { + kept = append(kept, h.Args[1]) + } + if !strings.HasPrefix(h.Path, toolkitDir) { + t.Fatalf("hook path %q should point at the mounted toolkit", h.Path) + } + } + } + if len(kept) != 1 || kept[0] != "create-symlinks" { + t.Fatalf("hooks = %v, want only [create-symlinks]", kept) + } +} + +// TestPrependLibraryPath covers the two cases that decide whether a CUDA program +// can load libcuda.so.1: an image that sets no LD_LIBRARY_PATH (must get one) and +// an NVIDIA image that already lists the driver directory (must be left alone). +func TestPrependLibraryPath(t *testing.T) { + dirs := []string{"/usr/local/nvidia/lib64"} + for _, tc := range []struct { + name string + env []string + want string + }{ + {"no existing value", []string{"PATH=/bin"}, "LD_LIBRARY_PATH=/usr/local/nvidia/lib64"}, + {"template value is kept after the driver dir", []string{"LD_LIBRARY_PATH=/opt/app/lib"}, + "LD_LIBRARY_PATH=/usr/local/nvidia/lib64:/opt/app/lib"}, + {"already present is left alone", []string{"LD_LIBRARY_PATH=/usr/local/nvidia/lib64:/x"}, + "LD_LIBRARY_PATH=/usr/local/nvidia/lib64:/x"}, + } { + t.Run(tc.name, func(t *testing.T) { + got := prependLibraryPath(tc.env, dirs) + var found string + for _, e := range got { + if strings.HasPrefix(e, "LD_LIBRARY_PATH=") { + found = e + } + } + if found != tc.want { + t.Fatalf("got %q, want %q", found, tc.want) + } + if n := strings.Count(strings.Join(got, " "), "LD_LIBRARY_PATH="); n != 1 { + t.Fatalf("want exactly one LD_LIBRARY_PATH entry, got %d in %v", n, got) + } + }) + } +} + +// TestInjectGPUIntoBundle_Idempotent guards the invariant that atelet re-unpacks the +// bundle before each Run/Restore. The edits are appends against config.json on disk, +// so without the guard a second injection doubles every device, mount, env entry and +// hook — silently, since nothing errors. +func TestInjectGPUIntoBundle_Idempotent(t *testing.T) { + dir := t.TempDir() + // Explicit major/minor so the device need not exist on the test host. + specJSON := `{ + "cdiVersion": "0.6.0", + "kind": "nvidia.com/gpu", + "devices": [ + { + "name": "all", + "containerEdits": { + "deviceNodes": [{"path": "/dev/nvidia0", "major": 195, "minor": 0, "type": "c"}], + "env": ["NVIDIA_TEST=1"], + "hooks": [ + {"hookName": "createContainer", "path": "/x/nvidia-cdi-hook", + "args": ["nvidia-cdi-hook", "create-symlinks", "--link", "a::b"]} + ] + } + } + ] +}` + specDir := filepath.Join(dir, "cdi") + os.MkdirAll(specDir, 0o755) + os.WriteFile(filepath.Join(specDir, "nvidia.json"), []byte(specJSON), 0o644) + + bundle := filepath.Join(dir, "bundle") + os.MkdirAll(filepath.Join(bundle, "rootfs"), 0o755) + data, _ := json.Marshal(&specs.Spec{Version: "1.0.0", Process: &specs.Process{Args: []string{"true"}}}) + os.WriteFile(filepath.Join(bundle, "config.json"), data, 0o644) + + // devices, mounts, env, hooks. + shape := func() [4]int { + out, _ := os.ReadFile(filepath.Join(bundle, "config.json")) + var s specs.Spec + if err := json.Unmarshal(out, &s); err != nil { + t.Fatal(err) + } + hooks := 0 + if s.Hooks != nil { + hooks = len(s.Hooks.CreateContainer) + } + return [4]int{len(s.Linux.Devices), len(s.Mounts), len(s.Process.Env), hooks} + } + + if err := injectGPUIntoBundle(context.Background(), bundle, specDir); err != nil { + t.Fatalf("first inject: %v", err) + } + first := shape() + if first[0] == 0 { + t.Fatalf("first inject added no devices: %v", first) + } + if err := injectGPUIntoBundle(context.Background(), bundle, specDir); err != nil { + t.Fatalf("second inject: %v", err) + } + if second := shape(); second != first { + t.Fatalf("second injection changed the bundle: %v -> %v (devices, mounts, env, hooks)", first, second) + } +} + +// TestStageSonameSymlinks_ConfinedToRootfs covers the two ways the staging writes +// could escape into ateom's mount namespace, where the shared image cache and other +// actors' bundles live: a directory the image redirects out of the rootfs, and a +// SONAME read out of a library that is not a bare filename. +func TestStageSonameSymlinks_ConfinedToRootfs(t *testing.T) { + for _, tc := range []struct { + name string + soname string + // dest is the CDI mount destination inside the actor. + dest string + // escape, when set, is planted in the rootfs as a symlink to the outside dir. + escape string + }{ + { + name: "image redirects the driver dir out of the rootfs", + soname: "libcuda.so.1", + dest: "/usr/lib/gpu/libcuda.so.580.65.06", + escape: "usr/lib/gpu", + }, + { + name: "SONAME climbs out with ../", + soname: "../../../../escaped.so.1", + dest: "/usr/lib/gpu/libcuda.so.580.65.06", + }, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + rootfs := filepath.Join(dir, "rootfs") + outside := filepath.Join(dir, "outside") + if err := os.MkdirAll(filepath.Join(rootfs, "usr/lib"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(outside, 0o755); err != nil { + t.Fatal(err) + } + victim := filepath.Join(outside, "libcuda.so.1") + if err := os.WriteFile(victim, []byte("do not delete"), 0o644); err != nil { + t.Fatal(err) + } + if tc.escape != "" { + if err := os.Symlink(outside, filepath.Join(rootfs, tc.escape)); err != nil { + t.Fatal(err) + } + } + + old := elfSonameFn + elfSonameFn = func(string) (string, error) { return tc.soname, nil } + defer func() { elfSonameFn = old }() + + // Refusing outright and skipping are both acceptable; writing outside is not. + _ = stageSonameSymlinks(context.Background(), rootfs, []specs.Mount{{ + Source: "/host/libcuda.so.580.65.06", + Destination: tc.dest, + }}) + + if b, err := os.ReadFile(victim); err != nil || string(b) != "do not delete" { + t.Fatalf("a file outside the rootfs was modified: err=%v content=%q", err, b) + } + if entries, err := os.ReadDir(outside); err == nil && len(entries) != 1 { + t.Fatalf("something was created outside the rootfs: %v", entries) + } + }) + } +} + +func TestInjectGPUIntoBundle_MissingSpecFails(t *testing.T) { + dir := t.TempDir() + bundle := filepath.Join(dir, "bundle") + os.MkdirAll(bundle, 0o755) + base := &specs.Spec{Version: "1.0.0", Process: &specs.Process{}} + data, _ := json.Marshal(base) + os.WriteFile(filepath.Join(bundle, "config.json"), data, 0o644) + + // An empty spec dir has no nvidia.json, so injection fails to read the spec. + emptyDir := filepath.Join(dir, "cdi-empty") + os.MkdirAll(emptyDir, 0o755) + if err := injectGPUIntoBundle(context.Background(), bundle, emptyDir); err == nil { + t.Fatal("expected error when the CDI spec is missing") + } +} diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index 80cd2b793..6c55a2fae 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -124,11 +124,15 @@ func do(ctx context.Context) error { } // Prepare the pod cgroup so runsc can create per-actor-container leaves under - // it with real accounting, instead of ignoring cgroups entirely. + // it with real accounting. if err := setupCgroupDelegation(ctx); err != nil { return fmt.Errorf("while setting up cgroup delegation: %w", err) } + if gpuPresent() { + slog.InfoContext(ctx, "GPU detected; enabling runsc nvproxy for all sandboxes") + } + // TODO: Consider whether we want to fork, so that we have an "init" process // as PID 1 that does nothing but reap processes that get reparented to it. // Then we won't have to mess about with locking the reaper while we do our @@ -349,6 +353,9 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload return nil, fmt.Errorf("while composing %q rootfs: %w", ac.GetName(), err) } containersToDelete = append(containersToDelete, ac.GetName()) + if err := maybeInjectGPU(ctx, req.GetActorUid(), ac.GetName()); err != nil { + return nil, fmt.Errorf("while injecting GPU for %q: %w", ac.GetName(), err) + } if err := rcmd.cmdCreate(ctx, pw, ac.GetName(), nil); err != nil { return nil, fmt.Errorf("while creating %q application container: %w", ac.GetName(), err) } @@ -594,6 +601,9 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore if err := imagecache.SetupBundleRootfs(ateompath.OCIBundlePath(req.GetActorUid(), ac.GetName())); err != nil { return nil, fmt.Errorf("while composing %q rootfs: %w", ac.GetName(), err) } + if err := maybeInjectGPU(ctx, req.GetActorUid(), ac.GetName()); err != nil { + return nil, fmt.Errorf("while injecting GPU for %q: %w", ac.GetName(), err) + } switch req.GetScope() { case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: containersToDelete = append(containersToDelete, ac.GetName()) diff --git a/cmd/ateom-gvisor/runsc.go b/cmd/ateom-gvisor/runsc.go index 003bd861a..f750ddfca 100644 --- a/cmd/ateom-gvisor/runsc.go +++ b/cmd/ateom-gvisor/runsc.go @@ -36,6 +36,23 @@ type runsc struct { actorUID string } +// nvproxyGlobalArgs returns the runsc global flags for GPU sandboxes, enabling +// gVisor's GPU ioctl proxy when the worker has a GPU. --nvproxy must be set when +// the sandbox is created (the pause/root container) so the sentry initializes GPU +// support up front — like enabling nvproxy in the containerd runtime config on +// normal Kubernetes. Otherwise nvproxy would try to initialize late, when the app +// subcontainer joins carrying the CDI /dev/nvidia* devices, and the running sentry +// crashes (StartSubcontainer: EOF). +// +// Only create and restore need it: both boot a sentry. `runsc start` acts on a +// sandbox that already exists, so the flag has no effect there. +func nvproxyGlobalArgs() []string { + if gpuPresent() { + return []string{"--nvproxy"} + } + return nil +} + // ensureContainerCgroupsPath sets the OCI spec's cgroupsPath so runsc creates a // per-container cgroup leaf under the worker pod's own cgroup (see // setupCgroupDelegation). atelet emits a runtime-agnostic spec with no @@ -90,10 +107,13 @@ func (r *runsc) cmdCreate(ctx context.Context, out io.Writer, containerName stri // "-log-packets", // "-strace", "-root", ateompath.RunSCStateDir(r.actorUID), + } + args = append(args, nvproxyGlobalArgs()...) + args = append(args, "create", "-bundle", ateompath.OCIBundlePath(r.actorUID, containerName), "-pid-file", ateompath.PIDFilePath(r.actorUID, containerName), - } + ) args = append(args, additionalArgs...) args = append(args, containerName) // Name of the container @@ -119,9 +139,7 @@ func (r *runsc) cmdStart(ctx context.Context, out io.Writer, containerName strin slog.InfoContext(ctx, "About to run runsc start", slog.String("container", containerName)) - cmd := exec.CommandContext( - ctx, - r.path, + startArgs := []string{ "-log-format", "json", "--alsologtostderr", // "-debug", @@ -131,9 +149,9 @@ func (r *runsc) cmdStart(ctx context.Context, out io.Writer, containerName strin // "-strace", "-allow-connected-on-save", "-root", ateompath.RunSCStateDir(r.actorUID), - "start", - containerName, // Name of the container - ) + } + startArgs = append(startArgs, "start", containerName) + cmd := exec.CommandContext(ctx, r.path, startArgs...) cmd.Stdout = out cmd.Stderr = out @@ -226,9 +244,7 @@ func (r *runsc) cmdRestore(ctx context.Context, out io.Writer, containerName, ch return fmt.Errorf("while setting cgroups path for %q: %w", containerName, err) } - cmd := exec.CommandContext( - ctx, - r.path, + restoreArgs := []string{ "-log-format", "json", "--alsologtostderr", // "-debug", @@ -237,6 +253,9 @@ func (r *runsc) cmdRestore(ctx context.Context, out io.Writer, containerName, ch // "-log-packets", // "-strace", "-root", ateompath.RunSCStateDir(r.actorUID), + } + restoreArgs = append(restoreArgs, nvproxyGlobalArgs()...) + restoreArgs = append(restoreArgs, "restore", "-bundle", ateompath.OCIBundlePath(r.actorUID, containerName), "-image-path", checkpointPath, @@ -246,6 +265,7 @@ func (r *runsc) cmdRestore(ctx context.Context, out io.Writer, containerName, ch "-detach", containerName, ) + cmd := exec.CommandContext(ctx, r.path, restoreArgs...) cmd.Stdout = out cmd.Stderr = out if err := cmd.Run(); err != nil { diff --git a/cmd/ateom-gvisor/runsc_test.go b/cmd/ateom-gvisor/runsc_test.go new file mode 100644 index 000000000..360a6ce89 --- /dev/null +++ b/cmd/ateom-gvisor/runsc_test.go @@ -0,0 +1,45 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "path/filepath" + "testing" +) + +// TestNvproxyGlobalArgs checks that runsc is told to enable nvproxy exactly when the +// worker has a GPU. The flag must be present on sandbox creation so the sentry +// initializes GPU support up front; without it the GPU subcontainer crashes. +func TestNvproxyGlobalArgs(t *testing.T) { + dir := t.TempDir() + old := gpuDeviceGlob + gpuDeviceGlob = filepath.Join(dir, "nvidia[0-9]*") + defer func() { gpuDeviceGlob = old }() + + if got := nvproxyGlobalArgs(); len(got) != 0 { + t.Fatalf("no GPU: want no flags, got %v", got) + } + + if err := os.WriteFile(filepath.Join(dir, "nvidia0"), nil, 0o644); err != nil { + t.Fatal(err) + } + got := nvproxyGlobalArgs() + if len(got) != 1 || got[0] != "--nvproxy" { + t.Fatalf("GPU: want [--nvproxy], got %v", got) + } +} From f9e18fa10e371939cb57941d1ad40ca62cb6d467 Mon Sep 17 00:00:00 2001 From: Eliran Wolff Date: Sun, 2 Aug 2026 11:55:10 +0300 Subject: [PATCH 3/7] docs: document GPU passthrough into gVisor actors Signed-off-by: Eliran Wolff --- docs/api-guide.md | 63 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 54 insertions(+), 9 deletions(-) diff --git a/docs/api-guide.md b/docs/api-guide.md index 97c30c668..ffc2a1ce8 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -43,7 +43,13 @@ spec: # gvisor SandboxConfig unless sandboxConfigName is set. ``` -### Example with GPU node scheduling +### GPU worker pools + +A GPU pool needs two things: (1) scheduling onto GPU nodes, and (2) a +`nvidia.com/gpu` request in `template.resources`. The request does double duty — +it makes the device plugin assign a GPU to the worker pod **and** triggers +Substrate to pass that GPU **through to each actor's sandbox**. No per-actor +configuration is needed. ```yaml apiVersion: ate.dev/v1alpha1 @@ -53,8 +59,10 @@ metadata: namespace: ate-demo spec: replicas: 5 - ateomImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + # GPU pools need a glibc ateom-gvisor build — see Requirements below. + ateomImage: /ateom-gvisor-glibc@sha256:... template: + # (1) schedule onto GPU nodes nodeSelector: cloud.google.com/gke-accelerator: nvidia-tesla-t4 tolerations: @@ -62,13 +70,6 @@ spec: operator: Exists effect: NoSchedule priorityClassName: substrate-workers - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: workload - operator: In - values: [substrate] resources: requests: cpu: 500m @@ -76,8 +77,52 @@ spec: limits: cpu: "1" memory: 2Gi + # (2) claim a GPU — this request is what triggers GPU passthrough + nvidia.com/gpu: "1" ``` +`atecontroller` propagates the request onto the `ateom` container and mounts the +host NVIDIA toolkit into the pod. `ateom-gvisor` then generates a CDI spec with +`nvidia-ctk` and injects the GPU device nodes, driver libraries, and env into +each actor container's OCI spec, and runs `runsc` with `--nvproxy` so CUDA and NVML +work inside the sandbox. A worker requesting `nvidia.com/gpu: N` passes all N +through. + +Every container in the actor gets the GPU. An actor's containers share one sandbox +and the worker's whole device set, and `ActorTemplate` has no per-container resource +fields, so GPUs are shared at the actor level rather than assigned to one container, +the same as cpu and memory. This differs from a Kubernetes Pod, where the GPU goes +only to the container that requests it. If per-container resource limits are added +later, GPU assignment should follow them. + +The driver library directory is prepended to each container's `LD_LIBRARY_PATH` so an +image does not have to set it to find `libcuda.so.1`; any existing value is kept after +it rather than replaced. + +**Requirements** + +- **A glibc `ateom-gvisor` image**, set as `spec.ateomImage`. The distroless default + cannot exec `nvidia-ctk`. Build one with + `KO_DEFAULTBASEIMAGE=debian:stable-slim ko build ./cmd/ateom-gvisor`. +- **`nvidia-ctk` on the node**, at the path mounted into the worker — by default + `/usr/local/nvidia/toolkit`, overridable with the controller's + `ATE_NVIDIA_TOOLKIT_HOST_PATH`. gpu-operator installs it; GKE's built-in GPU + support does not. +- **The driver mounted into the pod by the device plugin**, at `/usr/local/nvidia` + by default. `nvidia-ctk` needs its libraries to enumerate the GPUs at all, so a + cluster whose plugin uses a different layout must set the controller's + `ATE_NVIDIA_DRIVER_ROOT`. +- **`atelet` must run on the GPU nodes**, so add a matching toleration to its + DaemonSet if those nodes are tainted (for example `nvidia.com/gpu`). +- **gVisor only.** `microvm` pools would need VFIO PCI passthrough instead. + +**Known limitation: a GPU actor can only be suspended while no CUDA context is +open.** gVisor cannot serialize GPU state, so a checkpoint taken while the workload +holds a context fails with `can't save with live nvproxy clients` and terminates the +sandbox. Workloads that run CUDA and exit snapshot normally; one that keeps a +context alive (a model resident in device memory, say) cannot be suspended or have a +golden snapshot taken. + ### Status (`WorkerPoolStatus`) | Field | Type | Description | From 915f382b16a3096ca454cf3e545a1a3836c421c8 Mon Sep 17 00:00:00 2001 From: Eliran Wolff Date: Tue, 4 Aug 2026 18:56:51 +0300 Subject: [PATCH 4/7] feat(api): reject nvidia.com/gpu on non-gVisor WorkerPools GPU passthrough is implemented only for the gVisor runtime, but the pod template's resources are applied before the sandbox-class check, so a micro-VM pool requesting nvidia.com/gpu still got the request on its worker pod. The pod then scheduled onto a GPU node and held a device that no actor could use. Reject the combination at apply time with a CEL rule on WorkerPoolSpec, matching the cross-field rules ActorTemplate already carries. The rule keys off limits or requests, mirroring the pod-shaping check, and is written positively so a future sandbox class must opt in rather than silently inherit GPU support. Signed-off-by: Eliran Wolff --- .../generated/ate.dev_workerpools.yaml | 6 +++ pkg/api/v1alpha1/workerpool_types.go | 1 + .../v1alpha1/workerpool_validation_test.go | 39 +++++++++++++++++++ 3 files changed, 46 insertions(+) diff --git a/manifests/ate-install/generated/ate.dev_workerpools.yaml b/manifests/ate-install/generated/ate.dev_workerpools.yaml index b9d8c14a6..8941467b2 100644 --- a/manifests/ate-install/generated/ate.dev_workerpools.yaml +++ b/manifests/ate-install/generated/ate.dev_workerpools.yaml @@ -416,6 +416,12 @@ spec: - ateomImage - replicas type: object + x-kubernetes-validations: + - message: nvidia.com/gpu is only supported when sandboxClass is 'gvisor' + rule: '!has(self.sandboxClass) || self.sandboxClass == ''gvisor'' || + !has(self.template) || !has(self.template.resources) || !((has(self.template.resources.limits) + && ''nvidia.com/gpu'' in self.template.resources.limits) || (has(self.template.resources.requests) + && ''nvidia.com/gpu'' in self.template.resources.requests))' status: description: status is the observed state of WorkerPool properties: diff --git a/pkg/api/v1alpha1/workerpool_types.go b/pkg/api/v1alpha1/workerpool_types.go index ecd386f2f..17404b8d3 100644 --- a/pkg/api/v1alpha1/workerpool_types.go +++ b/pkg/api/v1alpha1/workerpool_types.go @@ -51,6 +51,7 @@ type WorkerPoolPodTemplate struct { Resources *corev1.ResourceRequirements `json:"resources,omitempty"` } +// +kubebuilder:validation:XValidation:rule="!has(self.sandboxClass) || self.sandboxClass == 'gvisor' || !has(self.template) || !has(self.template.resources) || !((has(self.template.resources.limits) && 'nvidia.com/gpu' in self.template.resources.limits) || (has(self.template.resources.requests) && 'nvidia.com/gpu' in self.template.resources.requests))",message="nvidia.com/gpu is only supported when sandboxClass is 'gvisor'" type WorkerPoolSpec struct { // Replicas is the number of worker pods to run. // +required diff --git a/pkg/api/v1alpha1/workerpool_validation_test.go b/pkg/api/v1alpha1/workerpool_validation_test.go index ab6dc0abf..417f074cf 100644 --- a/pkg/api/v1alpha1/workerpool_validation_test.go +++ b/pkg/api/v1alpha1/workerpool_validation_test.go @@ -24,6 +24,14 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +const gpuResourceName = corev1.ResourceName("nvidia.com/gpu") + +func gpuTemplate(limits, requests corev1.ResourceList) *WorkerPoolPodTemplate { + return &WorkerPoolPodTemplate{ + Resources: &corev1.ResourceRequirements{Limits: limits, Requests: requests}, + } +} + func TestWorkerPoolValidation(t *testing.T) { ctx := context.Background() @@ -99,6 +107,37 @@ func TestWorkerPoolValidation(t *testing.T) { }, wantErr: true, errMsg: "spec.template.tolerations: Too many", + }, { + name: "gpu on a gvisor pool", + mutate: func(wp *WorkerPool) { + wp.Spec.SandboxClass = SandboxClassGvisor + wp.Spec.Template = gpuTemplate(corev1.ResourceList{gpuResourceName: resource.MustParse("1")}, nil) + }, + wantErr: false, + }, { + name: "gpu limit on a micro-VM pool", + mutate: func(wp *WorkerPool) { + wp.Spec.SandboxClass = SandboxClassMicroVM + wp.Spec.Template = gpuTemplate(corev1.ResourceList{gpuResourceName: resource.MustParse("1")}, nil) + }, + wantErr: true, + errMsg: "nvidia.com/gpu is only supported when sandboxClass is 'gvisor'", + }, { + // The pod shape keys off limits OR requests, so the rule must reject both. + name: "gpu request on a micro-VM pool", + mutate: func(wp *WorkerPool) { + wp.Spec.SandboxClass = SandboxClassMicroVM + wp.Spec.Template = gpuTemplate(nil, corev1.ResourceList{gpuResourceName: resource.MustParse("1")}) + }, + wantErr: true, + errMsg: "nvidia.com/gpu is only supported when sandboxClass is 'gvisor'", + }, { + name: "micro-VM pool without a gpu", + mutate: func(wp *WorkerPool) { + wp.Spec.SandboxClass = SandboxClassMicroVM + wp.Spec.Template = gpuTemplate(corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, nil) + }, + wantErr: false, }} for _, tt := range tests { From c8fb3e9d1f4a62dfa89bc66d67cc80fcf94e5847 Mon Sep 17 00:00:00 2001 From: Eliran Wolff Date: Thu, 6 Aug 2026 11:02:25 +0300 Subject: [PATCH 5/7] fix(api): require nvidia.com/gpu in limits on a WorkerPool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kubernetes does not admit a pod that requests an extended resource without a matching limit ("Limit must be set for non overcommitable resources"), but the GPU pod shaping keyed off limits or requests. A pool that set only a request was accepted, got the toolkit mount and driver-root env, and then had its Deployment's pods refused — surfacing the error on the Deployment rather than on the WorkerPool the user wrote. Reject it with a CEL rule alongside the sandbox-class one, so both land on the same object at apply time. The message names the Kubernetes rule behind it. The controller still keys off limits or requests: with this rule the requests-only branch is unreachable, but it should not assume CRD validation is installed. Signed-off-by: Eliran Wolff --- .../generated/ate.dev_workerpools.yaml | 5 +++++ pkg/api/v1alpha1/workerpool_types.go | 1 + pkg/api/v1alpha1/workerpool_validation_test.go | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/manifests/ate-install/generated/ate.dev_workerpools.yaml b/manifests/ate-install/generated/ate.dev_workerpools.yaml index 8941467b2..f19a40894 100644 --- a/manifests/ate-install/generated/ate.dev_workerpools.yaml +++ b/manifests/ate-install/generated/ate.dev_workerpools.yaml @@ -422,6 +422,11 @@ spec: !has(self.template) || !has(self.template.resources) || !((has(self.template.resources.limits) && ''nvidia.com/gpu'' in self.template.resources.limits) || (has(self.template.resources.requests) && ''nvidia.com/gpu'' in self.template.resources.requests))' + - message: 'nvidia.com/gpu must be set in limits: Kubernetes does not + admit a request for an extended resource without a matching limit' + rule: '!has(self.template) || !has(self.template.resources) || !has(self.template.resources.requests) + || !(''nvidia.com/gpu'' in self.template.resources.requests) || (has(self.template.resources.limits) + && ''nvidia.com/gpu'' in self.template.resources.limits)' status: description: status is the observed state of WorkerPool properties: diff --git a/pkg/api/v1alpha1/workerpool_types.go b/pkg/api/v1alpha1/workerpool_types.go index 17404b8d3..d7a52b485 100644 --- a/pkg/api/v1alpha1/workerpool_types.go +++ b/pkg/api/v1alpha1/workerpool_types.go @@ -52,6 +52,7 @@ type WorkerPoolPodTemplate struct { } // +kubebuilder:validation:XValidation:rule="!has(self.sandboxClass) || self.sandboxClass == 'gvisor' || !has(self.template) || !has(self.template.resources) || !((has(self.template.resources.limits) && 'nvidia.com/gpu' in self.template.resources.limits) || (has(self.template.resources.requests) && 'nvidia.com/gpu' in self.template.resources.requests))",message="nvidia.com/gpu is only supported when sandboxClass is 'gvisor'" +// +kubebuilder:validation:XValidation:rule="!has(self.template) || !has(self.template.resources) || !has(self.template.resources.requests) || !('nvidia.com/gpu' in self.template.resources.requests) || (has(self.template.resources.limits) && 'nvidia.com/gpu' in self.template.resources.limits)",message="nvidia.com/gpu must be set in limits: Kubernetes does not admit a request for an extended resource without a matching limit" type WorkerPoolSpec struct { // Replicas is the number of worker pods to run. // +required diff --git a/pkg/api/v1alpha1/workerpool_validation_test.go b/pkg/api/v1alpha1/workerpool_validation_test.go index 417f074cf..ebd5d0a70 100644 --- a/pkg/api/v1alpha1/workerpool_validation_test.go +++ b/pkg/api/v1alpha1/workerpool_validation_test.go @@ -131,6 +131,24 @@ func TestWorkerPoolValidation(t *testing.T) { }, wantErr: true, errMsg: "nvidia.com/gpu is only supported when sandboxClass is 'gvisor'", + }, { + // Kubernetes refuses a pod that requests an extended resource without a + // matching limit, so a pool like this would shape a worker the Deployment + // then rejects. Catch it on the WorkerPool the user wrote. + name: "gpu in requests only", + mutate: func(wp *WorkerPool) { + wp.Spec.Template = gpuTemplate(nil, corev1.ResourceList{gpuResourceName: resource.MustParse("1")}) + }, + wantErr: true, + errMsg: "nvidia.com/gpu must be set in limits", + }, { + name: "gpu in both limits and requests", + mutate: func(wp *WorkerPool) { + wp.Spec.Template = gpuTemplate( + corev1.ResourceList{gpuResourceName: resource.MustParse("1")}, + corev1.ResourceList{gpuResourceName: resource.MustParse("1")}) + }, + wantErr: false, }, { name: "micro-VM pool without a gpu", mutate: func(wp *WorkerPool) { From e8719b03bc62455ad9a973f8b0f7e118485ffcb5 Mon Sep 17 00:00:00 2001 From: Eliran Wolff Date: Fri, 7 Aug 2026 23:22:12 +0300 Subject: [PATCH 6/7] fix(ateom-gvisor): build on a glibc base for the NVIDIA toolkit ateom-gvisor execs the NVIDIA container toolkit binaries it mounts from the host (nvidia-ctk to generate the CDI spec, nvidia-cdi-hook to run the createContainer hooks). Those are glibc-dynamic, so the distroless static default cannot load them and GPU injection fails at exec. Also stop quoting a runsc-version-specific error string in the docs: the message for checkpointing a live CUDA context changed between release-20260622 and release-20260803, though the failure is the same. Signed-off-by: Eliran Wolff --- .ko.yaml | 4 ++++ docs/api-guide.md | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.ko.yaml b/.ko.yaml index fd8e85968..ac1711624 100644 --- a/.ko.yaml +++ b/.ko.yaml @@ -24,3 +24,7 @@ baseImageOverrides: # (to bind the image into the virtiofsd shared dir) — both in debian:stable-slim but # not in the distroless static default. github.com/agent-substrate/substrate/cmd/ateom-microvm: debian:stable-slim + # ateom-gvisor runs the NVIDIA container toolkit binaries it mounts from the host + # for GPU passthrough (nvidia-ctk, nvidia-cdi-hook). Those are glibc-dynamic, and + # the distroless static default ships no libc to load them. + github.com/agent-substrate/substrate/cmd/ateom-gvisor: debian:stable-slim diff --git a/docs/api-guide.md b/docs/api-guide.md index ffc2a1ce8..c1b5cc398 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -118,7 +118,7 @@ it rather than replaced. **Known limitation: a GPU actor can only be suspended while no CUDA context is open.** gVisor cannot serialize GPU state, so a checkpoint taken while the workload -holds a context fails with `can't save with live nvproxy clients` and terminates the +holds a context fails with an nvproxy encoding error and terminates the sandbox. Workloads that run CUDA and exit snapshot normally; one that keeps a context alive (a model resident in device memory, say) cannot be suspended or have a golden snapshot taken. From 1c8c2c135f54f87f80e3a850d7c298c08979ef2b Mon Sep 17 00:00:00 2001 From: Eliran Wolff Date: Fri, 7 Aug 2026 23:28:00 +0300 Subject: [PATCH 7/7] Revert "build ateom-gvisor on a glibc base" GPU pools already get a glibc ateom through spec.ateomImage, which is what docs/api-guide.md documents and what the hardware testing used. Overriding the default base image would have widened every gVisor worker from distroless static to debian for a case that is opt-in and already served. The docs change from the reverted commit stands: the checkpoint error string for a live CUDA context differs between runsc releases. Signed-off-by: Eliran Wolff --- .ko.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.ko.yaml b/.ko.yaml index ac1711624..fd8e85968 100644 --- a/.ko.yaml +++ b/.ko.yaml @@ -24,7 +24,3 @@ baseImageOverrides: # (to bind the image into the virtiofsd shared dir) — both in debian:stable-slim but # not in the distroless static default. github.com/agent-substrate/substrate/cmd/ateom-microvm: debian:stable-slim - # ateom-gvisor runs the NVIDIA container toolkit binaries it mounts from the host - # for GPU passthrough (nvidia-ctk, nvidia-cdi-hook). Those are glibc-dynamic, and - # the distroless static default ships no libc to load them. - github.com/agent-substrate/substrate/cmd/ateom-gvisor: debian:stable-slim