diff --git a/lib/system/init/mode_exec.go b/lib/system/init/mode_exec.go index 41f80ed2..be1c06ec 100644 --- a/lib/system/init/mode_exec.go +++ b/lib/system/init/mode_exec.go @@ -54,6 +54,7 @@ func runExecMode(log *Logger, cfg *vmconfig.Config) { // Start guest-agent in background (skip if guest-agent was not copied) // Pass environment variables so they're available via hypeman exec + knownChildren := newKnownChildPIDs() var agentCmd *exec.Cmd if cfg.SkipGuestAgent { log.Info("hypeman-init:setup", "skipping guest-agent (skip_guest_agent=true)") @@ -88,9 +89,13 @@ func runExecMode(log *Logger, cfg *vmconfig.Config) { _ = readyPipeWriter.Close() agentExited := make(chan error, 1) - go func() { - agentExited <- agentCmd.Wait() - }() + agentPID := agentCmd.Process.Pid + knownChildren.add(agentPID) + go func(cmd *exec.Cmd, pid int) { + err := cmd.Wait() + knownChildren.remove(pid) + agentExited <- err + }(agentCmd, agentPID) // Strict startup gate: do not launch the guest program until agent is ready. if err := waitForGuestAgentReady(readyPipeReader, guestAgentReadyTimeout, agentExited); err != nil { @@ -139,6 +144,11 @@ func runExecMode(log *Logger, cfg *vmconfig.Config) { log.Info("hypeman-init:entrypoint", formatProgramStartSentinel("exec")) log.Info("hypeman-init:entrypoint", fmt.Sprintf("container app started (PID %d)", appCmd.Process.Pid)) + // exec.Cmd.Wait owns these direct children; the orphan reaper must not consume their statuses. + knownChildren.add(appCmd.Process.Pid) + stopOrphanReaper := startOrphanReaper(knownChildren) + defer stopOrphanReaper() + // Set up signal forwarding: when init receives a signal (e.g. from guest-agent // Shutdown RPC), forward it to the entrypoint child process so it can gracefully // shut down. This is how Docker/containerd works -- SIGTERM to PID 1 gets @@ -155,6 +165,7 @@ func runExecMode(log *Logger, cfg *vmconfig.Config) { // Wait for app to exit err := appCmd.Wait() + knownChildren.remove(appCmd.Process.Pid) signal.Stop(sigCh) exitCode := 0 diff --git a/lib/system/init/reaper.go b/lib/system/init/reaper.go new file mode 100644 index 00000000..dca13fb5 --- /dev/null +++ b/lib/system/init/reaper.go @@ -0,0 +1,133 @@ +package main + +import ( + "bytes" + "os" + "os/signal" + "path/filepath" + "sort" + "strconv" + "sync" + "syscall" +) + +type wait4Func func(int, *syscall.WaitStatus, int, *syscall.Rusage) (int, error) + +type knownChildPIDs struct { + mu sync.RWMutex + pids map[int]struct{} + changed chan struct{} +} + +func newKnownChildPIDs() *knownChildPIDs { + return &knownChildPIDs{ + pids: make(map[int]struct{}), + changed: make(chan struct{}, 1), + } +} + +func (k *knownChildPIDs) add(pid int) { + k.mu.Lock() + k.pids[pid] = struct{}{} + k.mu.Unlock() +} + +func (k *knownChildPIDs) remove(pid int) { + k.mu.Lock() + _, existed := k.pids[pid] + delete(k.pids, pid) + k.mu.Unlock() + if existed { + select { + case k.changed <- struct{}{}: + default: + } + } +} + +func (k *knownChildPIDs) contains(pid int) bool { + k.mu.RLock() + _, known := k.pids[pid] + k.mu.RUnlock() + return known +} + +func startOrphanReaper(knownChildren *knownChildPIDs) func() { + sigCh := make(chan os.Signal, 1) + stopCh := make(chan struct{}) + doneCh := make(chan struct{}) + signal.Notify(sigCh, syscall.SIGCHLD) + + go func() { + defer close(doneCh) + reapAdoptedZombies("/proc", os.Getpid(), knownChildren, syscall.Wait4) + for { + select { + case <-sigCh: + reapAdoptedZombies("/proc", os.Getpid(), knownChildren, syscall.Wait4) + case <-knownChildren.changed: + reapAdoptedZombies("/proc", os.Getpid(), knownChildren, syscall.Wait4) + case <-stopCh: + return + } + } + }() + + return func() { + signal.Stop(sigCh) + close(stopCh) + <-doneCh + } +} + +func reapAdoptedZombies(procRoot string, parentPID int, knownChildren *knownChildPIDs, wait4 wait4Func) { + for _, pid := range adoptedZombiePIDs(procRoot, parentPID, knownChildren) { + var status syscall.WaitStatus + _, _ = wait4(pid, &status, syscall.WNOHANG, nil) + } +} + +func adoptedZombiePIDs(procRoot string, parentPID int, knownChildren *knownChildPIDs) []int { + entries, err := os.ReadDir(procRoot) + if err != nil { + return nil + } + + pids := make([]int, 0) + for _, entry := range entries { + pid, err := strconv.Atoi(entry.Name()) + if err != nil { + continue + } + if knownChildren.contains(pid) { + continue + } + + stat, err := os.ReadFile(filepath.Join(procRoot, entry.Name(), "stat")) + if err != nil { + continue + } + state, ppid, ok := parseProcStat(stat) + if ok && state == 'Z' && ppid == parentPID { + pids = append(pids, pid) + } + } + sort.Ints(pids) + return pids +} + +func parseProcStat(stat []byte) (byte, int, bool) { + commEnd := bytes.LastIndexByte(stat, ')') + if commEnd < 0 || commEnd+1 >= len(stat) { + return 0, 0, false + } + fields := bytes.Fields(stat[commEnd+1:]) + if len(fields) < 2 || len(fields[0]) != 1 { + return 0, 0, false + } + ppid, err := strconv.Atoi(string(fields[1])) + if err != nil { + return 0, 0, false + } + return fields[0][0], ppid, true +} diff --git a/lib/system/init/reaper_linux_test.go b/lib/system/init/reaper_linux_test.go new file mode 100644 index 00000000..fbc90a4d --- /dev/null +++ b/lib/system/init/reaper_linux_test.go @@ -0,0 +1,56 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +func TestOrphanReaperIntegration(t *testing.T) { + if os.Getenv("HYPEMAN_REAPER_HELPER") == "1" { + runOrphanReaperHelper(t) + return + } + + cmd := exec.Command(os.Args[0], "-test.run=^TestOrphanReaperIntegration$") + cmd.Env = append(os.Environ(), "HYPEMAN_REAPER_HELPER=1") + output, err := cmd.CombinedOutput() + require.NoError(t, err, string(output)) +} + +func runOrphanReaperHelper(t *testing.T) { + require.NoError(t, unix.Prctl(unix.PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0)) + + pidFile := filepath.Join(t.TempDir(), "grandchild.pid") + appCmd := exec.Command("/bin/sh", "-c", `sleep 0.05 & echo $! > "$1"`, "sh", pidFile) + require.NoError(t, appCmd.Start()) + + knownChildren := newKnownChildPIDs() + knownChildren.add(appCmd.Process.Pid) + stopReaper := startOrphanReaper(knownChildren) + defer stopReaper() + require.NoError(t, appCmd.Wait()) + knownChildren.remove(appCmd.Process.Pid) + + pidBytes, err := os.ReadFile(pidFile) + require.NoError(t, err) + grandchildPID, err := strconv.Atoi(strings.TrimSpace(string(pidBytes))) + require.NoError(t, err) + + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if _, err := os.Stat(fmt.Sprintf("/proc/%d", grandchildPID)); os.IsNotExist(err) { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("adopted child %d was not reaped", grandchildPID) +} diff --git a/lib/system/init/reaper_test.go b/lib/system/init/reaper_test.go new file mode 100644 index 00000000..1bbf3c8d --- /dev/null +++ b/lib/system/init/reaper_test.go @@ -0,0 +1,88 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "syscall" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseProcStat(t *testing.T) { + state, ppid, ok := parseProcStat([]byte("123 (chromium helper) Z 1 123 123 0")) + require.True(t, ok) + assert.Equal(t, byte('Z'), state) + assert.Equal(t, 1, ppid) + + state, ppid, ok = parseProcStat([]byte("123 (name with ) parenthesis) S 42 123 123 0")) + require.True(t, ok) + assert.Equal(t, byte('S'), state) + assert.Equal(t, 42, ppid) + + _, _, ok = parseProcStat([]byte("malformed")) + assert.False(t, ok) +} + +func TestAdoptedZombiePIDs(t *testing.T) { + procRoot := t.TempDir() + writeProcStat(t, procRoot, 101, "chromium", 'Z', 1) + writeProcStat(t, procRoot, 102, "known app", 'Z', 1) + writeProcStat(t, procRoot, 103, "running child", 'S', 1) + writeProcStat(t, procRoot, 104, "other parent", 'Z', 99) + require.NoError(t, os.Mkdir(filepath.Join(procRoot, "105"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(procRoot, "105", "stat"), []byte("malformed"), 0o644)) + + knownChildren := newKnownChildPIDs() + knownChildren.add(102) + got := adoptedZombiePIDs(procRoot, 1, knownChildren) + assert.Equal(t, []int{101}, got) + + knownChildren.remove(102) + got = adoptedZombiePIDs(procRoot, 1, knownChildren) + assert.Equal(t, []int{101, 102}, got) +} + +func TestKnownChildRemovalSignalsRescan(t *testing.T) { + knownChildren := newKnownChildPIDs() + knownChildren.add(102) + knownChildren.remove(102) + + assert.False(t, knownChildren.contains(102)) + signaled := false + select { + case <-knownChildren.changed: + signaled = true + default: + } + assert.True(t, signaled) +} + +func TestReapAdoptedZombies(t *testing.T) { + procRoot := t.TempDir() + writeProcStat(t, procRoot, 201, "first", 'Z', 1) + writeProcStat(t, procRoot, 202, "second", 'Z', 1) + writeProcStat(t, procRoot, 203, "known", 'Z', 1) + + var reaped []int + wait4 := func(pid int, _ *syscall.WaitStatus, options int, _ *syscall.Rusage) (int, error) { + assert.Equal(t, syscall.WNOHANG, options) + reaped = append(reaped, pid) + return pid, nil + } + + knownChildren := newKnownChildPIDs() + knownChildren.add(203) + reapAdoptedZombies(procRoot, 1, knownChildren, wait4) + assert.Equal(t, []int{201, 202}, reaped) +} + +func writeProcStat(t *testing.T, procRoot string, pid int, name string, state byte, ppid int) { + t.Helper() + dir := filepath.Join(procRoot, fmt.Sprintf("%d", pid)) + require.NoError(t, os.Mkdir(dir, 0o755)) + stat := fmt.Sprintf("%d (%s) %c %d 0 0 0", pid, name, state, ppid) + require.NoError(t, os.WriteFile(filepath.Join(dir, "stat"), []byte(stat), 0o644)) +}