Reap orphaned guest processes

lib/system/init/mode_exec.go

@@ -54,6 +54,7 @@ func runExecMode(log *Logger, cfg *vmconfig.Config) {
 
 	// Start guest-agent in background (skip if guest-agent was not copied)
 	// Pass environment variables so they're available via hypeman exec
+	knownChildren := newKnownChildPIDs()
 	var agentCmd *exec.Cmd
 	if cfg.SkipGuestAgent {
 		log.Info("hypeman-init:setup", "skipping guest-agent (skip_guest_agent=true)")
@@ -88,9 +89,13 @@ func runExecMode(log *Logger, cfg *vmconfig.Config) {
 		_ = readyPipeWriter.Close()
 
 		agentExited := make(chan error, 1)
-		go func() {
-			agentExited <- agentCmd.Wait()
-		}()
+		agentPID := agentCmd.Process.Pid
+		knownChildren.add(agentPID)
+		go func(cmd *exec.Cmd, pid int) {
+			err := cmd.Wait()
+			knownChildren.remove(pid)
+			agentExited <- err
+		}(agentCmd, agentPID)
 
 		// Strict startup gate: do not launch the guest program until agent is ready.
 		if err := waitForGuestAgentReady(readyPipeReader, guestAgentReadyTimeout, agentExited); err != nil {
@@ -139,6 +144,11 @@ func runExecMode(log *Logger, cfg *vmconfig.Config) {
 	log.Info("hypeman-init:entrypoint", formatProgramStartSentinel("exec"))
 	log.Info("hypeman-init:entrypoint", fmt.Sprintf("container app started (PID %d)", appCmd.Process.Pid))
 
+	// exec.Cmd.Wait owns these direct children; the orphan reaper must not consume their statuses.
+	knownChildren.add(appCmd.Process.Pid)
+	stopOrphanReaper := startOrphanReaper(knownChildren)
+	defer stopOrphanReaper()
+
 	// Set up signal forwarding: when init receives a signal (e.g. from guest-agent
 	// Shutdown RPC), forward it to the entrypoint child process so it can gracefully
 	// shut down. This is how Docker/containerd works -- SIGTERM to PID 1 gets
@@ -155,6 +165,7 @@ func runExecMode(log *Logger, cfg *vmconfig.Config) {
 
 	// Wait for app to exit
 	err := appCmd.Wait()
+	knownChildren.remove(appCmd.Process.Pid)
 	signal.Stop(sigCh)
 
 	exitCode := 0

lib/system/init/reaper.go

@@ -0,0 +1,133 @@
+package main
+
+import (
+	"bytes"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"sync"
+	"syscall"
+)
+
+type wait4Func func(int, *syscall.WaitStatus, int, *syscall.Rusage) (int, error)
+
+type knownChildPIDs struct {
+	mu      sync.RWMutex
+	pids    map[int]struct{}
+	changed chan struct{}
+}
+
+func newKnownChildPIDs() *knownChildPIDs {
+	return &knownChildPIDs{
+		pids:    make(map[int]struct{}),
+		changed: make(chan struct{}, 1),
+	}
+}
+
+func (k *knownChildPIDs) add(pid int) {
+	k.mu.Lock()
+	k.pids[pid] = struct{}{}
+	k.mu.Unlock()
+}
+
+func (k *knownChildPIDs) remove(pid int) {
+	k.mu.Lock()
+	_, existed := k.pids[pid]
+	delete(k.pids, pid)
+	k.mu.Unlock()
+	if existed {
+		select {
+		case k.changed <- struct{}{}:
+		default:
+		}
+	}
+}
+
+func (k *knownChildPIDs) contains(pid int) bool {
+	k.mu.RLock()
+	_, known := k.pids[pid]
+	k.mu.RUnlock()
+	return known
+}
+
+func startOrphanReaper(knownChildren *knownChildPIDs) func() {
+	sigCh := make(chan os.Signal, 1)
+	stopCh := make(chan struct{})
+	doneCh := make(chan struct{})
+	signal.Notify(sigCh, syscall.SIGCHLD)
+
+	go func() {
+		defer close(doneCh)
+		reapAdoptedZombies("/proc", os.Getpid(), knownChildren, syscall.Wait4)
+		for {
+			select {
+			case <-sigCh:
+				reapAdoptedZombies("/proc", os.Getpid(), knownChildren, syscall.Wait4)
+			case <-knownChildren.changed:
+				reapAdoptedZombies("/proc", os.Getpid(), knownChildren, syscall.Wait4)
+			case <-stopCh:
+				return
+			}
+		}
+	}()
+
+	return func() {
+		signal.Stop(sigCh)
+		close(stopCh)
+		<-doneCh
+	}
+}
+
+func reapAdoptedZombies(procRoot string, parentPID int, knownChildren *knownChildPIDs, wait4 wait4Func) {
+	for _, pid := range adoptedZombiePIDs(procRoot, parentPID, knownChildren) {
+		var status syscall.WaitStatus
+		_, _ = wait4(pid, &status, syscall.WNOHANG, nil)
+	}
+}
+
+func adoptedZombiePIDs(procRoot string, parentPID int, knownChildren *knownChildPIDs) []int {
+	entries, err := os.ReadDir(procRoot)
+	if err != nil {
+		return nil
+	}
+
+	pids := make([]int, 0)
+	for _, entry := range entries {
+		pid, err := strconv.Atoi(entry.Name())
+		if err != nil {
+			continue
+		}
+		if knownChildren.contains(pid) {
+			continue
+		}
+
+		stat, err := os.ReadFile(filepath.Join(procRoot, entry.Name(), "stat"))
+		if err != nil {
+			continue
+		}
+		state, ppid, ok := parseProcStat(stat)
+		if ok && state == 'Z' && ppid == parentPID {
+			pids = append(pids, pid)
+		}
+	}
+	sort.Ints(pids)
+	return pids
+}
+
+func parseProcStat(stat []byte) (byte, int, bool) {
+	commEnd := bytes.LastIndexByte(stat, ')')
+	if commEnd < 0 || commEnd+1 >= len(stat) {
+		return 0, 0, false
+	}
+	fields := bytes.Fields(stat[commEnd+1:])
+	if len(fields) < 2 || len(fields[0]) != 1 {
+		return 0, 0, false
+	}
+	ppid, err := strconv.Atoi(string(fields[1]))
+	if err != nil {
+		return 0, 0, false
+	}
+	return fields[0][0], ppid, true
+}

lib/system/init/reaper_linux_test.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
+)
+
+func TestOrphanReaperIntegration(t *testing.T) {
+	if os.Getenv("HYPEMAN_REAPER_HELPER") == "1" {
+		runOrphanReaperHelper(t)
+		return
+	}
+
+	cmd := exec.Command(os.Args[0], "-test.run=^TestOrphanReaperIntegration$")
+	cmd.Env = append(os.Environ(), "HYPEMAN_REAPER_HELPER=1")
+	output, err := cmd.CombinedOutput()
+	require.NoError(t, err, string(output))
+}
+
+func runOrphanReaperHelper(t *testing.T) {
+	require.NoError(t, unix.Prctl(unix.PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0))
+
+	pidFile := filepath.Join(t.TempDir(), "grandchild.pid")
+	appCmd := exec.Command("/bin/sh", "-c", `sleep 0.05 & echo $! > "$1"`, "sh", pidFile)
+	require.NoError(t, appCmd.Start())
+
+	knownChildren := newKnownChildPIDs()
+	knownChildren.add(appCmd.Process.Pid)
+	stopReaper := startOrphanReaper(knownChildren)
+	defer stopReaper()
+	require.NoError(t, appCmd.Wait())
+	knownChildren.remove(appCmd.Process.Pid)
+
+	pidBytes, err := os.ReadFile(pidFile)
+	require.NoError(t, err)
+	grandchildPID, err := strconv.Atoi(strings.TrimSpace(string(pidBytes)))
+	require.NoError(t, err)
+
+	deadline := time.Now().Add(2 * time.Second)
+	for time.Now().Before(deadline) {
+		if _, err := os.Stat(fmt.Sprintf("/proc/%d", grandchildPID)); os.IsNotExist(err) {
+			return
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	t.Fatalf("adopted child %d was not reaped", grandchildPID)
+}

lib/system/init/reaper_test.go

@@ -0,0 +1,88 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"syscall"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseProcStat(t *testing.T) {
+	state, ppid, ok := parseProcStat([]byte("123 (chromium helper) Z 1 123 123 0"))
+	require.True(t, ok)
+	assert.Equal(t, byte('Z'), state)
+	assert.Equal(t, 1, ppid)
+
+	state, ppid, ok = parseProcStat([]byte("123 (name with ) parenthesis) S 42 123 123 0"))
+	require.True(t, ok)
+	assert.Equal(t, byte('S'), state)
+	assert.Equal(t, 42, ppid)
+
+	_, _, ok = parseProcStat([]byte("malformed"))
+	assert.False(t, ok)
+}
+
+func TestAdoptedZombiePIDs(t *testing.T) {
+	procRoot := t.TempDir()
+	writeProcStat(t, procRoot, 101, "chromium", 'Z', 1)
+	writeProcStat(t, procRoot, 102, "known app", 'Z', 1)
+	writeProcStat(t, procRoot, 103, "running child", 'S', 1)
+	writeProcStat(t, procRoot, 104, "other parent", 'Z', 99)
+	require.NoError(t, os.Mkdir(filepath.Join(procRoot, "105"), 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(procRoot, "105", "stat"), []byte("malformed"), 0o644))
+
+	knownChildren := newKnownChildPIDs()
+	knownChildren.add(102)
+	got := adoptedZombiePIDs(procRoot, 1, knownChildren)
+	assert.Equal(t, []int{101}, got)
+
+	knownChildren.remove(102)
+	got = adoptedZombiePIDs(procRoot, 1, knownChildren)
+	assert.Equal(t, []int{101, 102}, got)
+}
+
+func TestKnownChildRemovalSignalsRescan(t *testing.T) {
+	knownChildren := newKnownChildPIDs()
+	knownChildren.add(102)
+	knownChildren.remove(102)
+
+	assert.False(t, knownChildren.contains(102))
+	signaled := false
+	select {
+	case <-knownChildren.changed:
+		signaled = true
+	default:
+	}
+	assert.True(t, signaled)
+}
+
+func TestReapAdoptedZombies(t *testing.T) {
+	procRoot := t.TempDir()
+	writeProcStat(t, procRoot, 201, "first", 'Z', 1)
+	writeProcStat(t, procRoot, 202, "second", 'Z', 1)
+	writeProcStat(t, procRoot, 203, "known", 'Z', 1)
+
+	var reaped []int
+	wait4 := func(pid int, _ *syscall.WaitStatus, options int, _ *syscall.Rusage) (int, error) {
+		assert.Equal(t, syscall.WNOHANG, options)
+		reaped = append(reaped, pid)
+		return pid, nil
+	}
+
+	knownChildren := newKnownChildPIDs()
+	knownChildren.add(203)
+	reapAdoptedZombies(procRoot, 1, knownChildren, wait4)
+	assert.Equal(t, []int{201, 202}, reaped)
+}
+
+func writeProcStat(t *testing.T, procRoot string, pid int, name string, state byte, ppid int) {
+	t.Helper()
+	dir := filepath.Join(procRoot, fmt.Sprintf("%d", pid))
+	require.NoError(t, os.Mkdir(dir, 0o755))
+	stat := fmt.Sprintf("%d (%s) %c %d 0 0 0", pid, name, state, ppid)
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "stat"), []byte(stat), 0o644))
+}