feat(release): ship a DWARF debug companion for prod kernels

.github/workflows/release.yml

@@ -82,8 +82,27 @@ jobs:
             exit 1
           fi
 
+  # Gate: enabling DWARF on a prod config must be code-neutral (only non-loadable
+  # .debug_* sections added, boot image unchanged). Builds the x86_64 DWARF config
+  # twice (with/without DWARF) and compares loadable segments; `publish` depends on
+  # it, so a non-neutral kernel can never be released. Runs only on the publish path
+  # (PRs are covered by verify-dwarf-neutral.yml).
+  verify-dwarf:
+    needs: matrix
+    if: github.event_name == 'workflow_dispatch'
+    name: Verify DWARF code-neutral (x86_64)
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - name: Verify DWARF is code-neutral
+        # No version: verifies every x86_64 config that enables DWARF (auto-tracks
+        # whichever kernel currently carries it), so a bump can't bypass the gate.
+        run: sudo --preserve-env ./scripts/verify-dwarf-code-neutral.sh
+
   publish:
-    needs: build
+    needs: [build, verify-dwarf]
     if: github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-24.04
     permissions:
@@ -108,6 +127,9 @@ jobs:
             name=$(basename "$dir")
             [ -f "$dir/amd64/vmlinux.bin" ] && cp "$dir/amd64/vmlinux.bin" "release-assets/${name}-amd64.bin"
             [ -f "$dir/arm64/vmlinux.bin" ] && cp "$dir/arm64/vmlinux.bin" "release-assets/${name}-arm64.bin"
+            # DWARF debug companions (present when the config builds with debug info).
+            [ -f "$dir/amd64/vmlinux.debug" ] && cp "$dir/amd64/vmlinux.debug" "release-assets/${name}-amd64.debug"
+            [ -f "$dir/arm64/vmlinux.debug" ] && cp "$dir/arm64/vmlinux.debug" "release-assets/${name}-arm64.debug"
             # Legacy non-arch-suffixed asset (= amd64) for backwards compat.
             [ -f "$dir/vmlinux.bin" ] && cp "$dir/vmlinux.bin" "release-assets/${name}.bin"
           done

.github/workflows/verify-dwarf-neutral.yml

@@ -0,0 +1,31 @@
+name: Verify DWARF code-neutral
+
+# Guards that enabling DWARF on a prod kernel config does not change the boot
+# image (loadable segments). Runs on changes to the kernel build inputs and on
+# manual dispatch. Two full kernel builds — keep separate from the release path.
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Kernel version to verify (default: all DWARF-enabled x86_64 configs)'
+        required: false
+        type: string
+        default: ''
+  pull_request:
+    paths:
+      - 'configs/x86_64/**'
+      - 'build.sh'
+      - 'scripts/check-loadable-sections.sh'
+      - 'scripts/verify-dwarf-code-neutral.sh'
+
+permissions:
+  contents: read
+
+jobs:
+  verify:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Verify DWARF is code-neutral (x86_64)
+        # Empty version => verify every x86_64 config that enables DWARF.
+        run: sudo --preserve-env ./scripts/verify-dwarf-code-neutral.sh "${{ inputs.version }}"

build.sh

@@ -25,7 +25,7 @@ normalize_arch() {
 install_dependencies() {
   local target_arch="$1"
   local packages=(
-    bc bison busybox-static cpio curl flex gcc libelf-dev libssl-dev make patch squashfs-tools tree
+    bc binutils bison busybox-static cpio curl flex gcc libelf-dev libssl-dev make patch squashfs-tools tree
   )
 
   [[ "$target_arch" == "arm64" && "$HOST_ARCH" != "aarch64" ]] && packages+=( gcc-aarch64-linux-gnu )
@@ -93,16 +93,23 @@ build_version() {
 
   echo "Copying finished build to builds directory"
   local out_dir="$SCRIPT_DIR/builds/vmlinux-${version}/${output_arch}"
+  local legacy_dir="$SCRIPT_DIR/builds/vmlinux-${version}"
   mkdir -p "$out_dir"
   if [[ "$target_arch" == "arm64" ]]; then
     cp arch/arm64/boot/Image "$out_dir/vmlinux.bin"
+  elif readelf -S vmlinux | grep -q '\.debug_info'; then
+    # The config builds with DWARF. Ship a lean boot image (loadable segments +
+    # symtab, DWARF stripped) plus a split vmlinux.debug companion. --strip-debug
+    # only removes non-loadable .debug_* sections, so the boot image's loadable
+    # segments are unchanged vs a no-DWARF build.
+    objcopy --only-keep-debug vmlinux "$out_dir/vmlinux.debug"
+    objcopy --strip-debug vmlinux "$out_dir/vmlinux.bin"
+    # legacy path (x86_64, no arch subdir) for backwards compat
+    cp "$out_dir/vmlinux.bin" "$legacy_dir/vmlinux.bin"
+    cp "$out_dir/vmlinux.debug" "$legacy_dir/vmlinux.debug"
   else
     cp vmlinux "$out_dir/vmlinux.bin"
-  fi
-
-  # x86_64: also copy to legacy path (no arch subdir) for backwards compat.
-  if [[ "$target_arch" == "x86_64" ]]; then
-    cp vmlinux "$SCRIPT_DIR/builds/vmlinux-${version}/vmlinux.bin"
+    cp vmlinux "$legacy_dir/vmlinux.bin"
   fi
 }
 

configs/x86_64/6.1.158.config

@@ -3044,10 +3044,10 @@ CONFIG_DEBUG_MISC=y
 # Compile-time checks and compiler options
 #
 CONFIG_AS_HAS_NON_CONST_LEB128=y
-CONFIG_DEBUG_INFO_NONE=y
+# CONFIG_DEBUG_INFO_NONE is not set
 # CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT is not set
 # CONFIG_DEBUG_INFO_DWARF4 is not set
-# CONFIG_DEBUG_INFO_DWARF5 is not set
+CONFIG_DEBUG_INFO_DWARF5=y
 CONFIG_FRAME_WARN=2048
 CONFIG_STRIP_ASM_SYMS=y
 # CONFIG_READABLE_ASM is not set

scripts/check-loadable-sections.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# Verify two kernel ELF images have byte-identical *executable code*.
+#
+# Enabling DWARF on a prod config is meant to be codegen-neutral: it only adds
+# non-loadable .debug_* sections, so the machine code must be unchanged versus a
+# no-DWARF build (see scripts/verify-dwarf-code-neutral.sh).
+#
+# We compare the executable (SHF_EXECINSTR) sections — .text and friends — rather
+# than the whole loadable image, because the non-code loadable data legitimately
+# differs between two independent builds and is not codegen:
+#   - the GNU build-id note (.notes) is a hash over the build (it covers .debug_*,
+#     so it changes when DWARF is toggled even though the code does not);
+#   - the ".version" build counter ("#N" in linux_banner, .rodata/.data) increments
+#     on every relink.
+# The caller (verify-dwarf-code-neutral.sh) additionally builds with CONFIG_IKCONFIG
+# disabled, so the embedded /proc/config.gz blob — whose gzip size depends on the
+# config text, which differs by the debug-info lines — does not shift .init.data and
+# the addresses that .init.text references.
+set -euo pipefail
+
+if [[ $# -ne 2 ]]; then
+  echo "usage: $0 <vmlinux-a> <vmlinux-b>" >&2
+  exit 2
+fi
+
+tmp=$(mktemp -d)
+trap 'rm -rf "$tmp"' EXIT
+
+# Executable sections (flags "AX") of the first image. DWARF only adds non-loadable
+# .debug_* sections, so the executable section set is identical between the two.
+# (Plain read loop rather than mapfile/readarray to stay bash 3.2 compatible.)
+secs=()
+while IFS= read -r sec; do
+  secs+=("$sec")
+done < <(readelf -SW "$1" | grep -E ' AX ' | sed -E 's/.*\] (\.[^ ]+) .*/\1/' | sort -u)
+if [[ ${#secs[@]} -eq 0 ]]; then
+  echo "no executable sections found in $1" >&2
+  exit 2
+fi
+
+rc=0
+for s in "${secs[@]}"; do
+  objcopy -O binary --only-section="$s" "$1" "$tmp/a" 2>/dev/null
+  objcopy -O binary --only-section="$s" "$2" "$tmp/b" 2>/dev/null
+  if ! cmp -s "$tmp/a" "$tmp/b"; then
+    echo "MISMATCH: executable section $s differs between '$1' and '$2'" >&2
+    rc=1
+  fi
+done
+
+if [[ "$rc" -eq 0 ]]; then
+  echo "OK: executable code is byte-identical (${#secs[@]} sections)"
+else
+  echo "Enabling DWARF must not change codegen; investigate before releasing." >&2
+fi
+exit "$rc"

scripts/upload-release-to-gcs.sh

@@ -77,7 +77,7 @@ ASSETS=()
 while IFS= read -r line || [[ -n "$line" ]]; do
   [[ -n "$line" ]] && ASSETS+=("$line")
 done < <(gh release view "$RELEASE_TAG" --repo "$REPO" --json assets \
-  --jq '.assets[] | select(.name | test("^vmlinux-.*\\.bin$")) | .name')
+  --jq '.assets[] | select(.name | test("^vmlinux-.*\\.(bin|debug)$")) | .name')
 
 if [[ "${#ASSETS[@]}" -eq 0 ]]; then
   echo "ERROR: release $RELEASE_TAG has no vmlinux-*.bin assets" >&2
@@ -90,13 +90,19 @@ trap 'rm -rf "$TMP_DIR"' EXIT
 uploaded=0
 skipped=0
 for asset in "${ASSETS[@]}"; do
-  if [[ ! "$asset" =~ ^vmlinux-(.+)-(amd64|arm64)\.bin$ ]]; then
+  if [[ "$asset" =~ ^vmlinux-(.+)-(amd64|arm64)\.bin$ ]]; then
+    version="${BASH_REMATCH[1]}"
+    arch="${BASH_REMATCH[2]}"
+    dst="${BUCKET_URI}/vmlinux-${version}_${SHORT_HASH}/${arch}/vmlinux.bin"
+  elif [[ "$asset" =~ ^vmlinux-(.+)-(amd64|arm64)\.debug$ ]]; then
+    # DWARF debug companion for the boot image. Fetched only when debugging.
+    version="${BASH_REMATCH[1]}"
+    arch="${BASH_REMATCH[2]}"
+    dst="${BUCKET_URI}/vmlinux-${version}_${SHORT_HASH}/${arch}/vmlinux.debug"
+  else
     # Legacy non-arch release asset or unrecognized name — not uploaded.
     continue
   fi
-  version="${BASH_REMATCH[1]}"
-  arch="${BASH_REMATCH[2]}"
-  dst="${BUCKET_URI}/vmlinux-${version}_${SHORT_HASH}/${arch}/vmlinux.bin"
 
   if gcloud storage ls "$dst" >/dev/null 2>&1; then
     echo "  EXISTS  $dst"

scripts/verify-dwarf-code-neutral.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+# Self-contained guard that enabling DWARF on a prod kernel config is code-neutral.
+#
+# For each verified version, builds the kernel twice from the same source/tag/patches
+# — WITH DWARF (the committed config) and WITHOUT DWARF (a generated reference) — with
+# build metadata pinned, and asserts the boot image's loadable segments are
+# byte-identical. DWARF lives only in non-loadable .debug_* sections, so a clean build
+# must produce the same loadable image; a mismatch means enabling DWARF perturbed
+# codegen.
+#
+# Usage: verify-dwarf-code-neutral.sh [version] [arch]
+# With no version, verifies every x86_64 config that enables CONFIG_DEBUG_INFO_DWARF5,
+# so the gate auto-tracks whatever kernel currently carries DWARF (no hardcoded
+# version). x86_64 only. Two full kernel builds per version — intended for CI /
+# occasional local runs, not every build.
+set -euo pipefail
+
+arch="${2:-x86_64}"
+if [[ "$arch" != "x86_64" ]]; then
+  echo "verify-dwarf-code-neutral: x86_64 only, skipping $arch"
+  exit 0
+fi
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+
+# Pin build metadata so linux_banner (.rodata, loadable) is identical across both
+# builds; otherwise the embedded timestamp/builder would differ.
+export KBUILD_BUILD_TIMESTAMP="2024-01-01"
+export KBUILD_BUILD_USER="ci"
+export KBUILD_BUILD_HOST="ci"
+
+work_root=$(mktemp -d)
+trap 'rm -rf "$work_root"' EXIT
+
+# Build vmlinux in the (already checked-out, patched) linux tree from a config
+# derived by applying $1 (sed program) to the resolved DWARF config, and copy the
+# result to $2.
+build_variant() {
+  local sed_prog="$1" out="$2" work
+  work="$(dirname "$out")"
+  sed -e "$sed_prog" "$work/dwarf.config" >"$work/variant.config"
+  ( cd "$SCRIPT_DIR/linux"
+    cp "$work/variant.config" .config
+    make olddefconfig
+    make vmlinux -j "$(nproc)"
+    cp vmlinux "$out" )
+}
+
+# Verify one version: build a WITH-DWARF and a WITHOUT-DWARF kernel that differ
+# only in the debug-info toggle, and assert their executable code is identical.
+verify_one() {
+  local version="$1"
+  local work="$work_root/$version"
+  mkdir -p "$work"
+
+  # Real build: validates the shipped (IKCONFIG=y) kernel compiles, and leaves the
+  # linux tree checked out at the right tag with patches applied + the resolved
+  # DWARF config in linux/.config.
+  echo ">>> [$version] build WITH DWARF (committed config, real build)"
+  "$SCRIPT_DIR/build.sh" "$version" "$arch"
+  cp "$SCRIPT_DIR/linux/.config" "$work/dwarf.config"
+
+  # check-loadable-sections.sh compares the executable code, which is immune to the
+  # per-build GNU build-id and the "#N" build counter. CONFIG_IKCONFIG must still be
+  # disabled on both, though: with it on, each kernel embeds its own .config
+  # (/proc/config.gz), the two configs differ by the debug-info lines, so the gzip
+  # blob differs in size and shifts .init.data — moving symbols that .init.text
+  # references and thus perturbing the compared code. Disabling it keeps .init.data
+  # layout identical, so the only remaining difference is DWARF (non-loadable
+  # .debug_*) and the executable code matches exactly.
+  local ikconfig_off='s/^CONFIG_IKCONFIG=y$/# CONFIG_IKCONFIG is not set/'
+  local dwarf_off='s/^CONFIG_DEBUG_INFO_DWARF5=y$/# CONFIG_DEBUG_INFO_DWARF5 is not set/;s/^# CONFIG_DEBUG_INFO_NONE is not set$/CONFIG_DEBUG_INFO_NONE=y/'
+
+  echo ">>> [$version] build WITH DWARF, IKCONFIG off (A)"
+  build_variant "$ikconfig_off" "$work/dwarf.bin"
+
+  echo ">>> [$version] build WITHOUT DWARF, IKCONFIG off (B)"
+  build_variant "${ikconfig_off};${dwarf_off}" "$work/nodwarf.bin"
+
+  echo ">>> [$version] compare executable code"
+  "$SCRIPT_DIR/scripts/check-loadable-sections.sh" "$work/dwarf.bin" "$work/nodwarf.bin"
+  echo "OK: DWARF is code-neutral for ${version} (${arch})."
+}
+
+# Versions to verify: the explicit arg, else every x86_64 config enabling DWARF.
+versions=()
+if [[ -n "${1:-}" ]]; then
+  versions=("$1")
+else
+  for cfg in "$SCRIPT_DIR"/configs/"$arch"/*.config; do
+    [[ -e "$cfg" ]] || continue
+    grep -q '^CONFIG_DEBUG_INFO_DWARF5=y' "$cfg" || continue
+    v="$(basename "$cfg" .config)"
+    # Skip variant configs (e.g. 6.1.158-numaemu); only plain version names ship.
+    [[ "$v" =~ ^[0-9]+(\.[0-9]+)+$ ]] || continue
+    versions+=("$v")
+  done
+fi
+
+if [[ ${#versions[@]} -eq 0 ]]; then
+  echo "no ${arch} config enables CONFIG_DEBUG_INFO_DWARF5; nothing to verify"
+  exit 0
+fi
+
+echo "verifying DWARF code-neutrality for: ${versions[*]}"
+for v in "${versions[@]}"; do
+  verify_one "$v"
+done