From 4c4e6acd3591aff15af3e9e7e93b4ddbf043a167 Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Sun, 16 Aug 2026 09:44:06 -0400 Subject: [PATCH] fix(config): harden production DaemonSet manifests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A production-readiness pass over config/ against the CIS Benchmark and Pod Security Standards flagged several gaps across galactic-cni, galactic-router, galactic-gateway, fabric-router, and vmtap-cni. This closes the mechanical ones. - dnsPolicy: ClusterFirstWithHostNet on every hostNetwork: true pod (cni, gateway, router, vmtap; fabric already had it) — without it a hostNetwork pod silently falls back to the host's own /etc/resolv.conf instead of cluster DNS, with no admission error, just a kubelet-log warning. - priorityClassName: system-node-critical on cni, gateway, router, and vmtap, matching fabric-router. galactic-cni in particular is more fundamental than any other node component: if evicted under node pressure, no pod (including the CNI's own peers) can schedule on that node at all. - capabilities.drop: ["ALL"] paired with every capabilities.add across cni, gateway, and router. Without an explicit drop, containers retain the runtime default capability set on top of whatever they actually add. - seccompProfile: RuntimeDefault at pod level on all five DaemonSets. - readOnlyRootFilesystem: true on every container across cni, gateway, router, and vmtap, verified none of the four binaries write anywhere outside their already-mounted hostPath volumes. - startupProbe on cni, router, and both gateway containers, reusing the existing gRPC health checks with a longer failureThreshold, since BPF map creation and GoBGP's cold start can outrun the short initialDelaySeconds on readinessProbe/livenessProbe and trigger a restart loop before the process finishes initializing. - Explicit updateStrategy: RollingUpdate maxUnavailable: 1 on cni, gateway, router, and vmtap, matching fabric. - pod-security.kubernetes.io/audit and /warn: baseline alongside the existing enforce: privileged on the galactic-system namespace, so drift now surfaces in events/audit logs instead of nowhere. - automountServiceAccountToken: false on vmtap-cni, which has no RBAC at all — internal/vmtap has no Kubernetes client, so mounting a ServiceAccount token there granted zero capability for no reason. - Every DaemonSet's actual listening ports are now declared explicitly via containerPort, even though hostNetwork: true makes this documentation rather than something Kubernetes enforces. Every BGP-speaking container previously had zero containerPort entries for its real BGP port, and galactic-cni's metrics port had no trace anywhere in the manifest at all. Making those ports explicit meant actually looking at the numbers, which surfaced two collision-prone defaults. galactic-router's metrics port moved from 8080 to 9179, next to its own grpc-health port (5179) instead of the generic controller-runtime default. galactic-router's grpc-health code default moved from 5000 to 5179: 5000 collides with Talos's /sbin/dashboard, macOS AirPlay Receiver, Flask's dev server, and Docker Registry, and every deployed manifest already had to override it for exactly that reason. galactic-cni's metrics port moved from 9091 (Prometheus Pushgateway's own reserved default) to 9180, next to galactic-router's 9179. Verified via kubectl kustomize on every affected kustomization, and task lint, task build, task test:unit. Co-Authored-By: Claude Sonnet 5 --- cmd/galactic-cni/main.go | 3 +- config/cni/daemonset.yaml | 42 ++++++++++++++++ config/fabric/daemonset.yaml | 13 +++++ config/gateway/base/daemonset.yaml | 49 ++++++++++++++++--- config/router/base/daemonset.yaml | 41 ++++++++++++++-- .../tenant-control/daemonset-patch.yaml | 11 +++++ config/router/tenant/daemonset-patch.yaml | 4 ++ config/system/namespace.yaml | 6 +++ config/vmtap/daemonset.yaml | 16 ++++++ docs/agent-startup.md | 4 +- docs/agents/ARCHITECTURE-GATEWAY.md | 6 +-- docs/agents/ARCHITECTURE-ROUTER.md | 24 ++++----- docs/router/configuration.md | 19 ++++--- internal/config/gateway.go | 7 ++- internal/config/router.go | 10 ++-- 15 files changed, 208 insertions(+), 47 deletions(-) diff --git a/cmd/galactic-cni/main.go b/cmd/galactic-cni/main.go index 0b1cbd00..9444ce5e 100644 --- a/cmd/galactic-cni/main.go +++ b/cmd/galactic-cni/main.go @@ -68,7 +68,8 @@ func newRunCommand() *cobra.Command { }, } runCmd.Flags().IntVar(&grpcHealthPort, "grpc-health-port", 5180, "gRPC health check port") - runCmd.Flags().IntVar(&metricsPort, "metrics-port", 9091, "Prometheus metrics HTTP port") + // 9180 sits next to galactic-router's own metrics port (9179). + runCmd.Flags().IntVar(&metricsPort, "metrics-port", 9180, "Prometheus metrics HTTP port") return runCmd } diff --git a/config/cni/daemonset.yaml b/config/cni/daemonset.yaml index 3ccc1ec3..8e86ec63 100644 --- a/config/cni/daemonset.yaml +++ b/config/cni/daemonset.yaml @@ -9,6 +9,10 @@ spec: selector: matchLabels: app.kubernetes.io/name: galactic-cni + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 template: metadata: labels: @@ -16,6 +20,14 @@ spec: spec: serviceAccountName: galactic-cni hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + # galactic-cni is more fundamental than any other node component: if + # evicted under node pressure, no pod (including the CNI's own peers) + # can schedule on this node at all. + priorityClassName: system-node-critical + securityContext: + seccompProfile: + type: RuntimeDefault tolerations: - operator: Exists # Only worker nodes run VPC pods; galactic-cni has nothing to do on @@ -50,7 +62,10 @@ spec: fieldPath: spec.nodeName securityContext: runAsUser: 0 + capabilities: + drop: ["ALL"] allowPrivilegeEscalation: false + readOnlyRootFilesystem: true resources: requests: cpu: 10m @@ -72,6 +87,19 @@ spec: # placeholder, not a published tag. image: ghcr.io/datum-cloud/galactic-cni:latest command: ["/galactic-cni", "run"] + # Documentation only: hostNetwork: true means these are the + # process's actual listen ports on the node, not something + # Kubernetes publishes -- containerPort here doesn't change + # behavior, it just makes both ports (--grpc-health-port/ + # --metrics-port defaults) visible to `kubectl describe pod` + # instead of only the gRPC one showing up via the probes below. + ports: + - name: grpc-health + containerPort: 5180 + protocol: TCP + - name: metrics + containerPort: 9180 + protocol: TCP # --- PRIVILEGE EXPANSION (Milestone 3.1 of # .local/implementation-plan-ebpf-xdp-usid-datapath.md) --- # This container previously ran with allowPrivilegeEscalation: @@ -89,10 +117,12 @@ spec: securityContext: runAsUser: 0 capabilities: + drop: ["ALL"] add: - BPF - NET_ADMIN allowPrivilegeEscalation: false + readOnlyRootFilesystem: true resources: requests: cpu: 5m @@ -109,6 +139,18 @@ spec: mountPath: /host/var/log/galactic - name: bpf-fs mountPath: /sys/fs/bpf + # BPF map creation under load (image pull + verifier work on a + # slow node) can outrun readinessProbe/livenessProbe's short + # initialDelaySeconds, risking a liveness-triggered restart loop + # before the process finishes initializing. startupProbe reuses + # the same gRPC health check with a much longer failureThreshold + # so liveness doesn't start counting until this succeeds once. + startupProbe: + grpc: + port: 5180 + service: ebpf-datapath + periodSeconds: 5 + failureThreshold: 30 livenessProbe: grpc: port: 5180 diff --git a/config/fabric/daemonset.yaml b/config/fabric/daemonset.yaml index 08a35d97..62358b74 100644 --- a/config/fabric/daemonset.yaml +++ b/config/fabric/daemonset.yaml @@ -21,6 +21,9 @@ spec: hostNetwork: true dnsPolicy: ClusterFirstWithHostNet priorityClassName: system-node-critical + securityContext: + seccompProfile: + type: RuntimeDefault # FRR here establishes the underlay eBGP session to the physical fabric # and brings up the node's lo address, both of which galactic-router # depends on before it can start — so this must tolerate NotReady the @@ -110,6 +113,16 @@ spec: containers: - name: frr image: ghcr.io/datum-cloud/fabric-router:latest + # Documentation only (hostNetwork: true, so this doesn't change + # behavior): the standard BGP port, opened by bgpd itself once + # the per-node frr.conf. (see fabric-config's + # frr-init above) enables it -- nothing here actually declares + # or gates it, this just makes it visible without reading FRR's + # own config. + ports: + - name: bgp + containerPort: 179 + protocol: TCP lifecycle: preStop: exec: diff --git a/config/gateway/base/daemonset.yaml b/config/gateway/base/daemonset.yaml index d6f33534..b17b1f21 100644 --- a/config/gateway/base/daemonset.yaml +++ b/config/gateway/base/daemonset.yaml @@ -9,6 +9,10 @@ spec: selector: matchLabels: app.kubernetes.io/name: galactic-gateway + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 template: metadata: labels: @@ -36,6 +40,14 @@ spec: # bindings grant the union of what each container needs. serviceAccountName: galactic-gateway hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + # Loss of galactic-gateway blackholes edge traffic through this + # node, so the scheduler/kubelet must never preempt it for ordinary + # workloads under node pressure. + priorityClassName: system-node-critical + securityContext: + seccompProfile: + type: RuntimeDefault tolerations: - operator: Exists containers: @@ -67,23 +79,32 @@ spec: - name: GALACTIC_ROUTER_GC_NAMESPACE value: galactic-system - name: GALACTIC_ROUTER_BGP_LISTEN_PORT + # Same as config/router/tenant/daemonset-patch.yaml: -1 + # disables the inbound BGP listener, so there's no BGP + # containerPort to declare on this container either -- this + # role only dials out to iBGP peers. value: "-1" - # Talos nodes permanently bind 127.0.0.1:5000 for the built-in - # /sbin/dashboard; since this DaemonSet runs hostNetwork: true, - # the default gRPC health port (5000) always collides on Talos - # — see docs/router/configuration.md. This pod's galactic-gateway - # container shares the same network namespace (hostNetwork: - # true), so its own grpc-health port (below) must also differ - # from this one, not just from 5000. + # 5179 is also the binary's own default. Set explicitly here + # anyway so it can't silently drift, and — more importantly — + # this pod's galactic-gateway container shares the same + # network namespace (hostNetwork: true), so its own + # grpc-health port (below) must always differ from this one. - name: GALACTIC_ROUTER_GRPC_HEALTH_PORT value: "5179" ports: - name: metrics - containerPort: 8080 + containerPort: 9179 protocol: TCP - name: grpc-health containerPort: 5179 protocol: TCP + # See config/router/base/daemonset.yaml: same GoBGP cold-start + # rationale for startupProbe. + startupProbe: + grpc: + port: 5179 + periodSeconds: 5 + failureThreshold: 30 livenessProbe: grpc: port: 5179 @@ -97,9 +118,11 @@ spec: securityContext: runAsUser: 0 capabilities: + drop: ["ALL"] add: - NET_ADMIN allowPrivilegeEscalation: false + readOnlyRootFilesystem: true resources: requests: cpu: 50m @@ -149,6 +172,14 @@ spec: - name: grpc-health containerPort: 5181 protocol: TCP + # The eBPF XDP loader (map creation/attach) has the same + # slow-cold-start shape as galactic-cni's — see that manifest's + # startupProbe comment. + startupProbe: + grpc: + port: 5181 + periodSeconds: 5 + failureThreshold: 30 livenessProbe: grpc: port: 5181 @@ -176,11 +207,13 @@ spec: securityContext: runAsUser: 0 capabilities: + drop: ["ALL"] add: - NET_ADMIN - BPF - PERFMON allowPrivilegeEscalation: false + readOnlyRootFilesystem: true resources: requests: cpu: 50m diff --git a/config/router/base/daemonset.yaml b/config/router/base/daemonset.yaml index c117ce95..9e51ecb9 100644 --- a/config/router/base/daemonset.yaml +++ b/config/router/base/daemonset.yaml @@ -9,6 +9,10 @@ spec: selector: matchLabels: app.kubernetes.io/name: galactic-router + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 template: metadata: labels: @@ -16,6 +20,14 @@ spec: spec: serviceAccountName: galactic-router hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + # Loss of galactic-router blackholes every VPC route through this + # node, so the scheduler/kubelet must never preempt it for ordinary + # workloads under node pressure. + priorityClassName: system-node-critical + securityContext: + seccompProfile: + type: RuntimeDefault tolerations: - operator: Exists containers: @@ -39,19 +51,36 @@ spec: value: tenant - name: GALACTIC_ROUTER_GC_NAMESPACE value: galactic-system - # Talos nodes permanently bind 127.0.0.1:5000 for the built-in - # /sbin/dashboard; since this DaemonSet runs hostNetwork: true, - # the default gRPC health port (5000) always collides on Talos. - # See docs/router/configuration.md for details. + # 5179 is also the binary's own default. Set explicitly here + # anyway, matching this manifest's convention elsewhere, so it + # can't silently drift if the binary's default ever changes. - name: GALACTIC_ROUTER_GRPC_HEALTH_PORT value: "5179" ports: - name: metrics - containerPort: 8080 + containerPort: 9179 protocol: TCP - name: grpc-health containerPort: 5179 protocol: TCP + # No BGP containerPort here: GALACTIC_ROUTER_BGP_LISTEN_PORT + # isn't set at this (never-applied-directly) base level at all, + # and its real value is role-dependent -- see + # ../tenant-control/daemonset-patch.yaml (1790, the one role + # that actually listens) and ../tenant/daemonset-patch.yaml + # (-1, disabled). Declaring one here would just be wrong for + # whichever role doesn't match it. + # GoBGP's cold start (peer session setup, RIB population) can + # outrun readinessProbe/livenessProbe's short initialDelaySeconds + # on a slow node, risking a liveness-triggered restart loop before + # the process finishes initializing. startupProbe reuses the same + # gRPC health check with a much longer failureThreshold so + # liveness doesn't start counting until this succeeds once. + startupProbe: + grpc: + port: 5179 + periodSeconds: 5 + failureThreshold: 30 livenessProbe: grpc: port: 5179 @@ -65,9 +94,11 @@ spec: securityContext: runAsUser: 0 capabilities: + drop: ["ALL"] add: - NET_ADMIN allowPrivilegeEscalation: false + readOnlyRootFilesystem: true resources: requests: cpu: 50m diff --git a/config/router/tenant-control/daemonset-patch.yaml b/config/router/tenant-control/daemonset-patch.yaml index c8d91c1c..1f6c9e83 100644 --- a/config/router/tenant-control/daemonset-patch.yaml +++ b/config/router/tenant-control/daemonset-patch.yaml @@ -41,3 +41,14 @@ spec: # host's lo interface by default (the stable peering address # other tenant routers reflect through). Set it explicitly here # only if lo doesn't carry the desired address on this cluster. + # Documentation only (hostNetwork: true, so this doesn't change + # behavior): the reflector role is the one place galactic-router + # actually listens for inbound BGP -- tenant nodes (see + # ../tenant/daemonset-patch.yaml) and the router container in + # config/gateway/base/daemonset.yaml both set + # GALACTIC_ROUTER_BGP_LISTEN_PORT=-1 and open no BGP port at all, + # so this entry only belongs on this patch, not the shared base. + ports: + - name: bgp + containerPort: 1790 + protocol: TCP diff --git a/config/router/tenant/daemonset-patch.yaml b/config/router/tenant/daemonset-patch.yaml index f010d444..77cd6f2e 100644 --- a/config/router/tenant/daemonset-patch.yaml +++ b/config/router/tenant/daemonset-patch.yaml @@ -34,4 +34,8 @@ spec: - name: galactic-router env: - name: GALACTIC_ROUTER_BGP_LISTEN_PORT + # Tenant nodes only dial *out* to iBGP peers (the reflector, + # see ../tenant-control/daemonset-patch.yaml); -1 disables + # the inbound listener entirely, so no BGP containerPort + # belongs on this patch -- there's no port here to document. value: "-1" diff --git a/config/system/namespace.yaml b/config/system/namespace.yaml index 8be820e1..41cf42bd 100644 --- a/config/system/namespace.yaml +++ b/config/system/namespace.yaml @@ -9,3 +9,9 @@ metadata: # against a cluster with the restricted/baseline PodSecurity default # fails admission with no pointer to the fix. pod-security.kubernetes.io/enforce: privileged + # audit/warn don't change enforcement (enforce: privileged already + # allows everything) — they just surface drift from the baseline + # profile in events/audit logs as the manifest set evolves, at no + # cost since nothing is actually blocked. + pod-security.kubernetes.io/audit: baseline + pod-security.kubernetes.io/warn: baseline diff --git a/config/vmtap/daemonset.yaml b/config/vmtap/daemonset.yaml index 92cb5b48..0cab8898 100644 --- a/config/vmtap/daemonset.yaml +++ b/config/vmtap/daemonset.yaml @@ -9,13 +9,27 @@ spec: selector: matchLabels: app.kubernetes.io/name: vmtap-cni + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 template: metadata: labels: app.kubernetes.io/name: vmtap-cni spec: serviceAccountName: vmtap-cni + # There's no rbac.yaml for vmtap-cni at all (internal/vmtap has no + # k8s client — it only patches a local conflist file and manages + # TAP/TC state), so the default ServiceAccount token would grant this + # pod zero capability while still being mounted for no reason. + automountServiceAccountToken: false hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + priorityClassName: system-node-critical + securityContext: + seccompProfile: + type: RuntimeDefault tolerations: - operator: Exists # Placeholder node signal: only nodes actually running kraftlet-managed @@ -48,6 +62,7 @@ spec: securityContext: runAsUser: 0 allowPrivilegeEscalation: false + readOnlyRootFilesystem: true resources: requests: cpu: 10m @@ -71,6 +86,7 @@ spec: securityContext: runAsUser: 0 allowPrivilegeEscalation: false + readOnlyRootFilesystem: true resources: requests: cpu: 5m diff --git a/docs/agent-startup.md b/docs/agent-startup.md index 326975da..34022543 100644 --- a/docs/agent-startup.md +++ b/docs/agent-startup.md @@ -8,8 +8,8 @@ sequenceDiagram Router->>Router: validate config (--node-name, --mode: transit/fabric/tenant, ...) Router->>Router: select RuntimeFactory: tenant→GoBGP, fabric→FRR stub, transit→error (unsupported) - Router->>Kubernetes: build controller-runtime manager (metrics on :8080, no HTTP health) - Router->>Router: start gRPC health server (:5000, SERVING immediately) + Router->>Kubernetes: build controller-runtime manager (metrics on :9179, no HTTP health) + Router->>Router: start gRPC health server (:5179, SERVING immediately) Router->>Kubernetes: RBAC pre-flight — SelfSubjectAccessReview per watched resource (logs if watch denied) Router->>Kubernetes: register field indexes (BGPPeer×2, BGPPolicy, BGPAdvertisement, BGPVRFInstance, BGPRouter) Router->>Kubernetes: register controllers — BGPRouter, BGPPeer, BGPAdvertisement, BGPVRFInstance, BGPPolicy, Secret, Node, GC diff --git a/docs/agents/ARCHITECTURE-GATEWAY.md b/docs/agents/ARCHITECTURE-GATEWAY.md index 2544f4b5..6b73d580 100644 --- a/docs/agents/ARCHITECTURE-GATEWAY.md +++ b/docs/agents/ARCHITECTURE-GATEWAY.md @@ -296,8 +296,8 @@ degraded. The metrics/gRPC-health port defaults (`8081`/`5181`) deliberately differ from every other `galactic-*` process's own defaults -(`galactic-router`'s `8080`/`5000`, overridden to `5179` on gateway nodes; -`galactic-cni`'s credential-refresh health port `5180`) because +(`galactic-router`'s `9179`/`5179`; `galactic-cni`'s credential-refresh +health port `5180`, metrics port `9180`) because `galactic-gateway` runs as a second container in the same `hostNetwork: true` pod as `galactic-router` — every port it binds shares that node's network namespace with every other `galactic-*` process already @@ -306,7 +306,7 @@ pod's port table: | Container | Metrics | gRPC health | | ---------------------------------------------- | ------- | ----------- | -| `galactic-router` (this pod's tenant-BGP side) | `8080` | `5179` | +| `galactic-router` (this pod's tenant-BGP side) | `9179` | `5179` | | `galactic-gateway` | `8081` | `5181` | ### Two-container pod (`config/gateway/base/daemonset.yaml`) diff --git a/docs/agents/ARCHITECTURE-ROUTER.md b/docs/agents/ARCHITECTURE-ROUTER.md index 20501ce3..aacd15b9 100644 --- a/docs/agents/ARCHITECTURE-ROUTER.md +++ b/docs/agents/ARCHITECTURE-ROUTER.md @@ -142,9 +142,9 @@ lives in `root.go`'s `runCmd`: `GALACTIC_ROUTER_GC_INTERVAL`, `GALACTIC_ROUTER_REFLECTOR`. 2. Select `RuntimeFactory`: `tenant` → GoBGP, `fabric` → FRR stub, `transit` → returns an error ("not yet supported"). -3. Build controller-runtime manager (metrics on configurable port, default `:8080`; +3. Build controller-runtime manager (metrics on configurable port, default `:9179`; no HTTP health endpoint). -4. Start gRPC health server on a configurable port (default `:5000`). +4. Start gRPC health server on a configurable port (default `:5179`). 5. RBAC pre-flight: `checkWatchPermissions` (in `main.go`) issues a `SelfSubjectAccessReview` for every watched resource type and logs an actionable error if watch RBAC is missing (informer caches would otherwise silently never sync). @@ -177,18 +177,20 @@ sibling container instead. | `GALACTIC_ROUTER_REFLECTOR` | No | `false` | Enable route reflector mode; only valid for `fabric`/`tenant` | | `GALACTIC_ROUTER_BGP_LISTEN_PORT` | No | `179` | BGP TCP listen port; `-1` disables inbound connections (outbound-only) | | `GALACTIC_ROUTER_BGP_LOCAL_ADDRESS` | No | — | Source address for outgoing BGP TCP connections (numbered underlay use) | -| `GALACTIC_ROUTER_METRICS_PORT` | No | `8080` | controller-runtime Prometheus metrics port | -| `GALACTIC_ROUTER_GRPC_HEALTH_PORT` | No | `5000` | gRPC health check port (liveness/readiness probes) | +| `GALACTIC_ROUTER_METRICS_PORT` | No | `9179` | controller-runtime Prometheus metrics port | +| `GALACTIC_ROUTER_GRPC_HEALTH_PORT` | No | `5179` | gRPC health check port (liveness/readiness probes) | | `GALACTIC_ROUTER_GC_NAMESPACE` | No | `galactic-system` | Namespace the GC controller scans for orphaned CRDs | | `GALACTIC_ROUTER_GC_INTERVAL` | No | `5m` | GC controller sweep interval | See [docs/router/configuration.md](../router/configuration.md) for the full reference, including CLI flags and precedence. -On a gateway-role node, `GALACTIC_ROUTER_BGP_LISTEN_PORT=-1` and -`GALACTIC_ROUTER_GRPC_HEALTH_PORT=5179` are set by -`config/gateway/base/daemonset.yaml` specifically to avoid colliding with -the co-located `galactic-gateway` container's own ports on the same -`hostNetwork: true` pod — see +On a gateway-role node, `GALACTIC_ROUTER_BGP_LISTEN_PORT=-1` is set by +`config/gateway/base/daemonset.yaml` — the tenant-BGP container only dials +out to iBGP peers there, same as the plain tenant role. +`GALACTIC_ROUTER_GRPC_HEALTH_PORT=5179` is set explicitly too, matching +the binary's own default, so it can't silently start colliding with the +co-located `galactic-gateway` container's own health port on the same +`hostNetwork: true` pod if either default ever changes — see [ARCHITECTURE-GATEWAY.md#configuration](ARCHITECTURE-GATEWAY.md#configuration). --- @@ -222,7 +224,7 @@ the co-located `galactic-gateway` container's own ports on the same | `github.com/spf13/cobra` | v1.10.2 | CLI command/flag handling | | `github.com/spf13/viper` | v1.21.0 | Config resolution (flags/env/defaults) — unlike the CNI-chain binaries (see [ARCHITECTURE-CNI.md](ARCHITECTURE-CNI.md)), which resolve config themselves and don't import viper | | `github.com/vishvananda/netlink` | pinned pseudo-version | Linux netlink: VRF, SRv6 routes (GC's kernel-state sweep) | -| `google.golang.org/grpc` | v1.82.0 | gRPC health server (default `:5000`) | +| `google.golang.org/grpc` | v1.82.0 | gRPC health server (default `:5179`) | | `k8s.io/api`, `k8s.io/client-go` | v0.36.0 | Kubernetes client, Node/Secret API types | --- @@ -236,7 +238,7 @@ the co-located `galactic-gateway` container's own ports on the same - **Hash-based no-op suppression.** SHA-256 over the sorted `DesiredRouter` prevents redundant GoBGP Apply calls on every CRD event. - **RuntimeFactory pattern.** `--mode=tenant` (`GALACTIC_ROUTER_ROUTER_MODE=tenant`) selects GoBGP; `--mode=fabric` selects the FRR stub; `--mode=transit` is accepted by validation but returns an error at startup (not yet implemented). The mode is selected at startup; no controller changes are needed to add a new mode. - **DEL is intentionally minimal everywhere in the CNI chain; GC reclaims shared state asynchronously.** See [ARCHITECTURE-CNI.md](ARCHITECTURE-CNI.md#key-design-decisions) for the CNI-side half of this decision. `galactic-router`'s GC controller (ticker-driven, default every 5m) reclaims orphaned CRDs, stale kernel VRFs, and stale eBPF entries once no live container still references them. -- **gRPC health, configurable port.** Liveness and readiness probes use the gRPC health protocol (`google.golang.org/grpc/health`) on a configurable port (default `5000`). No HTTP health endpoint. `galactic-gateway` follows the identical convention on its own port — see [ARCHITECTURE-GATEWAY.md](ARCHITECTURE-GATEWAY.md). +- **gRPC health, configurable port.** Liveness and readiness probes use the gRPC health protocol (`google.golang.org/grpc/health`) on a configurable port (default `5179`). No HTTP health endpoint. `galactic-gateway` follows the identical convention on its own port — see [ARCHITECTURE-GATEWAY.md](ARCHITECTURE-GATEWAY.md). - **`galactic-router` carries no gateway-role code.** The edge XDP NAT+LB gateway used to be a `galactic-router` mode; it was split into its own `galactic-gateway` binary specifically so a crash on either side no longer takes the other down with it — see [ARCHITECTURE-GATEWAY.md](ARCHITECTURE-GATEWAY.md) for the full rationale and design. --- diff --git a/docs/router/configuration.md b/docs/router/configuration.md index 731a3125..58827812 100644 --- a/docs/router/configuration.md +++ b/docs/router/configuration.md @@ -22,8 +22,8 @@ exact name from the table below. | Route reflector | `GALACTIC_ROUTER_REFLECTOR` | `--reflector` | `false` | | BGP listen port | `GALACTIC_ROUTER_BGP_LISTEN_PORT` | `--bgp-listen-port` | `179` | | BGP local address | `GALACTIC_ROUTER_BGP_LOCAL_ADDRESS` | `--bgp-local-address` | _(auto-detected from `lo`)_ | -| Metrics port | `GALACTIC_ROUTER_METRICS_PORT` | `--metrics-port` | `8080` | -| gRPC health port | `GALACTIC_ROUTER_GRPC_HEALTH_PORT` | `--grpc-health-port` | `5000` | +| Metrics port | `GALACTIC_ROUTER_METRICS_PORT` | `--metrics-port` | `9179` | +| gRPC health port | `GALACTIC_ROUTER_GRPC_HEALTH_PORT` | `--grpc-health-port` | `5179` | | Orphan-cleanup namespace | `GALACTIC_ROUTER_GC_NAMESPACE` | `--gc-namespace` | `galactic-system` | | Orphan-cleanup interval | `GALACTIC_ROUTER_GC_INTERVAL` | `--gc-interval` | `5m` | @@ -96,7 +96,7 @@ TCP port for the controller-runtime metrics HTTP server. Exposes Prometheus metrics for monitoring. **Type:** integer -**Default:** `8080` +**Default:** `9179` **Valid values:** `1`–`65535` ### `--grpc-health-port` / `GALACTIC_ROUTER_GRPC_HEALTH_PORT` @@ -105,15 +105,14 @@ TCP port for the gRPC health check server. Used by Kubernetes liveness and readiness probes. **Type:** integer -**Default:** `5000` +**Default:** `5179` **Valid values:** `1`–`65535` -> **Talos:** `/sbin/dashboard` permanently binds `127.0.0.1:5000` on every -> Talos node. Since `galactic-router` runs with `hostNetwork: true`, the -> default `5000` always collides on Talos-based clusters. The shipped -> `config/router/base/daemonset.yaml` sets this to `5179` for exactly this -> reason; if you run `galactic-router` outside those manifests on Talos, set -> it to something other than `5000` yourself. +> **Why not 5000:** `5000` is one of the most overloaded dev ports in +> existence (macOS AirPlay Receiver, Flask's dev server, Docker Registry), +> and on Talos specifically `/sbin/dashboard` permanently binds +> `127.0.0.1:5000` on every node. Since `galactic-router` runs with +> `hostNetwork: true`, that would collide on Talos-based clusters. ### `--gc-namespace` / `GALACTIC_ROUTER_GC_NAMESPACE` diff --git a/internal/config/gateway.go b/internal/config/gateway.go index 6f79be52..d8abd323 100644 --- a/internal/config/gateway.go +++ b/internal/config/gateway.go @@ -17,10 +17,9 @@ import ( const ( // DefaultGatewayMetricsPort/DefaultGatewayGRPCHealthPort deliberately - // differ from DefaultRouterMetricsPort (8080) and - // DefaultRouterGRPCHealthPort (5000, overridden to 5179 on Talos by - // config/router/base/daemonset.yaml) and from galactic-cni's - // credential-refresh grpc-health port (5180, + // differ from DefaultRouterMetricsPort (9179) and + // DefaultRouterGRPCHealthPort (5179) and from galactic-cni's + // credential-refresh ports (grpc-health 5180, metrics 9180, // config/cni/daemonset.yaml): galactic-gateway is deployed as a // second container in the same hostNetwork: true pod as // galactic-router, so every port it binds is on the same network diff --git a/internal/config/router.go b/internal/config/router.go index 6cda0ffe..18d8ecb7 100644 --- a/internal/config/router.go +++ b/internal/config/router.go @@ -16,9 +16,13 @@ import ( // --- Router defaults ------------------------------------------------------- const ( - DefaultRouterBGPListenPort = 179 - DefaultRouterMetricsPort = 8080 - DefaultRouterGRPCHealthPort = 5000 + DefaultRouterBGPListenPort = 179 + DefaultRouterMetricsPort = 9179 + // DefaultRouterGRPCHealthPort avoids 5000: one of the most overloaded + // dev ports in existence (macOS AirPlay Receiver, Flask's dev server, + // Docker Registry), and on Talos specifically /sbin/dashboard + // permanently binds 127.0.0.1:5000 -- see docs/router/configuration.md. + DefaultRouterGRPCHealthPort = 5179 DefaultRouterGCNamespace = "galactic-system" DefaultRouterGCInterval = 5 * time.Minute