Define a production-ready default tracing policy

cmd/ateapi/main.go

@@ -47,7 +47,6 @@ import (
 	"github.com/spf13/pflag"
 	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
 	"go.opentelemetry.io/otel"
-	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 	"golang.org/x/oauth2/google"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
@@ -107,7 +106,7 @@ func main() {
 
 	tp, err := serverboot.InitTracing(ctx, serverboot.TracingOptions{
 		ServiceName: "ateapi",
-		Sampler:     sdktrace.ParentBased(sdktrace.AlwaysSample()),
+		Sampling:    serverboot.ResolveTraceSampling(ctx, serverboot.ParentRatioSampling(serverboot.ControlPlaneTraceRatio)),
 	})
 	if err != nil {
 		serverboot.Fatal(ctx, "Failed to initialize tracing", err)

cmd/atecontroller/internal/controllers/workerpool_apply.go

@@ -46,6 +46,12 @@ type ateomOTelSettings struct {
 	// MetricExportTimeout overrides the SDK's  per-export timeout, in the same
 	// whole-millisecond form as MetricExportInterval. Empty keeps the default.
 	MetricExportTimeout string
+	// TracesSampler and TracesSamplerArg are the raw OTEL_TRACES_SAMPLER and
+	// OTEL_TRACES_SAMPLER_ARG values, passed through untouched: ateom's own
+	// serverboot resolution validates them. Empty sampler keeps the worker's
+	// default and drops the arg, which is dead config on its own.
+	TracesSampler    string
+	TracesSamplerArg string
 }
 
 const (
@@ -183,6 +189,16 @@ func ateomContainerEnv(otel ateomOTelSettings) []*corev1ac.EnvVarApplyConfigurat
 			WithName("OTEL_METRIC_EXPORT_TIMEOUT").
 			WithValue(otel.MetricExportTimeout))
 	}
+	if otel.TracesSampler != "" {
+		envs = append(envs, corev1ac.EnvVar().
+			WithName("OTEL_TRACES_SAMPLER").
+			WithValue(otel.TracesSampler))
+		if otel.TracesSamplerArg != "" {
+			envs = append(envs, corev1ac.EnvVar().
+				WithName("OTEL_TRACES_SAMPLER_ARG").
+				WithValue(otel.TracesSamplerArg))
+		}
+	}
 	return envs
 }
 

cmd/atecontroller/internal/controllers/workerpool_apply_test.go

@@ -447,6 +447,63 @@ func TestBuildDeploymentApplyConfigMetricExportTuning(t *testing.T) {
 	}
 }
 
+// TestBuildDeploymentApplyConfigTracesSamplerPropagation asserts the sampler
+// env pair reaches the ateom container only alongside an endpoint, and the arg
+// only alongside a sampler: an arg without a sampler name is dead config the
+// SDK ignores.
+func TestBuildDeploymentApplyConfigTracesSamplerPropagation(t *testing.T) {
+	const endpoint = "http://collector.otel-system.svc:4317"
+	tests := []struct {
+		name string
+		otel ateomOTelSettings
+		want map[string]string // value by env name; absent key means must not be set
+	}{
+		{
+			name: "unset keeps binary default",
+			otel: ateomOTelSettings{Endpoint: endpoint},
+			want: nil,
+		},
+		{
+			name: "sampler and arg with endpoint",
+			otel: ateomOTelSettings{Endpoint: endpoint, TracesSampler: "parentbased_traceidratio", TracesSamplerArg: "0.25"},
+			want: map[string]string{"OTEL_TRACES_SAMPLER": "parentbased_traceidratio", "OTEL_TRACES_SAMPLER_ARG": "0.25"},
+		},
+		{
+			name: "sampler alone",
+			otel: ateomOTelSettings{Endpoint: endpoint, TracesSampler: "parentbased_always_on"},
+			want: map[string]string{"OTEL_TRACES_SAMPLER": "parentbased_always_on"},
+		},
+		{
+			name: "arg alone stays unset",
+			otel: ateomOTelSettings{Endpoint: endpoint, TracesSamplerArg: "0.25"},
+			want: nil,
+		},
+		{
+			name: "ignored without endpoint",
+			otel: ateomOTelSettings{TracesSampler: "parentbased_traceidratio", TracesSamplerArg: "0.25"},
+			want: nil,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), tt.otel).
+				Spec.Template.Spec.Containers[0]
+			env := envByName(c.Env)
+			for _, k := range []string{"OTEL_TRACES_SAMPLER", "OTEL_TRACES_SAMPLER_ARG"} {
+				got, ok := env[k]
+				want, wantSet := tt.want[k]
+				if ok != wantSet {
+					t.Errorf("%s present = %v, want %v", k, ok, wantSet)
+					continue
+				}
+				if ok && got.value != want {
+					t.Errorf("%s = %q, want %q", k, got.value, want)
+				}
+			}
+		})
+	}
+}
+
 type envInfo struct {
 	index int
 	value string

cmd/atecontroller/internal/controllers/workerpool_controller.go

@@ -43,6 +43,12 @@ type WorkerPoolReconciler struct {
 	// OTelMetricExportTimeout is the OTEL_METRIC_EXPORT_TIMEOUT propagated to
 	// ateom pods. Empty keeps the SDK's default.
 	OTelMetricExportTimeout string
+	// OTelTracesSampler is the OTEL_TRACES_SAMPLER propagated to ateom pods.
+	// Empty keeps the ateom binary's default.
+	OTelTracesSampler string
+	// OTelTracesSamplerArg is the OTEL_TRACES_SAMPLER_ARG propagated to ateom
+	// pods. Ignored unless OTelTracesSampler is set.
+	OTelTracesSamplerArg string
 }
 
 //+kubebuilder:rbac:groups=ate.dev,resources=workerpools,verbs=get;list;watch;create;update;patch;delete
@@ -102,6 +108,8 @@ func (r *WorkerPoolReconciler) applyDeployment(ctx context.Context, wp *atev1alp
 		Endpoint:             r.OTelEndpoint,
 		MetricExportInterval: r.OTelMetricExportInterval,
 		MetricExportTimeout:  r.OTelMetricExportTimeout,
+		TracesSampler:        r.OTelTracesSampler,
+		TracesSamplerArg:     r.OTelTracesSamplerArg,
 	})
 	if err := r.Apply(ctx, depAC, client.FieldOwner(workerPoolFieldOwner), client.ForceOwnership); err != nil {
 		return fmt.Errorf("failed to apply Deployment: %w", err)

cmd/atecontroller/main.go

@@ -48,6 +48,12 @@ var (
 	otelMetricExportTimeout = pflag.String("otel-metric-export-timeout", os.Getenv("OTEL_METRIC_EXPORT_TIMEOUT"),
 		"Per-export timeout in milliseconds set on ateom worker pods. Empty keeps the OTel SDK's 30s default. Defaults to the controller's own OTEL_METRIC_EXPORT_TIMEOUT.")
 
+	otelTracesSampler = pflag.String("otel-traces-sampler", os.Getenv("OTEL_TRACES_SAMPLER"),
+		"Trace sampler set on ateom worker pods. Empty keeps the ateom binary's default. Defaults to the controller's own OTEL_TRACES_SAMPLER.")
+
+	otelTracesSamplerArg = pflag.String("otel-traces-sampler-arg", os.Getenv("OTEL_TRACES_SAMPLER_ARG"),
+		"Trace sampler argument set on ateom worker pods, ignored unless --otel-traces-sampler is set. Defaults to the controller's own OTEL_TRACES_SAMPLER_ARG.")
+
 	ateapiCAFile     = pflag.String("ateapi-ca-file", ateapiauth.DefaultServiceAccountCAFile, "PEM file with CAs trusted to verify the ateapi server cert.")
 	ateapiServerName = pflag.String("ateapi-server-name", "", "SNI / hostname expected on the ateapi server cert. Optional.")
 	ateapiTokenAuth  = pflag.Bool("ateapi-use-token-auth", false, "Authenticate to ateapi with the Bearer token from --ateapi-token-file instead of the client certificate from --ateapi-client-cert.")
@@ -98,6 +104,8 @@ func main() {
 		OTelEndpoint:             *otelEndpoint,
 		OTelMetricExportInterval: *otelMetricExportInterval,
 		OTelMetricExportTimeout:  *otelMetricExportTimeout,
+		OTelTracesSampler:        *otelTracesSampler,
+		OTelTracesSamplerArg:     *otelTracesSamplerArg,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "WorkerPool")
 		os.Exit(1)

cmd/atelet/main.go

@@ -51,7 +51,6 @@ import (
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/metric"
-	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/api/option"
 	"google.golang.org/grpc"
@@ -94,7 +93,7 @@ func main() {
 
 	tp, err := serverboot.InitTracing(ctx, serverboot.TracingOptions{
 		ServiceName: "atelet",
-		Sampler:     sdktrace.ParentBased(sdktrace.NeverSample()),
+		Sampling:    serverboot.ResolveTraceSampling(ctx, serverboot.ParentRatioSampling(serverboot.ControlPlaneTraceRatio)),
 	})
 	if err != nil {
 		serverboot.Fatal(ctx, "Failed to initialize tracing", err)

cmd/atenet/internal/router/dataplane.go

@@ -44,10 +44,10 @@ func (r atenetRouter) healthCheck() dataplaneHealthCheck {
 	}
 }
 
-func (s *RouterServer) startDataplane(ctx context.Context, g *errgroup.Group, parkCfg ParkedRequestConfig) error {
+func (s *RouterServer) startDataplane(ctx context.Context, g *errgroup.Group, parkCfg ParkedRequestConfig, traceRootSamplingPercent float64) error {
 	switch s.cfg.atenetRouter() {
 	case atenetRouterEnvoy:
-		s.startEnvoyDataplane(ctx, g, parkCfg)
+		s.startEnvoyDataplane(ctx, g, parkCfg, traceRootSamplingPercent)
 	case atenetRouterAgentgateway:
 		// Agentgateway receives all routing configuration from its static file.
 	default:
@@ -56,10 +56,11 @@ func (s *RouterServer) startDataplane(ctx context.Context, g *errgroup.Group, pa
 	return nil
 }
 
-func (s *RouterServer) startEnvoyDataplane(ctx context.Context, g *errgroup.Group, parkCfg ParkedRequestConfig) {
+func (s *RouterServer) startEnvoyDataplane(ctx context.Context, g *errgroup.Group, parkCfg ParkedRequestConfig, traceRootSamplingPercent float64) {
 	xdsSrv := NewXdsServer(s.cfg.XdsPort)
 	xdsSrv.SetConfig(s.cfg.HttpPort, s.cfg.ExtprocPort, s.cfg.ExtprocAddr)
 	setOtlpCollector(ctx, xdsSrv, s.cfg.OtlpCollectorAddress)
+	xdsSrv.SetTraceRootSamplingPercent(traceRootSamplingPercent)
 
 	xdsSrv.SetExtProcMaxRequests(s.cfg.extProcMaxRequests())
 	if parkCfg.enabled() {

cmd/atenet/internal/router/router.go

@@ -28,7 +28,6 @@ import (
 	"github.com/spf13/cobra"
 	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
 	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
-	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/grpc"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -45,6 +44,10 @@ import (
 	"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
 )
 
+// dataPlaneTraceRatio is the default root sampling fraction for parentless
+// data plane requests; OTEL_TRACES_SAMPLER / OTEL_TRACES_SAMPLER_ARG override it.
+const dataPlaneTraceRatio = 0.01
+
 var (
 	scheme = runtime.NewScheme()
 )
@@ -141,10 +144,12 @@ func (s *RouterServer) Run(ctx context.Context) error {
 
 	// Tracing must be initialized before constructing the ateapi gRPC client
 	// below, because otelgrpc.NewClientHandler captures the global
-	// TracerProvider at construction time.
+	// TracerProvider at construction time. Resolved once so the router's SDK
+	// sampler and Envoy's RandomSampling percent cannot drift.
+	sampling := serverboot.ResolveTraceSampling(ctx, serverboot.ParentRatioSampling(dataPlaneTraceRatio))
 	tp, err := serverboot.InitTracing(ctx, serverboot.TracingOptions{
 		ServiceName: routerServiceName,
-		Sampler:     sdktrace.ParentBased(sdktrace.NeverSample()),
+		Sampling:    sampling,
 	})
 	if err != nil {
 		return fmt.Errorf("failed to initialize tracing: %w", err)
@@ -199,7 +204,7 @@ func (s *RouterServer) Run(ctx context.Context) error {
 	}
 	s.health = newRouterHealth(s.cfg.HealthInterval, s.clientset, s.apiClient, s.cfg)
 
-	if err := s.startDataplane(ctx, g, parkCfg); err != nil {
+	if err := s.startDataplane(ctx, g, parkCfg, sampling.RootSamplingPercent()); err != nil {
 		return err
 	}
 

cmd/atenet/internal/router/xds.go

@@ -126,6 +126,10 @@ type XdsServer struct {
 	otlpHost string
 	otlpPort uint32
 
+	// traceRootSamplingPercent mirrors the router's resolved sampling policy
+	// into Envoy's RandomSampling. Zero until Run sets it.
+	traceRootSamplingPercent float64
+
 	// extProcMessageTimeout bounds how long Envoy waits for the router's ext_proc
 	// response. Must be >= the parking budget so parked requests aren't cut short.
 	extProcMessageTimeout time.Duration
@@ -240,6 +244,15 @@ func (x *XdsServer) DisableOtlpCollector() {
 	x.otlpPort = 0
 }
 
+// SetTraceRootSamplingPercent sets the RandomSampling percent Envoy applies to
+// requests arriving without a traceparent. Derived from the router's resolved
+// OTel sampling policy so the two root decisions cannot drift.
+func (x *XdsServer) SetTraceRootSamplingPercent(p float64) {
+	x.mu.Lock()
+	defer x.mu.Unlock()
+	x.traceRootSamplingPercent = p
+}
+
 // normalizeOtlpCollector resolves a collector endpoint to the bare host and
 // numeric port an xDS SocketAddress requires (buildOtlpCollectorCluster).
 //
@@ -685,12 +698,10 @@ func (x *XdsServer) buildHcm(statPrefix string) *anypb.Any {
 // configured OTLP gRPC collector. Returns nil when no collector is set,
 // in which case Envoy emits no spans on its own.
 //
-// `RandomSampling: 100%` makes Envoy ALWAYS sample requests that arrive
-// with no parent traceparent. We rely on upstream clients (locust, etc.)
-// to gate sampling: requests without a sampled parent are still tagged
-// `sampled` here but downstream services in this repo use
-// `ParentBased(NeverSample)` so unsampled-by-client requests stay
-// unsampled overall.
+// RandomSampling is the root decision for requests arriving without a
+// traceparent. Requests already sampled by the caller (kubectl-ate --trace,
+// load generators) are continued regardless of the percent, and downstream
+// ParentBased samplers keep the decision end to end.
 func (x *XdsServer) buildTracing() *hcmv3.HttpConnectionManager_Tracing {
 	if x.otlpHost == "" {
 		return nil
@@ -706,7 +717,7 @@ func (x *XdsServer) buildTracing() *hcmv3.HttpConnectionManager_Tracing {
 		ServiceName: "atenet-router-envoy",
 	})
 	return &hcmv3.HttpConnectionManager_Tracing{
-		RandomSampling: &typev3.Percent{Value: 100},
+		RandomSampling: &typev3.Percent{Value: x.traceRootSamplingPercent},
 		Provider: &tracev3.Tracing_Http{
 			Name: "envoy.tracers.opentelemetry",
 			ConfigType: &tracev3.Tracing_Http_TypedConfig{

cmd/atenet/internal/router/xds_test.go

@@ -649,3 +649,67 @@ func TestXdsServer_SetOtlpCollector_EmptyDisablesTracing(t *testing.T) {
 		t.Errorf("snapshot contains cluster %q, want it omitted when tracing is disabled", OtlpClusterName)
 	}
 }
+
+func TestXdsServer_BuildTracingRandomSamplingFromPolicy(t *testing.T) {
+	const collectorAddr = "collector.otel-system.svc:4317"
+
+	tests := []struct {
+		name        string
+		collector   string
+		percent     float64
+		setPercent  bool
+		wantTracing bool
+		wantPercent float64
+	}{
+		{
+			name:        "percent mirrors the resolved policy",
+			collector:   collectorAddr,
+			percent:     1,
+			setPercent:  true,
+			wantTracing: true,
+			wantPercent: 1,
+		},
+		{
+			name:        "full sampling",
+			collector:   collectorAddr,
+			percent:     100,
+			setPercent:  true,
+			wantTracing: true,
+			wantPercent: 100,
+		},
+		{
+			// A caller that never threads in a policy must fail toward no
+			// root sampling, not toward 100%.
+			name:        "setter never called defaults to zero",
+			collector:   collectorAddr,
+			wantTracing: true,
+			wantPercent: 0,
+		},
+		{
+			name:       "no collector yields no tracing block",
+			percent:    100,
+			setPercent: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			x := NewXdsServer(0)
+			if err := x.SetOtlpCollector(tt.collector); err != nil {
+				t.Fatalf("SetOtlpCollector(%q) failed: %v", tt.collector, err)
+			}
+			if tt.setPercent {
+				x.SetTraceRootSamplingPercent(tt.percent)
+			}
+			tr := x.buildTracing()
+			if (tr != nil) != tt.wantTracing {
+				t.Fatalf("buildTracing() = %v, want tracing block: %v", tr, tt.wantTracing)
+			}
+			if !tt.wantTracing {
+				return
+			}
+			if got := tr.GetRandomSampling().GetValue(); got != tt.wantPercent {
+				t.Errorf("RandomSampling = %v, want %v", got, tt.wantPercent)
+			}
+		})
+	}
+}

cmd/ateom-gvisor/main.go

@@ -45,7 +45,6 @@ import (
 	"github.com/spf13/pflag"
 	"github.com/vishvananda/netns"
 	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
-	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 	"golang.org/x/sys/unix"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/reflection"
@@ -103,7 +102,7 @@ func do(ctx context.Context) error {
 	const serviceName = "ateom-gvisor"
 	tp, err := serverboot.InitTracing(ctx, serverboot.TracingOptions{
 		ServiceName: serviceName,
-		Sampler:     sdktrace.ParentBased(sdktrace.NeverSample()),
+		Sampling:    serverboot.ResolveTraceSampling(ctx, serverboot.ParentRatioSampling(serverboot.ControlPlaneTraceRatio)),
 	})
 	if err != nil {
 		serverboot.Fatal(ctx, "Failed to initialize tracing", err)

cmd/ateom-microvm/main.go

@@ -46,7 +46,6 @@ import (
 	"github.com/agent-substrate/substrate/internal/version"
 	"github.com/vishvananda/netns"
 	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
-	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 	"golang.org/x/sys/unix"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/reflection"
@@ -103,7 +102,7 @@ func do(ctx context.Context) error {
 	const serviceName = "ateom-microvm"
 	tp, err := serverboot.InitTracing(ctx, serverboot.TracingOptions{
 		ServiceName: serviceName,
-		Sampler:     sdktrace.ParentBased(sdktrace.NeverSample()),
+		Sampling:    serverboot.ResolveTraceSampling(ctx, serverboot.ParentRatioSampling(serverboot.ControlPlaneTraceRatio)),
 	})
 	if err != nil {
 		serverboot.Fatal(ctx, "Failed to initialize tracing", err)