diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 683ece541..57860bb58 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -47,7 +47,6 @@ import ( "github.com/spf13/pflag" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.opentelemetry.io/otel" - sdktrace "go.opentelemetry.io/otel/sdk/trace" "golang.org/x/oauth2/google" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -107,7 +106,7 @@ func main() { tp, err := serverboot.InitTracing(ctx, serverboot.TracingOptions{ ServiceName: "ateapi", - Sampler: sdktrace.ParentBased(sdktrace.AlwaysSample()), + Sampling: serverboot.ResolveTraceSampling(ctx, serverboot.ParentRatioSampling(serverboot.ControlPlaneTraceRatio)), }) if err != nil { serverboot.Fatal(ctx, "Failed to initialize tracing", err) diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply.go b/cmd/atecontroller/internal/controllers/workerpool_apply.go index 8a52a93b9..cf159b498 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply.go @@ -46,6 +46,12 @@ type ateomOTelSettings struct { // MetricExportTimeout overrides the SDK's per-export timeout, in the same // whole-millisecond form as MetricExportInterval. Empty keeps the default. MetricExportTimeout string + // TracesSampler and TracesSamplerArg are the raw OTEL_TRACES_SAMPLER and + // OTEL_TRACES_SAMPLER_ARG values, passed through untouched: ateom's own + // serverboot resolution validates them. Empty sampler keeps the worker's + // default and drops the arg, which is dead config on its own. + TracesSampler string + TracesSamplerArg string } const ( @@ -183,6 +189,16 @@ func ateomContainerEnv(otel ateomOTelSettings) []*corev1ac.EnvVarApplyConfigurat WithName("OTEL_METRIC_EXPORT_TIMEOUT"). WithValue(otel.MetricExportTimeout)) } + if otel.TracesSampler != "" { + envs = append(envs, corev1ac.EnvVar(). + WithName("OTEL_TRACES_SAMPLER"). + WithValue(otel.TracesSampler)) + if otel.TracesSamplerArg != "" { + envs = append(envs, corev1ac.EnvVar(). + WithName("OTEL_TRACES_SAMPLER_ARG"). + WithValue(otel.TracesSamplerArg)) + } + } return envs } diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go index 45dc9ca49..c818f7af9 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go @@ -447,6 +447,63 @@ func TestBuildDeploymentApplyConfigMetricExportTuning(t *testing.T) { } } +// TestBuildDeploymentApplyConfigTracesSamplerPropagation asserts the sampler +// env pair reaches the ateom container only alongside an endpoint, and the arg +// only alongside a sampler: an arg without a sampler name is dead config the +// SDK ignores. +func TestBuildDeploymentApplyConfigTracesSamplerPropagation(t *testing.T) { + const endpoint = "http://collector.otel-system.svc:4317" + tests := []struct { + name string + otel ateomOTelSettings + want map[string]string // value by env name; absent key means must not be set + }{ + { + name: "unset keeps binary default", + otel: ateomOTelSettings{Endpoint: endpoint}, + want: nil, + }, + { + name: "sampler and arg with endpoint", + otel: ateomOTelSettings{Endpoint: endpoint, TracesSampler: "parentbased_traceidratio", TracesSamplerArg: "0.25"}, + want: map[string]string{"OTEL_TRACES_SAMPLER": "parentbased_traceidratio", "OTEL_TRACES_SAMPLER_ARG": "0.25"}, + }, + { + name: "sampler alone", + otel: ateomOTelSettings{Endpoint: endpoint, TracesSampler: "parentbased_always_on"}, + want: map[string]string{"OTEL_TRACES_SAMPLER": "parentbased_always_on"}, + }, + { + name: "arg alone stays unset", + otel: ateomOTelSettings{Endpoint: endpoint, TracesSamplerArg: "0.25"}, + want: nil, + }, + { + name: "ignored without endpoint", + otel: ateomOTelSettings{TracesSampler: "parentbased_traceidratio", TracesSamplerArg: "0.25"}, + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), tt.otel). + Spec.Template.Spec.Containers[0] + env := envByName(c.Env) + for _, k := range []string{"OTEL_TRACES_SAMPLER", "OTEL_TRACES_SAMPLER_ARG"} { + got, ok := env[k] + want, wantSet := tt.want[k] + if ok != wantSet { + t.Errorf("%s present = %v, want %v", k, ok, wantSet) + continue + } + if ok && got.value != want { + t.Errorf("%s = %q, want %q", k, got.value, want) + } + } + }) + } +} + type envInfo struct { index int value string diff --git a/cmd/atecontroller/internal/controllers/workerpool_controller.go b/cmd/atecontroller/internal/controllers/workerpool_controller.go index 27318c81c..c01068d58 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_controller.go +++ b/cmd/atecontroller/internal/controllers/workerpool_controller.go @@ -43,6 +43,12 @@ type WorkerPoolReconciler struct { // OTelMetricExportTimeout is the OTEL_METRIC_EXPORT_TIMEOUT propagated to // ateom pods. Empty keeps the SDK's default. OTelMetricExportTimeout string + // OTelTracesSampler is the OTEL_TRACES_SAMPLER propagated to ateom pods. + // Empty keeps the ateom binary's default. + OTelTracesSampler string + // OTelTracesSamplerArg is the OTEL_TRACES_SAMPLER_ARG propagated to ateom + // pods. Ignored unless OTelTracesSampler is set. + OTelTracesSamplerArg string } //+kubebuilder:rbac:groups=ate.dev,resources=workerpools,verbs=get;list;watch;create;update;patch;delete @@ -102,6 +108,8 @@ func (r *WorkerPoolReconciler) applyDeployment(ctx context.Context, wp *atev1alp Endpoint: r.OTelEndpoint, MetricExportInterval: r.OTelMetricExportInterval, MetricExportTimeout: r.OTelMetricExportTimeout, + TracesSampler: r.OTelTracesSampler, + TracesSamplerArg: r.OTelTracesSamplerArg, }) if err := r.Apply(ctx, depAC, client.FieldOwner(workerPoolFieldOwner), client.ForceOwnership); err != nil { return fmt.Errorf("failed to apply Deployment: %w", err) diff --git a/cmd/atecontroller/main.go b/cmd/atecontroller/main.go index d7d49324b..112d4de5a 100644 --- a/cmd/atecontroller/main.go +++ b/cmd/atecontroller/main.go @@ -48,6 +48,12 @@ var ( otelMetricExportTimeout = pflag.String("otel-metric-export-timeout", os.Getenv("OTEL_METRIC_EXPORT_TIMEOUT"), "Per-export timeout in milliseconds set on ateom worker pods. Empty keeps the OTel SDK's 30s default. Defaults to the controller's own OTEL_METRIC_EXPORT_TIMEOUT.") + otelTracesSampler = pflag.String("otel-traces-sampler", os.Getenv("OTEL_TRACES_SAMPLER"), + "Trace sampler set on ateom worker pods. Empty keeps the ateom binary's default. Defaults to the controller's own OTEL_TRACES_SAMPLER.") + + otelTracesSamplerArg = pflag.String("otel-traces-sampler-arg", os.Getenv("OTEL_TRACES_SAMPLER_ARG"), + "Trace sampler argument set on ateom worker pods, ignored unless --otel-traces-sampler is set. Defaults to the controller's own OTEL_TRACES_SAMPLER_ARG.") + ateapiCAFile = pflag.String("ateapi-ca-file", ateapiauth.DefaultServiceAccountCAFile, "PEM file with CAs trusted to verify the ateapi server cert.") ateapiServerName = pflag.String("ateapi-server-name", "", "SNI / hostname expected on the ateapi server cert. Optional.") ateapiTokenAuth = pflag.Bool("ateapi-use-token-auth", false, "Authenticate to ateapi with the Bearer token from --ateapi-token-file instead of the client certificate from --ateapi-client-cert.") @@ -98,6 +104,8 @@ func main() { OTelEndpoint: *otelEndpoint, OTelMetricExportInterval: *otelMetricExportInterval, OTelMetricExportTimeout: *otelMetricExportTimeout, + OTelTracesSampler: *otelTracesSampler, + OTelTracesSamplerArg: *otelTracesSamplerArg, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "WorkerPool") os.Exit(1) diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index b3f0241c9..1829ad2da 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -51,7 +51,6 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" - sdktrace "go.opentelemetry.io/otel/sdk/trace" "golang.org/x/sync/errgroup" "google.golang.org/api/option" "google.golang.org/grpc" @@ -94,7 +93,7 @@ func main() { tp, err := serverboot.InitTracing(ctx, serverboot.TracingOptions{ ServiceName: "atelet", - Sampler: sdktrace.ParentBased(sdktrace.NeverSample()), + Sampling: serverboot.ResolveTraceSampling(ctx, serverboot.ParentRatioSampling(serverboot.ControlPlaneTraceRatio)), }) if err != nil { serverboot.Fatal(ctx, "Failed to initialize tracing", err) diff --git a/cmd/atenet/internal/router/dataplane.go b/cmd/atenet/internal/router/dataplane.go index 7af7bbcca..e77a0b274 100644 --- a/cmd/atenet/internal/router/dataplane.go +++ b/cmd/atenet/internal/router/dataplane.go @@ -44,10 +44,10 @@ func (r atenetRouter) healthCheck() dataplaneHealthCheck { } } -func (s *RouterServer) startDataplane(ctx context.Context, g *errgroup.Group, parkCfg ParkedRequestConfig) error { +func (s *RouterServer) startDataplane(ctx context.Context, g *errgroup.Group, parkCfg ParkedRequestConfig, traceRootSamplingPercent float64) error { switch s.cfg.atenetRouter() { case atenetRouterEnvoy: - s.startEnvoyDataplane(ctx, g, parkCfg) + s.startEnvoyDataplane(ctx, g, parkCfg, traceRootSamplingPercent) case atenetRouterAgentgateway: // Agentgateway receives all routing configuration from its static file. default: @@ -56,10 +56,11 @@ func (s *RouterServer) startDataplane(ctx context.Context, g *errgroup.Group, pa return nil } -func (s *RouterServer) startEnvoyDataplane(ctx context.Context, g *errgroup.Group, parkCfg ParkedRequestConfig) { +func (s *RouterServer) startEnvoyDataplane(ctx context.Context, g *errgroup.Group, parkCfg ParkedRequestConfig, traceRootSamplingPercent float64) { xdsSrv := NewXdsServer(s.cfg.XdsPort) xdsSrv.SetConfig(s.cfg.HttpPort, s.cfg.ExtprocPort, s.cfg.ExtprocAddr) setOtlpCollector(ctx, xdsSrv, s.cfg.OtlpCollectorAddress) + xdsSrv.SetTraceRootSamplingPercent(traceRootSamplingPercent) xdsSrv.SetExtProcMaxRequests(s.cfg.extProcMaxRequests()) if parkCfg.enabled() { diff --git a/cmd/atenet/internal/router/router.go b/cmd/atenet/internal/router/router.go index 13d633660..d8607b72f 100644 --- a/cmd/atenet/internal/router/router.go +++ b/cmd/atenet/internal/router/router.go @@ -28,7 +28,6 @@ import ( "github.com/spf13/cobra" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" - sdktrace "go.opentelemetry.io/otel/sdk/trace" "golang.org/x/sync/errgroup" "google.golang.org/grpc" "k8s.io/apimachinery/pkg/runtime" @@ -45,6 +44,10 @@ import ( "github.com/agent-substrate/substrate/pkg/proto/ateapipb" ) +// dataPlaneTraceRatio is the default root sampling fraction for parentless +// data plane requests; OTEL_TRACES_SAMPLER / OTEL_TRACES_SAMPLER_ARG override it. +const dataPlaneTraceRatio = 0.01 + var ( scheme = runtime.NewScheme() ) @@ -141,10 +144,12 @@ func (s *RouterServer) Run(ctx context.Context) error { // Tracing must be initialized before constructing the ateapi gRPC client // below, because otelgrpc.NewClientHandler captures the global - // TracerProvider at construction time. + // TracerProvider at construction time. Resolved once so the router's SDK + // sampler and Envoy's RandomSampling percent cannot drift. + sampling := serverboot.ResolveTraceSampling(ctx, serverboot.ParentRatioSampling(dataPlaneTraceRatio)) tp, err := serverboot.InitTracing(ctx, serverboot.TracingOptions{ ServiceName: routerServiceName, - Sampler: sdktrace.ParentBased(sdktrace.NeverSample()), + Sampling: sampling, }) if err != nil { return fmt.Errorf("failed to initialize tracing: %w", err) @@ -199,7 +204,7 @@ func (s *RouterServer) Run(ctx context.Context) error { } s.health = newRouterHealth(s.cfg.HealthInterval, s.clientset, s.apiClient, s.cfg) - if err := s.startDataplane(ctx, g, parkCfg); err != nil { + if err := s.startDataplane(ctx, g, parkCfg, sampling.RootSamplingPercent()); err != nil { return err } diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index e422376d9..315d7adf9 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -126,6 +126,10 @@ type XdsServer struct { otlpHost string otlpPort uint32 + // traceRootSamplingPercent mirrors the router's resolved sampling policy + // into Envoy's RandomSampling. Zero until Run sets it. + traceRootSamplingPercent float64 + // extProcMessageTimeout bounds how long Envoy waits for the router's ext_proc // response. Must be >= the parking budget so parked requests aren't cut short. extProcMessageTimeout time.Duration @@ -240,6 +244,15 @@ func (x *XdsServer) DisableOtlpCollector() { x.otlpPort = 0 } +// SetTraceRootSamplingPercent sets the RandomSampling percent Envoy applies to +// requests arriving without a traceparent. Derived from the router's resolved +// OTel sampling policy so the two root decisions cannot drift. +func (x *XdsServer) SetTraceRootSamplingPercent(p float64) { + x.mu.Lock() + defer x.mu.Unlock() + x.traceRootSamplingPercent = p +} + // normalizeOtlpCollector resolves a collector endpoint to the bare host and // numeric port an xDS SocketAddress requires (buildOtlpCollectorCluster). // @@ -685,12 +698,10 @@ func (x *XdsServer) buildHcm(statPrefix string) *anypb.Any { // configured OTLP gRPC collector. Returns nil when no collector is set, // in which case Envoy emits no spans on its own. // -// `RandomSampling: 100%` makes Envoy ALWAYS sample requests that arrive -// with no parent traceparent. We rely on upstream clients (locust, etc.) -// to gate sampling: requests without a sampled parent are still tagged -// `sampled` here but downstream services in this repo use -// `ParentBased(NeverSample)` so unsampled-by-client requests stay -// unsampled overall. +// RandomSampling is the root decision for requests arriving without a +// traceparent. Requests already sampled by the caller (kubectl-ate --trace, +// load generators) are continued regardless of the percent, and downstream +// ParentBased samplers keep the decision end to end. func (x *XdsServer) buildTracing() *hcmv3.HttpConnectionManager_Tracing { if x.otlpHost == "" { return nil @@ -706,7 +717,7 @@ func (x *XdsServer) buildTracing() *hcmv3.HttpConnectionManager_Tracing { ServiceName: "atenet-router-envoy", }) return &hcmv3.HttpConnectionManager_Tracing{ - RandomSampling: &typev3.Percent{Value: 100}, + RandomSampling: &typev3.Percent{Value: x.traceRootSamplingPercent}, Provider: &tracev3.Tracing_Http{ Name: "envoy.tracers.opentelemetry", ConfigType: &tracev3.Tracing_Http_TypedConfig{ diff --git a/cmd/atenet/internal/router/xds_test.go b/cmd/atenet/internal/router/xds_test.go index 23f9ffdac..07d9e1cc3 100644 --- a/cmd/atenet/internal/router/xds_test.go +++ b/cmd/atenet/internal/router/xds_test.go @@ -649,3 +649,67 @@ func TestXdsServer_SetOtlpCollector_EmptyDisablesTracing(t *testing.T) { t.Errorf("snapshot contains cluster %q, want it omitted when tracing is disabled", OtlpClusterName) } } + +func TestXdsServer_BuildTracingRandomSamplingFromPolicy(t *testing.T) { + const collectorAddr = "collector.otel-system.svc:4317" + + tests := []struct { + name string + collector string + percent float64 + setPercent bool + wantTracing bool + wantPercent float64 + }{ + { + name: "percent mirrors the resolved policy", + collector: collectorAddr, + percent: 1, + setPercent: true, + wantTracing: true, + wantPercent: 1, + }, + { + name: "full sampling", + collector: collectorAddr, + percent: 100, + setPercent: true, + wantTracing: true, + wantPercent: 100, + }, + { + // A caller that never threads in a policy must fail toward no + // root sampling, not toward 100%. + name: "setter never called defaults to zero", + collector: collectorAddr, + wantTracing: true, + wantPercent: 0, + }, + { + name: "no collector yields no tracing block", + percent: 100, + setPercent: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + x := NewXdsServer(0) + if err := x.SetOtlpCollector(tt.collector); err != nil { + t.Fatalf("SetOtlpCollector(%q) failed: %v", tt.collector, err) + } + if tt.setPercent { + x.SetTraceRootSamplingPercent(tt.percent) + } + tr := x.buildTracing() + if (tr != nil) != tt.wantTracing { + t.Fatalf("buildTracing() = %v, want tracing block: %v", tr, tt.wantTracing) + } + if !tt.wantTracing { + return + } + if got := tr.GetRandomSampling().GetValue(); got != tt.wantPercent { + t.Errorf("RandomSampling = %v, want %v", got, tt.wantPercent) + } + }) + } +} diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index 5e1a6440a..752acf2d3 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -45,7 +45,6 @@ import ( "github.com/spf13/pflag" "github.com/vishvananda/netns" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" - sdktrace "go.opentelemetry.io/otel/sdk/trace" "golang.org/x/sys/unix" "google.golang.org/grpc" "google.golang.org/grpc/reflection" @@ -103,7 +102,7 @@ func do(ctx context.Context) error { const serviceName = "ateom-gvisor" tp, err := serverboot.InitTracing(ctx, serverboot.TracingOptions{ ServiceName: serviceName, - Sampler: sdktrace.ParentBased(sdktrace.NeverSample()), + Sampling: serverboot.ResolveTraceSampling(ctx, serverboot.ParentRatioSampling(serverboot.ControlPlaneTraceRatio)), }) if err != nil { serverboot.Fatal(ctx, "Failed to initialize tracing", err) diff --git a/cmd/ateom-microvm/main.go b/cmd/ateom-microvm/main.go index 246726997..057863ac8 100644 --- a/cmd/ateom-microvm/main.go +++ b/cmd/ateom-microvm/main.go @@ -46,7 +46,6 @@ import ( "github.com/agent-substrate/substrate/internal/version" "github.com/vishvananda/netns" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" - sdktrace "go.opentelemetry.io/otel/sdk/trace" "golang.org/x/sys/unix" "google.golang.org/grpc" "google.golang.org/grpc/reflection" @@ -103,7 +102,7 @@ func do(ctx context.Context) error { const serviceName = "ateom-microvm" tp, err := serverboot.InitTracing(ctx, serverboot.TracingOptions{ ServiceName: serviceName, - Sampler: sdktrace.ParentBased(sdktrace.NeverSample()), + Sampling: serverboot.ResolveTraceSampling(ctx, serverboot.ParentRatioSampling(serverboot.ControlPlaneTraceRatio)), }) if err != nil { serverboot.Fatal(ctx, "Failed to initialize tracing", err) diff --git a/cmd/benchmarking/glutton/main.go b/cmd/benchmarking/glutton/main.go index 90079bb81..4ea0479b1 100644 --- a/cmd/benchmarking/glutton/main.go +++ b/cmd/benchmarking/glutton/main.go @@ -39,7 +39,6 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" - sdktrace "go.opentelemetry.io/otel/sdk/trace" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" @@ -79,7 +78,7 @@ func main() { tp, err := serverboot.InitTracing(ctx, serverboot.TracingOptions{ ServiceName: "glutton", - Sampler: sdktrace.ParentBased(sdktrace.NeverSample()), + Sampling: serverboot.ResolveTraceSampling(ctx, serverboot.ParentNeverSampling()), }) if err != nil { serverboot.Fatal(ctx, "Failed to initialize tracing", err) diff --git a/docs/dev/best-practices/tracing.md b/docs/dev/best-practices/tracing.md index af94296e0..125033007 100644 --- a/docs/dev/best-practices/tracing.md +++ b/docs/dev/best-practices/tracing.md @@ -34,7 +34,7 @@ All servers need to initialize an OpenTelemetry exporter and tracer provider. S ```go tp, err := serverboot.InitTracing(ctx, serverboot.TracingOptions{ ServiceName: "ateapi", - Sampler: sdktrace.ParentBased(sdktrace.AlwaysSample()), + Sampling: serverboot.ResolveTraceSampling(ctx, serverboot.ParentRatioSampling(serverboot.ControlPlaneTraceRatio)), }) if err != nil { serverboot.Fatal(ctx, "Failed to initialize tracing", err) @@ -48,8 +48,31 @@ Note the following important features: * We are not validating the TLS certs of the collector * We provide a service name to exporter to identify which process is emitting the spans -* We only trace on-demand when desired by the client, as determined by the presence of tracing metadata/headers in the request - * For production, we will want to gate who/how tracing can be enabled for security purposes +* Every component samples by default at a per-component ratio (see below); a client that arrives with a sampled trace context is always traced end to end + * For production, we will want to gate who/how client-forced tracing can be enabled for security purposes + +### Sampling defaults and overrides + +Samplers are resolved with `serverboot.ResolveTraceSampling`, which applies the standard `OTEL_TRACES_SAMPLER` / `OTEL_TRACES_SAMPLER_ARG` environment variables on top of a per-component default. Pass the component default; never pass a raw sampler to the provider, because an explicit sampler silences the env vars. + +| Component | Default | +|---|---| +| ateapi, atelet, ateom-gvisor, ateom-microvm | `parentbased_traceidratio` 0.1 | +| atenet router (data plane root) | `parentbased_traceidratio` 0.01, mirrored into Envoy's `RandomSampling` | +| glutton (benchmarking) | `parentbased_always_off` | +| boomer (benchmarking) | runtime-controlled via dynconfig, ignores the env vars | + +All defaults are `ParentBased`, so a request that arrives already sampled stays sampled on every hop, and one that arrives explicitly unsampled stays unsampled. Only parentless requests are subject to the ratio, at whichever component roots the trace. + +An invalid sampler name, or a missing or unparsable ratio arg, keeps the component default and logs a warning. This deliberately diverges from the OTel SDK's own env handling, which falls back to 100% sampling on invalid input and reads a missing arg as ratio 1.0. + +In agentgateway mode the data plane root fraction lives in the agentgateway ConfigMap (`randomSampling`, same 0.01 default). Unlike Envoy's `RandomSampling`, it is static config that env overrides on the router do not reach, so adjust both together. + +These are head sampling ratios that bound what leaves the process. Keep decisions based on request outcome (errors, latency) belong in a collector pipeline, not in substrate binaries. + +### Disabling tracing (perf/load tests) + +Set `OTEL_TRACES_SAMPLER=always_off` on the components under test (for ateom workers, via the controller's `--otel-traces-sampler` flag). `parentbased_always_off` is not enough under a load generator: boomer and locust send ratio-sampled trace context, and parent based samplers honor it. Alternatively set the generator's `trace_probability` to 0 and leave the servers alone. On kind, also override ateapi's `parentbased_always_on` pin. The YAML manifest for your server should include the `OTEL_EXPORTER_OTLP_ENDPOINT` environment variable to point the exporter to GKE's managed traces collector, e.g.: @@ -117,10 +140,14 @@ tracing metadata in their requests to give users the ability to initiate a trace #### Golang -Like for servers, the tracer provider must be initialized and shutdown, but no exporter is required (note the sampler toggle): +Like for servers, the tracer provider must be initialized and shutdown, but no exporter is required. When tracing is not requested, install nothing: a provider with a `NeverSample` sampler would inject an explicitly unsampled trace context, which pins every `ParentBased` server sampler downstream to not sampled and defeats the server side ratios. With the OTel globals left as noop, no context is injected and the server roots the trace itself. ```go func initTracing(ctx context.Context, enabled bool) (*sdktrace.TracerProvider, error) { + if !enabled { + return nil, nil + } + res, err := resource.New(ctx, resource.WithAttributes( semconv.UserAgentOriginal("my-client-name"), @@ -130,14 +157,9 @@ func initTracing(ctx context.Context, enabled bool) (*sdktrace.TracerProvider, e return nil, fmt.Errorf("failed to create resource: %w", err) } - sampler := sdktrace.NeverSample() - if enabled { - sampler = sdktrace.AlwaysSample() - } - tp := sdktrace.NewTracerProvider( sdktrace.WithResource(res), - sdktrace.WithSampler(sampler), + sdktrace.WithSampler(sdktrace.AlwaysSample()), ) otel.SetTracerProvider(tp) otel.SetTextMapPropagator(propagation.TraceContext{}) diff --git a/docs/observability.md b/docs/observability.md index 78ab8e922..2f07892c6 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -145,7 +145,7 @@ To explore metrics locally: Distributed tracing tracks the end-to-end flow of requests as they pass through the Agent Substrate gateway, router, worker pods, and external services. -Currently, Agent Substrate supports on-demand request tracing. When initiated by a client (e.g., via the `--trace` flag), Agent Substrate leverages OpenTelemetry (OTel) for context propagation across the call stack. Each traced request generates a unique trace hash/ID, which you can use to inspect the detailed request lifecycle and span hierarchy inside Google Cloud Trace or Jaeger. +Agent Substrate samples traces by default. Each component roots parentless requests at a per-component ratio (10% on the control plane components, 1% at the atenet router), overridable per component through the standard `OTEL_TRACES_SAMPLER` / `OTEL_TRACES_SAMPLER_ARG` environment variables. Every component uses a parent based sampler, so a client can also force a request to be traced end to end (e.g. via the `--trace` flag). Agent Substrate leverages OpenTelemetry (OTel) for context propagation across the call stack. Each traced request generates a unique trace hash/ID, which you can use to inspect the detailed request lifecycle and span hierarchy inside Google Cloud Trace or Jaeger. See the per-component defaults table in [Tracing Best Practices](dev/best-practices/tracing.md). ### Local Tracing with Jaeger (Kind Cluster) @@ -167,6 +167,7 @@ To visualize traces locally: # or kubectl ate suspend actor -a --trace ``` + The kind overlay pins `ateapi` to `parentbased_always_on`, so API calls show up even without `--trace`; the flag additionally prints the trace ID and forces sampling on every hop. 4. **Search and Inspect**: Copy the printed Trace ID from the CLI output and paste it into the Jaeger search box (top right), or select `ateapi` or `atelet` under the **Service** dropdown and click **Find Traces** to inspect detailed call stacks, DB transactions, state updates, and worker pod handoffs. @@ -185,7 +186,7 @@ Telemetry is emitted the same way everywhere; only the backend differs between a | Traces | Jaeger UI | Google Cloud Trace | | Dashboards | Not supported | Google Cloud Monitoring (see [Dashboards](#5-dashboards)) | -> In Kind only `ateapi` and `atelet` are pointed at the in-cluster collector; `atenet-router` still targets the GKE collector endpoint, so `atenet.router.route.duration` is emitted but not collected locally. +> In Kind, `ateapi`, `atelet`, `ate-controller`, and `atenet-router` are pointed at the in-cluster collector, and the controller propagates the endpoint to the ateom worker pods it creates, so all component telemetry lands locally. --- diff --git a/internal/ateclient/builder.go b/internal/ateclient/builder.go index 528f6af7c..621d75327 100644 --- a/internal/ateclient/builder.go +++ b/internal/ateclient/builder.go @@ -90,7 +90,9 @@ func NewClient(ctx context.Context, kubeconfigPath, k8sContext, endpoint string, } if err != nil { - _ = tp.Shutdown(ctx) + if tp != nil { + _ = tp.Shutdown(ctx) + } return nil, err } @@ -258,7 +260,15 @@ func (c bearerTokenCreds) GetRequestMetadata(_ context.Context, _ ...string) (ma func (c bearerTokenCreds) RequireTransportSecurity() bool { return true } +// initTracing returns (nil, nil) when tracing is disabled: the OTel globals +// stay noop so no traceparent is injected, the server roots the trace, and the +// server side sampling ratio applies. A NeverSample provider here would +// instead pin every ParentBased sampler downstream to not sampled. func initTracing(ctx context.Context, enabled bool) (*sdktrace.TracerProvider, error) { + if !enabled { + return nil, nil + } + res, err := resource.New(ctx, resource.WithSchemaURL(semconv.SchemaURL), resource.WithAttributes( @@ -269,14 +279,9 @@ func initTracing(ctx context.Context, enabled bool) (*sdktrace.TracerProvider, e return nil, fmt.Errorf("failed to create resource: %w", err) } - sampler := sdktrace.NeverSample() - if enabled { - sampler = sdktrace.AlwaysSample() - } - tp := sdktrace.NewTracerProvider( sdktrace.WithResource(res), - sdktrace.WithSampler(sampler), + sdktrace.WithSampler(sdktrace.AlwaysSample()), ) otel.SetTracerProvider(tp) otel.SetTextMapPropagator(propagation.TraceContext{}) diff --git a/internal/ateclient/builder_test.go b/internal/ateclient/builder_test.go index 3c9f43393..f696051e4 100644 --- a/internal/ateclient/builder_test.go +++ b/internal/ateclient/builder_test.go @@ -32,6 +32,16 @@ import ( "k8s.io/client-go/kubernetes/fake" ) +func TestInitTracingDisabledReturnsNoProvider(t *testing.T) { + tp, err := initTracing(context.Background(), false) + if err != nil { + t.Fatalf("initTracing(disabled): %v", err) + } + if tp != nil { + t.Error("initTracing(disabled) returned a provider; want nil so no traceparent is injected") + } +} + func TestBearerTokenCreds(t *testing.T) { md, err := bearerTokenCreds("some-token").GetRequestMetadata(context.Background()) if err != nil { diff --git a/internal/serverboot/sampling.go b/internal/serverboot/sampling.go new file mode 100644 index 000000000..d5a8c8592 --- /dev/null +++ b/internal/serverboot/sampling.go @@ -0,0 +1,144 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package serverboot + +import ( + "context" + "fmt" + "log/slog" + "os" + "strconv" + "strings" + + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +const ( + tracesSamplerEnv = "OTEL_TRACES_SAMPLER" + tracesSamplerArgEnv = "OTEL_TRACES_SAMPLER_ARG" +) + +// ControlPlaneTraceRatio is the default root sampling ratio for the control +// plane binaries: low volume, and their traces are what lifecycle debugging +// needs, so the default leans generous. +const ControlPlaneTraceRatio = 0.1 + +// TraceSampling is a resolved sampling policy. The root fraction is kept +// alongside the sampler because the atenet router must mirror it into Envoy's +// RandomSampling percent, which roots the data plane traces. +type TraceSampling struct { + sampler sdktrace.Sampler + rootRatio float64 +} + +func (s TraceSampling) Sampler() sdktrace.Sampler { return s.sampler } + +// RootSamplingPercent is the percentage (0 to 100) of parentless requests +// sampled at the root. +func (s TraceSampling) RootSamplingPercent() float64 { return s.rootRatio * 100 } + +// ParentRatioSampling is ParentBased(TraceIDRatioBased(ratio)), clamped to [0, 1]. +func ParentRatioSampling(ratio float64) TraceSampling { + ratio = min(max(ratio, 0), 1) + return TraceSampling{ + sampler: sdktrace.ParentBased(sdktrace.TraceIDRatioBased(ratio)), + rootRatio: ratio, + } +} + +// ParentNeverSampling is ParentBased(NeverSample()). +func ParentNeverSampling() TraceSampling { + return TraceSampling{sampler: sdktrace.ParentBased(sdktrace.NeverSample())} +} + +// ResolveTraceSampling applies OTEL_TRACES_SAMPLER / OTEL_TRACES_SAMPLER_ARG on +// top of the component default. serverboot resolves the env itself because an +// explicit WithSampler silences the SDK's env handling, and the SDK falls open +// to 100% sampling on invalid values where this keeps the default and logs. +// That includes a ratio sampler without an arg, which the spec reads as 1.0. +func ResolveTraceSampling(ctx context.Context, def TraceSampling) TraceSampling { + name, nameSet := os.LookupEnv(tracesSamplerEnv) + arg, argSet := os.LookupEnv(tracesSamplerArgEnv) + resolved, err := resolveTraceSampling(name, nameSet, arg, argSet, def) + if err != nil { + slog.WarnContext(ctx, "Invalid trace sampler environment, keeping the component default", + slog.String("sampler", name), + slog.String("arg", arg), + slog.String("default", def.description()), + slog.Any("err", err)) + } + return resolved +} + +func (s TraceSampling) description() string { + if s.sampler == nil { + return "" + } + return s.sampler.Description() +} + +func isRatioSampler(name string) bool { + switch strings.ToLower(strings.TrimSpace(name)) { + case "traceidratio", "parentbased_traceidratio": + return true + } + return false +} + +// resolveTraceSampling accepts the sampler names the OTel SDK understands, +// minus the remote ones substrate does not vendor. Any error means def was kept. +func resolveTraceSampling(name string, nameSet bool, arg string, argSet bool, def TraceSampling) (TraceSampling, error) { + if !nameSet { + return def, nil + } + name = strings.ToLower(strings.TrimSpace(name)) + // Unlike the SDK, treat set-but-empty as unset: templated manifests can + // render empty env vars. + if name == "" { + return def, nil + } + + var ratio float64 + if isRatioSampler(name) { + trimmed := strings.TrimSpace(arg) + if !argSet || trimmed == "" { + return def, fmt.Errorf("%s %q requires %s to be a ratio in [0, 1]", tracesSamplerEnv, name, tracesSamplerArgEnv) + } + parsed, err := strconv.ParseFloat(trimmed, 64) + if err != nil { + return def, fmt.Errorf("parse %s %q: %w", tracesSamplerArgEnv, arg, err) + } + if parsed < 0 || parsed > 1 { + return def, fmt.Errorf("%s %q outside [0, 1]", tracesSamplerArgEnv, arg) + } + ratio = parsed + } + + switch name { + case "always_on": + return TraceSampling{sampler: sdktrace.AlwaysSample(), rootRatio: 1}, nil + case "always_off": + return TraceSampling{sampler: sdktrace.NeverSample()}, nil + case "traceidratio": + return TraceSampling{sampler: sdktrace.TraceIDRatioBased(ratio), rootRatio: ratio}, nil + case "parentbased_always_on": + return TraceSampling{sampler: sdktrace.ParentBased(sdktrace.AlwaysSample()), rootRatio: 1}, nil + case "parentbased_always_off": + return ParentNeverSampling(), nil + case "parentbased_traceidratio": + return ParentRatioSampling(ratio), nil + } + return def, fmt.Errorf("unsupported %s %q", tracesSamplerEnv, name) +} diff --git a/internal/serverboot/sampling_test.go b/internal/serverboot/sampling_test.go new file mode 100644 index 000000000..88dc23997 --- /dev/null +++ b/internal/serverboot/sampling_test.go @@ -0,0 +1,248 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package serverboot + +import ( + "context" + "testing" + + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +const testDefaultRatio = 0.25 + +func TestResolveTraceSampling(t *testing.T) { + def := ParentRatioSampling(testDefaultRatio) + + tests := []struct { + name string + sampler string + samplerSet bool + arg string + argSet bool + def TraceSampling + want sdktrace.Sampler + wantPercent float64 + wantErr bool + }{ + { + name: "unset keeps ratio default", + def: def, + want: def.Sampler(), + wantPercent: testDefaultRatio * 100, + }, + { + name: "unset keeps never default", + def: ParentNeverSampling(), + want: ParentNeverSampling().Sampler(), + wantPercent: 0, + }, + { + name: "set but empty keeps default", + samplerSet: true, + def: def, + want: def.Sampler(), + wantPercent: testDefaultRatio * 100, + }, + { + name: "always_on", + sampler: "always_on", + samplerSet: true, + def: def, + want: sdktrace.AlwaysSample(), + wantPercent: 100, + }, + { + name: "always_off", + sampler: "always_off", + samplerSet: true, + def: def, + want: sdktrace.NeverSample(), + wantPercent: 0, + }, + { + name: "traceidratio with arg", + sampler: "traceidratio", + samplerSet: true, + arg: "0.5", + argSet: true, + def: def, + want: sdktrace.TraceIDRatioBased(0.5), + wantPercent: 50, + }, + { + name: "parentbased_always_on", + sampler: "parentbased_always_on", + samplerSet: true, + def: def, + want: sdktrace.ParentBased(sdktrace.AlwaysSample()), + wantPercent: 100, + }, + { + name: "parentbased_always_off", + sampler: "parentbased_always_off", + samplerSet: true, + def: def, + want: sdktrace.ParentBased(sdktrace.NeverSample()), + wantPercent: 0, + }, + { + name: "parentbased_traceidratio with arg", + sampler: "parentbased_traceidratio", + samplerSet: true, + arg: "0.5", + argSet: true, + def: def, + want: sdktrace.ParentBased(sdktrace.TraceIDRatioBased(0.5)), + wantPercent: 50, + }, + { + name: "mixed case and whitespace normalize", + sampler: " ParentBased_TraceIDRatio ", + samplerSet: true, + arg: " 0.5 ", + argSet: true, + def: def, + want: sdktrace.ParentBased(sdktrace.TraceIDRatioBased(0.5)), + wantPercent: 50, + }, + { + name: "ratio without arg keeps default", + sampler: "parentbased_traceidratio", + samplerSet: true, + def: def, + want: def.Sampler(), + wantPercent: testDefaultRatio * 100, + wantErr: true, + }, + { + name: "ratio with empty arg keeps default", + sampler: "traceidratio", + samplerSet: true, + arg: " ", + argSet: true, + def: def, + want: def.Sampler(), + wantPercent: testDefaultRatio * 100, + wantErr: true, + }, + { + name: "unknown sampler keeps default", + sampler: "jaeger_remote", + samplerSet: true, + def: def, + want: def.Sampler(), + wantPercent: testDefaultRatio * 100, + wantErr: true, + }, + { + name: "unparsable arg keeps default", + sampler: "parentbased_traceidratio", + samplerSet: true, + arg: "abc", + argSet: true, + def: def, + want: def.Sampler(), + wantPercent: testDefaultRatio * 100, + wantErr: true, + }, + { + name: "arg above 1 keeps default", + sampler: "parentbased_traceidratio", + samplerSet: true, + arg: "1.5", + argSet: true, + def: def, + want: def.Sampler(), + wantPercent: testDefaultRatio * 100, + wantErr: true, + }, + { + name: "negative arg keeps default", + sampler: "traceidratio", + samplerSet: true, + arg: "-0.1", + argSet: true, + def: def, + want: def.Sampler(), + wantPercent: testDefaultRatio * 100, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := resolveTraceSampling(tt.sampler, tt.samplerSet, tt.arg, tt.argSet, tt.def) + if (err != nil) != tt.wantErr { + t.Fatalf("resolveTraceSampling() error = %v, wantErr %v", err, tt.wantErr) + } + if got.Sampler().Description() != tt.want.Description() { + t.Errorf("sampler = %s, want %s", got.Sampler().Description(), tt.want.Description()) + } + if got.RootSamplingPercent() != tt.wantPercent { + t.Errorf("RootSamplingPercent() = %v, want %v", got.RootSamplingPercent(), tt.wantPercent) + } + }) + } +} + +func TestResolveTraceSamplingReadsEnv(t *testing.T) { + def := ParentNeverSampling() + + t.Setenv("OTEL_TRACES_SAMPLER", "parentbased_traceidratio") + t.Setenv("OTEL_TRACES_SAMPLER_ARG", "0.5") + got := ResolveTraceSampling(context.Background(), def) + want := sdktrace.ParentBased(sdktrace.TraceIDRatioBased(0.5)) + if got.Sampler().Description() != want.Description() { + t.Errorf("sampler = %s, want %s", got.Sampler().Description(), want.Description()) + } + + t.Setenv("OTEL_TRACES_SAMPLER", "not_a_sampler") + got = ResolveTraceSampling(context.Background(), def) + if got.Sampler().Description() != def.Sampler().Description() { + t.Errorf("sampler = %s, want the default %s", got.Sampler().Description(), def.Sampler().Description()) + } +} + +func TestResolveTraceSamplingZeroValueDefault(t *testing.T) { + t.Setenv("OTEL_TRACES_SAMPLER", "not_a_sampler") + got := ResolveTraceSampling(context.Background(), TraceSampling{}) + if got.Sampler() != nil { + t.Errorf("Sampler() = %v, want nil from a zero-value default", got.Sampler()) + } + if got.RootSamplingPercent() != 0 { + t.Errorf("RootSamplingPercent() = %v, want 0", got.RootSamplingPercent()) + } +} + +func TestParentSamplingConstructors(t *testing.T) { + if got := ParentRatioSampling(0.01).RootSamplingPercent(); got != 1 { + t.Errorf("ParentRatioSampling(0.01).RootSamplingPercent() = %v, want 1", got) + } + if got := ParentNeverSampling().RootSamplingPercent(); got != 0 { + t.Errorf("ParentNeverSampling().RootSamplingPercent() = %v, want 0", got) + } + if got := ParentRatioSampling(1.5).RootSamplingPercent(); got != 100 { + t.Errorf("ParentRatioSampling(1.5).RootSamplingPercent() = %v, want 100 (clamped)", got) + } + if got := ParentRatioSampling(-1).RootSamplingPercent(); got != 0 { + t.Errorf("ParentRatioSampling(-1).RootSamplingPercent() = %v, want 0 (clamped)", got) + } +} + +func TestInitTracingRequiresSampling(t *testing.T) { + if _, err := InitTracing(context.Background(), TracingOptions{ServiceName: "x"}); err == nil { + t.Error("InitTracing() with zero Sampling: expected error, got nil") + } +} diff --git a/internal/serverboot/serverboot.go b/internal/serverboot/serverboot.go index bfe649995..e83f39aa4 100644 --- a/internal/serverboot/serverboot.go +++ b/internal/serverboot/serverboot.go @@ -107,9 +107,10 @@ func newResource(ctx context.Context, serviceName string) (*resource.Resource, e type TracingOptions struct { // ServiceName is required; populates resource.semconv ServiceName. ServiceName string - // Sampler is required. ateapi typically uses ParentBased(AlwaysSample); - // atelet/ateom-gvisor use ParentBased(NeverSample). - Sampler sdktrace.Sampler + // Sampling is required. Build it with ResolveTraceSampling so + // OTEL_TRACES_SAMPLER / OTEL_TRACES_SAMPLER_ARG override the component + // default. + Sampling TraceSampling } // InitTracing registers a global TracerProvider with the given options @@ -118,14 +119,24 @@ func InitTracing(ctx context.Context, opts TracingOptions) (*sdktrace.TracerProv if opts.ServiceName == "" { return nil, fmt.Errorf("TracingOptions.ServiceName is required") } + if opts.Sampling.sampler == nil { + return nil, fmt.Errorf("TracingOptions.Sampling is required") + } res, err := newResource(ctx, opts.ServiceName) if err != nil { return nil, fmt.Errorf("create tracer resource: %w", err) } + // The SDK's default handler writes to stderr, bypassing the JSON logs. + // Registered before NewTracerProvider so its env parsing complaints land + // in slog too. + otel.SetErrorHandler(otel.ErrorHandlerFunc(func(err error) { + slog.Warn("OpenTelemetry SDK error", slog.Any("err", err)) + })) + tpOpts := []sdktrace.TracerProviderOption{ sdktrace.WithResource(res), - sdktrace.WithSampler(opts.Sampler), + sdktrace.WithSampler(opts.Sampling.Sampler()), } exporter, err := otlptracegrpc.New(ctx, // GKE managed traces doesn't support validating the TLS certs of the collector. @@ -139,6 +150,7 @@ func InitTracing(ctx context.Context, opts TracingOptions) (*sdktrace.TracerProv tp := sdktrace.NewTracerProvider(tpOpts...) otel.SetTracerProvider(tp) otel.SetTextMapPropagator(propagation.TraceContext{}) + slog.InfoContext(ctx, "Tracing initialized", slog.String("sampler", opts.Sampling.Sampler().Description())) return tp, nil } diff --git a/manifests/ate-install/components/agentgateway/configmap.yaml b/manifests/ate-install/components/agentgateway/configmap.yaml index 9876b6c8a..4ca8a9f2c 100644 --- a/manifests/ate-install/components/agentgateway/configmap.yaml +++ b/manifests/ate-install/components/agentgateway/configmap.yaml @@ -30,7 +30,9 @@ data: tracing: host: $AGENTGATEWAY_OTLP_ADDRESS protocol: grpc - randomSampling: true + # Matches the router's dataPlaneTraceRatio; static config, so a router + # OTEL_TRACES_SAMPLER override must adjust it in step. + randomSampling: 0.01 gateways: http: diff --git a/manifests/ate-install/kind/kustomization.yaml b/manifests/ate-install/kind/kustomization.yaml index adc7858f4..ba668e079 100644 --- a/manifests/ate-install/kind/kustomization.yaml +++ b/manifests/ate-install/kind/kustomization.yaml @@ -40,6 +40,10 @@ resources: # On ate-controller both do double duty: the controller also propagates them to # the ateom worker pods it creates, which have no other way to receive them. # Kind only; production keeps the SDK defaults. +# +# OTEL_TRACES_SAMPLER pins ate-api-server to parentbased_always_on so every API +# call it roots is traced; the other components keep their binary defaults and +# follow its sampled flag via their ParentBased samplers. patches: - patch: |- apiVersion: apps/v1 @@ -59,6 +63,8 @@ patches: value: "10000" - name: OTEL_METRIC_EXPORT_TIMEOUT value: "10000" + - name: OTEL_TRACES_SAMPLER + value: parentbased_always_on - patch: |- apiVersion: apps/v1 kind: DaemonSet