diff --git a/cmd/ateapi/internal/controlapi/workflow.go b/cmd/ateapi/internal/controlapi/workflow.go index a62e954f6..afb4b62ad 100644 --- a/cmd/ateapi/internal/controlapi/workflow.go +++ b/cmd/ateapi/internal/controlapi/workflow.go @@ -158,7 +158,7 @@ func NewActorWorkflow( return &ActorWorkflow{ store: store, workerCache: workerCache, - scheduler: scheduling.New(workerCache), + scheduler: scheduling.New(workerCache, scheduling.WithMeter(otel.Meter("ateapi"))), dialer: dialer, actorTemplateLister: actorTemplateLister, workerPoolLister: workerPoolLister, diff --git a/cmd/ateapi/internal/scheduling/metrics.go b/cmd/ateapi/internal/scheduling/metrics.go new file mode 100644 index 000000000..cd4e87897 --- /dev/null +++ b/cmd/ateapi/internal/scheduling/metrics.go @@ -0,0 +1,109 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scheduling + +import ( + "context" + "fmt" + "log/slog" + + "github.com/agent-substrate/substrate/internal/ateattr" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.opentelemetry.io/otel/metric" +) + +// Metric identifier for tracking eligible worker counts. +const eligibleWorkersMetric = "ate.scheduler.eligible_workers" + +// RegisterEligibleWorkers registers the ate.scheduler.eligible_workers histogram instrument against meter. +// Returns (nil, nil) if meter is nil. If registration fails, an error is returned. +func RegisterEligibleWorkers(meter metric.Meter) (metric.Int64Histogram, error) { + if meter == nil { + return nil, nil + } + hist, err := meter.Int64Histogram( + eligibleWorkersMetric, + metric.WithUnit("{worker}"), + metric.WithDescription("Number of eligible workers available during scheduling given the constraint filters."), + metric.WithExplicitBucketBoundaries(0, 1, 2, 3, 5, 10, 20, 50, 100, 250), + ) + if err != nil { + return nil, fmt.Errorf("create %s histogram: %w", eligibleWorkersMetric, err) + } + return hist, nil +} + +// WithMeter configures the meter used to create telemetry instruments for the scheduler. +// If meter is nil, the option is a no-op. If instrument creation fails, an error is explicitly logged. +func WithMeter(meter metric.Meter) Option { + return func(s *scheduler) { + hist, err := RegisterEligibleWorkers(meter) + if err != nil { + slog.Error("Failed to register ate.scheduler.eligible_workers histogram", "metric", eligibleWorkersMetric, "error", err) + return + } + s.eligibleWorkers = hist + } +} + +// Records candidate worker counts grouped by WorkerPool namespace and WorkerPool name, +// stamping sandbox class and scheduling constraint attributes on all histogram datapoints. +func (s *scheduler) recordEligibleWorkers(ctx context.Context, matching []*ateapipb.Worker, constraints Constraints) { + if s.eligibleWorkers == nil { + return + } + + // Sandbox class and constraint are constant across every key: Applies requires + // an exact class match, and the classification is per call. They belong on the + // Record call, not in the key. + type key struct{ namespace, pool string } + eligibleByPool := make(map[key]int64) + for _, w := range matching { + k := key{namespace: w.GetWorkerNamespace(), pool: w.GetWorkerPool()} + if _, ok := eligibleByPool[k]; !ok { + eligibleByPool[k] = 0 + } + if w.GetAssignment() == nil { + eligibleByPool[k]++ + } + } + + // No pool matched the constraints at all. Emit a single zero-valued series so + // "nothing is schedulable" stays visible; empty namespace/pool marks it. The + // label set matches the per-pool series, so dashboards need no special case. + if len(eligibleByPool) == 0 { + eligibleByPool[key{}] = 0 + } + + constraintStr := classifyConstraint(constraints) + for k, count := range eligibleByPool { + s.eligibleWorkers.Record(ctx, count, metric.WithAttributes( + ateattr.WorkerPoolNamespaceKey.String(k.namespace), + ateattr.WorkerPoolNameKey.String(k.pool), + ateattr.SandboxClassKey.String(constraints.SandboxClass), + ateattr.SchedulingConstraintKey.String(constraintStr), + )) + } +} + +func classifyConstraint(c Constraints) string { + if len(c.RequiredNodes) > 0 { + return ateattr.ConstraintRequiredNodes + } + if (c.TemplateSelector != nil && !c.TemplateSelector.Empty()) || (c.ActorSelector != nil && !c.ActorSelector.Empty()) { + return ateattr.ConstraintSelector + } + return ateattr.ConstraintNone +} diff --git a/cmd/ateapi/internal/scheduling/scheduling.go b/cmd/ateapi/internal/scheduling/scheduling.go index a68b6d7c9..0c69d6b15 100644 --- a/cmd/ateapi/internal/scheduling/scheduling.go +++ b/cmd/ateapi/internal/scheduling/scheduling.go @@ -23,6 +23,7 @@ import ( "slices" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.opentelemetry.io/otel/metric" "k8s.io/apimachinery/pkg/labels" ) @@ -66,6 +67,8 @@ type scheduler struct { // intn returns a uniformly distributed random value in [0,n). // Defaults to the global math/rand source intn func(n int) int + // Records the number of eligible workers available during scheduling. + eligibleWorkers metric.Int64Histogram } // Option configures the Scheduler returned by New. @@ -86,22 +89,33 @@ func New(source WorkerSource, opts ...Option) Scheduler { return s } +// Schedule filters the current worker fleet to find unassigned candidates matching the given constraints. func (s *scheduler) Schedule(ctx context.Context, constraints Constraints) (*ateapipb.Worker, error) { workers, err := s.source.Workers() if err != nil { return nil, fmt.Errorf("while listing workers: %w", err) } + // Filter for candidate workers that are unassigned and meet all scheduling constraints + matching := make([]*ateapipb.Worker, 0, len(workers)) var candidates []*ateapipb.Worker for _, worker := range workers { - if worker.GetAssignment() == nil && s.Applies(worker, constraints) { + if !s.Applies(worker, constraints) { + continue + } + matching = append(matching, worker) + if worker.GetAssignment() == nil { candidates = append(candidates, worker) } } + // Record telemetry on the number of eligible workers per pool/namespace before returning + s.recordEligibleWorkers(ctx, matching, constraints) + if len(candidates) == 0 { return nil, ErrNoCapacity } + return candidates[s.intn(len(candidates))], nil } diff --git a/cmd/ateapi/internal/scheduling/scheduling_test.go b/cmd/ateapi/internal/scheduling/scheduling_test.go index 52062e47c..68a8000f0 100644 --- a/cmd/ateapi/internal/scheduling/scheduling_test.go +++ b/cmd/ateapi/internal/scheduling/scheduling_test.go @@ -19,7 +19,10 @@ import ( "errors" "testing" + "github.com/agent-substrate/substrate/internal/ateattr" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" "k8s.io/apimachinery/pkg/labels" ) @@ -267,3 +270,301 @@ func assigned(atespace, name string) func(*ateapipb.Worker) { // firstIntn always picks the first candidate, making Schedule deterministic. func firstIntn(int) int { return 0 } + +func workerWithPool(pod, ns, pool, class, node string, lbls map[string]string, opts ...func(*ateapipb.Worker)) *ateapipb.Worker { + w := worker(pod, class, node, lbls, opts...) + w.WorkerNamespace = ns + w.WorkerPool = pool + return w +} + +func TestSchedule_EligibleWorkersMetric(t *testing.T) { + t.Run("records histogram metric with namespaced attributes for candidates", func(t *testing.T) { + reader := sdkmetric.NewManualReader() + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + meter := provider.Meter("test") + + flt := fleet{ + workerWithPool("w-1", "ns-a", "pool-1", "gvisor", "node-a", nil), + workerWithPool("w-2", "ns-a", "pool-1", "gvisor", "node-a", nil, assigned("demo", "a")), + workerWithPool("w-3", "ns-b", "pool-2", "gvisor", "node-b", nil), + } + + s := New(flt, WithIntn(firstIntn), WithMeter(meter)) + _, err := s.Schedule(context.Background(), Constraints{SandboxClass: "gvisor"}) + if err != nil { + t.Fatalf("Schedule() error = %v", err) + } + + var rm metricdata.ResourceMetrics + if err := reader.Collect(context.Background(), &rm); err != nil { + t.Fatalf("reader.Collect() error = %v", err) + } + + foundMetric := false + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name == "ate.scheduler.eligible_workers" { + foundMetric = true + histogram, ok := m.Data.(metricdata.Histogram[int64]) + if !ok { + t.Fatalf("metric Data is %T, want metricdata.Histogram[int64]", m.Data) + } + if len(histogram.DataPoints) == 0 { + t.Fatalf("got 0 DataPoints for ate.scheduler.eligible_workers") + } + for _, dp := range histogram.DataPoints { + attrs := dp.Attributes + ns, _ := attrs.Value(ateattr.WorkerPoolNamespaceKey) + pool, _ := attrs.Value(ateattr.WorkerPoolNameKey) + class, _ := attrs.Value(ateattr.SandboxClassKey) + constraint, _ := attrs.Value(ateattr.SchedulingConstraintKey) + if class.AsString() != "gvisor" { + t.Errorf("got sandbox class %q, want %q", class.AsString(), "gvisor") + } + if constraint.AsString() != "none" { + t.Errorf("got constraint %q, want %q", constraint.AsString(), "none") + } + if ns.AsString() == "ns-a" && pool.AsString() == "pool-1" { + if dp.Count != 1 || dp.Sum != 1 { + t.Errorf("pool-1 datapoint count=%d sum=%d, want count=1 sum=1", dp.Count, dp.Sum) + } + } + if ns.AsString() == "ns-b" && pool.AsString() == "pool-2" { + if dp.Count != 1 || dp.Sum != 1 { + t.Errorf("pool-2 datapoint count=%d sum=%d, want count=1 sum=1", dp.Count, dp.Sum) + } + } + } + } + } + } + if !foundMetric { + t.Fatalf("ate.scheduler.eligible_workers metric not found") + } + }) + + t.Run("records 0 eligible workers when fleet has no capacity", func(t *testing.T) { + reader := sdkmetric.NewManualReader() + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + meter := provider.Meter("test") + + flt := fleet{ + workerWithPool("w-busy", "ns-a", "pool-1", "gvisor", "node-a", nil, assigned("demo", "a")), + } + + s := New(flt, WithIntn(firstIntn), WithMeter(meter)) + _, err := s.Schedule(context.Background(), Constraints{SandboxClass: "gvisor"}) + if !errors.Is(err, ErrNoCapacity) { + t.Fatalf("Schedule() error = %v, want ErrNoCapacity", err) + } + + var rm metricdata.ResourceMetrics + if err := reader.Collect(context.Background(), &rm); err != nil { + t.Fatalf("reader.Collect() error = %v", err) + } + + foundMetric := false + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name == "ate.scheduler.eligible_workers" { + foundMetric = true + histogram, ok := m.Data.(metricdata.Histogram[int64]) + if !ok { + t.Fatalf("metric Data is %T, want metricdata.Histogram[int64]", m.Data) + } + if len(histogram.DataPoints) == 0 { + t.Fatalf("got 0 DataPoints") + } + dp := histogram.DataPoints[0] + if dp.Sum != 0 { + t.Errorf("datapoint sum = %d, want 0", dp.Sum) + } + attrs := dp.Attributes + ns, _ := attrs.Value(ateattr.WorkerPoolNamespaceKey) + pool, _ := attrs.Value(ateattr.WorkerPoolNameKey) + constraint, _ := attrs.Value(ateattr.SchedulingConstraintKey) + if ns.AsString() != "ns-a" || pool.AsString() != "pool-1" { + t.Errorf("got namespace=%q pool=%q, want ns-a / pool-1", ns.AsString(), pool.AsString()) + } + if constraint.AsString() != "none" { + t.Errorf("got constraint=%q, want none", constraint.AsString()) + } + } + } + } + if !foundMetric { + t.Fatalf("ate.scheduler.eligible_workers metric not found") + } + }) + + t.Run("records constraint classification attributes correctly", func(t *testing.T) { + sel, _ := labels.Parse("env=prod") + tests := []struct { + name string + constraints Constraints + wantConstraint string + }{ + { + name: "none", + constraints: Constraints{SandboxClass: "gvisor"}, + wantConstraint: ateattr.ConstraintNone, + }, + { + name: "selector", + constraints: Constraints{SandboxClass: "gvisor", ActorSelector: sel}, + wantConstraint: ateattr.ConstraintSelector, + }, + { + name: "required_nodes", + constraints: Constraints{SandboxClass: "gvisor", ActorSelector: sel, RequiredNodes: []string{"node-a"}}, + wantConstraint: ateattr.ConstraintRequiredNodes, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + reader := sdkmetric.NewManualReader() + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + meter := provider.Meter("test") + + flt := fleet{ + workerWithPool("w-1", "ns-a", "pool-1", "gvisor", "node-a", map[string]string{"env": "prod"}), + } + + s := New(flt, WithIntn(firstIntn), WithMeter(meter)) + _, err := s.Schedule(context.Background(), tc.constraints) + if err != nil { + t.Fatalf("Schedule() error = %v", err) + } + + var rm metricdata.ResourceMetrics + _ = reader.Collect(context.Background(), &rm) + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name == "ate.scheduler.eligible_workers" { + histogram := m.Data.(metricdata.Histogram[int64]) + dp := histogram.DataPoints[0] + constraint, _ := dp.Attributes.Value(ateattr.SchedulingConstraintKey) + if constraint.AsString() != tc.wantConstraint { + t.Errorf("got constraint=%q, want %q", constraint.AsString(), tc.wantConstraint) + } + } + } + } + }) + } + }) + + t.Run("records 0 eligible workers when fleet is completely empty", func(t *testing.T) { + reader := sdkmetric.NewManualReader() + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + meter := provider.Meter("test") + + flt := fleet{} + + s := New(flt, WithIntn(firstIntn), WithMeter(meter)) + _, err := s.Schedule(context.Background(), Constraints{SandboxClass: "gvisor"}) + if !errors.Is(err, ErrNoCapacity) { + t.Fatalf("Schedule() error = %v, want ErrNoCapacity", err) + } + + var rm metricdata.ResourceMetrics + _ = reader.Collect(context.Background(), &rm) + foundMetric := false + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name == "ate.scheduler.eligible_workers" { + foundMetric = true + histogram := m.Data.(metricdata.Histogram[int64]) + dp := histogram.DataPoints[0] + if dp.Sum != 0 { + t.Errorf("datapoint sum = %d, want 0", dp.Sum) + } + class, _ := dp.Attributes.Value(ateattr.SandboxClassKey) + if class.AsString() != "gvisor" { + t.Errorf("got sandbox class %q, want gvisor", class.AsString()) + } + } + } + } + if !foundMetric { + t.Fatalf("ate.scheduler.eligible_workers metric not found") + } + }) + + t.Run("records 0 eligible workers on sandbox class mismatch", func(t *testing.T) { + reader := sdkmetric.NewManualReader() + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + meter := provider.Meter("test") + + flt := fleet{ + workerWithPool("w-1", "ns-a", "pool-1", "gvisor", "node-a", nil), + } + + s := New(flt, WithIntn(firstIntn), WithMeter(meter)) + _, err := s.Schedule(context.Background(), Constraints{SandboxClass: "kata"}) + if !errors.Is(err, ErrNoCapacity) { + t.Fatalf("Schedule() error = %v, want ErrNoCapacity", err) + } + + var rm metricdata.ResourceMetrics + _ = reader.Collect(context.Background(), &rm) + foundMetric := false + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name == "ate.scheduler.eligible_workers" { + foundMetric = true + histogram := m.Data.(metricdata.Histogram[int64]) + dp := histogram.DataPoints[0] + if dp.Sum != 0 { + t.Errorf("datapoint sum = %d, want 0", dp.Sum) + } + class, _ := dp.Attributes.Value(ateattr.SandboxClassKey) + if class.AsString() != "kata" { + t.Errorf("got sandbox class %q, want kata", class.AsString()) + } + } + } + } + if !foundMetric { + t.Fatalf("ate.scheduler.eligible_workers metric not found") + } + }) + + t.Run("excludes draining and inactive workers from eligible counts", func(t *testing.T) { + reader := sdkmetric.NewManualReader() + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + meter := provider.Meter("test") + + drainingWorker := workerWithPool("w-draining", "ns-a", "pool-1", "gvisor", "node-a", nil) + drainingWorker.State = ateapipb.Worker_STATE_DRAINING + + flt := fleet{drainingWorker} + + s := New(flt, WithIntn(firstIntn), WithMeter(meter)) + _, err := s.Schedule(context.Background(), Constraints{SandboxClass: "gvisor"}) + if !errors.Is(err, ErrNoCapacity) { + t.Fatalf("Schedule() error = %v, want ErrNoCapacity", err) + } + + var rm metricdata.ResourceMetrics + _ = reader.Collect(context.Background(), &rm) + foundMetric := false + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name == "ate.scheduler.eligible_workers" { + foundMetric = true + histogram := m.Data.(metricdata.Histogram[int64]) + dp := histogram.DataPoints[0] + if dp.Sum != 0 { + t.Errorf("datapoint sum = %d, want 0", dp.Sum) + } + } + } + } + if !foundMetric { + t.Fatalf("ate.scheduler.eligible_workers metric not found") + } + }) +} diff --git a/docs/observability.md b/docs/observability.md index d520a6b71..86ba8804b 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -112,6 +112,7 @@ Agent Substrate emits foundational OpenTelemetry system and server metrics to mo | `rpc.server.call.duration` | ateapi & atelet (gRPC servers, via `otelgrpc`) | histogram | per-method gRPC latency, request rate, and errors (labels `rpc.method`, `rpc.response.status_code`) | | `ate.actor.crashes` | ateapi | counter | Number of times actors transitioned to `STATUS_CRASHED` with failure reasons (labels `ate.actor.operation.name`, `ate.failure.reason`, `ate.template.namespace`, `ate.template.name`, `ate.workerpool.name`, `ate.sandbox.class`) | | `atenet.router.route.duration` | atenet-router | histogram | Substrate E2E — Envoy receiving a request to Envoy forwarding it to the resolved worker, excluding actor compute and the response (labels `ate.template.namespace`, `ate.template.name`, `ate.router.outcome`, `ate.router.resume`) | +| `ate.scheduler.eligible_workers` | ateapi | histogram | number of eligible unassigned workers available during scheduling given the constraint filters (labels `ate.workerpool.namespace`, `ate.workerpool.name`, `ate.sandbox.class`, `ate.scheduling.constraint`) | | `atelet.snapshot.size` | atelet | histogram | uncompressed size in bytes of each gVisor snapshot image written during checkpoint (labels `kind`, `actor_template_namespace`, `actor_template_name`) | | `ate.workerpool.workers` | ateapi | up/down counter | live worker count per pool, split by state (`idle`/`assigned`) and sandbox class to provide fleet capacity and saturation at a glance | | `ate.actor.lifecycle.operation.duration` | ateapi | histogram | how long each actor operation (create/resume/suspend/pause/delete) takes and whether it failed (`error.type` present = failure, absent = success); labeled by operation, template, pool, sandbox class, and snapshot kind on resume; already-running resume no-ops are not recorded so the histogram tracks actual activations, not router traffic | @@ -123,6 +124,9 @@ For `atenet.router.route.duration`: * `ate.router.outcome` categorizes the route attempt result: `ok`, `cancelled`, `timeout`, `no_capacity`, `lock_conflict`, `not_found`, `unavailable`, `rate_limited`, or `resume_error`. * `ate.router.resume` indicates the singleflight execution state of actor resumption: `none` (actor already running), `triggered` (initiated cold activation), or `joined` (parked on in-flight activation). +For `ate.scheduler.eligible_workers`: +* `ate.scheduling.constraint` categorizes the scheduling request constraint type: `none` (unconstrained), `selector` (actor or template label selectors specified), or `required_nodes` (pinned to specific node VMs). + The `ate.*` control-plane metric labels are either fixed value sets (operation, outcome, state, class, kind) or scoped to the deployment catalog (template and pool names are operator-created, never derived from request payloads), and the label set varies per operation: resume carries the most dimensions, delete only the operation and error type. `ate.sandbox.class` is derived from the template (each template has exactly one class), so it adds no extra series next to the template labels; it exists so dashboards can aggregate by class without enumerating template names. High-cardinality actor identity (name/uid/atespace) stays off metrics entirely and lives on logs and traces instead. ### Local Metrics with Prometheus (Kind Cluster) diff --git a/internal/ateattr/ateattr.go b/internal/ateattr/ateattr.go index b4300cb6a..9ce5d6edf 100644 --- a/internal/ateattr/ateattr.go +++ b/internal/ateattr/ateattr.go @@ -57,16 +57,24 @@ const ( // WorkerPoolNamespaceKey pairs with WorkerPoolNameKey: a WorkerPool is // namespaced, so the name alone does not identify one. const ( - ActorOperationNameKey = attribute.Key("ate.actor.operation.name") - WorkerPoolNamespaceKey = attribute.Key("ate.workerpool.namespace") - WorkerPoolNameKey = attribute.Key("ate.workerpool.name") - WorkerStateKey = attribute.Key("ate.worker.state") - SandboxClassKey = attribute.Key("ate.sandbox.class") - SnapshotKindKey = attribute.Key("ate.snapshot.kind") - SchedulerOutcomeKey = attribute.Key("ate.scheduler.outcome") - RouterResumeKey = attribute.Key("ate.router.resume") - RouterOutcomeKey = attribute.Key("ate.router.outcome") - FailureReasonKey = attribute.Key("ate.failure.reason") + ActorOperationNameKey = attribute.Key("ate.actor.operation.name") + WorkerPoolNamespaceKey = attribute.Key("ate.workerpool.namespace") + WorkerPoolNameKey = attribute.Key("ate.workerpool.name") + WorkerStateKey = attribute.Key("ate.worker.state") + SandboxClassKey = attribute.Key("ate.sandbox.class") + SnapshotKindKey = attribute.Key("ate.snapshot.kind") + SchedulerOutcomeKey = attribute.Key("ate.scheduler.outcome") + SchedulingConstraintKey = attribute.Key("ate.scheduling.constraint") + RouterResumeKey = attribute.Key("ate.router.resume") + RouterOutcomeKey = attribute.Key("ate.router.outcome") + FailureReasonKey = attribute.Key("ate.failure.reason") +) + +// Values for SchedulingConstraintKey. +const ( + ConstraintNone = "none" + ConstraintRequiredNodes = "required_nodes" + ConstraintSelector = "selector" ) // Control-plane failure reasons for ate.actor.crashes metric. diff --git a/internal/e2e/collector_metrics.go b/internal/e2e/collector_metrics.go index a30180c50..9f25259c0 100644 --- a/internal/e2e/collector_metrics.go +++ b/internal/e2e/collector_metrics.go @@ -44,6 +44,7 @@ var PlatformMetricPrefixes = []string{ "ate_actor_lifecycle_operation_duration", "ate_scheduler_assignment_duration", "atenet_router_route_duration", + "ate_scheduler_eligible_workers", } // ScrapeCollectorMetrics port-forwards the kind stack's OTel Collector and reads diff --git a/internal/e2e/suites/metrics/metrics_test.go b/internal/e2e/suites/metrics/metrics_test.go index ec3a61625..4f4d064c7 100644 --- a/internal/e2e/suites/metrics/metrics_test.go +++ b/internal/e2e/suites/metrics/metrics_test.go @@ -24,7 +24,6 @@ import ( "context" "fmt" "os" - "strings" "testing" "time" @@ -105,61 +104,126 @@ func TestPlatformMetricsEmitted(t *testing.T) { missing = e2e.MissingPlatformMetrics(scrape, e2e.PlatformMetricPrefixes) ateomSeen = e2e.CollectorHasService(scrape, "ateom-gvisor", "ateom-microvm") if len(missing) == 0 && ateomSeen { + var errs []string + + // Verify ate_scheduler_eligible_workers metric carries valid attributes: + // - Full labels (namespace, pool, class, constraint) for per-pool candidate lines. + // - Necessary base labels (class, constraint) for edge cases when no worker pools match. + foundEligibleLine := false + foundFullPoolLine := false + for _, line := range strings.Split(scrape, "\n") { + if strings.HasPrefix(line, "ate_scheduler_eligible_workers") { + foundEligibleLine = true + nsVal := extractLabelValue(line, "ate_workerpool_namespace") + poolVal := extractLabelValue(line, "ate_workerpool_name") + classVal := extractLabelValue(line, "ate_sandbox_class") + constraintVal := extractLabelValue(line, "ate_scheduling_constraint") + + var lineErrs []string + if classVal == "" { + lineErrs = append(lineErrs, "ate_sandbox_class label is missing or empty") + } + if constraintVal == "" { + lineErrs = append(lineErrs, "ate_scheduling_constraint label is missing or empty") + } else if constraintVal != ateattr.ConstraintNone && constraintVal != ateattr.ConstraintRequiredNodes && constraintVal != ateattr.ConstraintSelector { + lineErrs = append(lineErrs, fmt.Sprintf("ate_scheduling_constraint %q is invalid (must be one of {%s, %s, %s})", + constraintVal, ateattr.ConstraintNone, ateattr.ConstraintRequiredNodes, ateattr.ConstraintSelector)) + } + + // Determine line type for error reporting. + isPerPoolLine := poolVal != "" || nsVal != "" + caseType := "[NORMAL CASE: Per-Pool Candidates Expected]" + if !isPerPoolLine { + caseType = "[EDGE CASE: No Worker Pools Matched Constraints]" + } + + // If the line has pool or namespace labels, verify both are non-empty (full per-pool line). + if isPerPoolLine { + if nsVal == "" { + lineErrs = append(lineErrs, "ate_workerpool_namespace label is missing or empty") + } + if poolVal == "" { + lineErrs = append(lineErrs, "ate_workerpool_name label is missing or empty") + } + if len(lineErrs) == 0 { + foundFullPoolLine = true + } + } + + if len(lineErrs) > 0 { + errs = append(errs, fmt.Sprintf("%s line %q failed label validation:\n - %s\n (Extracted labels: ate_workerpool_namespace=%q, ate_workerpool_name=%q, ate_sandbox_class=%q, ate_scheduling_constraint=%q)", + caseType, line, strings.Join(lineErrs, "\n - "), nsVal, poolVal, classVal, constraintVal)) + } + } + } + if !foundEligibleLine { + errs = append(errs, "ate_scheduler_eligible_workers metric line not found in collector scrape output") + } else if !foundFullPoolLine { + errs = append(errs, "ate_scheduler_eligible_workers [NORMAL CASE] per-pool candidates was not found in collector scrape output; only edge-case 0-count histogram was present") + } + // Verify ate_actor_crashes metric carries valid, non-empty low-cardinality labels for all attributes. foundCrashLine := false for _, line := range strings.Split(scrape, "\n") { if strings.HasPrefix(line, "ate_actor_crashes") { foundCrashLine = true - opVal := extractPrometheusLabelValue(line, "ate_actor_operation_name") - reasonVal := extractPrometheusLabelValue(line, "ate_failure_reason") - tmplNSVal := extractPrometheusLabelValue(line, "ate_template_namespace") - tmplNameVal := extractPrometheusLabelValue(line, "ate_template_name") - workerPoolVal := extractPrometheusLabelValue(line, "ate_workerpool_name") - sandboxVal := extractPrometheusLabelValue(line, "ate_sandbox_class") - - var errs []string + opVal := extractLabelValue(line, "ate_actor_operation_name") + reasonVal := extractLabelValue(line, "ate_failure_reason") + tmplNSVal := extractLabelValue(line, "ate_template_namespace") + tmplNameVal := extractLabelValue(line, "ate_template_name") + workerPoolVal := extractLabelValue(line, "ate_workerpool_name") + sandboxVal := extractLabelValue(line, "ate_sandbox_class") + + var crashErrs []string if opVal == "" { - errs = append(errs, "ate_actor_operation_name label is missing or empty") + crashErrs = append(crashErrs, "ate_actor_operation_name label is missing or empty") } else if ateattr.NormalizeOperationName(opVal) != opVal { - errs = append(errs, fmt.Sprintf("ate_actor_operation_name %q is invalid (must be one of {create, resume, suspend, pause, delete, unknown})", opVal)) + crashErrs = append(crashErrs, fmt.Sprintf("ate_actor_operation_name %q is invalid (must be one of {create, resume, suspend, pause, delete, unknown})", opVal)) } if reasonVal == "" { - errs = append(errs, "ate_failure_reason label is missing or empty") + crashErrs = append(crashErrs, "ate_failure_reason label is missing or empty") } else if !ateerrors.IsValidReason(reasonVal) { - errs = append(errs, fmt.Sprintf("ate_failure_reason %q is invalid (must be a registered ateerrors reason enum like CORRUPTED_ASSIGNMENT, WORKER_POD_GONE, WORKER_REASSIGNED, UNKNOWN)", reasonVal)) + crashErrs = append(crashErrs, fmt.Sprintf("ate_failure_reason %q is invalid (must be a registered ateerrors reason enum like CORRUPTED_ASSIGNMENT, WORKER_POD_GONE, WORKER_REASSIGNED, UNKNOWN)", reasonVal)) } if tmplNSVal == "" { - errs = append(errs, "ate_template_namespace label is missing or empty") + crashErrs = append(crashErrs, "ate_template_namespace label is missing or empty") } if tmplNameVal == "" { - errs = append(errs, "ate_template_name label is missing or empty") + crashErrs = append(crashErrs, "ate_template_name label is missing or empty") } if workerPoolVal == "" { - errs = append(errs, "ate_workerpool_name label is missing or empty") + crashErrs = append(crashErrs, "ate_workerpool_name label is missing or empty") } if sandboxVal == "" { - errs = append(errs, "ate_sandbox_class label is missing or empty") + crashErrs = append(crashErrs, "ate_sandbox_class label is missing or empty") } - if len(errs) == 0 { - return + if len(crashErrs) > 0 { + errs = append(errs, fmt.Sprintf("ate_actor_crashes line %q failed label validation:\n - %s\n (Extracted labels: op=%q, reason=%q, tmplNS=%q, tmplName=%q, workerPool=%q, sandboxClass=%q)", + line, strings.Join(crashErrs, "\n - "), opVal, reasonVal, tmplNSVal, tmplNameVal, workerPoolVal, sandboxVal)) } - lastLabelErr = fmt.Errorf("scraped metric line %q failed label validation:\n - %s\n (Extracted labels: op=%q, reason=%q, tmplNS=%q, tmplName=%q, workerPool=%q, sandboxClass=%q)", - line, strings.Join(errs, "\n - "), opVal, reasonVal, tmplNSVal, tmplNameVal, workerPoolVal, sandboxVal) } } if !foundCrashLine { - lastLabelErr = fmt.Errorf("ate_actor_crashes metric line not found in collector scrape output") + errs = append(errs, "ate_actor_crashes metric line not found in collector scrape output") } + + if len(errs) == 0 { + return + } + + lastLabelErr = fmt.Errorf("platform metrics label validation failed:\n - %s", strings.Join(errs, "\n - ")) + time.Sleep(2 * time.Second) + continue } time.Sleep(3 * time.Second) } if lastLabelErr != nil { - t.Fatalf("platform telemetry validation failed: missing metrics %v, ateom pushed=%v, error detail: %v", missing, ateomSeen, lastLabelErr) + t.Fatalf("platform telemetry validation failed: missing metrics %v, ateom pushed=%v, detail: %v", missing, ateomSeen, lastLabelErr) } t.Fatalf("platform telemetry never reached the collector: missing metrics %v, ateom pushed=%v", missing, ateomSeen) } @@ -216,7 +280,7 @@ func waitForStatus(t *testing.T, ctx context.Context, clients *e2e.Clients, acto t.Fatalf("actor %q never reached %v", actorID, want) } -func extractPrometheusLabelValue(line, labelName string) string { +func extractLabelValue(line, labelName string) string { key := labelName + `="` idx := strings.Index(line, key) if idx == -1 {