Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/ateapi/internal/controlapi/workload_spec.go
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ func toAteletReadyz(in *atev1alpha1.ContainerReadyz) *ateletpb.Readyz {
if in == nil {
return nil
}
out := &ateletpb.Readyz{}
out := &ateletpb.Readyz{TimeoutSeconds: in.TimeoutSeconds}
if in.HTTPGet != nil {
out.HttpGet = &ateletpb.HTTPGetAction{
Path: in.HTTPGet.Path,
Expand Down
6 changes: 4 additions & 2 deletions cmd/ateapi/internal/controlapi/workload_spec_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -358,7 +358,8 @@ func TestWorkloadSpecFromActorTemplatePropagatesReadyz(t *testing.T) {
Name: "with-probe",
Image: "main",
Readyz: &atev1alpha1.ContainerReadyz{
HTTPGet: &atev1alpha1.HTTPGetAction{Path: "/health", Port: 8080},
HTTPGet: &atev1alpha1.HTTPGetAction{Path: "/health", Port: 8080},
TimeoutSeconds: 45,
},
},
{
Expand All @@ -378,7 +379,8 @@ func TestWorkloadSpecFromActorTemplatePropagatesReadyz(t *testing.T) {
Name: "with-probe",
Image: "main",
Readyz: &ateletpb.Readyz{
HttpGet: &ateletpb.HTTPGetAction{Path: "/health", Port: 8080},
HttpGet: &ateletpb.HTTPGetAction{Path: "/health", Port: 8080},
TimeoutSeconds: 45,
},
},
{
Expand Down
1 change: 1 addition & 0 deletions cmd/atelet/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -951,6 +951,7 @@ func toAteomReadyz(in *ateletpb.Readyz) *ateompb.Readyz {
Port: hg.GetPort(),
}
}
out.TimeoutSeconds = in.GetTimeoutSeconds()
return out
}

Expand Down
6 changes: 4 additions & 2 deletions cmd/atelet/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -588,7 +588,8 @@ func TestBuildAteomWorkloadSpecForwardsReadyz(t *testing.T) {
Name: "with-probe",
Image: "main",
Readyz: &ateletpb.Readyz{
HttpGet: &ateletpb.HTTPGetAction{Path: "/health", Port: 8080},
HttpGet: &ateletpb.HTTPGetAction{Path: "/health", Port: 8080},
TimeoutSeconds: 45,
},
},
{
Expand All @@ -601,7 +602,8 @@ func TestBuildAteomWorkloadSpecForwardsReadyz(t *testing.T) {
{
Name: "with-probe",
Readyz: &ateompb.Readyz{
HttpGet: &ateompb.HTTPGetAction{Path: "/health", Port: 8080},
HttpGet: &ateompb.HTTPGetAction{Path: "/health", Port: 8080},
TimeoutSeconds: 45,
},
},
{Name: "without-probe"},
Expand Down
9 changes: 9 additions & 0 deletions internal/e2e/fixtures/probe/probe.yaml.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,15 @@ spec:
- name: probe
image: ko://github.com/agent-substrate/substrate/internal/e2e/fixtures/probe
command: ["/ko-app/probe"]
# The probe binary binds :80 immediately, so this gates actor start on a
# readiness signal rather than a guess, and carries a non-default
# timeoutSeconds so e2e covers the value crossing ateapi -> atelet -> ateom
# instead of only the ateom's built-in default.
readyz:
httpGet:
path: /healthz
port: 80
timeoutSeconds: 60
workerSelector:
matchLabels:
workload: probe
Expand Down
23 changes: 17 additions & 6 deletions internal/proto/ateletpb/atelet.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions internal/proto/ateletpb/atelet.proto
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,9 @@ message EnvEntry {
// Only HTTP is supported today.
message Readyz {
HTTPGetAction http_get = 1;
// How long to keep polling before giving up and failing the actor start.
// Zero means the ateom's default.
int32 timeout_seconds = 2;
}

// HTTPGetAction performs an HTTP GET against the container.
Expand Down
23 changes: 17 additions & 6 deletions internal/proto/ateompb/ateom.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions internal/proto/ateompb/ateom.proto
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,9 @@ message DurableDirVolumeMount {
// Only HTTP is supported today.
message Readyz {
HTTPGetAction http_get = 1;
// How long to keep polling before giving up and failing the actor start.
// Zero means the ateom's default.
int32 timeout_seconds = 2;
}

// HTTPGetAction performs an HTTP GET against the container.
Expand Down
29 changes: 22 additions & 7 deletions internal/readyz/readyz.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,14 @@ import (
// a few seconds to bind; HTTPClient below is a var so tests can substitute
// a transport that targets a test server's loopback address.
const (
OverallTimeout = 30 * time.Second
RequestTimeout = 250 * time.Millisecond
PollInterval = 1 * time.Millisecond
DefaultPath = "/readyz"
maxIdleConnsHost = 1
// DefaultOverallTimeout applies to probes that do not set
// timeout_seconds. A workload that needs longer says so on its
// ActorTemplate rather than having every actor wait as long.
DefaultOverallTimeout = 30 * time.Second
RequestTimeout = 250 * time.Millisecond
PollInterval = 1 * time.Millisecond
DefaultPath = "/readyz"
maxIdleConnsHost = 1
)

// HTTPClient builds a keep-alive HTTP client tuned for fast, repeated
Expand Down Expand Up @@ -87,8 +90,9 @@ func Wait(ctx context.Context, containerName string, probe *ateompb.Readyz, acto
client := HTTPClient()
defer client.CloseIdleConnections()

timeout := overallTimeout(probe)
start := time.Now()
deadline := start.Add(OverallTimeout)
deadline := start.Add(timeout)
attempts := 0
var lastErr error
for {
Expand All @@ -98,7 +102,7 @@ func Wait(ctx context.Context, containerName string, probe *ateompb.Readyz, acto
}
if time.Now().After(deadline) {
return fmt.Errorf("readyz for %q never returned 200 within %s (%d attempts, last error: %v)",
containerName, OverallTimeout, attempts, lastErr)
containerName, timeout, attempts, lastErr)
}

attempts++
Expand Down Expand Up @@ -126,6 +130,17 @@ func Wait(ctx context.Context, containerName string, probe *ateompb.Readyz, acto
}
}

// overallTimeout resolves how long Wait polls before giving up. A
// non-positive timeout_seconds falls back to the default: unlike a warmup
// delay, a zero deadline is never a meaningful request, so it means "unset"
// rather than "fail immediately".
func overallTimeout(probe *ateompb.Readyz) time.Duration {
if s := probe.GetTimeoutSeconds(); s > 0 {
return time.Duration(s) * time.Second
}
return DefaultOverallTimeout
}

func tryOnce(ctx context.Context, client *http.Client, url string) (bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
Expand Down
58 changes: 58 additions & 0 deletions internal/readyz/readyz_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,64 @@ func TestWait_ContextCancellation(t *testing.T) {
}
}

func TestOverallTimeout(t *testing.T) {
tests := []struct {
name string
probe *ateompb.Readyz
want time.Duration
}{
{
name: "unset falls back to the default",
probe: &ateompb.Readyz{},
want: DefaultOverallTimeout,
},
{
name: "explicit value is honored",
probe: &ateompb.Readyz{TimeoutSeconds: 300},
want: 300 * time.Second,
},
{
// A zero deadline could never be met, so it means "unset"
// rather than "fail immediately".
name: "negative falls back to the default",
probe: &ateompb.Readyz{TimeoutSeconds: -1},
want: DefaultOverallTimeout,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := overallTimeout(tt.probe); got != tt.want {
t.Errorf("overallTimeout = %v, want %v", got, tt.want)
}
})
}
}

func TestWait_GivesUpAtProbeTimeout(t *testing.T) {
// Nothing ever binds this port, so the poll loop runs until the
// probe's own deadline rather than the package default.
port := pickFreePort(t)
probe := &ateompb.Readyz{
HttpGet: &ateompb.HTTPGetAction{Port: int32(port)},
TimeoutSeconds: 1,
}

ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
start := time.Now()
err := Wait(ctx, "main", probe, "127.0.0.1")
if err == nil {
t.Fatalf("Wait returned nil, expected a timeout error")
}
elapsed := time.Since(start)
if elapsed < time.Second {
t.Errorf("Wait gave up after %v, before the probe's 1s timeout", elapsed)
}
if elapsed > 5*time.Second {
t.Errorf("Wait took %v; the probe timeout was ignored in favor of the %v default", elapsed, DefaultOverallTimeout)
}
}

func TestWaitAll_SkipsContainersWithoutProbe(t *testing.T) {
// No server bound, but no probes => should return nil immediately.
containers := []*ateompb.Container{
Expand Down
20 changes: 20 additions & 0 deletions manifests/ate-install/generated/ate.dev_actortemplates.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,26 @@ spec:
required:
- port
type: object
timeoutSeconds:
default: 30
description: |-
TimeoutSeconds is how long to keep polling HTTPGet before giving up.
Exceeding it fails the actor start rather than proceeding with a
container that never reported ready.

How long a workload takes to become ready is a property of that workload,
which is why this is set per template rather than cluster-wide: a heavy
runtime that needs minutes should not force every other template to wait
as long before its failures surface.

Unset defaults to 30, applied by the API server so the effective value is
visible on the stored object rather than only in the ateom. A manifest
asking for 0 is rejected: unlike a warmup delay, a zero deadline could
never be met, so it is never what a template author means.
format: int32
maximum: 3600
minimum: 1
type: integer
required:
- httpGet
type: object
Expand Down
20 changes: 20 additions & 0 deletions pkg/api/v1alpha1/actortemplate_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,26 @@ type ContainerReadyz struct {
//
// +required
HTTPGet *HTTPGetAction `json:"httpGet"`

// TimeoutSeconds is how long to keep polling HTTPGet before giving up.
// Exceeding it fails the actor start rather than proceeding with a
// container that never reported ready.
//
// How long a workload takes to become ready is a property of that workload,
// which is why this is set per template rather than cluster-wide: a heavy
// runtime that needs minutes should not force every other template to wait
// as long before its failures surface.
//
// Unset defaults to 30, applied by the API server so the effective value is
// visible on the stored object rather than only in the ateom. A manifest
// asking for 0 is rejected: unlike a warmup delay, a zero deadline could
// never be met, so it is never what a template author means.
//
// +optional
Comment thread
mayawang marked this conversation as resolved.
// +kubebuilder:default=30
// +kubebuilder:validation:Minimum=1
// +kubebuilder:validation:Maximum=3600
TimeoutSeconds int32 `json:"timeoutSeconds,omitempty"`
}

// HTTPGetAction describes an HTTP GET request to perform against the
Expand Down
Loading
Loading