diff --git a/CHANGELOG.md b/CHANGELOG.md index dd279cfa131..487faf479fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -105,6 +105,7 @@ * [BUGFIX] Alertmanager: Reject the global `mattermost_webhook_url_file` setting in per-tenant configs, consistent with every other global `*_file` setting. #7768 * [BUGFIX] Alertmanager: Tighten per-tenant config validation to reject additional file-based settings. #7767 * [BUGFIX] Querier: Fix panic (`index out of range [-1]`) in the active request tracker when truncating a `match[]`/`query` value made entirely of invalid UTF-8 continuation bytes. The backwards scan for a rune boundary now stops at index 0 instead of underflowing. #7743 +* [BUGFIX] Tenant Federation: Fix regex tenant federation dropping tenants when `-blocks-storage.users-scanner.cache-ttl` is set. The regex resolver sorted the user list returned by the users scanner in place, corrupting the scanner cache and progressively losing tenants on every sync until the cache expired. #7812 ## 1.21.1 2026-06-04 diff --git a/pkg/querier/tenantfederation/regex_resolver.go b/pkg/querier/tenantfederation/regex_resolver.go index 9fb2bc33aed..7d60d7dc7d6 100644 --- a/pkg/querier/tenantfederation/regex_resolver.go +++ b/pkg/querier/tenantfederation/regex_resolver.go @@ -128,7 +128,9 @@ func (r *RegexResolver) running(ctx context.Context) error { continue } - newUsers := append(active, deleting...) + newUsers := make([]string, 0, len(active)+len(deleting)) + newUsers = append(newUsers, active...) + newUsers = append(newUsers, deleting...) sort.Strings(newUsers) r.Lock() diff --git a/pkg/querier/tenantfederation/regex_resolver_test.go b/pkg/querier/tenantfederation/regex_resolver_test.go index 92285e835eb..542b37b202b 100644 --- a/pkg/querier/tenantfederation/regex_resolver_test.go +++ b/pkg/querier/tenantfederation/regex_resolver_test.go @@ -266,6 +266,59 @@ func Test_RegexResolver_Cache(t *testing.T) { } } +func Test_RegexResolver_DoesNotMutateScannerResult(t *testing.T) { + reg := prometheus.NewRegistry() + + activeTenants := []string{"b-1", "b-2", "b-3"} + deletingTenant := "a-del" + expected := []string{"a-del", "b-1", "b-2", "b-3"} + + bucketClient := &bucket.ClientMock{} + bucketClient.MockIter("", append([]string{deletingTenant}, activeTenants...), nil) + bucketClient.MockIter("__markers__", []string{"__markers__/" + deletingTenant + "/"}, nil) + bucketClient.MockExists(users.GetGlobalDeletionMarkPath(deletingTenant), true, nil) + for _, tenant := range activeTenants { + bucketClient.MockExists(users.GetGlobalDeletionMarkPath(tenant), false, nil) + bucketClient.MockExists(users.GetLocalDeletionMarkPath(tenant), false, nil) + } + + bucketClientFactory := func(ctx context.Context) (objstore.InstrumentedBucket, error) { + return bucketClient, nil + } + + // Caching makes the users scanner return the very same slices on every scan, so + // mutating them in place would corrupt the result of all the following scans. + usersScannerConfig := users.UsersScannerConfig{Strategy: users.UserScanStrategyList, CacheTTL: time.Hour} + tenantFederationConfig := Config{UserSyncInterval: 50 * time.Millisecond, MaxTenant: 0, RegexCacheSize: 10} + regexResolver, err := NewRegexResolver(usersScannerConfig, tenantFederationConfig, reg, bucketClientFactory, log.NewNopLogger()) + require.NoError(t, err) + + require.NoError(t, services.StartAndAwaitRunning(context.Background(), regexResolver)) + defer services.StopAndAwaitTerminated(context.Background(), regexResolver) //nolint:errcheck + + // Wait for the first scan. + test.Poll(t, 10*time.Second, true, func() any { + return testutil.ToFloat64(regexResolver.lastUpdateUserRun) > 0 + }) + firstRun := testutil.ToFloat64(regexResolver.lastUpdateUserRun) + + // Wait for several more scans. The corruption only shows up from the second scan + // on, once the scanner starts returning the previously mutated slices. + test.Poll(t, 10*time.Second, true, func() any { + return testutil.ToFloat64(regexResolver.lastUpdateUserRun) > firstRun + }) + + regexResolver.RLock() + knownUsers := append([]string(nil), regexResolver.knownUsers...) + regexResolver.RUnlock() + require.Equal(t, expected, knownUsers) + + ctx := user.InjectOrgID(context.Background(), "b-.+") + orgIDs, err := regexResolver.TenantIDs(ctx) + require.NoError(t, err) + require.Equal(t, activeTenants, orgIDs) +} + func Test_RegexResolver_CacheInvalidation(t *testing.T) { reg := prometheus.NewRegistry() initialTenants := []string{"user-1", "user-2"}