perf(webapp,clickhouse): faster logs list and search queries

Mono-RevId: 7fd0191ab689a6a51bac2069f8297ff7e996e25a

Author: carderne

.server-changes/faster-logs-search.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: fix
+---
+
+The Logs page loads and searches faster, and busy environments no longer fail to load it. Searches that are still too expensive now show a message suggesting a narrower range instead of a generic error.

apps/webapp/app/env.server.ts

@@ -2300,8 +2300,8 @@ const EnvironmentSchema = z
       .int()
       .default(256_000_000),
     CLICKHOUSE_LOGS_LIST_MAX_THREADS: z.coerce.number().int().default(2),
-    CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ: z.coerce.number().int().default(10_000_000),
-    CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME: z.coerce.number().int().default(120),
+    // The HTTP request timeout is derived from this so ClickHouse gives up before the client does.
+    CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME: z.coerce.number().int().default(15),
     // Bound read-in-order memory on object-storage reads: each part opens a per-column read
     // stream, and the default ~1 MiB+ S3 buffers dominate peak memory. These two byte sizes
     // cap the per-stream buffers and exist on every supported ClickHouse, so they are always on.

apps/webapp/app/presenters/v3/LogsListPresenter.server.ts

@@ -1,4 +1,9 @@
-import { type ClickHouse, type WhereCondition } from "@internal/clickhouse";
+import {
+  type ClickHouse,
+  isClickhouseResourceLimitError,
+  TASK_EVENT_SEARCH_MAX_TRIGGERED_AFTER_INSERT_MS,
+  type WhereCondition,
+} from "@internal/clickhouse";
 import { type PrismaClientOrTransaction } from "@trigger.dev/database";
 import { z } from "zod";
 import { EVENT_STORE_TYPES, getConfiguredEventRepository } from "~/v3/eventRepository/index.server";
@@ -274,6 +279,14 @@ export class LogsListPresenter extends BasePresenter {
         queryBuilder.where("triggered_timestamp >= {triggeredAtStart: DateTime64(3)}", {
           triggeredAtStart: convertDateToClickhouseDateTime(effectiveFrom),
         });
+        // The table is partitioned by inserted_at, not event time. Writers clamp
+        // triggered_timestamp to at most inserted_at plus a fixed delay, so this bound only
+        // prunes partitions that cannot hold a matching row.
+        queryBuilder.where("inserted_at >= {insertedAtStart: DateTime64(3)}", {
+          insertedAtStart: convertDateToClickhouseDateTime(
+            new Date(effectiveFrom.getTime() - TASK_EVENT_SEARCH_MAX_TRIGGERED_AFTER_INSERT_MS)
+          ),
+        });
       }
 
       // Task filter (applies directly to ClickHouse)
@@ -354,6 +367,13 @@ export class LogsListPresenter extends BasePresenter {
 
     const [queryError, queryResult] = await runQuery();
     if (queryError) {
+      if (isClickhouseResourceLimitError(queryError)) {
+        throw new ServiceValidationError(
+          searchTerm === undefined
+            ? "These logs took too long to load. Try a shorter time range or add a filter."
+            : "This search took too long. Try a shorter time range, a more specific search, or add a filter."
+        );
+      }
       throw queryError;
     }
 

apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts

@@ -7,6 +7,9 @@ import { singleton } from "~/utils/singleton";
 import type { OrganizationDataStoresRegistry } from "~/services/dataStores/organizationDataStoresRegistry.server";
 import { type IEventRepository } from "~/v3/eventRepository/eventRepository.types";
 
+// Declared before the singletons below, which initialize clients at module load.
+const LOGS_LIST_REQUEST_TIMEOUT_MARGIN_SECONDS = 5;
+
 // ---------------------------------------------------------------------------
 // Default clients (singleton per process)
 // ---------------------------------------------------------------------------
@@ -70,15 +73,23 @@ function getLogsListClickhouseSettings() {
       filesystem_cache_prefer_bigger_buffer_size:
         env.CLICKHOUSE_LOGS_LIST_FILESYSTEM_CACHE_PREFER_BIGGER_BUFFER_SIZE,
     }),
-    ...(env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ && {
-      max_rows_to_read: env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ.toString(),
-    }),
+    // No max_rows_to_read: ClickHouse checks it against the estimated rows of every selected
+    // granule before reading, which rejects newest-first LIMIT queries on large environments that
+    // would stop after a few thousand rows. max_execution_time bounds the work instead.
     ...(env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME && {
       max_execution_time: env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME,
     }),
   };
 }
 
+/** Outlasts the server-side limit so ClickHouse cancels the query and reports why. */
+function getLogsListRequestTimeoutMs() {
+  if (!env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME) return undefined;
+  return (
+    (env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME + LOGS_LIST_REQUEST_TIMEOUT_MARGIN_SECONDS) * 1000
+  );
+}
+
 function initializeLogsClickhouseClient() {
   const url = new URL(
     env.LOGS_SEARCH_READER_CLICKHOUSE_URL ?? env.CLICKHOUSE_READER_URL ?? env.CLICKHOUSE_URL
@@ -95,6 +106,7 @@ function initializeLogsClickhouseClient() {
     logLevel: env.CLICKHOUSE_LOG_LEVEL,
     compression: { request: true },
     maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS,
+    requestTimeoutMs: getLogsListRequestTimeoutMs(),
     clickhouseSettings: getLogsListClickhouseSettings(),
   });
 }
@@ -592,6 +604,7 @@ function buildOrgClickhouseClient(url: string, clientType: ClientType): ClickHou
         logLevel: env.CLICKHOUSE_LOG_LEVEL,
         compression: { request: true },
         maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS,
+        requestTimeoutMs: getLogsListRequestTimeoutMs(),
         clickhouseSettings: getLogsListClickhouseSettings(),
       });
     case "engine":

internal-packages/clickhouse/src/taskEvents.ts

@@ -386,6 +386,12 @@ export function getLogsSearchListQueryBuilder(ch: ClickhouseReader) {
     ],
     settings: {
       use_query_condition_cache: 1,
+      // The ngram text index covers every organization's rows in a part, so reading it costs more
+      // than scanning one environment's time range, often by seconds when it is not cached.
+      ignore_data_skipping_indices: "idx_search_text",
+      // Hold the response until the query finishes so a limit error arrives as an HTTP error the
+      // client turns into a QueryError, not mid-stream. Pages are a few hundred small rows.
+      wait_end_of_query: 1,
     },
   });
 

internal-packages/clickhouse/src/taskEventsSearch.ts

@@ -25,6 +25,12 @@ export const TaskEventSearchV2Input = z.object({
 
 export type TaskEventSearchV2Input = z.input<typeof TaskEventSearchV2Input>;
 
+/**
+ * `triggered_timestamp` is clamped to at most `inserted_at` plus this delay, so readers can derive a
+ * safe `inserted_at` lower bound (the partition key) from an event-time lower bound.
+ */
+export const TASK_EVENT_SEARCH_MAX_TRIGGERED_AFTER_INSERT_MS = 5 * 60_000;
+
 export const TASK_EVENT_SEARCH_V2_INSERT_COLUMNS = [
   "environment_id",
   "organization_id",
@@ -138,7 +144,9 @@ function errorMessage(attributes: unknown): string {
 
 function triggeredTimestamp(startTime: string, duration: string, insertedAt: string): string {
   const completedAt = parseNanoseconds(startTime) + BigInt(duration);
-  const latestAllowed = parseMilliseconds(insertedAt) + BigInt(5 * 60_000) * BigInt(1_000_000);
+  const latestAllowed =
+    parseMilliseconds(insertedAt) +
+    BigInt(TASK_EVENT_SEARCH_MAX_TRIGGERED_AFTER_INSERT_MS) * BigInt(1_000_000);
   return formatNanoseconds(completedAt < latestAllowed ? completedAt : latestAllowed);
 }
 

internal-packages/clickhouse/src/taskEventsSearchProjector.ts

@@ -3,6 +3,7 @@ import type { Result } from "@trigger.dev/core/v3";
 import { z } from "zod";
 import type { QueryError } from "./client/errors.js";
 import type { ClickhouseWriter } from "./client/types.js";
+import { TASK_EVENT_SEARCH_MAX_TRIGGERED_AFTER_INSERT_MS } from "./taskEventsSearch.js";
 
 export type TaskEventsSearchV2ProjectionWindow = {
   start: Date;
@@ -74,7 +75,9 @@ FROM
         least(
           toInt128(toUnixTimestamp64Nano(start_time)) + toInt128(duration),
           toInt128(
-            toUnixTimestamp64Nano(inserted_at + INTERVAL 5 MINUTE)
+            toUnixTimestamp64Nano(
+              inserted_at + INTERVAL ${TASK_EVENT_SEARCH_MAX_TRIGGERED_AFTER_INSERT_MS / 1000} SECOND
+            )
           )
         )
       )