-
Notifications
You must be signed in to change notification settings - Fork 16
Fix/indexing behavior dependencies #1909
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| import config from "@/config"; | ||
|
|
||
| import { getENSRootChainId } from "@ensnode/datasources"; | ||
|
|
||
| import type { EnsIndexerConfig } from "@/config/types"; | ||
| import { ensDbClient } from "@/lib/ensdb/singleton"; | ||
|
|
||
| /** | ||
| * Indexing Behavior Dependencies | ||
| * | ||
| * The following values are indexing behavior dependencies: | ||
| * - ENSDb: Schema checksum | ||
| * - Different ENSDb Schema definitions may influence indexing behavior. | ||
| * - ENSIndexer: config fields | ||
| * - `namespace` | ||
| * - Changes the datasources used for indexing, which influences the indexing behavior. | ||
| * - `plugins` | ||
| * - Can change the indexed chains and contracts, which influences the indexing behavior. | ||
| * - `globalBlockrange` | ||
| * - Changes the blockrange of indexed chains, which influences the indexing behavior. | ||
| * - `isSubgraphCompatible` | ||
| * - Changes the indexing logic, which influences the indexing behavior. | ||
| * - `labelSet` | ||
| * - Changes the label set used for healing labels during indexing, which influences the indexing behavior. | ||
| */ | ||
| const indexingBehaviorDependencies = { | ||
| // while technically not necessary, since these configuration properties are reflected in the | ||
| // generated ponderConfig, we include them here for clarity | ||
| namespace: config.namespace, | ||
| plugins: config.plugins, | ||
| globalBlockrange: config.globalBlockrange, | ||
| // these config properties don't explicitly affect the generated ponderConfig and need to be | ||
| // injected here to ensure that, if they are configured differently, ponder generates a unique | ||
| // build id to differentiate between runs with otherwise-identical configs (see above). | ||
| isSubgraphCompatible: config.isSubgraphCompatible, | ||
| labelSet: config.labelSet, | ||
| ensDbSchemaChecksum: ensDbClient.ensDbSchemaChecksum, | ||
| } satisfies Pick< | ||
| EnsIndexerConfig, | ||
| "namespace" | "plugins" | "globalBlockrange" | "isSubgraphCompatible" | "labelSet" | ||
| > & { ensDbSchemaChecksum: string }; | ||
|
|
||
| // We use the root chain ID to build a minimal representation of a valid contract config for ENSIndexer. | ||
| const rootChainId = getENSRootChainId(config.namespace); | ||
|
|
||
| /** | ||
| * Build a special "indexing behavior injection" contracts config for ENSIndexer | ||
| * to inject into `contracts` field of the Ponder config. The contracts config | ||
| * does not represent any real contracts to index, but rather serves as | ||
| * a container for all indexing behavior dependencies that should be reflected | ||
| * in the Ponder build ID. | ||
| */ | ||
| export function buildIndexingBehaviorInjectionContracts() { | ||
| return { | ||
| IndexingBehaviorInjectionContract: { | ||
| chain: `${rootChainId}`, | ||
| indexingBehaviorDependencies, | ||
| }, | ||
|
Comment on lines
+53
to
+58
|
||
| } as const; | ||
| } | ||
|
Comment on lines
+53
to
+60
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🧹 Nitpick | 🔵 Trivial Module-level initialization occurs at import time. The 🤖 Prompt for AI Agents |
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -13,6 +13,7 @@ import { | |
| buildIndividualEnsDbSchemas, | ||
| type EnsDbDrizzleClient, | ||
| type EnsNodeSchema, | ||
| getDrizzleSchemaChecksum, | ||
| } from "../lib/drizzle"; | ||
| import { EnsNodeMetadataKeys } from "./ensnode-metadata"; | ||
| import type { | ||
|
|
@@ -94,6 +95,23 @@ export class EnsDbReader< | |
| return this.drizzleClient; | ||
| } | ||
|
|
||
| /** | ||
| * Getter for the ENSDb Schema Checksum | ||
| * | ||
| * The Schema Checksum is a hash based on the "concrete" ENSIndexer Schema and | ||
| * the ENSNode Schema definitions used in the Drizzle client for ENSDb. | ||
| * | ||
| * It can be used to verify that the {@link ensDb} Drizzle client | ||
| * is using the expected schema definitions while interacting with | ||
| * the ENSDb instance. | ||
| */ | ||
| get ensDbSchemaChecksum(): string { | ||
| return getDrizzleSchemaChecksum({ | ||
| ...this._concreteEnsIndexerSchema, | ||
| ...this._ensNodeSchema, | ||
| }); | ||
| } | ||
|
Comment on lines
+108
to
+113
|
||
|
|
||
| /** | ||
| * Getter for the "concrete" ENSIndexer Schema definition used in the Drizzle client | ||
| * for ENSDb instance. | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,9 @@ | ||
| /** | ||
| * Utilities for Drizzle ORM integration with ENSDb. | ||
| */ | ||
|
|
||
| import { createHash } from "node:crypto"; | ||
|
|
||
| import type { Logger as DrizzleLogger } from "drizzle-orm/logger"; | ||
| import { drizzle, type NodePgDatabase } from "drizzle-orm/node-postgres"; | ||
| import { isPgEnum } from "drizzle-orm/pg-core"; | ||
|
|
@@ -169,3 +172,39 @@ export function buildEnsDbDrizzleClient<ConcreteEnsIndexerSchema extends Abstrac | |
| logger, | ||
| }); | ||
| } | ||
|
|
||
| /** | ||
| * Safely stringify a Drizzle schema definition. | ||
| * | ||
| * Handles circular references in the Drizzle schema definition by replacing | ||
| * them with the string "[circular]". Thanks to this, we can safely stringify | ||
| * any Drizzle schema definition without running into errors due to inability | ||
| * of {@link JSON.stringify} to handle circular references by default. | ||
| * | ||
| * @param schema - The Drizzle schema definition to stringify. | ||
| * @returns A JSON string representation of the schema, with circular | ||
| * references replaced by "[circular]". | ||
| */ | ||
| function safeStringifyDrizzleSchema(schema: Record<string, unknown>): string { | ||
| const seen = new WeakSet(); | ||
|
|
||
| return JSON.stringify(schema, (_key, value) => { | ||
| if (typeof value === "object" && value !== null) { | ||
| if (seen.has(value)) return "[circular]"; | ||
| seen.add(value); | ||
| } | ||
|
|
||
| return value; | ||
| }); | ||
| } | ||
|
Comment on lines
+188
to
+199
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Non-deterministic key ordering may cause inconsistent checksums.
Consider sorting keys during serialization for deterministic output: 🔧 Proposed fix for deterministic serialization function safeStringifyDrizzleSchema(schema: Record<string, unknown>): string {
const seen = new WeakSet();
- return JSON.stringify(schema, (_key, value) => {
+ return JSON.stringify(schema, (_key, value) => {
if (typeof value === "object" && value !== null) {
if (seen.has(value)) return "[circular]";
seen.add(value);
+ // Sort object keys for deterministic output
+ if (!Array.isArray(value)) {
+ const sorted: Record<string, unknown> = {};
+ for (const k of Object.keys(value).sort()) {
+ sorted[k] = (value as Record<string, unknown>)[k];
+ }
+ return sorted;
+ }
}
return value;
});
}🤖 Prompt for AI Agents |
||
|
|
||
| /** | ||
| * Get a checksum for a Drizzle schema definition. | ||
| * @param schema - The Drizzle schema definition. | ||
| * @returns A 10-character checksum string for the schema. | ||
| */ | ||
| export function getDrizzleSchemaChecksum(schema: Record<string, unknown>): string { | ||
| const stringifiedSchema = safeStringifyDrizzleSchema(schema); | ||
|
|
||
| return createHash("sha256").update(stringifiedSchema).digest("hex").slice(0, 10); | ||
| } | ||
|
Comment on lines
+206
to
+210
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The IndexingBehaviorInjectionContract has an invalid chain property structure - it's a string instead of an object mapping chain IDs to configuration objects