diff --git a/README.md b/README.md index cd23928..e820e1c 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ $tenant->addBatch([ Events are request-level metrics like bandwidth, executions, API calls. They are summed when aggregated. Event-specific columns (see `Metric::EVENT_COLUMNS`): `path`, `method`, `status`, -`service`, `resource`, `resourceId`, `resourceInternalId`, `teamId`, +`service`, `resource`, `resourceId`, `resourceInternalId`, `ordinal`, `teamId`, `teamInternalId`, `country`, `region`, `hostname`, `osCode`, `osName`, `osVersion`, `clientType`, `clientCode`, `clientName`, `clientVersion`, `clientEngine`, `clientEngineVersion`, `deviceName`, `deviceBrand`, diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index 4500ac9..bbd416d 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -1797,7 +1797,7 @@ private function getColumnType(string $id, string $type = 'event'): string 'connectionUsageType', 'autonomousSystemNumber', // sdk identity 'sdk', 'sdkVersion', - // gauge replica ordinal + // replica ordinal (events and gauges) 'ordinal', ]; diff --git a/src/Usage/Adapter/Database.php b/src/Usage/Adapter/Database.php index b5c21de..d5649bb 100644 --- a/src/Usage/Adapter/Database.php +++ b/src/Usage/Adapter/Database.php @@ -76,9 +76,8 @@ public function setup(): void // Event schema is a superset of the gauge schema for the dimensions // that exist in both (resourceId, resourceInternalId, teamId, - // teamInternalId), so a single Database collection backed by the - // event schema works for both types. Gauge-only columns (ordinal) - // are appended below. + // teamInternalId, ordinal), so a single Database collection backed + // by the event schema works for both types. $attributes = $this->getAttributeDocuments('event'); $indexDocs = $this->getIndexDocuments('event'); @@ -99,22 +98,6 @@ public function setup(): void 'attributes' => ['type'], ]); - // Gauge-only replica ordinal dimension. - $attributes[] = new Document([ - '$id' => 'ordinal', - 'type' => 'string', - 'size' => 255, - 'required' => false, - 'signed' => true, - 'array' => false, - 'filters' => [], - ]); - $indexDocs[] = new Document([ - '$id' => 'index-ordinal', - 'type' => 'key', - 'attributes' => ['ordinal'], - ]); - try { $this->db->createCollection( $this->collection, diff --git a/src/Usage/Metric.php b/src/Usage/Metric.php index 334feff..6175792 100644 --- a/src/Usage/Metric.php +++ b/src/Usage/Metric.php @@ -52,6 +52,7 @@ class Metric extends ArrayObject public const EVENT_COLUMNS = [ 'path', 'method', 'status', 'service', 'resourceType', 'resourceId', 'resourceInternalId', + 'ordinal', 'teamId', 'teamInternalId', 'country', 'region', 'hostname', 'ip', // request attributes (firewall rule matching) @@ -89,6 +90,7 @@ class Metric extends ArrayObject * - path / method / status: HTTP shape * - service: API service segment (storage, databases, …) * - resourceType / resourceId / resourceInternalId: resource identity + * - ordinal: replica ordinal for multi-node resources (0 is the first member) * - teamId / teamInternalId: owning team identity * - country / region / hostname / ip: geographic + caller origin * - protocol / accept / acceptLanguage / queryKeys: request attributes (firewall rule matching) @@ -104,7 +106,7 @@ class Metric extends ArrayObject * * Gauge-only dimension columns (see GAUGE_COLUMNS): * - teamId / teamInternalId / resourceId / resourceInternalId - * - ordinal: replica ordinal for multi-node resources (0 is the primary) + * - ordinal: replica ordinal for multi-node resources (0 is the first member) * * @param array $input Metric data */ @@ -286,7 +288,7 @@ public function getResourceInternalId(): ?string } /** - * Get replica ordinal (gauge metrics). 0 is the primary; 1+ are replicas. + * Get replica ordinal (event and gauge metrics). 0 is the first member; 1+ are further members. */ public function getOrdinal(): ?string { @@ -647,6 +649,7 @@ public static function getEventSchema(): array $stringColumn('resourceType', 256), $stringColumn('resourceId', 255), $stringColumn('resourceInternalId', 255), + $stringColumn('ordinal', 255), $stringColumn('teamId', 255), $stringColumn('teamInternalId', 255), $stringColumn('country', 2), @@ -773,12 +776,13 @@ public static function getEventIndexes(): array $indexed = [ 'path', 'method', 'status', 'service', 'resourceType', 'resourceId', 'resourceInternalId', + 'ordinal', 'teamId', 'teamInternalId', 'country', 'region', 'hostname', 'ip', 'osName', 'clientType', 'clientName', 'deviceName', ]; - $setIndexed = ['status', 'method', 'country', 'service', 'clientType', 'osName']; + $setIndexed = ['status', 'method', 'country', 'service', 'clientType', 'osName', 'ordinal']; return array_map( static function (string $col) use ($setIndexed): array { diff --git a/tests/Usage/Adapter/ClickHouseColumnTypeTest.php b/tests/Usage/Adapter/ClickHouseColumnTypeTest.php index b6cf638..fecb337 100644 --- a/tests/Usage/Adapter/ClickHouseColumnTypeTest.php +++ b/tests/Usage/Adapter/ClickHouseColumnTypeTest.php @@ -88,8 +88,8 @@ public function testLowCardinalitySdkColumns(): void } /** - * The gauge replica ordinal holds a handful of distinct values, so it - * must map to LowCardinality(Nullable(String)). + * The replica ordinal holds a handful of distinct values, so it + * must map to LowCardinality(Nullable(String)) on both tables. */ public function testLowCardinalityOrdinalColumn(): void { @@ -98,5 +98,10 @@ public function testLowCardinalityOrdinalColumn(): void $this->columnType('ordinal', 'gauge'), 'ordinal should be LowCardinality(Nullable(String))' ); + $this->assertSame( + 'LowCardinality(Nullable(String))', + $this->columnType('ordinal', 'event'), + 'event ordinal should be LowCardinality(Nullable(String))' + ); } } diff --git a/tests/Usage/Adapter/ClickHouseTest.php b/tests/Usage/Adapter/ClickHouseTest.php index 1a0a161..395e7ca 100644 --- a/tests/Usage/Adapter/ClickHouseTest.php +++ b/tests/Usage/Adapter/ClickHouseTest.php @@ -383,6 +383,59 @@ public function testGaugeOrdinalSeparatesReplicaSeries(): void $this->assertEquals(['0' => 10, '1' => 20], $byOrdinal); } + /** + * Per-replica event rows for one resource stay distinct series via the + * ordinal dimension. Filtering isolates one node; grouping returns a + * sum per node. The daily MV is intentionally unsplit — billing still + * sums every member. + */ + public function testEventOrdinalSeparatesReplicaSeries(): void + { + $this->usage->purge('1', [], Usage::TYPE_EVENT); + + $this->assertTrue($this->usage->addBatch([ + [ + 'tenant' => '1', + 'metric' => 'event-ordinal-test', + 'value' => 512, + 'tags' => ['resourceType' => 'dedicatedDatabases', 'resourceId' => 'db1', 'ordinal' => '0'], + ], + [ + 'tenant' => '1', + 'metric' => 'event-ordinal-test', + 'value' => 128, + 'tags' => ['resourceType' => 'dedicatedDatabases', 'resourceId' => 'db1', 'ordinal' => '1'], + ], + ], Usage::TYPE_EVENT)); + + $primary = $this->usage->find('1', [ + \Utopia\Query\Query::equal('metric', ['event-ordinal-test']), + \Utopia\Query\Query::equal('ordinal', ['0']), + ], Usage::TYPE_EVENT); + + $this->assertCount(1, $primary); + $this->assertEquals(512, $primary[0]->getValue()); + $this->assertEquals('0', $primary[0]->getOrdinal()); + + $perNode = $this->usage->find('1', [ + \Utopia\Query\Query::equal('metric', ['event-ordinal-test']), + UsageQuery::groupBy('ordinal'), + ], Usage::TYPE_EVENT); + + $this->assertCount(2, $perNode); + $byOrdinal = []; + foreach ($perNode as $row) { + $ordinal = $row->getOrdinal(); + $this->assertNotNull($ordinal); + $byOrdinal[$ordinal] = (int) $row->getValue(); + } + $this->assertEquals(['0' => 512, '1' => 128], $byOrdinal); + + $this->assertSame(640, $this->usage->sum('1', [ + \Utopia\Query\Query::equal('metric', ['event-ordinal-test']), + ])); + } + public function testUnknownTagKeyThrows(): void { $this->expectException(\Exception::class); diff --git a/tests/Usage/MetricTest.php b/tests/Usage/MetricTest.php index 720d688..971185d 100644 --- a/tests/Usage/MetricTest.php +++ b/tests/Usage/MetricTest.php @@ -607,6 +607,7 @@ public function testEventColumnsConstant(): void $expected = [ 'path', 'method', 'status', 'service', 'resourceType', 'resourceId', 'resourceInternalId', + 'ordinal', 'teamId', 'teamInternalId', 'country', 'region', 'hostname', 'ip', 'protocol', 'accept', 'acceptLanguage', 'queryKeys', @@ -633,27 +634,35 @@ public function testGaugeColumnsConstant(): void } /** - * The replica ordinal is a gauge-only dimension: present in GAUGE_COLUMNS, - * the gauge schema and gauge indexes, extracted from tags into its column, - * and readable via the typed accessor. Events must not carry it. + * The replica ordinal is a shared dimension: present in both EVENT_COLUMNS + * and GAUGE_COLUMNS, extracted from tags into its column, and readable via + * the typed accessor. Daily event rollups stay unsplit — they still group + * on resource identity only. */ - public function testOrdinalIsGaugeOnly(): void + public function testOrdinalIsASharedDimension(): void { $this->assertContains('ordinal', Metric::GAUGE_COLUMNS); - $this->assertNotContains('ordinal', Metric::EVENT_COLUMNS); + $this->assertContains('ordinal', Metric::EVENT_COLUMNS); $gaugeIds = array_column(Metric::getGaugeSchema(), '$id'); $this->assertContains('ordinal', $gaugeIds); $eventIds = array_column(Metric::getEventSchema(), '$id'); - $this->assertNotContains('ordinal', $eventIds); + $this->assertContains('ordinal', $eventIds); - $indexIds = array_column(Metric::getGaugeIndexes(), '$id'); - $this->assertContains('index-ordinal', $indexIds); + $gaugeIndexIds = array_column(Metric::getGaugeIndexes(), '$id'); + $this->assertContains('index-ordinal', $gaugeIndexIds); - $columns = Metric::extractColumns(['resourceId' => 'db_a', 'ordinal' => 1], 'gauge'); - $this->assertSame('1', $columns['ordinal']); - $this->assertSame('db_a', $columns['resourceId']); + $eventIndexIds = array_column(Metric::getEventIndexes(), '$id'); + $this->assertContains('index-ordinal', $eventIndexIds); + + $gauge = Metric::extractColumns(['resourceId' => 'db_a', 'ordinal' => 1], 'gauge'); + $this->assertSame('1', $gauge['ordinal']); + $this->assertSame('db_a', $gauge['resourceId']); + + $event = Metric::extractColumns(['resourceId' => 'db_a', 'ordinal' => 0], 'event'); + $this->assertSame('0', $event['ordinal']); + $this->assertSame('db_a', $event['resourceId']); $metric = new Metric(['ordinal' => '2']); $this->assertSame('2', $metric->getOrdinal()); @@ -667,7 +676,7 @@ public function testEventSchemaHasAllNewColumns(): void { $ids = array_column(Metric::getEventSchema(), '$id'); foreach ([ - 'service', 'resourceInternalId', 'teamId', 'teamInternalId', + 'service', 'resourceInternalId', 'ordinal', 'teamId', 'teamInternalId', 'region', 'hostname', 'osCode', 'osName', 'osVersion', 'clientType', 'clientCode', 'clientName', 'clientVersion', 'clientEngine', 'clientEngineVersion',