Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ $tenant->addBatch([
Events are request-level metrics like bandwidth, executions, API calls. They are summed when aggregated.

Event-specific columns (see `Metric::EVENT_COLUMNS`): `path`, `method`, `status`,
`service`, `resource`, `resourceId`, `resourceInternalId`, `teamId`,
`service`, `resource`, `resourceId`, `resourceInternalId`, `ordinal`, `teamId`,
`teamInternalId`, `country`, `region`, `hostname`, `osCode`, `osName`,
`osVersion`, `clientType`, `clientCode`, `clientName`, `clientVersion`,
`clientEngine`, `clientEngineVersion`, `deviceName`, `deviceBrand`,
Expand Down
2 changes: 1 addition & 1 deletion src/Usage/Adapter/ClickHouse.php
Original file line number Diff line number Diff line change
Expand Up @@ -1797,7 +1797,7 @@ private function getColumnType(string $id, string $type = 'event'): string
'connectionUsageType', 'autonomousSystemNumber',
// sdk identity
'sdk', 'sdkVersion',
// gauge replica ordinal
// replica ordinal (events and gauges)
'ordinal',
];

Expand Down
21 changes: 2 additions & 19 deletions src/Usage/Adapter/Database.php
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,8 @@ public function setup(): void

// Event schema is a superset of the gauge schema for the dimensions
// that exist in both (resourceId, resourceInternalId, teamId,
// teamInternalId), so a single Database collection backed by the
// event schema works for both types. Gauge-only columns (ordinal)
// are appended below.
// teamInternalId, ordinal), so a single Database collection backed
// by the event schema works for both types.
$attributes = $this->getAttributeDocuments('event');
$indexDocs = $this->getIndexDocuments('event');

Expand All @@ -99,22 +98,6 @@ public function setup(): void
'attributes' => ['type'],
]);

// Gauge-only replica ordinal dimension.
$attributes[] = new Document([
'$id' => 'ordinal',
'type' => 'string',
'size' => 255,
'required' => false,
'signed' => true,
'array' => false,
'filters' => [],
]);
$indexDocs[] = new Document([
'$id' => 'index-ordinal',
'type' => 'key',
'attributes' => ['ordinal'],
]);

try {
$this->db->createCollection(
$this->collection,
Expand Down
10 changes: 7 additions & 3 deletions src/Usage/Metric.php
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ class Metric extends ArrayObject
public const EVENT_COLUMNS = [
'path', 'method', 'status',
'service', 'resourceType', 'resourceId', 'resourceInternalId',
'ordinal',
'teamId', 'teamInternalId',
'country', 'region', 'hostname', 'ip',
// request attributes (firewall rule matching)
Expand Down Expand Up @@ -89,6 +90,7 @@ class Metric extends ArrayObject
* - path / method / status: HTTP shape
* - service: API service segment (storage, databases, …)
* - resourceType / resourceId / resourceInternalId: resource identity
* - ordinal: replica ordinal for multi-node resources (0 is the first member)
* - teamId / teamInternalId: owning team identity
* - country / region / hostname / ip: geographic + caller origin
* - protocol / accept / acceptLanguage / queryKeys: request attributes (firewall rule matching)
Expand All @@ -104,7 +106,7 @@ class Metric extends ArrayObject
*
* Gauge-only dimension columns (see GAUGE_COLUMNS):
* - teamId / teamInternalId / resourceId / resourceInternalId
* - ordinal: replica ordinal for multi-node resources (0 is the primary)
* - ordinal: replica ordinal for multi-node resources (0 is the first member)
*
* @param array<string, mixed> $input Metric data
*/
Expand Down Expand Up @@ -286,7 +288,7 @@ public function getResourceInternalId(): ?string
}

/**
* Get replica ordinal (gauge metrics). 0 is the primary; 1+ are replicas.
* Get replica ordinal (event and gauge metrics). 0 is the first member; 1+ are further members.
*/
public function getOrdinal(): ?string
{
Expand Down Expand Up @@ -647,6 +649,7 @@ public static function getEventSchema(): array
$stringColumn('resourceType', 256),
$stringColumn('resourceId', 255),
$stringColumn('resourceInternalId', 255),
$stringColumn('ordinal', 255),
$stringColumn('teamId', 255),
$stringColumn('teamInternalId', 255),
$stringColumn('country', 2),
Expand Down Expand Up @@ -773,12 +776,13 @@ public static function getEventIndexes(): array
$indexed = [
'path', 'method', 'status',
'service', 'resourceType', 'resourceId', 'resourceInternalId',
'ordinal',
'teamId', 'teamInternalId',
'country', 'region', 'hostname', 'ip',
'osName', 'clientType', 'clientName', 'deviceName',
];

$setIndexed = ['status', 'method', 'country', 'service', 'clientType', 'osName'];
$setIndexed = ['status', 'method', 'country', 'service', 'clientType', 'osName', 'ordinal'];

return array_map(
static function (string $col) use ($setIndexed): array {
Expand Down
9 changes: 7 additions & 2 deletions tests/Usage/Adapter/ClickHouseColumnTypeTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,8 @@ public function testLowCardinalitySdkColumns(): void
}

/**
* The gauge replica ordinal holds a handful of distinct values, so it
* must map to LowCardinality(Nullable(String)).
* The replica ordinal holds a handful of distinct values, so it
* must map to LowCardinality(Nullable(String)) on both tables.
*/
public function testLowCardinalityOrdinalColumn(): void
{
Expand All @@ -98,5 +98,10 @@ public function testLowCardinalityOrdinalColumn(): void
$this->columnType('ordinal', 'gauge'),
'ordinal should be LowCardinality(Nullable(String))'
);
$this->assertSame(
'LowCardinality(Nullable(String))',
$this->columnType('ordinal', 'event'),
'event ordinal should be LowCardinality(Nullable(String))'
);
}
}
53 changes: 53 additions & 0 deletions tests/Usage/Adapter/ClickHouseTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -383,6 +383,59 @@ public function testGaugeOrdinalSeparatesReplicaSeries(): void
$this->assertEquals(['0' => 10, '1' => 20], $byOrdinal);
}

/**
* Per-replica event rows for one resource stay distinct series via the
* ordinal dimension. Filtering isolates one node; grouping returns a
* sum per node. The daily MV is intentionally unsplit — billing still
* sums every member.
*/
public function testEventOrdinalSeparatesReplicaSeries(): void
{
$this->usage->purge('1', [], Usage::TYPE_EVENT);

$this->assertTrue($this->usage->addBatch([
[
'tenant' => '1',
'metric' => 'event-ordinal-test',
'value' => 512,
'tags' => ['resourceType' => 'dedicatedDatabases', 'resourceId' => 'db1', 'ordinal' => '0'],
],
[
'tenant' => '1',
'metric' => 'event-ordinal-test',
'value' => 128,
'tags' => ['resourceType' => 'dedicatedDatabases', 'resourceId' => 'db1', 'ordinal' => '1'],
],
], Usage::TYPE_EVENT));

$primary = $this->usage->find('1', [
\Utopia\Query\Query::equal('metric', ['event-ordinal-test']),
\Utopia\Query\Query::equal('ordinal', ['0']),
], Usage::TYPE_EVENT);

$this->assertCount(1, $primary);
$this->assertEquals(512, $primary[0]->getValue());
$this->assertEquals('0', $primary[0]->getOrdinal());

$perNode = $this->usage->find('1', [
\Utopia\Query\Query::equal('metric', ['event-ordinal-test']),
UsageQuery::groupBy('ordinal'),
], Usage::TYPE_EVENT);

$this->assertCount(2, $perNode);
$byOrdinal = [];
foreach ($perNode as $row) {
$ordinal = $row->getOrdinal();
$this->assertNotNull($ordinal);
$byOrdinal[$ordinal] = (int) $row->getValue();
}
$this->assertEquals(['0' => 512, '1' => 128], $byOrdinal);

$this->assertSame(640, $this->usage->sum('1', [
\Utopia\Query\Query::equal('metric', ['event-ordinal-test']),
]));
}

public function testUnknownTagKeyThrows(): void
{
$this->expectException(\Exception::class);
Expand Down
33 changes: 21 additions & 12 deletions tests/Usage/MetricTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -607,6 +607,7 @@ public function testEventColumnsConstant(): void
$expected = [
'path', 'method', 'status',
'service', 'resourceType', 'resourceId', 'resourceInternalId',
'ordinal',
'teamId', 'teamInternalId',
'country', 'region', 'hostname', 'ip',
'protocol', 'accept', 'acceptLanguage', 'queryKeys',
Expand All @@ -633,27 +634,35 @@ public function testGaugeColumnsConstant(): void
}

/**
* The replica ordinal is a gauge-only dimension: present in GAUGE_COLUMNS,
* the gauge schema and gauge indexes, extracted from tags into its column,
* and readable via the typed accessor. Events must not carry it.
* The replica ordinal is a shared dimension: present in both EVENT_COLUMNS
* and GAUGE_COLUMNS, extracted from tags into its column, and readable via
* the typed accessor. Daily event rollups stay unsplit — they still group
* on resource identity only.
*/
public function testOrdinalIsGaugeOnly(): void
public function testOrdinalIsASharedDimension(): void
{
$this->assertContains('ordinal', Metric::GAUGE_COLUMNS);
$this->assertNotContains('ordinal', Metric::EVENT_COLUMNS);
$this->assertContains('ordinal', Metric::EVENT_COLUMNS);

$gaugeIds = array_column(Metric::getGaugeSchema(), '$id');
$this->assertContains('ordinal', $gaugeIds);

$eventIds = array_column(Metric::getEventSchema(), '$id');
$this->assertNotContains('ordinal', $eventIds);
$this->assertContains('ordinal', $eventIds);

$indexIds = array_column(Metric::getGaugeIndexes(), '$id');
$this->assertContains('index-ordinal', $indexIds);
$gaugeIndexIds = array_column(Metric::getGaugeIndexes(), '$id');
$this->assertContains('index-ordinal', $gaugeIndexIds);

$columns = Metric::extractColumns(['resourceId' => 'db_a', 'ordinal' => 1], 'gauge');
$this->assertSame('1', $columns['ordinal']);
$this->assertSame('db_a', $columns['resourceId']);
$eventIndexIds = array_column(Metric::getEventIndexes(), '$id');
$this->assertContains('index-ordinal', $eventIndexIds);

$gauge = Metric::extractColumns(['resourceId' => 'db_a', 'ordinal' => 1], 'gauge');
$this->assertSame('1', $gauge['ordinal']);
$this->assertSame('db_a', $gauge['resourceId']);

$event = Metric::extractColumns(['resourceId' => 'db_a', 'ordinal' => 0], 'event');
$this->assertSame('0', $event['ordinal']);
$this->assertSame('db_a', $event['resourceId']);

$metric = new Metric(['ordinal' => '2']);
$this->assertSame('2', $metric->getOrdinal());
Expand All @@ -667,7 +676,7 @@ public function testEventSchemaHasAllNewColumns(): void
{
$ids = array_column(Metric::getEventSchema(), '$id');
foreach ([
'service', 'resourceInternalId', 'teamId', 'teamInternalId',
'service', 'resourceInternalId', 'ordinal', 'teamId', 'teamInternalId',
'region', 'hostname', 'osCode', 'osName', 'osVersion',
'clientType', 'clientCode', 'clientName', 'clientVersion',
'clientEngine', 'clientEngineVersion',
Expand Down
Loading