From 39cf531ad5172bf4d36a325b8612a9204084077d Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Wed, 22 Apr 2026 14:37:55 +0200 Subject: [PATCH] hybrid: added system.hybrid_watermarks to inspect the Hybrid table watermarks --- .../engines/table-engines/special/hybrid.md | 4 + .../system-tables/hybrid_watermarks.md | 78 ++++++ src/Common/FailPoint.cpp | 1 + src/Storages/StorageDistributed.cpp | 7 + src/Storages/StorageDistributed.h | 8 + .../System/StorageSystemHybridWatermarks.cpp | 236 ++++++++++++++++++ .../System/StorageSystemHybridWatermarks.h | 39 +++ src/Storages/System/attachSystemTables.cpp | 2 + .../03682_system_hybrid_watermarks.reference | 28 +++ .../03682_system_hybrid_watermarks.sql | 176 +++++++++++++ 10 files changed, 579 insertions(+) create mode 100644 docs/en/operations/system-tables/hybrid_watermarks.md create mode 100644 src/Storages/System/StorageSystemHybridWatermarks.cpp create mode 100644 src/Storages/System/StorageSystemHybridWatermarks.h create mode 100644 tests/queries/0_stateless/03682_system_hybrid_watermarks.reference create mode 100644 tests/queries/0_stateless/03682_system_hybrid_watermarks.sql diff --git a/docs/en/engines/table-engines/special/hybrid.md b/docs/en/engines/table-engines/special/hybrid.md index 81dd4d67e364..982105f0ce6c 100644 --- a/docs/en/engines/table-engines/special/hybrid.md +++ b/docs/en/engines/table-engines/special/hybrid.md @@ -103,6 +103,10 @@ ALTER TABLE tiered MODIFY SETTING hybrid_watermark_cold = '2025-08-01'; ``` +### Inspecting current watermarks + +Current effective watermark values are exposed through [`system.hybrid_watermarks`](../../../operations/system-tables/hybrid_watermarks.md). One row per declared `hybridParam()` name; a diagnostic row with `last_exception` is emitted if the metadata read fails. + ### Restrictions - Only `hybrid_watermark_*` settings are accepted on Hybrid tables. Regular `DistributedSettings` (e.g. `bytes_to_delay_insert`) are rejected. diff --git a/docs/en/operations/system-tables/hybrid_watermarks.md b/docs/en/operations/system-tables/hybrid_watermarks.md new file mode 100644 index 000000000000..fa1f6b60e6eb --- /dev/null +++ b/docs/en/operations/system-tables/hybrid_watermarks.md @@ -0,0 +1,78 @@ +--- +description: 'System table exposing current effective watermark values for Hybrid-engine tables.' +keywords: ['system table', 'hybrid_watermarks', 'hybrid'] +slug: /operations/system-tables/hybrid_watermarks +title: 'system.hybrid_watermarks' +doc_type: 'reference' +--- + +Exposes the current effective watermark values for every attached [Hybrid](../../engines/table-engines/special/hybrid.md) table. Use this table for monitoring, alerting, and runbooks instead of parsing `SHOW CREATE TABLE` or `system.tables.create_table_query`. + +Columns: + + + + +## Row contract + +For every in-scope Hybrid table, `system.hybrid_watermarks` emits exactly one of: + +1. **N rows** — one per declared `hybridParam()` name. `value` is taken from the runtime snapshot, `type` from the declaration, `last_exception` is empty. +2. **Zero rows** — if the table's predicates contain no `hybridParam()` calls. +3. **One diagnostic row** — if reading the table's hybrid metadata raised an exception or a keyspace consistency check failed. `name`, `value`, and `type` are empty; `last_exception` is populated. + +The three cases are mutually exclusive for a given `(database, table)`: you never see, e.g., some healthy rows plus a diagnostic row for the same table. + +`SELECT * FROM system.hybrid_watermarks WHERE last_exception != ''` is a correct alert for "live Hybrid table I cannot introspect". + +## Scope + +Covers Hybrid tables visible to `getTablesIterator()` plus session-local temporary Hybrid tables — the same set that appears in `system.tables`. Temporary tables are emitted with `database = ''`, matching the `system.tables` convention. + +Out of scope: + +- **On-disk metadata that fails to load.** A `.sql` file that fails the factory-time validation of `hybridParam()` arity, literal types, or declared-type conflicts is rejected during startup and the table never attaches. Such failures surface in the server log, not in this table. +- **Detached tables.** Re-`ATTACH` re-runs factory validation. +- **Non-Hybrid `Distributed` tables.** Filtered out by `getName() == "Hybrid"`. + +## Access control + +`SHOW_TABLES` on `(database, table)` for persistent tables, identical to `system.distribution_queue`. Session-local temporary tables are only visible to their owning session and are not gated by `SHOW_TABLES`, matching `system.tables`. + +## Example + +```sql +SELECT * FROM system.hybrid_watermarks FORMAT Vertical; +``` + +```text +Row 1: +────── +database: default +table: hybrid_hot_cold +name: hybrid_watermark_hot +value: 2025-10-01 +type: DateTime +last_exception: +``` + +Find all tables with at least one watermark at a given value: + +```sql +SELECT database, table +FROM system.hybrid_watermarks +WHERE name = 'hybrid_watermark_hot' AND value = '2025-10-01'; +``` + +Alert on tables that currently cannot be introspected: + +```sql +SELECT database, table, last_exception +FROM system.hybrid_watermarks +WHERE last_exception != ''; +``` + +**See Also** + +- [Hybrid table engine](../../engines/table-engines/special/hybrid.md) +- [Distributed table engine](../../engines/table-engines/special/distributed.md) diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index 6cf84a54af19..b2bf44358ee0 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -40,6 +40,7 @@ static struct InitFiu REGULAR(use_delayed_remote_source) \ REGULAR(cluster_discovery_faults) \ REGULAR(stripe_log_sink_write_fallpoint) \ + REGULAR(hybrid_watermarks_read_fail) \ ONCE(smt_commit_merge_mutate_zk_fail_after_op) \ ONCE(smt_commit_merge_mutate_zk_fail_before_op) \ ONCE(smt_commit_write_zk_fail_after_op) \ diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 458a45110ba0..e6e197434c5a 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -1797,6 +1797,13 @@ static std::unordered_map collectHybridParamTypes( return result; } +std::unordered_map StorageDistributed::getDeclaredHybridParamTypes() const +{ + if (getName() != "Hybrid") + return {}; + return collectHybridParamTypes(base_segment_predicate, segments); +} + void StorageDistributed::checkAlterIsPossible(const AlterCommands & commands, ContextPtr local_context) const { std::optional name_deps{}; diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h index 138a49d9c992..298c5a206fac 100644 --- a/src/Storages/StorageDistributed.h +++ b/src/Storages/StorageDistributed.h @@ -180,6 +180,14 @@ class StorageDistributed final : public IStorage, WithContext MultiVersion::Version getHybridWatermarkParams() const { return hybrid_watermark_params.get(); } + /// Returns (name -> declared type) for every `hybridParam()` call in the + /// stored Hybrid predicates. Empty for non-Hybrid tables. + /// + /// For any attached Hybrid table this returns a consistent map, because + /// registerStorageHybrid() rejects conflicting types for the same name at + /// factory time. The result is used by system.hybrid_watermarks. + std::unordered_map getDeclaredHybridParamTypes() const; + void loadHybridWatermarkParams(SettingsChanges & changes); /// Getter methods for ClusterProxy::executeQuery diff --git a/src/Storages/System/StorageSystemHybridWatermarks.cpp b/src/Storages/System/StorageSystemHybridWatermarks.cpp new file mode 100644 index 000000000000..407f256e8767 --- /dev/null +++ b/src/Storages/System/StorageSystemHybridWatermarks.cpp @@ -0,0 +1,236 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int FAULT_INJECTED; +} + +namespace FailPoints +{ + extern const char hybrid_watermarks_read_fail[]; +} + + +ColumnsDescription StorageSystemHybridWatermarks::getColumnsDescription() +{ + return ColumnsDescription + { + {"database", std::make_shared(), "Name of the database."}, + {"table", std::make_shared(), "Name of the Hybrid table."}, + {"name", std::make_shared(), + "Watermark parameter name from hybridParam() (always starts with hybrid_watermark_). Empty on a diagnostic row."}, + {"value", std::make_shared(), + "Current effective watermark value from the runtime snapshot. Empty on a diagnostic row."}, + {"type", std::make_shared(), + "Declared type from hybridParam('name', 'type'). Empty on a diagnostic row."}, + {"last_exception", std::make_shared(), + "Empty on success. Populated when reading this table's hybrid metadata raised an exception or produced an inconsistent view."}, + }; +} + +Block StorageSystemHybridWatermarks::getFilterSampleBlock() const +{ + return { + { {}, std::make_shared(), "database" }, + { {}, std::make_shared(), "table" }, + }; +} + +void StorageSystemHybridWatermarks::fillData( + MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node * predicate, std::vector) const +{ + const auto access = context->getAccess(); + const bool check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES); + + /// Enumerate only Hybrid tables (StorageDistributed with getName() == "Hybrid"). + std::map> tables; + for (const auto & db : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) + { + if (db.second->isExternal()) + continue; + + /// Temp tables are surfaced via the session-local branch below; mirrors system.tables. + if (db.first == DatabaseCatalog::TEMPORARY_DATABASE) + continue; + + const bool check_access_for_tables = check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, db.first); + + for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) + { + StoragePtr table = iterator->table(); + if (!table) + continue; + + const auto * distributed = dynamic_cast(table.get()); + if (!distributed || distributed->getName() != "Hybrid") + continue; + + if (check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, db.first, iterator->name())) + continue; + + tables[db.first][iterator->name()] = table; + } + } + + /// Session-local temporary tables, mirroring `system.tables` (see + /// StorageSystemTables::read(), the temporary-table branch around line 371). + /// They are emitted with database = "" to match `system.tables`' convention. + /// No SHOW_TABLES gate: externals are session-scoped and only visible to the + /// owning session, the same as in `system.tables`. + if (context->hasSessionContext()) + { + for (auto & [name, storage] : context->getSessionContext()->getExternalTables()) + { + if (!storage) + continue; + + const auto * distributed = dynamic_cast(storage.get()); + if (!distributed || distributed->getName() != "Hybrid") + continue; + + tables[""][name] = storage; + } + } + + MutableColumnPtr col_database_mut = ColumnString::create(); + MutableColumnPtr col_table_mut = ColumnString::create(); + + for (auto & db : tables) + { + for (auto & table : db.second) + { + col_database_mut->insert(db.first); + col_table_mut->insert(table.first); + } + } + + ColumnPtr col_database_to_filter = std::move(col_database_mut); + ColumnPtr col_table_to_filter = std::move(col_table_mut); + + /// Apply pushed-down predicate on (database, table). + { + Block filtered_block + { + { col_database_to_filter, std::make_shared(), "database" }, + { col_table_to_filter, std::make_shared(), "table" }, + }; + + VirtualColumnUtils::filterBlockWithPredicate(predicate, filtered_block, context); + + if (!filtered_block.rows()) + return; + + col_database_to_filter = filtered_block.getByName("database").column; + col_table_to_filter = filtered_block.getByName("table").column; + } + + auto emit_diagnostic = [&](const String & database, const String & table, const String & message) + { + size_t c = 0; + res_columns[c++]->insert(database); + res_columns[c++]->insert(table); + res_columns[c++]->insertDefault(); /// name + res_columns[c++]->insertDefault(); /// value + res_columns[c++]->insertDefault(); /// type + res_columns[c++]->insert(message); + }; + + for (size_t i = 0, tables_size = col_database_to_filter->size(); i < tables_size; ++i) + { + String database = (*col_database_to_filter)[i].safeGet(); + String table = (*col_table_to_filter)[i].safeGet(); + + auto & distributed_table = dynamic_cast(*tables[database][table]); + + std::unordered_map types; + MultiVersion::Version snapshot; + + /// Per-table fault isolation: mirrors StorageSystemTables (see its lines 450-462). + /// In normal operation this try never throws, but we want one broken attached + /// Hybrid table never to take down the whole scan. + try + { + /// Test-only hook for exercising the diagnostic row path from SQL. + fiu_do_on(FailPoints::hybrid_watermarks_read_fail, + { + throw Exception(ErrorCodes::FAULT_INJECTED, + "Injected fault for system.hybrid_watermarks"); + }); + + types = distributed_table.getDeclaredHybridParamTypes(); + snapshot = distributed_table.getHybridWatermarkParams(); + } + catch (...) + { + tryLogCurrentException(getLogger("StorageSystemHybridWatermarks"), + fmt::format("Failed to read hybrid watermarks for {}.{}", database, table), + LogsLevel::information); + emit_diagnostic(database, table, getCurrentExceptionMessage(/*with_stacktrace=*/false)); + continue; + } + + /// Row contract case 2: zero declared watermarks → emit zero rows. + if (types.empty()) + continue; + + /// Row contract case 3 (defense in depth): keyspace mismatch. CREATE + /// enforces declared-keys == snapshot-keys at [StorageDistributed.cpp] + /// lines 3043-3051, so this only triggers on unexpected runtime drift. + bool consistent = snapshot && snapshot->size() == types.size(); + if (consistent) + { + for (const auto & [name, _] : types) + { + if (!snapshot->contains(name)) + { + consistent = false; + break; + } + } + } + + if (!consistent) + { + emit_diagnostic(database, table, fmt::format( + "Hybrid watermark keyspace mismatch: {} declared, {} in snapshot", + types.size(), snapshot ? snapshot->size() : 0)); + continue; + } + + /// Row contract case 1: N healthy rows, sorted by name for reference-file stability. + std::vector sorted_names; + sorted_names.reserve(types.size()); + for (const auto & [name, _] : types) + sorted_names.push_back(name); + std::sort(sorted_names.begin(), sorted_names.end()); + + for (const auto & name : sorted_names) + { + size_t c = 0; + res_columns[c++]->insert(database); + res_columns[c++]->insert(table); + res_columns[c++]->insert(name); + res_columns[c++]->insert(snapshot->at(name)); + res_columns[c++]->insert(types.at(name)); + res_columns[c++]->insertDefault(); /// last_exception + } + } +} + +} diff --git a/src/Storages/System/StorageSystemHybridWatermarks.h b/src/Storages/System/StorageSystemHybridWatermarks.h new file mode 100644 index 000000000000..58abf3233875 --- /dev/null +++ b/src/Storages/System/StorageSystemHybridWatermarks.h @@ -0,0 +1,39 @@ +#pragma once + +#include + + +namespace DB +{ + +class Context; + + +/** Implements the `system.hybrid_watermarks` table, which exposes the current + * effective watermark values for every attached Hybrid-engine table. + * + * For each live Hybrid table, emits exactly one of: + * - N rows, one per declared `hybridParam()` name; + * - 0 rows, if the table has no `hybridParam()` references; + * - 1 diagnostic row with `last_exception` populated, on read failure or + * post-read consistency violation. + * + * Non-Hybrid tables are filtered out. Tables that fail to load from metadata + * at startup never attach and therefore never appear here — they surface in + * server logs instead. + */ +class StorageSystemHybridWatermarks final : public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemHybridWatermarks"; } + + static ColumnsDescription getColumnsDescription(); + +protected: + using IStorageSystemOneBlock::IStorageSystemOneBlock; + + void fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node * predicate, std::vector) const override; + Block getFilterSampleBlock() const override; +}; + +} diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index 0f02959fe503..f3016ccda82c 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -53,6 +53,7 @@ #include #include #include +#include #include #include #include @@ -237,6 +238,7 @@ void attachSystemTablesServer(ContextPtr context, IDatabase & system_database, b attach(context, system_database, "replication_queue", "Contains information about tasks from replication queues stored in ClickHouse Keeper, or ZooKeeper, for each table replica."); attach(context, system_database, "distributed_ddl_queue", "Contains information about distributed DDL queries (ON CLUSTER clause) that were executed on a cluster."); attach(context, system_database, "distribution_queue", "Contains information about local files that are in the queue to be sent to the shards. These local files contain new parts that are created by inserting new data into the Distributed table in asynchronous mode."); + attach(context, system_database, "hybrid_watermarks", "Current effective watermark values for Hybrid-engine tables. One row per declared hybridParam() name, or one diagnostic row with last_exception on read failure."); attach(context, system_database, "dictionaries", "Contains information about dictionaries."); attach(context, system_database, "models", "Contains a list of CatBoost models loaded into a LibraryBridge's memory along with time when it was loaded."); attach(context, system_database, "clusters", "Contains information about clusters defined in the configuration file or generated by a Replicated database."); diff --git a/tests/queries/0_stateless/03682_system_hybrid_watermarks.reference b/tests/queries/0_stateless/03682_system_hybrid_watermarks.reference new file mode 100644 index 000000000000..cd637c93cb26 --- /dev/null +++ b/tests/queries/0_stateless/03682_system_hybrid_watermarks.reference @@ -0,0 +1,28 @@ +--- Test 1: Healthy N=1 +1 t hybrid_watermark_hot 2025-09-01 DateTime +1 +--- Test 2: ALTER refresh +hybrid_watermark_hot 2025-10-01 DateTime +1 +--- Test 3: Multi-watermark ordering +hybrid_watermark_cold 2025-01-01 DateTime +hybrid_watermark_hot 2025-10-01 DateTime +2 +--- Test 4: Zero-declared Hybrid emits zero rows +0 +--- Test 5: Non-Hybrid exclusion +0 +--- Test 6: No diagnostic rows on healthy cluster +0 +--- Test 7: last_exception path via failpoint +t 1 +t3 1 +t_no_params 1 +1 +1 +--- Test 8: Healthy after failpoint disabled +hybrid_watermark_hot 2025-10-01 DateTime +0 +--- Test 9: Temporary Hybrid table visibility + tmp_hybrid hybrid_watermark_hot 2025-11-15 DateTime +1 diff --git a/tests/queries/0_stateless/03682_system_hybrid_watermarks.sql b/tests/queries/0_stateless/03682_system_hybrid_watermarks.sql new file mode 100644 index 000000000000..7e52d5dd2e23 --- /dev/null +++ b/tests/queries/0_stateless/03682_system_hybrid_watermarks.sql @@ -0,0 +1,176 @@ +-- Tags: no-fasttest, no-parallel +-- Tag no-fasttest: requires remote() table function +-- Tag no-parallel: uses a process-wide failpoint + +SET allow_experimental_hybrid_table = 1; + +DROP TABLE IF EXISTS local_hot SYNC; +DROP TABLE IF EXISTS local_warm SYNC; +DROP TABLE IF EXISTS local_cold SYNC; +DROP TABLE IF EXISTS t SYNC; +DROP TABLE IF EXISTS t3 SYNC; +DROP TABLE IF EXISTS t_no_params SYNC; +DROP TABLE IF EXISTS dist_plain SYNC; +DROP TABLE IF EXISTS mt SYNC; + +CREATE TABLE local_hot (ts DateTime, value UInt64) ENGINE = MergeTree ORDER BY ts; +CREATE TABLE local_warm (ts DateTime, value UInt64) ENGINE = MergeTree ORDER BY ts; +CREATE TABLE local_cold (ts DateTime, value UInt64) ENGINE = MergeTree ORDER BY ts; + +-- ===================================================== +-- 1. Healthy N=1 case +-- ===================================================== +SELECT '--- Test 1: Healthy N=1'; +CREATE TABLE t +ENGINE = Hybrid( + remote('localhost:9000', currentDatabase(), 'local_hot'), + ts > hybridParam('hybrid_watermark_hot', 'DateTime'), + remote('localhost:9000', currentDatabase(), 'local_cold'), + ts <= hybridParam('hybrid_watermark_hot', 'DateTime') +) +SETTINGS hybrid_watermark_hot = '2025-09-01' +AS local_hot; + +SELECT database = currentDatabase(), table, name, value, type, last_exception +FROM system.hybrid_watermarks +WHERE database = currentDatabase() AND table = 't' +ORDER BY name; + +SELECT count() FROM system.hybrid_watermarks WHERE database = currentDatabase() AND table = 't'; + +-- ===================================================== +-- 2. ALTER refreshes value, row count stays at 1 +-- ===================================================== +SELECT '--- Test 2: ALTER refresh'; +ALTER TABLE t MODIFY SETTING hybrid_watermark_hot = '2025-10-01'; +SELECT name, value, type +FROM system.hybrid_watermarks +WHERE database = currentDatabase() AND table = 't' +ORDER BY name; + +SELECT count() FROM system.hybrid_watermarks WHERE database = currentDatabase() AND table = 't'; + +-- ===================================================== +-- 3. Multi-watermark ordering is stable (sorted by name) +-- ===================================================== +SELECT '--- Test 3: Multi-watermark ordering'; +CREATE TABLE t3 +ENGINE = Hybrid( + remote('localhost:9000', currentDatabase(), 'local_hot'), + ts > hybridParam('hybrid_watermark_hot', 'DateTime'), + remote('localhost:9000', currentDatabase(), 'local_warm'), + ts <= hybridParam('hybrid_watermark_hot', 'DateTime') + AND ts > hybridParam('hybrid_watermark_cold', 'DateTime'), + remote('localhost:9000', currentDatabase(), 'local_cold'), + ts <= hybridParam('hybrid_watermark_cold', 'DateTime') +) +SETTINGS hybrid_watermark_hot = '2025-10-01', hybrid_watermark_cold = '2025-01-01' +AS local_hot; + +SELECT name, value, type +FROM system.hybrid_watermarks +WHERE database = currentDatabase() AND table = 't3' +ORDER BY name; + +SELECT count() FROM system.hybrid_watermarks WHERE database = currentDatabase() AND table = 't3'; + +-- ===================================================== +-- 4. Zero-declared case: Hybrid with no hybridParam() emits zero rows +-- ===================================================== +SELECT '--- Test 4: Zero-declared Hybrid emits zero rows'; +CREATE TABLE t_no_params +ENGINE = Hybrid( + remote('localhost:9000', currentDatabase(), 'local_hot'), + ts > toDateTime('2025-09-01'), + remote('localhost:9000', currentDatabase(), 'local_cold'), + ts <= toDateTime('2025-09-01') +) +AS local_hot; + +SELECT count() FROM system.hybrid_watermarks WHERE database = currentDatabase() AND table = 't_no_params'; + +-- ===================================================== +-- 5. Non-Hybrid exclusion: MergeTree and plain Distributed never appear +-- ===================================================== +SELECT '--- Test 5: Non-Hybrid exclusion'; +CREATE TABLE mt (ts DateTime, value UInt64) ENGINE = MergeTree ORDER BY ts; +CREATE TABLE dist_plain AS remote('localhost:9000', currentDatabase(), 'local_hot'); + +SELECT count() FROM system.hybrid_watermarks +WHERE database = currentDatabase() AND table IN ('mt', 'dist_plain', 'local_hot', 'local_warm', 'local_cold'); + +-- ===================================================== +-- 6. Baseline: no stuck diagnostic rows on a healthy cluster +-- ===================================================== +SELECT '--- Test 6: No diagnostic rows on healthy cluster'; +SELECT count() FROM system.hybrid_watermarks +WHERE database = currentDatabase() AND last_exception != ''; + +-- ===================================================== +-- 7. Diagnostic row path: enable failpoint, see exactly one row with last_exception populated +-- ===================================================== +SELECT '--- Test 7: last_exception path via failpoint'; +SYSTEM ENABLE FAILPOINT hybrid_watermarks_read_fail; + +-- Every in-scope Hybrid table collapses to a single diagnostic row. +-- t: 1 diagnostic row; t3: 1 diagnostic row; t_no_params: still 1 diagnostic row +-- because the failpoint fires before the zero-declared check. +SELECT table, name, value, type, last_exception != '' AS has_exception +FROM system.hybrid_watermarks +WHERE database = currentDatabase() AND table IN ('t', 't3', 't_no_params') +ORDER BY table; + +SELECT count() FROM system.hybrid_watermarks +WHERE database = currentDatabase() AND table = 't' AND last_exception != ''; + +SELECT count() FROM system.hybrid_watermarks +WHERE database = currentDatabase() AND table = 't' AND name = '' AND value = '' AND type = ''; + +SYSTEM DISABLE FAILPOINT hybrid_watermarks_read_fail; + +-- ===================================================== +-- 8. Back to healthy state after failpoint disabled +-- ===================================================== +SELECT '--- Test 8: Healthy after failpoint disabled'; +SELECT name, value, type, last_exception +FROM system.hybrid_watermarks +WHERE database = currentDatabase() AND table = 't' +ORDER BY name; + +SELECT count() FROM system.hybrid_watermarks +WHERE database = currentDatabase() AND last_exception != ''; + +-- ===================================================== +-- 9. Temporary Hybrid tables are visible (emitted with database = '') +-- ===================================================== +SELECT '--- Test 9: Temporary Hybrid table visibility'; +CREATE TEMPORARY TABLE tmp_hybrid +ENGINE = Hybrid( + remote('localhost:9000', currentDatabase(), 'local_hot'), + ts > hybridParam('hybrid_watermark_hot', 'DateTime'), + remote('localhost:9000', currentDatabase(), 'local_cold'), + ts <= hybridParam('hybrid_watermark_hot', 'DateTime') +) +SETTINGS hybrid_watermark_hot = '2025-11-15' +AS local_hot; + +SELECT database, table, name, value, type, last_exception +FROM system.hybrid_watermarks +WHERE database = '' AND table = 'tmp_hybrid' +ORDER BY name; + +SELECT count() FROM system.hybrid_watermarks WHERE database = '' AND table = 'tmp_hybrid'; + +DROP TEMPORARY TABLE tmp_hybrid; + +-- ===================================================== +-- Cleanup +-- ===================================================== +DROP TABLE IF EXISTS t SYNC; +DROP TABLE IF EXISTS t3 SYNC; +DROP TABLE IF EXISTS t_no_params SYNC; +DROP TABLE IF EXISTS dist_plain SYNC; +DROP TABLE IF EXISTS mt SYNC; +DROP TABLE IF EXISTS local_hot SYNC; +DROP TABLE IF EXISTS local_warm SYNC; +DROP TABLE IF EXISTS local_cold SYNC;