From bfde4a371593327864af2e910d3b90f80306bbe8 Mon Sep 17 00:00:00 2001 From: shuke <37901441+shuke987@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:48:21 +0800 Subject: [PATCH 1/2] [fix](regression) Stabilize delete bitmap cache metrics test ### What problem does this PR solve? Issue Number: N/A Related PR: #64515 Problem Summary: The case warmed aggregate delete-bitmap cache entries before iterating over replicas, then fetched the large global brpc metrics payload inside that loop. On a loaded multi-replica P0 cluster, later replicas could be checked tens of seconds after warm-up even though the cache is a capacity-bounded LRU. Pin and query each replica immediately before inspecting its cache. Sort the tablet rows by ReplicaId to match use_fix_replica ordering, restore the session variable in finally, and fetch the global metrics payload once after all per-replica checks. ### Release note None ### Check List (For Author) - Test - [x] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [x] No. - [ ] Yes. - Does this need documentation? - [x] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- .../test_delete_bitmap_metrics.groovy | 129 ++++++++++-------- 1 file changed, 70 insertions(+), 59 deletions(-) diff --git a/regression-test/suites/metrics_p0/test_delete_bitmap_metrics.groovy b/regression-test/suites/metrics_p0/test_delete_bitmap_metrics.groovy index 80f7da957ef20e..c4a76cca5f7c64 100644 --- a/regression-test/suites/metrics_p0/test_delete_bitmap_metrics.groovy +++ b/regression-test/suites/metrics_p0/test_delete_bitmap_metrics.groovy @@ -184,79 +184,90 @@ suite("test_delete_bitmap_metrics", "p0") { def tablets = sql_return_maparray """ show tablets from ${testTable}; """ + tablets.sort { lhs, rhs -> lhs.ReplicaId.toLong() <=> rhs.ReplicaId.toLong() } logger.info("tablets: " + tablets) def local_delete_bitmap_count = 0 def ms_delete_bitmap_count = 0 def local_delete_bitmap_cardinality = 0; def ms_delete_bitmap_cardinality = 0; - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """ - logger.info("tablet: " + tablet_info) - String trigger_backend_id = tablet.BackendId + try { + for (int ri = 0; ri < tablets.size(); ri++) { + // use_fix_replica selects replicas in ReplicaId order. Populate and + // inspect each replica immediately because the agg cache is an LRU. + sql "set use_fix_replica=${ri};" + sql "select * from ${testTable};" - // before compaction, delete_bitmap_count is (rowsets num - 1) - local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality - logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) - logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality) - assertTrue(local_delete_bitmap_count == 7) - assertTrue(local_delete_bitmap_cardinality == 7) + def tablet = tablets[ri] + String tablet_id = tablet.TabletId + def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """ + logger.info("tablet: " + tablet_info) + String trigger_backend_id = tablet.BackendId - if (isCloudMode()) { - ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality - logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count) - logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality) - assertTrue(ms_delete_bitmap_count == 7) - assertTrue(ms_delete_bitmap_cardinality == 7) - } + // before compaction, delete_bitmap_count is (rowsets num - 1) + local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count + local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality + logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) + logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality) + assertTrue(local_delete_bitmap_count == 7) + assertTrue(local_delete_bitmap_cardinality == 7) + + if (isCloudMode()) { + ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count + ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality + logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count) + logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality) + assertTrue(ms_delete_bitmap_count == 7) + assertTrue(ms_delete_bitmap_cardinality == 7) + } - def status = getAggCacheDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id) - logger.info("agg cache status: ${status}") - assert status.delete_bitmap_count == 8 - assert status.cardinality == 7 - assert status.size > 0 + def status = getAggCacheDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id) + logger.info("agg cache status: ${status}") + assert status.delete_bitmap_count == 8 + assert status.cardinality == 7 + assert status.size > 0 - status = getAggCacheDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id, true) - logger.info("agg cache verbose status: ${status}") + status = getAggCacheDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id, true) + logger.info("agg cache verbose status: ${status}") + } + } finally { + sql "set use_fix_replica=-1;" + } - def tablet_delete_bitmap_count = 0; - def base_rowset_delete_bitmap_count = 0; - int retry_time = 0; - while (retry_time < 10) { - log.info("retry_time: ${retry_time}") - getMetricsMethod.call() { - respCode, body -> - logger.info("test get delete bitmap count resp Code {}", "${respCode}".toString()) - assertEquals("${respCode}".toString(), "200") - String out = "${body}".toString() - def strs = out.split('\n') - for (String line in strs) { - if (line.startsWith("tablet_max_delete_bitmap_score")) { - logger.info("find: {}", line) - tablet_delete_bitmap_count = line.replaceAll("tablet_max_delete_bitmap_score ", "").toInteger() - break - } + def tablet_delete_bitmap_count = 0; + def base_rowset_delete_bitmap_count = 0; + int retry_time = 0; + while (retry_time < 10) { + log.info("retry_time: ${retry_time}") + getMetricsMethod.call() { + respCode, body -> + logger.info("test get delete bitmap count resp Code {}", "${respCode}".toString()) + assertEquals("${respCode}".toString(), "200") + String out = "${body}".toString() + def strs = out.split('\n') + for (String line in strs) { + if (line.startsWith("tablet_max_delete_bitmap_score")) { + logger.info("find: {}", line) + tablet_delete_bitmap_count = line.replaceAll("tablet_max_delete_bitmap_score ", "").toInteger() + break } - for (String line in strs) { - if (line.startsWith("tablet_max_base_rowset_delete_bitmap_score")) { - logger.info("find: {}", line) - base_rowset_delete_bitmap_count = line.replaceAll("tablet_max_base_rowset_delete_bitmap_score ", "").toInteger() - break - } + } + for (String line in strs) { + if (line.startsWith("tablet_max_base_rowset_delete_bitmap_score")) { + logger.info("find: {}", line) + base_rowset_delete_bitmap_count = line.replaceAll("tablet_max_base_rowset_delete_bitmap_score ", "").toInteger() + break } - } - if (tablet_delete_bitmap_count > 0 && base_rowset_delete_bitmap_count > 0) { - break; - } else { - Thread.sleep(10000) - retry_time++; - } + } + } + if (tablet_delete_bitmap_count > 0 && base_rowset_delete_bitmap_count > 0) { + break; + } else { + Thread.sleep(10000) + retry_time++; } - assertTrue(tablet_delete_bitmap_count > 0) - assertTrue(base_rowset_delete_bitmap_count > 0) } + assertTrue(tablet_delete_bitmap_count > 0) + assertTrue(base_rowset_delete_bitmap_count > 0) } finally { reset_be_param("check_tablet_delete_bitmap_interval_seconds") } From 10ef49067a833bf2aa6415cadb0bab64f9968b20 Mon Sep 17 00:00:00 2001 From: shuke <37901441+shuke987@users.noreply.github.com> Date: Fri, 17 Jul 2026 10:07:43 +0800 Subject: [PATCH 2/2] [fix](regression) Pin cloud delete bitmap cache checks ### What problem does this PR solve? Issue Number: N/A Related PR: #65535 Problem Summary: When cloud multi-replica placement is enabled, SHOW TABLETS and a later scan can resolve the same logical CloudReplica to different physical backends. The test then warms one backend but probes another backend's process-local aggregate delete-bitmap cache. Run the global FE-config change as a non-concurrent case. In cloud mode, temporarily disable cloud multi-replica only around SHOW TABLETS, the pinned warm-up scan, and the per-BE cache assertions. Reuse setFeConfigTemporary so the original FE config is restored on every exit, while the existing nested finally restores use_fix_replica before the FE config and BE configs. ### Release note None ### Check List (For Author) - Test: Groovy parse check, diff check, and assertion-preservation audit. The authorized agent5 environment was unreachable for a post-review runtime rerun; the previous PR head passed the normal-mode single case 3/3. - Behavior changed: No. - Does this need documentation: No. --- .../test_delete_bitmap_metrics.groovy | 100 ++++++++++-------- 1 file changed, 57 insertions(+), 43 deletions(-) diff --git a/regression-test/suites/metrics_p0/test_delete_bitmap_metrics.groovy b/regression-test/suites/metrics_p0/test_delete_bitmap_metrics.groovy index c4a76cca5f7c64..72d08c1056ff03 100644 --- a/regression-test/suites/metrics_p0/test_delete_bitmap_metrics.groovy +++ b/regression-test/suites/metrics_p0/test_delete_bitmap_metrics.groovy @@ -17,7 +17,7 @@ import org.codehaus.groovy.runtime.IOGroovyMethods -suite("test_delete_bitmap_metrics", "p0") { +suite("test_delete_bitmap_metrics", "p0,nonConcurrent") { def backendId_to_backendIP = [:] def backendId_to_backendHttpPort = [:] def backendId_to_params = [string: [:]] @@ -183,54 +183,68 @@ suite("test_delete_bitmap_metrics", "p0") { qt_sql "select * from ${testTable} order by plan_id" - def tablets = sql_return_maparray """ show tablets from ${testTable}; """ - tablets.sort { lhs, rhs -> lhs.ReplicaId.toLong() <=> rhs.ReplicaId.toLong() } - logger.info("tablets: " + tablets) - def local_delete_bitmap_count = 0 - def ms_delete_bitmap_count = 0 - def local_delete_bitmap_cardinality = 0; - def ms_delete_bitmap_cardinality = 0; - try { - for (int ri = 0; ri < tablets.size(); ri++) { - // use_fix_replica selects replicas in ReplicaId order. Populate and - // inspect each replica immediately because the agg cache is an LRU. - sql "set use_fix_replica=${ri};" - sql "select * from ${testTable};" + def checkPerReplicaDeleteBitmap = { + def tablets = sql_return_maparray """ show tablets from ${testTable}; """ + tablets.sort { lhs, rhs -> lhs.ReplicaId.toLong() <=> rhs.ReplicaId.toLong() } + logger.info("tablets: " + tablets) + def local_delete_bitmap_count = 0 + def ms_delete_bitmap_count = 0 + def local_delete_bitmap_cardinality = 0; + def ms_delete_bitmap_cardinality = 0; + try { + for (int ri = 0; ri < tablets.size(); ri++) { + // use_fix_replica selects replicas in ReplicaId order. Populate and + // inspect each replica immediately because the agg cache is an LRU. + sql "set use_fix_replica=${ri};" + sql "select * from ${testTable};" - def tablet = tablets[ri] - String tablet_id = tablet.TabletId - def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """ - logger.info("tablet: " + tablet_info) - String trigger_backend_id = tablet.BackendId + def tablet = tablets[ri] + String tablet_id = tablet.TabletId + def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """ + logger.info("tablet: " + tablet_info) + String trigger_backend_id = tablet.BackendId - // before compaction, delete_bitmap_count is (rowsets num - 1) - local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality - logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) - logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality) - assertTrue(local_delete_bitmap_count == 7) - assertTrue(local_delete_bitmap_cardinality == 7) + // before compaction, delete_bitmap_count is (rowsets num - 1) + local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count + local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality + logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) + logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality) + assertTrue(local_delete_bitmap_count == 7) + assertTrue(local_delete_bitmap_cardinality == 7) - if (isCloudMode()) { - ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality - logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count) - logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality) - assertTrue(ms_delete_bitmap_count == 7) - assertTrue(ms_delete_bitmap_cardinality == 7) - } + if (isCloudMode()) { + ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count + ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality + logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count) + logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality) + assertTrue(ms_delete_bitmap_count == 7) + assertTrue(ms_delete_bitmap_cardinality == 7) + } + + def status = getAggCacheDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id) + logger.info("agg cache status: ${status}") + assert status.delete_bitmap_count == 8 + assert status.cardinality == 7 + assert status.size > 0 - def status = getAggCacheDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id) - logger.info("agg cache status: ${status}") - assert status.delete_bitmap_count == 8 - assert status.cardinality == 7 - assert status.size > 0 + status = getAggCacheDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id, true) + logger.info("agg cache verbose status: ${status}") + } + } finally { + sql "set use_fix_replica=-1;" + } + } - status = getAggCacheDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id, true) - logger.info("agg cache verbose status: ${status}") + if (isCloudMode()) { + // Cloud multi-replica can resolve the same logical CloudReplica to a + // different physical BE for SHOW TABLETS and the warm-up scan. Disable + // it while binding the warm-up and per-BE cache assertion; the helper + // restores the original FE config even when an assertion fails. + setFeConfigTemporary([enable_cloud_multi_replica: false]) { + checkPerReplicaDeleteBitmap() } - } finally { - sql "set use_fix_replica=-1;" + } else { + checkPerReplicaDeleteBitmap() } def tablet_delete_bitmap_count = 0;