From c96f4ca2c39fff291e19f355e9a9a5b1f55bb377 Mon Sep 17 00:00:00 2001 From: linzhenqi Date: Sun, 12 Jul 2026 12:59:56 +0800 Subject: [PATCH] [Test](bitmap) Harden BitmapValue serialization test --- be/test/core/value/bitmap_value_test.cpp | 124 ++++++++++++++++-- .../exprs/function/function_bitmap_test.cpp | 31 ++++- .../bitmap_functions/test_bitmap_function.out | 12 ++ .../doc_bitmap_functions_test.groovy | 11 +- .../test_bitmap_function.groovy | 6 + 5 files changed, 163 insertions(+), 21 deletions(-) diff --git a/be/test/core/value/bitmap_value_test.cpp b/be/test/core/value/bitmap_value_test.cpp index 5b25551ae0c85c..1f89539c366cc7 100644 --- a/be/test/core/value/bitmap_value_test.cpp +++ b/be/test/core/value/bitmap_value_test.cpp @@ -21,12 +21,15 @@ #include #include +#include #include #include +#include #include "gtest/gtest.h" #include "gtest/gtest_pred_impl.h" #include "util/coding.h" +#include "util/url_coding.h" namespace doris { using roaring::Roaring; @@ -503,6 +506,20 @@ void check_bitmap_equal(const BitmapValue& left, const BitmapValue& right) { } } +std::vector sorted_bitmap_values(const BitmapValue& bitmap) { + std::vector values; + for (auto v : bitmap) { + values.emplace_back(v); + } + std::sort(values.begin(), values.end()); + return values; +} + +void check_bitmap_values(const BitmapValue& bitmap, std::vector expected) { + std::sort(expected.begin(), expected.end()); + EXPECT_EQ(sorted_bitmap_values(bitmap), expected); +} + TEST(BitmapValueTest, write_read) { config::enable_set_in_bitmap_value = true; BitmapValue bitmap_empty; @@ -966,14 +983,31 @@ std::string convert_bitmap_to_string(BitmapValue& bitmap) { return buf; } +std::string decode_base64_to_string(const std::string& base64) { + std::string decoded; + EXPECT_TRUE(base64_decode(base64, &decoded)); + return decoded; +} + +BitmapValue deserialize_bitmap_from_string(const std::string& buffer) { + BitmapValue bitmap; + EXPECT_TRUE(bitmap.deserialize(buffer.data())); + return bitmap; +} + TEST(BitmapValueTest, bitmap_serde) { + auto old_enable_set = config::enable_set_in_bitmap_value; + auto old_serialize_version = config::bitmap_serialize_version; + config::enable_set_in_bitmap_value = false; + config::bitmap_serialize_version = 1; + { // EMPTY BitmapValue empty; std::string buffer = convert_bitmap_to_string(empty); std::string expect_buffer(1, BitmapTypeCode::EMPTY); EXPECT_EQ(expect_buffer, buffer); - BitmapValue out(buffer.data()); + BitmapValue out = deserialize_bitmap_from_string(buffer); EXPECT_EQ(0, out.cardinality()); } { // SINGLE32 @@ -984,18 +1018,19 @@ TEST(BitmapValueTest, bitmap_serde) { put_fixed32_le(&expect_buffer, i); EXPECT_EQ(expect_buffer, buffer); - BitmapValue out(buffer.data()); + BitmapValue out = deserialize_bitmap_from_string(buffer); EXPECT_EQ(1, out.cardinality()); EXPECT_TRUE(out.contains(i)); } { // BITMAP32 - BitmapValue bitmap32({0, UINT32_MAX}); + BitmapValue bitmap32({1, 9999999}); std::string buffer = convert_bitmap_to_string(bitmap32); + EXPECT_EQ(decode_base64_to_string("AjowAAACAAAAAAAAAJgAAAAYAAAAGgAAAAEAf5Y="), buffer); - BitmapValue out(buffer.data()); + BitmapValue out = deserialize_bitmap_from_string(buffer); EXPECT_EQ(2, out.cardinality()); - EXPECT_TRUE(out.contains(0)); - EXPECT_TRUE(out.contains(UINT32_MAX)); + EXPECT_TRUE(out.contains(1)); + EXPECT_TRUE(out.contains(9999999)); } { // SINGLE64 uint64_t i = static_cast(UINT32_MAX) + 1; @@ -1005,19 +1040,88 @@ TEST(BitmapValueTest, bitmap_serde) { put_fixed64_le(&expect_buffer, i); EXPECT_EQ(expect_buffer, buffer); - BitmapValue out(buffer.data()); + BitmapValue out = deserialize_bitmap_from_string(buffer); EXPECT_EQ(1, out.cardinality()); EXPECT_TRUE(out.contains(i)); } { // BITMAP64 - BitmapValue bitmap64({0, static_cast(UINT32_MAX) + 1}); + BitmapValue bitmap64({1, static_cast(UINT32_MAX) + 1}); std::string buffer = convert_bitmap_to_string(bitmap64); + EXPECT_EQ(decode_base64_to_string( + "BAIAAAAAOjAAAAEAAAAAAAAAEAAAAAEAAQAAADowAAABAAAAAAAAABAAAAAAAA=="), + buffer); - BitmapValue out(buffer.data()); + BitmapValue out = deserialize_bitmap_from_string(buffer); EXPECT_EQ(2, out.cardinality()); - EXPECT_TRUE(out.contains(0)); + EXPECT_TRUE(out.contains(1)); EXPECT_TRUE(out.contains(static_cast(UINT32_MAX) + 1)); } + { // BITMAP32_V2 + config::bitmap_serialize_version = 2; + std::vector bits32 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32}; + BitmapValue bitmap32(bits32); + std::string buffer = convert_bitmap_to_string(bitmap32); + EXPECT_EQ(decode_base64_to_string("DAI7MAAAAQAAIAABAAAAIAA="), buffer); + + BitmapValue out = deserialize_bitmap_from_string(buffer); + check_bitmap_values(out, bits32); + } + { // BITMAP64_V2 + std::vector bits64 { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, static_cast(UINT32_MAX) + 1}; + BitmapValue bitmap64(bits64); + std::string buffer = convert_bitmap_to_string(bitmap64); + EXPECT_EQ(decode_base64_to_string("DQIAAAAAAjswAAABAAAfAAEAAAAfAAEAAAABAQAAAAAAAAA="), + buffer); + + BitmapValue out = deserialize_bitmap_from_string(buffer); + check_bitmap_values(out, bits64); + } + { // SET serialization is semantic-only because hash-set iteration order is not stable. + config::enable_set_in_bitmap_value = true; + config::bitmap_serialize_version = 1; + std::vector values {UINT64_MAX, 0, static_cast(UINT32_MAX) + 1, 7, 1}; + BitmapValue set_bitmap; + for (auto value : values) { + set_bitmap.add(value); + } + EXPECT_EQ(set_bitmap.get_type_code(), BitmapTypeCode::SET); + + std::string buffer = convert_bitmap_to_string(set_bitmap); + BitmapValue out = deserialize_bitmap_from_string(buffer); + check_bitmap_values(out, values); + } + { // Deserializing historical SET bytes must not depend on their serialized payload order. + config::enable_set_in_bitmap_value = true; + std::vector values {1, 9999999}; + std::string set_v1_insert_order = decode_base64_to_string("BQIBAAAAAAAAAH+WmAAAAAAA"); + std::string set_v1_reverse_order = decode_base64_to_string("BQJ/lpgAAAAAAAEAAAAAAAAA"); + std::string set_v2_reverse_order = decode_base64_to_string("CgIAAAB/lpgAAAAAAAEAAAAAAAAA"); + + BitmapValue set_in_insert_order = deserialize_bitmap_from_string(set_v1_insert_order); + BitmapValue set_in_reverse_order = deserialize_bitmap_from_string(set_v1_reverse_order); + BitmapValue set_v2_in_reverse_order = deserialize_bitmap_from_string(set_v2_reverse_order); + + check_bitmap_values(set_in_insert_order, values); + check_bitmap_values(set_in_reverse_order, values); + check_bitmap_values(set_v2_in_reverse_order, values); + } + { // Historical SET bytes with mixed 32-bit and 64-bit values must remain readable. + config::enable_set_in_bitmap_value = true; + std::vector values {UINT64_MAX, 0, static_cast(UINT32_MAX) + 1, 7, 1}; + std::string set_v1_mixed = + decode_base64_to_string("BQX//////////wAAAAAAAAAAAAAAAAEAAAAHAAAAAAAAAAEAAAAAAAAA"); + + BitmapValue out = deserialize_bitmap_from_string(set_v1_mixed); + check_bitmap_values(out, values); + } + + config::enable_set_in_bitmap_value = old_enable_set; + config::bitmap_serialize_version = old_serialize_version; } // Forked from CRoaring's UT of Roaring64Map diff --git a/be/test/exprs/function/function_bitmap_test.cpp b/be/test/exprs/function/function_bitmap_test.cpp index 6e18e10d2e354f..958132894277a4 100644 --- a/be/test/exprs/function/function_bitmap_test.cpp +++ b/be/test/exprs/function/function_bitmap_test.cpp @@ -29,6 +29,7 @@ #include "core/types.h" #include "core/value/bitmap_value.h" #include "exprs/function/function_test_util.h" +#include "util/url_coding.h" namespace doris { @@ -98,6 +99,14 @@ namespace doris::config { DECLARE_Bool(enable_set_in_bitmap_value); } +std::string encode_bitmap_to_base64(const BitmapValue& bitmap) { + std::string serialized(bitmap.getSizeInBytes(), '\0'); + bitmap.write_to(serialized.data()); + std::string encoded; + base64_encode(serialized, &encoded); + return encoded; +} + TEST(function_bitmap_test, function_bitmap_to_base64) { config::Register::Field field("bool", "enable_set_in_bitmap_value", &config::enable_set_in_bitmap_value, "false", false); @@ -175,10 +184,8 @@ TEST(function_bitmap_test, function_bitmap_to_base64) { { DataSet data_set = { {{&bitmap32_1}, std::string("AQEAAAA=")}, - {{&bitmap32_2}, std::string("BQIBAAAAAAAAAH+WmAAAAAAA")}, {{&bitmap32_3}, std::string("AjswAAABAAAgAAEAAAAgAA==")}, {{&bitmap64_1}, std::string("AwAAAAABAAAA")}, - {{&bitmap64_2}, std::string("BQIAAAAAAQAAAAEAAAAAAAAA")}, {{&bitmap64_3}, std::string("BAIAAAAAOzAAAAEAAB8AAQAAAB8AAQAAADowAAABAAAAAAAAABAAAAAAAA==")}, {{&empty_bitmap}, std::string("AA==")}, @@ -188,13 +195,20 @@ TEST(function_bitmap_test, function_bitmap_to_base64) { } { - std::string base64("BQQAAAAAAAAAAAEAAAAAAAAAAgAAAAAAAAADAAAAAAAAAA=="); + // SET serialization depends on hash-set iteration order, so do not pin a hard-coded + // base64 string here. The contract is that the current serialization can round-trip. + DataSet data_set = {{{&bitmap32_2}, encode_bitmap_to_base64(bitmap32_2)}, + {{&bitmap64_2}, encode_bitmap_to_base64(bitmap64_2)}}; + static_cast(check_function(func_name, input_types, data_set)); + } + + { BitmapValue bitmap; bitmap.add(0); bitmap.add(1); bitmap.add(2); bitmap.add(3); - DataSet data_set = {{{&bitmap}, base64}}; + DataSet data_set = {{{&bitmap}, encode_bitmap_to_base64(bitmap)}}; static_cast(check_function(func_name, input_types, data_set)); } @@ -291,6 +305,15 @@ TEST(function_bitmap_test, function_bitmap_from_base64) { DataSet data_set = {{{base64}, &bitmap}}; static_cast(check_function(func_name, input_types, data_set)); } + { + // Historical SET bytes may store the same elements in different orders; decoding must + // preserve the bitmap contents instead of relying on the serialized payload order. + std::string set_v1_reversed("BQJ/lpgAAAAAAAEAAAAAAAAA"); + std::string set_v2_reversed("CgIAAAB/lpgAAAAAAAEAAAAAAAAA"); + BitmapValue bitmap({1, 9999999}); + DataSet data_set = {{{set_v1_reversed}, &bitmap}, {{set_v2_reversed}, &bitmap}}; + static_cast(check_function(func_name, input_types, data_set)); + } { EXPECT_TRUE(config::set_config("bitmap_serialize_version", "1", false, true).ok()); diff --git a/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out b/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out index 766b791c4ea6b6..a162f381a4af7e 100644 --- a/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out +++ b/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out @@ -537,6 +537,18 @@ true -- !sql_bitmap_base64_nereids8 -- 1 +-- !sql_bitmap_base64_history_set_v1 -- +1,9999999 + +-- !sql_bitmap_base64_history_set_v1_reversed -- +1,9999999 + +-- !sql_bitmap_base64_history_set_v2_reversed -- +1,9999999 + +-- !sql_bitmap_base64_history_set_v1_mixed -- +0,1,7,4294967296,18446744073709551615 + -- !sql_bitmap_base64_nereids9 -- 0 diff --git a/regression-test/suites/doc/sql-manual/sql-functions/doc_bitmap_functions_test.groovy b/regression-test/suites/doc/sql-manual/sql-functions/doc_bitmap_functions_test.groovy index 06247016e7d848..627ee6b8411f7b 100644 --- a/regression-test/suites/doc/sql-manual/sql-functions/doc_bitmap_functions_test.groovy +++ b/regression-test/suites/doc/sql-manual/sql-functions/doc_bitmap_functions_test.groovy @@ -31,21 +31,18 @@ suite("doc_bitmap_functions_test") { ''') def bitmapToBase64Single = sql ''' - SELECT bitmap_to_base64(to_bitmap(1)); + SELECT bitmap_to_string(bitmap_from_base64(bitmap_to_base64(to_bitmap(1)))); ''' - def bitmapToBase64SingleExpected = ["BQEBAAAAAAAAAA==", "AQEAAAA="] - assertTrue(bitmapToBase64SingleExpected.contains(bitmapToBase64Single[0][0].toString()), + assertEquals("1", bitmapToBase64Single[0][0].toString(), "Unexpected bitmap_to_base64 single result: ${bitmapToBase64Single}") testFoldConst(''' SELECT bitmap_to_base64(to_bitmap(1)); ''') def bitmapToBase64Multi = sql ''' - SELECT bitmap_to_base64(bitmap_from_string("1,9999999")); + SELECT bitmap_to_string(bitmap_from_base64(bitmap_to_base64(bitmap_from_string("1,9999999")))); ''' - def bitmapToBase64MultiExpected = ["BQIBAAAAAAAAAH+WmAAAAAAA", - "AjowAAACAAAAAAAAAJgAAAAYAAAAGgAAAAEAf5Y="] - assertTrue(bitmapToBase64MultiExpected.contains(bitmapToBase64Multi[0][0].toString()), + assertEquals("1,9999999", bitmapToBase64Multi[0][0].toString(), "Unexpected bitmap_to_base64 multi result: ${bitmapToBase64Multi}") testFoldConst(''' SELECT bitmap_to_base64(bitmap_from_string("1,9999999")); diff --git a/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy b/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy index e09f6e02506cbc..fabfef875d31ba 100644 --- a/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy +++ b/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy @@ -777,6 +777,12 @@ suite("test_bitmap_function") { qt_sql_bitmap_base64_nereids6 """ select bitmap_to_string(bitmap_from_base64(bitmap_to_base64(bitmap_from_string("0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32")))) """ qt_sql_bitmap_base64_nereids7 """ select bitmap_to_string(bitmap_from_base64(bitmap_to_base64(bitmap_from_string("0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,4294967296")))) """ qt_sql_bitmap_base64_nereids8 """ select bitmap_to_string(bitmap_from_base64(bitmap_to_base64(to_bitmap(1)))); """ + // SET serialization order is not stable because it depends on hash-set iteration order. + // These fixed historical bytes verify from_base64 semantics rather than bitmap_to_base64 bytes. + qt_sql_bitmap_base64_history_set_v1 """ select bitmap_to_string(bitmap_from_base64("BQIBAAAAAAAAAH+WmAAAAAAA")); """ + qt_sql_bitmap_base64_history_set_v1_reversed """ select bitmap_to_string(bitmap_from_base64("BQJ/lpgAAAAAAAEAAAAAAAAA")); """ + qt_sql_bitmap_base64_history_set_v2_reversed """ select bitmap_to_string(bitmap_from_base64("CgIAAAB/lpgAAAAAAAEAAAAAAAAA")); """ + qt_sql_bitmap_base64_history_set_v1_mixed """ select bitmap_to_string(bitmap_from_base64("BQX//////////wAAAAAAAAAAAAAAAAEAAAAHAAAAAAAAAAEAAAAAAAAA")); """ // test nullable sql """ DROP TABLE IF EXISTS test_bitmap_base64 """