diff --git a/be/src/exprs/function/function_split_by_regexp.cpp b/be/src/exprs/function/function_split_by_regexp.cpp index d889b0fc8622b8..292bcb64559747 100644 --- a/be/src/exprs/function/function_split_by_regexp.cpp +++ b/be/src/exprs/function/function_split_by_regexp.cpp @@ -27,6 +27,7 @@ #include "core/types.h" #include "exprs/function/function.h" #include "exprs/function/simple_function_factory.h" +#include "util/simd/vstring_function.h" namespace doris { @@ -115,7 +116,8 @@ bool RegexpSplit::get(const char*& token_begin, const char*& token_end) { } } - _pos += 1; + const auto utf8_byte_length = get_utf8_byte_length(static_cast(*_pos)); + _pos += std::min(utf8_byte_length, _end - _pos); token_end = _pos; ++_splits; } else { diff --git a/regression-test/data/query_p0/sql_functions/string_functions/test_split_by_regexp.out b/regression-test/data/query_p0/sql_functions/string_functions/test_split_by_regexp.out index fa6e31abd94054..d9184915ab7b0e 100644 --- a/regression-test/data/query_p0/sql_functions/string_functions/test_split_by_regexp.out +++ b/regression-test/data/query_p0/sql_functions/string_functions/test_split_by_regexp.out @@ -2,6 +2,12 @@ -- !select1 -- ["a", "b", "c", "d", "e"] +-- !select_utf8_empty_pattern -- +["中", "a", "😀"] + +-- !select_utf8_empty_pattern_with_limit -- +["中", "a😀"] + -- !select2 -- ["a", "bc", "de", "f"] diff --git a/regression-test/suites/query_p0/sql_functions/string_functions/test_split_by_regexp.groovy b/regression-test/suites/query_p0/sql_functions/string_functions/test_split_by_regexp.groovy index 678f72216421ce..9754ca6e592f3a 100644 --- a/regression-test/suites/query_p0/sql_functions/string_functions/test_split_by_regexp.groovy +++ b/regression-test/suites/query_p0/sql_functions/string_functions/test_split_by_regexp.groovy @@ -17,6 +17,8 @@ suite("test_split_by_regexp") { qt_select1 "select split_by_regexp('abcde','');" + order_qt_select_utf8_empty_pattern "select split_by_regexp('中a😀', '');" + order_qt_select_utf8_empty_pattern_with_limit "select split_by_regexp('中a😀', '', 2);" qt_select2 "select split_by_regexp('a12bc23de345f','\\\\d+');" qt_select3 "select split_by_regexp('a12bc23de345f',NULL);" qt_select4 "select split_by_regexp(NULL, 'a12bc23de345f');"