[test](regression) Add ORC reader V2 Hive coverage

be/src/format_v2/orc/orc_reader.cpp

@@ -36,6 +36,7 @@
 #include <orc/sargs/Literal.hh>
 #include <orc/sargs/SearchArgument.hh>
 #include <set>
+#include <sstream>
 #include <string>
 #include <string_view>
 #include <system_error>
@@ -718,6 +719,7 @@ struct OrcReaderScanState {
     size_t orc_lazy_input_rows = 0;
     bool enable_lazy_materialization = true;
     bool enable_filter_by_min_max = true;
+    bool check_orc_init_sargs_success = false;
     bool orc_lazy_read_enabled = false;
     bool orc_lazy_selection_valid = false;
 
@@ -851,6 +853,7 @@ Status OrcReader::init(RuntimeState* state) {
     if (state != nullptr) {
         _state->enable_lazy_materialization = state->query_options().enable_orc_lazy_mat;
         _state->enable_filter_by_min_max = state->query_options().enable_orc_filter_by_min_max;
+        _state->check_orc_init_sargs_success = state->query_options().check_orc_init_sargs_success;
         _state->timezone = state->timezone();
         _state->timezone_obj = state->timezone_obj();
     }
@@ -1356,6 +1359,19 @@ Status OrcReader::_init_search_argument_from_local_filters() {
                     has_pushdown;
         }
         if (!has_pushdown) {
+            if (_state->check_orc_init_sargs_success) {
+                std::stringstream ss;
+                for (const auto& conjunct : _request->conjuncts) {
+                    if (conjunct != nullptr) {
+                        ss << conjunct->root()->debug_string() << "\n";
+                    }
+                }
+                return Status::InternalError(
+                        "Session variable check_orc_init_sargs_success is set, but "
+                        "_init_search_argument_from_local_filters returns false because all exprs "
+                        "can not be pushed down:\n {}",
+                        ss.str());
+            }
             return Status::OK();
         }
         builder->end();

regression-test/suites/external_table_p0/hive/test_hive_orc_predicate.groovy

@@ -15,14 +15,110 @@
 // specific language governing permissions and limitations
 // under the License.
 
+import groovy.json.JsonSlurper
+
 suite("test_hive_orc_predicate", "p0,external") {
+    def getProfileList = {
+        def dst = 'http://' + context.config.feHttpAddress
+        def conn = new URL(dst + "/rest/v1/query_profile").openConnection()
+        conn.setRequestMethod("GET")
+        def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" +
+                (context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8"))
+        conn.setRequestProperty("Authorization", "Basic ${encoding}")
+        return conn.getInputStream().getText()
+    }
+
+    def getProfile = { id ->
+        def dst = 'http://' + context.config.feHttpAddress
+        def conn = new URL(dst + "/api/profile/text/?query_id=$id").openConnection()
+        conn.setRequestMethod("GET")
+        def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" +
+                (context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8"))
+        conn.setRequestProperty("Authorization", "Basic ${encoding}")
+        return conn.getInputStream().getText()
+    }
+
+    def getProfileWithToken = { token ->
+        String profileId = ""
+        int attempts = 0
+        while (attempts < 10 && (profileId == null || profileId == "")) {
+            List profileData = new JsonSlurper().parseText(getProfileList()).data.rows
+            for (def profileItem in profileData) {
+                if (profileItem["Sql Statement"].toString().contains(token)) {
+                    profileId = profileItem["Profile ID"].toString()
+                    break
+                }
+            }
+            if (profileId == null || profileId == "") {
+                Thread.sleep(300)
+            }
+            attempts++
+        }
+        assertTrue(profileId != null && profileId != "")
+        Thread.sleep(800)
+        return getProfile(profileId).toString()
+    }
+
+    def extractProfileValue = { String profileText, String keyName ->
+        def matcher = profileText =~ /(?m)^\s*-\s*${keyName}:\s*sum\s+(\S+),/
+        return matcher.find() ? matcher.group(1).trim() : null
+    }
+
+    def assertOrcV2SargProfile = { String profileText ->
+        assertTrue(profileText.contains("UseScannerV2: true"), "Profile does not use ScannerV2")
+        assertTrue(profileText.contains("OrcReader"), "Profile does not contain OrcReader")
+        assertTrue(extractProfileValue(profileText, "EvaluatedRowGroupCount") != null)
+        assertTrue(extractProfileValue(profileText, "SelectedRowGroupCount") != null)
+        assertTrue(extractProfileValue(profileText, "RowGroupsFilteredByMinMax") != null)
+    }
 
     String enabled = context.config.otherConfigs.get("enableHiveTest")
     if (enabled == null || !enabled.equalsIgnoreCase("true")) {
         logger.info("diable Hive test.")
         return;
     }
 
+    def normalizeRows = { rows ->
+        rows.collect { row ->
+            row.collect { value -> value == null ? null : value.toString() }
+        }
+    }
+    def assertRepeatedRows = { expectedRow, rows ->
+        def normalizedRows = normalizeRows(rows)
+        assertEquals(10, normalizedRows.size())
+        assertTrue(normalizedRows.every { it == expectedRow })
+    }
+    def assertProfiledRepeatedRows = { expectedRow, queryBody ->
+        def profileToken = UUID.randomUUID().toString()
+        try {
+            sql """ set check_orc_init_sargs_success = true; """
+            def rows = sql("""
+                select * from (
+                    ${queryBody}
+                ) profile_query where '${profileToken}' = '${profileToken}';
+            """)
+            assertRepeatedRows(expectedRow, rows)
+            assertOrcV2SargProfile(getProfileWithToken(profileToken))
+        } finally {
+            sql """ set check_orc_init_sargs_success = false; """
+        }
+    }
+    def assertProfiledRows = { expectedRows, queryBody ->
+        def profileToken = UUID.randomUUID().toString()
+        try {
+            sql """ set check_orc_init_sargs_success = true; """
+            def rows = sql("""
+                select * from (
+                    ${queryBody}
+                ) profile_query where '${profileToken}' = '${profileToken}';
+            """)
+            assertEquals(expectedRows, normalizeRows(rows))
+            assertOrcV2SargProfile(getProfileWithToken(profileToken))
+        } finally {
+            sql """ set check_orc_init_sargs_success = false; """
+        }
+    }
+
     for (String hivePrefix : ["hive2", "hive3"]) {
         String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort")
         String catalog_name = "${hivePrefix}_test_predicate"
@@ -34,6 +130,10 @@ suite("test_hive_orc_predicate", "p0,external") {
             'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}'
         );"""
         sql """use `${catalog_name}`.`multi_catalog`"""
+        sql """set enable_file_scanner_v2 = true;"""
+        sql """set enable_orc_filter_by_min_max = true;"""
+        sql """set profile_level=2;"""
+        sql """set enable_profile=true;"""
 
         qt_predicate_fixed_char1 """ select * from fixed_char_table where c = 'a';"""
         qt_predicate_fixed_char2 """ select * from fixed_char_table where c = 'a ';"""
@@ -44,6 +144,62 @@ suite("test_hive_orc_predicate", "p0,external") {
 
         qt_predicate_null_aware_equal_in_rt """select * from table_a inner join table_b on table_a.age <=> table_b.age and table_b.id in (1,3) order by table_a.id;"""
 
+        assertProfiledRows([["1", null], ["3", null]], """
+            select id, age from table_a where age <=> null order by id
+        """)
+        assertProfiledRows([["2", "18"]], """
+            select id, age from table_a where age <=> 18 order by id
+        """)
+
+        sql """use `${catalog_name}`.`default`"""
+        assertProfiledRepeatedRows(["3"], """
+            select t_int from `${catalog_name}`.`default`.orc_all_types_t
+            where t_int >= 3 and t_int < 4 order by t_int
+        """)
+        assertProfiledRepeatedRows(["3", "-1234567890.12345678"], """
+            select t_int, t_decimal_precision_18
+            from `${catalog_name}`.`default`.orc_all_types_t
+            where t_decimal_precision_18 =
+                    cast('-1234567890.12345678' as decimal(18,8))
+            order by t_int
+        """)
+        assertProfiledRepeatedRows(["3", "-1234567890.12345678"], """
+            select t_int, t_decimal_precision_18
+            from `${catalog_name}`.`default`.orc_all_types_t
+            where t_decimal_precision_18 >=
+                    cast('-1234567890.12345678' as decimal(18,8))
+              and t_decimal_precision_18 <
+                    cast('-1234567890.12345677' as decimal(18,8))
+            order by t_int
+        """)
+        assertProfiledRepeatedRows(["3", "2011-05-06"], """
+            select t_int, t_date from `${catalog_name}`.`default`.orc_all_types_t
+            where t_date = date '2011-05-06' order by t_int
+        """)
+        assertProfiledRepeatedRows(["3", "2011-05-06"], """
+            select t_int, t_date from `${catalog_name}`.`default`.orc_all_types_t
+            where t_date >= date '2011-05-06'
+              and t_date < date '2011-05-07'
+            order by t_int
+        """)
+        assertProfiledRepeatedRows(["3", "2011-05-06T07:08:09.123"], """
+            select t_int, t_timestamp
+            from `${catalog_name}`.`default`.orc_all_types_t
+            where t_timestamp =
+                    cast('2011-05-06 07:08:09.123' as datetime(6))
+            order by t_int
+        """)
+        assertProfiledRepeatedRows(["3", "2011-05-06T07:08:09.123"], """
+            select t_int, t_timestamp
+            from `${catalog_name}`.`default`.orc_all_types_t
+            where t_timestamp >=
+                    cast('2011-05-06 07:08:09.123' as datetime(6))
+              and t_timestamp <
+                    cast('2011-05-06 07:08:10' as datetime(6))
+            order by t_int
+        """)
+
+        sql """use `${catalog_name}`.`multi_catalog`"""
         qt_lazy_materialization_for_list_type """ select l from complex_data_orc where id > 2 order by id; """
         qt_lazy_materialization_for_map_type """ select m from complex_data_orc where id > 2 order by id; """
         qt_lazy_materialization_for_list_and_map_type """ select * from complex_data_orc where id > 2 order by id; """

regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy

@@ -195,6 +195,7 @@ suite("test_hive_topn_lazy_mat", "p0,external") {
 
         sql """ use global_lazy_mat_db; """
 
+        sql """ set enable_file_scanner_v2 = true; """
 
         sql """
         set topn_lazy_materialization_threshold=1024;

regression-test/suites/external_table_p0/hive/test_orc_lazy_mat_profile.groovy

@@ -97,6 +97,11 @@ suite("test_orc_lazy_mat_profile", "p0,external") {
         return matcher.find() ? matcher.group(1).trim() : null
     }
 
+    String enabled = context.config.otherConfigs.get("enableHiveTest")
+    if (!"true".equalsIgnoreCase(enabled)) {
+        return;
+    }
+
     // session vars
     sql "unset variable all;"
     sql "set profile_level=2;"
@@ -105,11 +110,7 @@ suite("test_orc_lazy_mat_profile", "p0,external") {
     sql " set file_split_size = 10000000;"
     sql """set max_file_scanners_concurrency =  1; """
     sql """set enable_condition_cache = false; """
-
-    String enabled = context.config.otherConfigs.get("enableHiveTest")
-    if (!"true".equalsIgnoreCase(enabled)) {
-        return;
-    }
+    sql """set enable_file_scanner_v2 = true; """
 
     for (String hivePrefix : ["hive2"]) {
         String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
@@ -134,9 +135,14 @@ suite("test_orc_lazy_mat_profile", "p0,external") {
             def t1 = UUID.randomUUID().toString()
 
             def sql_result = sql """
-                select *, "${t1}" from orc_topn_lazy_mat_table where file_id = 1 and id = 1;
+                select id, name, value, active, score, file_id from (
+                    select *, "${t1}" as profile_token from orc_topn_lazy_mat_table
+                    where file_id = 1 and id = 1
+                ) profile_query;
             """
             logger.info("sql_result = ${sql_result}");
+            assertEquals(["1"], sql_result.collect { it[0].toString() }.sort())
+            assertTrue(sql_result.every { it[5].toString() == "1" })
             return getProfileWithToken(t1);             
         }
 
@@ -145,29 +151,43 @@ suite("test_orc_lazy_mat_profile", "p0,external") {
             def t1 = UUID.randomUUID().toString()
 
             def sql_result = sql """
-                select *, "${t1}" from orc_topn_lazy_mat_table where file_id = 1 and id <= 2;
+                select id, name, value, active, score, file_id from (
+                    select *, "${t1}" as profile_token from orc_topn_lazy_mat_table
+                    where file_id = 1 and id <= 2
+                ) profile_query;
             """
             logger.info("sql_result = ${sql_result}");
+            assertEquals(["1", "2"], sql_result.collect { it[0].toString() }.sort())
+            assertTrue(sql_result.every { it[5].toString() == "1" })
             return getProfileWithToken(t1);             
         }
 
         def q3 = {
             def t1 = UUID.randomUUID().toString()
 
             def sql_result = sql """
-                select *, "${t1}" from orc_topn_lazy_mat_table where file_id = 1 and id <= 3;
+                select id, name, value, active, score, file_id from (
+                    select *, "${t1}" as profile_token from orc_topn_lazy_mat_table
+                    where file_id = 1 and id <= 3
+                ) profile_query;
             """
             logger.info("sql_result = ${sql_result}");
+            assertEquals(["1", "2", "3"], sql_result.collect { it[0].toString() }.sort())
+            assertTrue(sql_result.every { it[5].toString() == "1" })
             return getProfileWithToken(t1);             
         }
 
         def q4 = {
             def t1 = UUID.randomUUID().toString()
 
             def sql_result = sql """
-                select *, "${t1}" from orc_topn_lazy_mat_table where file_id = 1 and id < 0;
+                select id, name, value, active, score, file_id from (
+                    select *, "${t1}" as profile_token from orc_topn_lazy_mat_table
+                    where file_id = 1 and id < 0
+                ) profile_query;
             """
             logger.info("sql_result = ${sql_result}");
+            assertEquals(0, sql_result.size())
             return getProfileWithToken(t1);             
         }