[fix](be) Fix missing decimal precision and scale for legacy decimal data (#65419)

be/src/olap/olap_common.h

@@ -195,6 +195,14 @@ constexpr bool field_is_slice_type(const FieldType& field_type) {
            field_type == FieldType::OLAP_FIELD_TYPE_STRING;
 }
 
+constexpr bool field_is_decimal_type(const FieldType& field_type) {
+    return field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL ||
+           field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL32 ||
+           field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL64 ||
+           field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL128I ||
+           field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL256;
+}
+
 constexpr bool field_is_numeric_type(const FieldType& field_type) {
     return field_type == FieldType::OLAP_FIELD_TYPE_INT ||
            field_type == FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT ||

be/src/olap/rowset/segment_v2/column_reader.cpp

@@ -721,7 +721,7 @@ Status ColumnReader::_load_index(const std::shared_ptr<IndexFileReader>& index_f
                         "create StringTypeInvertedIndexReader error: {}", e.what());
             }
         }
-    } else if (is_numeric_type(type)) {
+    } else if (field_is_numeric_type(type)) {
         try {
             index_reader = BkdIndexReader::create_shared(index_meta, index_file_reader);
         } catch (const CLuceneError& e) {

be/src/olap/rowset/segment_v2/inverted_index_iterator.cpp

@@ -273,7 +273,7 @@ Result<InvertedIndexReaderPtr> InvertedIndexIterator::select_best_reader(
         return select_for_text(match, query_type, normalized_key);
     }
 
-    if (is_numeric_type(field_type)) {
+    if (field_is_numeric_type(field_type)) {
         return select_for_numeric(match, query_type);
     }
 

be/src/olap/rowset/segment_v2/segment.cpp

@@ -23,6 +23,7 @@
 #include <gen_cpp/olap_file.pb.h>
 #include <gen_cpp/segment_v2.pb.h>
 
+#include <algorithm>
 #include <cstring>
 #include <memory>
 #include <sstream>
@@ -84,6 +85,53 @@ namespace doris::segment_v2 {
 
 class InvertedIndexIterator;
 
+namespace {
+
+void fill_missing_decimal_precision(const TabletColumn& column, ColumnMetaPB* meta) {
+    auto meta_type = static_cast<FieldType>(meta->type());
+    if (meta_type != column.type()) {
+        return;
+    }
+
+    if (field_is_decimal_type(meta_type)) {
+        if ((!meta->has_precision() || meta->precision() <= 0) && column.precision() > 0) {
+            meta->set_precision(column.precision());
+        }
+        if ((!meta->has_frac() || meta->frac() < 0) && column.frac() >= 0) {
+            meta->set_frac(column.frac());
+        }
+    }
+
+    // Complex column meta may also include storage helper children, such as
+    // array offsets. Only schema children have matching TabletColumn subtypes.
+    int child_count =
+            std::min(meta->children_columns_size(), static_cast<int>(column.get_subtype_count()));
+    for (int i = 0; i < child_count; ++i) {
+        fill_missing_decimal_precision(column.get_sub_column(i), meta->mutable_children_columns(i));
+    }
+}
+
+void fill_missing_decimal_precision_from_schema(const TabletSchemaSPtr& tablet_schema,
+                                                ColumnMetaPB* meta) {
+    if (!meta->has_unique_id()) {
+        return;
+    }
+    int32_t col_idx = tablet_schema->field_index(static_cast<int32_t>(meta->unique_id()));
+    if (col_idx < 0) {
+        return;
+    }
+    fill_missing_decimal_precision(tablet_schema->column(col_idx), meta);
+}
+
+void fill_footer_missing_decimal_precision(const TabletSchemaSPtr& tablet_schema,
+                                           SegmentFooterPB* footer) {
+    for (int i = 0; i < footer->columns_size(); ++i) {
+        fill_missing_decimal_precision_from_schema(tablet_schema, footer->mutable_columns(i));
+    }
+}
+
+} // namespace
+
 Status Segment::open(io::FileSystemSPtr fs, const std::string& path, int64_t tablet_id,
                      uint32_t segment_id, RowsetId rowset_id, TabletSchemaSPtr tablet_schema,
                      const io::FileReaderOptions& reader_options, std::shared_ptr<Segment>* output,
@@ -479,6 +527,10 @@ Status Segment::_parse_footer(std::shared_ptr<SegmentFooterPB>& footer,
                 _file_reader->path().native(), file_size,
                 file_cache_key_str(_file_reader->path().native()));
     }
+    // Segments written before #26572 do not persist decimal precision/frac in
+    // ColumnMetaPB, so recover the logical p/s from TabletSchema before
+    // ColumnReader builds DataTypeDecimal.
+    fill_footer_missing_decimal_precision(_tablet_schema, footer.get());
 
     VLOG_DEBUG << fmt::format("Loading segment footer from {} finished",
                               _file_reader->path().native());

be/src/olap/utils.h

@@ -182,32 +182,6 @@ constexpr bool is_string_type(const FieldType& field_type) {
            field_type == FieldType::OLAP_FIELD_TYPE_STRING;
 }
 
-constexpr bool is_numeric_type(const FieldType& field_type) {
-    return field_type == FieldType::OLAP_FIELD_TYPE_INT ||
-           field_type == FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT ||
-           field_type == FieldType::OLAP_FIELD_TYPE_BIGINT ||
-           field_type == FieldType::OLAP_FIELD_TYPE_SMALLINT ||
-           field_type == FieldType::OLAP_FIELD_TYPE_UNSIGNED_TINYINT ||
-           field_type == FieldType::OLAP_FIELD_TYPE_UNSIGNED_SMALLINT ||
-           field_type == FieldType::OLAP_FIELD_TYPE_TINYINT ||
-           field_type == FieldType::OLAP_FIELD_TYPE_DOUBLE ||
-           field_type == FieldType::OLAP_FIELD_TYPE_FLOAT ||
-           field_type == FieldType::OLAP_FIELD_TYPE_DATE ||
-           field_type == FieldType::OLAP_FIELD_TYPE_DATEV2 ||
-           field_type == FieldType::OLAP_FIELD_TYPE_DATETIME ||
-           field_type == FieldType::OLAP_FIELD_TYPE_DATETIMEV2 ||
-           field_type == FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ ||
-           field_type == FieldType::OLAP_FIELD_TYPE_LARGEINT ||
-           field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL ||
-           field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL32 ||
-           field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL64 ||
-           field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL128I ||
-           field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL256 ||
-           field_type == FieldType::OLAP_FIELD_TYPE_BOOL ||
-           field_type == FieldType::OLAP_FIELD_TYPE_IPV4 ||
-           field_type == FieldType::OLAP_FIELD_TYPE_IPV6;
-}
-
 // Util used to get string name of thrift enum item
 #define EnumToString(enum_type, index, out)                   \
     do {                                                      \

be/src/vec/data_types/data_type_decimal.h

@@ -125,10 +125,15 @@ class DataTypeDecimal final : public IDataType {
 
     static constexpr size_t max_precision() { return max_decimal_precision<T>(); }
 
-    DataTypeDecimal(UInt32 precision = max_decimal_precision<T>(),
-                    UInt32 scale = default_decimal_scale<T>(),
-                    UInt32 arg_original_precision = UINT32_MAX,
-                    UInt32 arg_original_scale = UINT32_MAX)
+    DataTypeDecimal(
+            UInt32 precision = max_decimal_precision<T>(),
+            UInt32 scale = default_decimal_scale<T>(),
+            // For decimalv2 only, record the original(schema) precision and scale.
+            // UINT32_MAX means original precision and scale are unknown.
+            // Decimalv2 will be converted to Decimal(27, 9) in memory when doing any calculations,
+            // but when casting decimalv2 to string, it's better to keep the presion and
+            // scale of it's original value in schema.
+            UInt32 arg_original_precision = UINT32_MAX, UInt32 arg_original_scale = UINT32_MAX)
             : precision(precision),
               scale(scale),
               original_precision(arg_original_precision),

be/src/vec/data_types/data_type_factory.cpp

@@ -385,23 +385,8 @@ DataTypePtr DataTypeFactory::create_data_type(const segment_v2::ColumnMetaPB& pc
         nested = std::make_shared<DataTypeStruct>(dataTypes, names);
     } else {
         // TODO add precision and frac
-        auto meta_precision = pcolumn.precision();
-        auto meta_scale = pcolumn.frac();
-        if (pcolumn.type() == static_cast<int>(FieldType::OLAP_FIELD_TYPE_DECIMAL)) {
-            // Segments written by Doris < 2.1.0 (before #26572) do not persist
-            // precision/frac in ColumnMetaPB, so they default to 0 when read back.
-            // Pass UINT32_MAX to DataTypeDecimalV2 to signal that the original
-            // precision/scale are unknown; otherwise check_type_precision(0) throws
-            // "meet invalid precision: real_precision=0".
-            UInt32 orig_precision =
-                    meta_precision > 0 ? static_cast<UInt32>(meta_precision) : UINT32_MAX;
-            UInt32 orig_scale = meta_precision > 0 ? static_cast<UInt32>(meta_scale) : UINT32_MAX;
-            nested = _create_primitive_data_type(static_cast<FieldType>(pcolumn.type()),
-                                                 orig_precision, orig_scale, -1);
-        } else {
-            nested = _create_primitive_data_type(static_cast<FieldType>(pcolumn.type()),
-                                                 meta_precision, meta_scale, -1);
-        }
+        nested = _create_primitive_data_type(static_cast<FieldType>(pcolumn.type()),
+                                             pcolumn.precision(), pcolumn.frac(), -1);
     }
 
     if (pcolumn.is_nullable() && nested) {