branch-3.0: [Fix](Variant) variant should not implicit be short key c…

…olumn when create mv #46444 (#46538) Cherry-picked from #46444 Co-authored-by: lihangyu <lihangyu@selectdb.com>
apache · Jan 8, 2025 · b4f1d7d · b4f1d7d
1 parent 3eeb15c
commit b4f1d7d
Show file tree

Hide file tree

Showing 12 changed files with 62 additions and 29 deletions.
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
@@ -382,6 +382,18 @@ public boolean supportSubType(Type subType) {
         return false;
     }
 
+    /**
+    * Return true if this type can be as short key
+    */
+    public boolean couldBeShortKey() {
+        return !(isFloatingPointType()
+                        || getPrimitiveType() == PrimitiveType.STRING
+                        || isJsonbType()
+                        || isComplexType()
+                        || isObjectStored()
+                        || isVariantType());
+    }
+
     /**
      * The output of this is stored directly in the hive metastore as the column type.
      * The string must match exactly.

diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
@@ -795,10 +795,9 @@ public List<Column> checkAndPrepareMaterializedView(AddRollupClause addRollupCla
                             break;
                         }
                     }
-                    if (column.getType().isFloatingPointType()) {
+                    if (!column.getType().couldBeShortKey()) {
                         break;
                     }
-
                     column.setIsKey(true);
 
                     if (column.getType().getPrimitiveType() == PrimitiveType.VARCHAR) {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
@@ -475,7 +475,7 @@ private void supplyOrderColumn() throws AnalysisException {
                     }
                     break;
                 }
-                if (column.getType().isFloatingPointType()) {
+                if (!column.getType().couldBeShortKey()) {
                     break;
                 }
                 if (column.getType().getPrimitiveType() == PrimitiveType.VARCHAR) {
@@ -486,7 +486,9 @@ private void supplyOrderColumn() throws AnalysisException {
                 column.setIsKey(true);
             }
             if (theBeginIndexOfValue == 0) {
-                throw new AnalysisException("The first column could not be float or double type, use decimal instead");
+                throw new AnalysisException(
+                    "The first column could not be float, double or complex "
+                    + "type like array, struct, map, json, variant.");
             }
             // supply value
             for (; theBeginIndexOfValue < mvColumnItemList.size(); theBeginIndexOfValue++) {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
@@ -374,16 +374,7 @@ public void analyze(Analyzer analyzer) throws UserException {
                                 }
                                 break;
                             }
-                            if (columnDef.getType().isFloatingPointType()) {
-                                break;
-                            }
-                            if (columnDef.getType().getPrimitiveType() == PrimitiveType.STRING) {
-                                break;
-                            }
-                            if (columnDef.getType().getPrimitiveType() == PrimitiveType.JSONB) {
-                                break;
-                            }
-                            if (columnDef.getType().isComplexType()) {
+                            if (!columnDef.getType().couldBeShortKey()) {
                                 break;
                             }
                             if (columnDef.getType().getPrimitiveType() == PrimitiveType.VARCHAR) {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -4679,7 +4679,7 @@ public static short calcShortKeyColumnCount(List<Column> columns, Map<String, St
                     }
                     break;
                 }
-                if (column.getType().isFloatingPointType()) {
+                if (!column.getType().couldBeShortKey()) {
                     break;
                 }
                 if (column.getDataType() == PrimitiveType.VARCHAR) {

diff --git a/...core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/...core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
@@ -313,10 +313,10 @@ private void analyzeKeys() {
                     }
                     break;
                 }
-                if (type.isFloatLikeType() || type.isStringType() || type.isJsonType()
-                        || catalogType.isComplexType() || type.isBitmapType() || type.isHllType()
-                        || type.isQuantileStateType() || type.isJsonType() || type.isStructType()
-                        || column.getAggType() != null || type.isVariantType()) {
+                if (column.getAggType() != null) {
+                    break;
+                }
+                if (!catalogType.couldBeShortKey()) {
                     break;
                 }
                 keys.add(column.getName());

diff --git a/...ore/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/...ore/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
@@ -376,8 +376,7 @@ public void validate(ConnectContext ctx) {
                                 }
                                 break;
                             }
-                            if (type.isFloatLikeType() || type.isStringType() || type.isJsonType()
-                                    || catalogType.isComplexType() || catalogType.isVariantType()) {
+                            if (!catalogType.couldBeShortKey()) {
                                 break;
                             }
                             keys.add(column.getName());

diff --git a/regression-test/data/variant_p0/mv/multi_slot.out b/regression-test/data/variant_p0/mv/multi_slot.out
@@ -26,3 +26,6 @@
 456	\N
 789	\N
 
+-- !sql --
+2021-01-01T11:11:11	"http://xxx.xxx.xxx"	12
+
diff --git a/regression-test/suites/mv_p0/test_create_mv_complex_type/test_create_mv_complex_type.groovy b/regression-test/suites/mv_p0/test_create_mv_complex_type/test_create_mv_complex_type.groovy
@@ -47,7 +47,7 @@ suite ("create_mv_complex_type") {
         sql """create materialized view mv as select c_jsonb, c_int from base_table;"""
         success = true
     } catch (Exception e) {
-        assertTrue(e.getMessage().contains("not support to create materialized view"), e.getMessage())
+        assertTrue(e.getMessage().contains("The first column could not be"), e.getMessage())
     }
     assertFalse(success)
 
@@ -65,7 +65,7 @@ suite ("create_mv_complex_type") {
         sql """create materialized view mv as select c_array, c_int from base_table;"""
         success = true
     } catch (Exception e) {
-        assertTrue(e.getMessage().contains("not support to create materialized view"), e.getMessage())
+        assertTrue(e.getMessage().contains("The first column could not be"), e.getMessage())
     }
     assertFalse(success)
 
@@ -83,7 +83,7 @@ suite ("create_mv_complex_type") {
         sql """create materialized view mv as select c_map, c_int from base_table;"""
         success = true
     } catch (Exception e) {
-        assertTrue(e.getMessage().contains("not support to create materialized view"), e.getMessage())
+        assertTrue(e.getMessage().contains("The first column could not be"), e.getMessage())
     }
     assertFalse(success)
 
@@ -101,7 +101,7 @@ suite ("create_mv_complex_type") {
         sql """create materialized view mv as select c_struct, c_int from base_table;"""
         success = true
     } catch (Exception e) {
-        assertTrue(e.getMessage().contains("not support to create materialized view"), e.getMessage())
+        assertTrue(e.getMessage().contains("The first column could not be"), e.getMessage())
     }
     assertFalse(success)
 

diff --git a/regression-test/suites/rollup_p0/test_materialized_view_array.groovy b/regression-test/suites/rollup_p0/test_materialized_view_array.groovy
@@ -67,7 +67,7 @@ suite("test_materialized_view_array", "rollup") {
         create_test_table.call(tableName)
         test {
             sql "CREATE MATERIALIZED VIEW idx AS select k2,k1, k3, k4, k5 from ${tableName}"
-            exception "errCode = 2, detailMessage = The ARRAY column[`mv_k2` array<smallint> NULL] not support to create materialized view"
+            exception "errCode = 2, detailMessage = The first column could not be float, double or complex type like array, struct, map, json, variant"
         }
     } finally {
         try_sql("DROP TABLE IF EXISTS ${tableName}")

diff --git a/regression-test/suites/rollup_p0/test_materialized_view_struct.groovy b/regression-test/suites/rollup_p0/test_materialized_view_struct.groovy
@@ -59,7 +59,7 @@ suite("test_materialized_view_struct", "rollup") {
         create_test_table.call(tableName)
         test {
             sql "CREATE MATERIALIZED VIEW idx AS select k2,k1, k3, k4, k5 from ${tableName}"
-            exception "errCode = 2, detailMessage = The STRUCT column[`mv_k2` struct<f1:smallint> NULL] not support to create materialized view"
+            exception "errCode = 2, detailMessage = The first column could not be float, double or complex type like array, struct, map, json, variant."
         }
     } finally {
         try_sql("DROP TABLE IF EXISTS ${tableName}")

diff --git a/regression-test/suites/variant_p0/mv/multi_slot.groovy b/regression-test/suites/variant_p0/mv/multi_slot.groovy
@@ -50,8 +50,35 @@ suite ("multi_slot") {
 
     order_qt_select_star "select abs(cast(v['k1'] as int))+cast(v['k2'] as int)+1,abs(cast(v['k2'] as int)+2)+cast(v['k3'] as int)+3 from multi_slot;"
     order_qt_select_star "select * from multi_slot order by cast(v['k1'] as int);"
-    // TODO fix and remove enable_rewrite_element_at_to_slot
-    order_qt_select_star "select /*+SET_VAR(enable_rewrite_element_at_to_slot=false) */ abs(cast(v['k4']['k44'] as int)), sum(abs(cast(v['k2'] as int)+2)+cast(v['k3'] as int)+3) from multi_slot group by abs(cast(v['k4']['k44'] as int))"
+    order_qt_select_star "select abs(cast(v['k4']['k44'] as int)), sum(abs(cast(v['k2'] as int)+2)+cast(v['k3'] as int)+3) from multi_slot group by abs(cast(v['k4']['k44'] as int))"
+
+    sql "drop table if exists test_mv"
+    sql """
+    CREATE TABLE `test_mv` (
+        `handle_time` datetime NOT NULL ,
+        `client_request` variant NULL,
+        `status` int NULL
+    )  
+    DISTRIBUTED BY HASH(`handle_time`)
+    BUCKETS 10 PROPERTIES (
+      "is_being_synced" = "false",
+      "storage_medium" = "hdd",
+      "storage_format" = "V2",
+      "inverted_index_storage_format" = "V1",
+      "light_schema_change" = "true",
+      "disable_auto_compaction" = "false",
+      "enable_single_replica_compaction" = "false",
+      "replication_num" = "1"
+    );  
+    """
+    sql """insert into test_mv values ('2021-01-01 11:11:11', '{"url" : "http://xxx.xxx.xxx"}', 12)"""
+    createMV("create materialized view mv_1 as select `handle_time`, `client_request`['url'] as `uri`, `status` from test_mv")
+    qt_sql "select `handle_time`, `client_request`['url'] as `uri`, `status` from test_mv"
+    test {
+        sql "create materialized view mv_x as select `client_request`['url'] as `uri`, `status` from test_mv"
+        exception("The first column could not be float, double or complex type like array, struct, map, json, variant.")
+    }
+
 
     // def retry_times = 60
     // for (def i = 0; i < retry_times; ++i) {
-Original file line number
+Diff line change
@@ Expand Up @@
                         }
                         break;
                     }
-                    if (column.getType().isFloatingPointType()) {
+                    if (!column.getType().couldBeShortKey()) {
                         break;
                     }
                     if (column.getDataType() == PrimitiveType.VARCHAR) {
@@ Expand Down @@