[fix](mtmv)Fix high level materialized view not hit because group by eliminate fail (#36888)

this depends on 
#36839
#36886

Such as low level materialized view contains 5 group by dimension, and
query also has 5 group by dimension, they are equals.In this scene,
would not add aggregate on mv when try to rewrite query by materialized
view.
But if query only use 4 group by dimension and the remain demension is
can be eliminated, then the query will change to 4 group by dimension.
this will cause add aggregate on mv and will cause high level
materialize rewrite fail later.

Solution:
in aggregate rewrite by materialized view, we try to eliminate mv group
by dimension by query used dimension. if eliminate successfully. then
high level will rewrite continue.


such as 
low level mv def sql is as following:

    def join_mv_1 = """
        select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, cast(sum(IFNULL(o_orderkey, 0) * IFNULL(o_custkey, 0)) as decimal(28, 8)) as agg1,
        sum(o_totalprice) as sum_total, 
        max(o_totalprice) as max_total, 
        min(o_totalprice) as min_total, 
        count(*) as count_all, 
        bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, 
        bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 
        from lineitem_1
        inner join orders_1
        on lineitem_1.l_orderkey = orders_1.o_orderkey
        where lineitem_1.l_shipdate >= "2023-10-17"
        group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey
        """
    def join_mv_2 = """
        select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey,
        t.agg1 as agg1, 
        t.sum_total as agg3,
        t.max_total as agg4,
        t.min_total as agg5,
        t.count_all as agg6,
        cast(sum(IFNULL(ps_suppkey, 0) * IFNULL(ps_partkey, 0)) as decimal(28, 8)) as agg2
        from ${mv_1} as t
        inner join partsupp_1
        on t.l_partkey = partsupp_1.ps_partkey and t.l_suppkey = partsupp_1.ps_suppkey
        where partsupp_1.ps_suppkey > 1
        group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, agg1, agg3, agg4, agg5, agg6
        """

high level mv def sql is as following:

 def join_mv_3 = """
        select t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, >t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6 
        from ${mv_2} as t1
        left join ${mv_2} as t2
        on t1.l_orderkey = t2.l_orderkey
        where t1.l_orderkey > 1
        group by t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, >t2.agg1, >t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6
        """

if we run the query as following, it can hit the mv3

select t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, >t1.agg2, >t2.agg3, t1.agg4, t2.agg5, t1.agg6 
        from (
            select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, 
            t.agg1 as agg1, 
            t.sum_total as agg3,
            t.max_total as agg4,
            t.min_total as agg5,
            t.count_all as agg6,
            cast(sum(IFNULL(ps_suppkey, 0) * IFNULL(ps_partkey, 0)) as decimal(28, 8)) as agg2
            from (
                select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, cast(sum(IFNULL(o_orderkey, 0) * >IFNULL(o_custkey, 0)) as decimal(28, 8)) as agg1,
                sum(o_totalprice) as sum_total, 
                max(o_totalprice) as max_total, 
                min(o_totalprice) as min_total, 
                count(*) as count_all, 
                bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) >cnt_1, 
                bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as >cnt_2 
                from lineitem_1
                inner join orders_1
                on lineitem_1.l_orderkey = orders_1.o_orderkey
                where lineitem_1.l_shipdate >= "2023-10-17"
                group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey
            ) as t
            inner join partsupp_1
            on t.l_partkey = partsupp_1.ps_partkey and t.l_suppkey = partsupp_1.ps_suppkey
            where partsupp_1.ps_suppkey > 1
            group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, agg1, agg3, agg4, >agg5, >agg6
        ) as t1
        left join (
            select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, 
            t.agg1 as agg1, 
            t.sum_total as agg3,
            t.max_total as agg4,
            t.min_total as agg5,
            t.count_all as agg6,
            cast(sum(IFNULL(ps_suppkey, 0) * IFNULL(ps_partkey, 0)) as decimal(28, 8)) as agg2
            from (
                select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, cast(sum(IFNULL(o_orderkey, 0) * >IFNULL(o_custkey, 0)) as decimal(28, 8)) as agg1,
                sum(o_totalprice) as sum_total, 
                max(o_totalprice) as max_total, 
                min(o_totalprice) as min_total, 
                count(*) as count_all, 
                bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) >cnt_1, 
                bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as >cnt_2 
                from lineitem_1
                inner join orders_1
                on lineitem_1.l_orderkey = orders_1.o_orderkey
                where lineitem_1.l_shipdate >= "2023-10-17"
                group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey
            ) as t
            inner join partsupp_1
            on t.l_partkey = partsupp_1.ps_partkey and t.l_suppkey = partsupp_1.ps_suppkey
            where partsupp_1.ps_suppkey > 1
            group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, agg1, agg3, agg4, agg5, >agg6
        ) as t2
        on t1.l_orderkey = t2.l_orderkey
        where t1.l_orderkey > 1
        group by t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, >t2.agg1, >t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6

---------

Co-authored-by: xiejiann <[email protected]>

Author: seawinde

fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java

@@ -291,6 +291,10 @@ public MTMVCache getOrGenerateCache(ConnectContext connectionContext) throws Ana
         return cache;
     }
 
+    public MTMVCache getCache() {
+        return cache;
+    }
+
     public Map<String, String> getMvProperties() {
         readMvLock();
         try {

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java renamed to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewScanRule.java

@@ -34,7 +34,7 @@
 /**
  * This is responsible for single table rewriting according to different pattern
  * */
-public abstract class MaterializedViewScanRule extends AbstractMaterializedViewRule {
+public abstract class AbstractMaterializedViewScanRule extends AbstractMaterializedViewRule {
 
     @Override
     protected Plan rewriteQueryByView(MatchMode matchMode,

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectScanRule.java

@@ -30,7 +30,7 @@
 /**
  * MaterializedViewFilterProjectScanRule
  */
-public class MaterializedViewFilterProjectScanRule extends MaterializedViewScanRule {
+public class MaterializedViewFilterProjectScanRule extends AbstractMaterializedViewScanRule {
 
     public static final MaterializedViewFilterProjectScanRule INSTANCE = new MaterializedViewFilterProjectScanRule();
 

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterScanRule.java

@@ -29,7 +29,7 @@
 /**
  * MaterializedViewFilterScanRule
  */
-public class MaterializedViewFilterScanRule extends MaterializedViewScanRule {
+public class MaterializedViewFilterScanRule extends AbstractMaterializedViewScanRule {
 
     public static final MaterializedViewFilterScanRule INSTANCE = new MaterializedViewFilterScanRule();
 

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyScanRule.java

@@ -27,7 +27,7 @@
 /**
  * MaterializedViewOnlyScanRule
  */
-public class MaterializedViewOnlyScanRule extends MaterializedViewScanRule {
+public class MaterializedViewOnlyScanRule extends AbstractMaterializedViewScanRule {
 
     public static final MaterializedViewOnlyScanRule INSTANCE = new MaterializedViewOnlyScanRule();
 

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterScanRule.java

@@ -30,7 +30,7 @@
 /**
  * MaterializedViewProjectFilterScanRule
  */
-public class MaterializedViewProjectFilterScanRule extends MaterializedViewScanRule {
+public class MaterializedViewProjectFilterScanRule extends AbstractMaterializedViewScanRule {
 
     public static final MaterializedViewProjectFilterScanRule INSTANCE = new MaterializedViewProjectFilterScanRule();
 

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectScanRule.java

@@ -29,7 +29,7 @@
 /**
  * MaterializedViewProjectScanRule
  */
-public class MaterializedViewProjectScanRule extends MaterializedViewScanRule {
+public class MaterializedViewProjectScanRule extends AbstractMaterializedViewScanRule {
 
     public static final MaterializedViewProjectScanRule INSTANCE = new MaterializedViewProjectScanRule();
 

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java

@@ -18,8 +18,10 @@
 package org.apache.doris.nereids.trees.plans.logical;
 
 import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.MTMV;
 import org.apache.doris.catalog.OlapTable;
 import org.apache.doris.catalog.Table;
+import org.apache.doris.mtmv.MTMVCache;
 import org.apache.doris.nereids.memo.GroupExpression;
 import org.apache.doris.nereids.properties.DataTrait;
 import org.apache.doris.nereids.properties.LogicalProperties;
@@ -47,6 +49,7 @@
 import org.json.JSONObject;
 
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -495,7 +498,16 @@ AGGREGATE KEY (siteid,citycode,username)
             return;
         }
         Set<Slot> outputSet = Utils.fastToImmutableSet(getOutputSet());
-        if (getTable().getKeysType().isAggregationFamily()) {
+        if (getTable() instanceof MTMV) {
+            MTMV mtmv = (MTMV) getTable();
+            MTMVCache cache = mtmv.getCache();
+            if (cache == null) {
+                return;
+            }
+            Plan originalPlan = cache.getOriginalPlan();
+            builder.addUniqueSlot(originalPlan.getLogicalProperties().getTrait());
+            builder.replaceUniqueBy(constructReplaceMap(mtmv));
+        } else if (getTable().getKeysType().isAggregationFamily()) {
             ImmutableSet.Builder<Slot> uniqSlots = ImmutableSet.builderWithExpectedSize(outputSet.size());
             for (Slot slot : outputSet) {
                 if (!(slot instanceof SlotReference)) {
@@ -509,4 +521,58 @@ AGGREGATE KEY (siteid,citycode,username)
             builder.addUniqueSlot(uniqSlots.build());
         }
     }
+
+    @Override
+    public void computeUniform(DataTrait.Builder builder) {
+        if (getTable() instanceof MTMV) {
+            MTMV mtmv = (MTMV) getTable();
+            MTMVCache cache = mtmv.getCache();
+            if (cache == null) {
+                return;
+            }
+            Plan originalPlan = cache.getOriginalPlan();
+            builder.addUniformSlot(originalPlan.getLogicalProperties().getTrait());
+            builder.replaceUniformBy(constructReplaceMap(mtmv));
+        }
+    }
+
+    @Override
+    public void computeEqualSet(DataTrait.Builder builder) {
+        if (getTable() instanceof MTMV && getTable().getName().equals("mv1")) {
+            System.out.println();
+        }
+        if (getTable() instanceof MTMV) {
+            MTMV mtmv = (MTMV) getTable();
+            MTMVCache cache = mtmv.getCache();
+            if (cache == null) {
+                return;
+            }
+            Plan originalPlan = cache.getOriginalPlan();
+            builder.addEqualSet(originalPlan.getLogicalProperties().getTrait());
+            builder.replaceEqualSetBy(constructReplaceMap(mtmv));
+        }
+    }
+
+    @Override
+    public void computeFd(DataTrait.Builder builder) {
+        if (getTable() instanceof MTMV) {
+            MTMV mtmv = (MTMV) getTable();
+            MTMVCache cache = mtmv.getCache();
+            if (cache == null) {
+                return;
+            }
+            Plan originalPlan = cache.getOriginalPlan();
+            builder.addFuncDepsDG(originalPlan.getLogicalProperties().getTrait());
+            builder.replaceFuncDepsBy(constructReplaceMap(mtmv));
+        }
+    }
+
+    Map<Slot, Slot> constructReplaceMap(MTMV mtmv) {
+        Map<Slot, Slot> replaceMap = new HashMap<>();
+        List<Slot> originOutputs = mtmv.getCache().getOriginalPlan().getOutput();
+        for (int i = 0; i < getOutput().size(); i++) {
+            replaceMap.put(originOutputs.get(i), getOutput().get(i));
+        }
+        return replaceMap;
+    }
 }

regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy

@@ -839,23 +839,22 @@ suite("nested_mtmv") {
     }
     compare_res(sql_2 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13")
 
-    // tmp and will fix soon
-//    explain {
-//        sql("${sql_3}")
-//        contains "${mv_3}(${mv_3})"
-//    }
-//    compare_res(sql_3 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13")
-//
-//    explain {
-//        sql("${sql_4}")
-//        contains "${mv_4}(${mv_4})"
-//    }
-//    compare_res(sql_4 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13")
-//
-//    explain {
-//        sql("${sql_5}")
-//        contains "${mv_5}(${mv_5})"
-//    }
-//    compare_res(sql_5 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13")
+    explain {
+        sql("${sql_3}")
+        contains "${mv_3}(${mv_3})"
+    }
+    compare_res(sql_3 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13")
+
+    explain {
+        sql("${sql_4}")
+        contains "${mv_4}(${mv_4})"
+    }
+    compare_res(sql_4 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13")
+
+    explain {
+        sql("${sql_5}")
+        contains "${mv_5}(${mv_5})"
+    }
+    compare_res(sql_5 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13")
 
 }