Skip to content

Commit f30d623

Browse files
Jibing-LiJibing-Li
committed
[improvement](statistics)Agg table set preagg on when doing sample analyzing. (#49918)
This pr includes 3 changes. 1. Nereids support set ScanNode preagg on by hint, like this: select * from table1 /*+PREAGGOPEN*/ 2. When sample analyze agg table and mor unique table, set preagg on to improve performance. 3. Skip sample analyzing agg table and mor unique table's value columns.
1 parent 0f05378 commit f30d623

22 files changed

+748
-157
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,7 @@ public enum TableFrom {
161161
private final List<Expression> joinFilters = new ArrayList<>();
162162

163163
private final List<Hint> hints = new ArrayList<>();
164+
private boolean hintForcePreAggOn = false;
164165

165166
// Map slot to its relation, currently used in SlotReference to find its original
166167
// Relation for example LogicalOlapScan
@@ -268,6 +269,14 @@ public void setNeedLockTables(boolean needLockTables) {
268269
this.needLockTables = needLockTables;
269270
}
270271

272+
public void setHintForcePreAggOn(boolean preAggOn) {
273+
this.hintForcePreAggOn = preAggOn;
274+
}
275+
276+
public boolean isHintForcePreAggOn() {
277+
return hintForcePreAggOn;
278+
}
279+
271280
/**
272281
* cache view info to avoid view's def and sql mode changed before lock it.
273282
*

‎fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.apache.doris.nereids.rules.analysis.CollectJoinConstraint;
3131
import org.apache.doris.nereids.rules.analysis.CollectSubQueryAlias;
3232
import org.apache.doris.nereids.rules.analysis.EliminateDistinctConstant;
33+
import org.apache.doris.nereids.rules.analysis.EliminateLogicalPreAggOnHint;
3334
import org.apache.doris.nereids.rules.analysis.EliminateLogicalSelectHint;
3435
import org.apache.doris.nereids.rules.analysis.FillUpMissingSlots;
3536
import org.apache.doris.nereids.rules.analysis.HavingToFilter;
@@ -92,7 +93,8 @@ private static List<RewriteJob> buildAnalyzerJobs() {
9293
return jobs(
9394
// we should eliminate hint before "Subquery unnesting".
9495
topDown(new AnalyzeCTE()),
95-
topDown(new EliminateLogicalSelectHint()),
96+
topDown(new EliminateLogicalSelectHint(),
97+
new EliminateLogicalPreAggOnHint()),
9698
bottomUp(
9799
new BindRelation(),
98100
new CheckPolicy()

‎fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java‎

Lines changed: 98 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -445,6 +445,7 @@
445445
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
446446
import org.apache.doris.nereids.trees.plans.logical.LogicalLimit;
447447
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
448+
import org.apache.doris.nereids.trees.plans.logical.LogicalPreAggOnHint;
448449
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
449450
import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
450451
import org.apache.doris.nereids.trees.plans.logical.LogicalSelectHint;
@@ -1412,12 +1413,15 @@ public LogicalPlan visitRegularQuerySpecification(RegularQuerySpecificationConte
14121413
return selectPlan;
14131414
}
14141415
List<ParserRuleContext> selectHintContexts = Lists.newArrayList();
1416+
List<ParserRuleContext> preAggOnHintContexts = Lists.newArrayList();
14151417
for (Integer key : selectHintMap.keySet()) {
14161418
if (key > selectCtx.getStart().getStopIndex() && key < selectCtx.getStop().getStartIndex()) {
14171419
selectHintContexts.add(selectHintMap.get(key));
1420+
} else {
1421+
preAggOnHintContexts.add(selectHintMap.get(key));
14181422
}
14191423
}
1420-
return withSelectHint(selectPlan, selectHintContexts);
1424+
return withHints(selectPlan, selectHintContexts, preAggOnHintContexts);
14211425
});
14221426
}
14231427

@@ -3195,91 +3199,111 @@ private LogicalPlan withJoinRelations(LogicalPlan input, RelationContext ctx) {
31953199
return last;
31963200
}
31973201

3198-
private LogicalPlan withSelectHint(LogicalPlan logicalPlan, List<ParserRuleContext> hintContexts) {
3199-
if (hintContexts.isEmpty()) {
3202+
private LogicalPlan withHints(LogicalPlan logicalPlan, List<ParserRuleContext> selectHintContexts,
3203+
List<ParserRuleContext> preAggOnHintContexts) {
3204+
if (selectHintContexts.isEmpty() && preAggOnHintContexts.isEmpty()) {
32003205
return logicalPlan;
32013206
}
3202-
ImmutableList.Builder<SelectHint> hints = ImmutableList.builder();
3203-
for (ParserRuleContext hintContext : hintContexts) {
3204-
SelectHintContext selectHintContext = (SelectHintContext) hintContext;
3205-
for (HintStatementContext hintStatement : selectHintContext.hintStatements) {
3206-
String hintName = hintStatement.hintName.getText().toLowerCase(Locale.ROOT);
3207-
switch (hintName) {
3208-
case "set_var":
3209-
Map<String, Optional<String>> parameters = Maps.newLinkedHashMap();
3210-
for (HintAssignmentContext kv : hintStatement.parameters) {
3211-
if (kv.key != null) {
3212-
String parameterName = visitIdentifierOrText(kv.key);
3213-
Optional<String> value = Optional.empty();
3214-
if (kv.constantValue != null) {
3215-
Literal literal = (Literal) visit(kv.constantValue);
3216-
value = Optional.ofNullable(literal.toLegacyLiteral().getStringValue());
3217-
} else if (kv.identifierValue != null) {
3218-
// maybe we should throw exception when the identifierValue is quoted identifier
3219-
value = Optional.ofNullable(kv.identifierValue.getText());
3207+
LogicalPlan newPlan = logicalPlan;
3208+
if (!selectHintContexts.isEmpty()) {
3209+
ImmutableList.Builder<SelectHint> hints = ImmutableList.builder();
3210+
for (ParserRuleContext hintContext : selectHintContexts) {
3211+
SelectHintContext selectHintContext = (SelectHintContext) hintContext;
3212+
for (HintStatementContext hintStatement : selectHintContext.hintStatements) {
3213+
String hintName = hintStatement.hintName.getText().toLowerCase(Locale.ROOT);
3214+
switch (hintName) {
3215+
case "set_var":
3216+
Map<String, Optional<String>> parameters = Maps.newLinkedHashMap();
3217+
for (HintAssignmentContext kv : hintStatement.parameters) {
3218+
if (kv.key != null) {
3219+
String parameterName = visitIdentifierOrText(kv.key);
3220+
Optional<String> value = Optional.empty();
3221+
if (kv.constantValue != null) {
3222+
Literal literal = (Literal) visit(kv.constantValue);
3223+
value = Optional.ofNullable(literal.toLegacyLiteral().getStringValue());
3224+
} else if (kv.identifierValue != null) {
3225+
// maybe we should throw exception when the identifierValue is quoted identifier
3226+
value = Optional.ofNullable(kv.identifierValue.getText());
3227+
}
3228+
parameters.put(parameterName, value);
32203229
}
3221-
parameters.put(parameterName, value);
32223230
}
3223-
}
3224-
SelectHintSetVar setVar = new SelectHintSetVar(hintName, parameters);
3225-
setVar.setVarOnceInSql(ConnectContext.get().getStatementContext());
3226-
hints.add(setVar);
3227-
break;
3228-
case "leading":
3229-
List<String> leadingParameters = new ArrayList<String>();
3230-
for (HintAssignmentContext kv : hintStatement.parameters) {
3231-
if (kv.key != null) {
3231+
SelectHintSetVar setVar = new SelectHintSetVar(hintName, parameters);
3232+
setVar.setVarOnceInSql(ConnectContext.get().getStatementContext());
3233+
hints.add(setVar);
3234+
break;
3235+
case "leading":
3236+
List<String> leadingParameters = new ArrayList<String>();
3237+
for (HintAssignmentContext kv : hintStatement.parameters) {
3238+
if (kv.key != null) {
3239+
String parameterName = visitIdentifierOrText(kv.key);
3240+
leadingParameters.add(parameterName);
3241+
}
3242+
}
3243+
hints.add(new SelectHintLeading(hintName, leadingParameters));
3244+
break;
3245+
case "ordered":
3246+
hints.add(new SelectHintOrdered(hintName));
3247+
break;
3248+
case "use_cbo_rule":
3249+
List<String> useRuleParameters = new ArrayList<String>();
3250+
for (HintAssignmentContext kv : hintStatement.parameters) {
32323251
String parameterName = visitIdentifierOrText(kv.key);
3233-
leadingParameters.add(parameterName);
3252+
useRuleParameters.add(parameterName);
32343253
}
3235-
}
3236-
hints.add(new SelectHintLeading(hintName, leadingParameters));
3237-
break;
3238-
case "ordered":
3239-
hints.add(new SelectHintOrdered(hintName));
3240-
break;
3241-
case "use_cbo_rule":
3242-
List<String> useRuleParameters = new ArrayList<String>();
3243-
for (HintAssignmentContext kv : hintStatement.parameters) {
3244-
String parameterName = visitIdentifierOrText(kv.key);
3245-
useRuleParameters.add(parameterName);
3246-
}
3247-
hints.add(new SelectHintUseCboRule(hintName, useRuleParameters, false));
3248-
break;
3249-
case "no_use_cbo_rule":
3250-
List<String> noUseRuleParameters = new ArrayList<String>();
3251-
for (HintAssignmentContext kv : hintStatement.parameters) {
3252-
String parameterName = visitIdentifierOrText(kv.key);
3253-
noUseRuleParameters.add(parameterName);
3254-
}
3255-
hints.add(new SelectHintUseCboRule(hintName, noUseRuleParameters, true));
3256-
break;
3257-
case "use_mv":
3258-
List<String> useIndexParameters = new ArrayList<String>();
3259-
for (HintAssignmentContext kv : hintStatement.parameters) {
3260-
String parameterName = visitIdentifierOrText(kv.key);
3261-
if (kv.key != null) {
3262-
useIndexParameters.add(parameterName);
3254+
hints.add(new SelectHintUseCboRule(hintName, useRuleParameters, false));
3255+
break;
3256+
case "no_use_cbo_rule":
3257+
List<String> noUseRuleParameters = new ArrayList<String>();
3258+
for (HintAssignmentContext kv : hintStatement.parameters) {
3259+
String parameterName = visitIdentifierOrText(kv.key);
3260+
noUseRuleParameters.add(parameterName);
32633261
}
3264-
}
3265-
hints.add(new SelectHintUseMv(hintName, useIndexParameters, true));
3266-
break;
3267-
case "no_use_mv":
3268-
List<String> noUseIndexParameters = new ArrayList<String>();
3269-
for (HintAssignmentContext kv : hintStatement.parameters) {
3270-
String parameterName = visitIdentifierOrText(kv.key);
3271-
if (kv.key != null) {
3272-
noUseIndexParameters.add(parameterName);
3262+
hints.add(new SelectHintUseCboRule(hintName, noUseRuleParameters, true));
3263+
break;
3264+
case "use_mv":
3265+
List<String> useIndexParameters = new ArrayList<String>();
3266+
for (HintAssignmentContext kv : hintStatement.parameters) {
3267+
String parameterName = visitIdentifierOrText(kv.key);
3268+
if (kv.key != null) {
3269+
useIndexParameters.add(parameterName);
3270+
}
3271+
}
3272+
hints.add(new SelectHintUseMv(hintName, useIndexParameters, true));
3273+
break;
3274+
case "no_use_mv":
3275+
List<String> noUseIndexParameters = new ArrayList<String>();
3276+
for (HintAssignmentContext kv : hintStatement.parameters) {
3277+
String parameterName = visitIdentifierOrText(kv.key);
3278+
if (kv.key != null) {
3279+
noUseIndexParameters.add(parameterName);
3280+
}
32733281
}
3282+
hints.add(new SelectHintUseMv(hintName, noUseIndexParameters, false));
3283+
break;
3284+
default:
3285+
break;
3286+
}
3287+
}
3288+
}
3289+
newPlan = new LogicalSelectHint<>(hints.build(), newPlan);
3290+
}
3291+
if (!preAggOnHintContexts.isEmpty()) {
3292+
for (ParserRuleContext hintContext : preAggOnHintContexts) {
3293+
if (hintContext instanceof SelectHintContext) {
3294+
SelectHintContext preAggOnHintContext = (SelectHintContext) hintContext;
3295+
if (preAggOnHintContext.hintStatement != null
3296+
&& preAggOnHintContext.hintStatement.hintName != null) {
3297+
String text = preAggOnHintContext.hintStatement.hintName.getText();
3298+
if (text.equalsIgnoreCase("PREAGGOPEN")) {
3299+
newPlan = new LogicalPreAggOnHint<>(newPlan);
3300+
break;
32743301
}
3275-
hints.add(new SelectHintUseMv(hintName, noUseIndexParameters, false));
3276-
break;
3277-
default:
3278-
break;
3302+
}
32793303
}
32803304
}
32813305
}
3282-
return new LogicalSelectHint<>(hints.build(), logicalPlan);
3306+
return newPlan;
32833307
}
32843308

32853309
@Override

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,7 @@ public enum RuleType {
120120
ELIMINATE_GROUP_BY_CONSTANT(RuleTypeClass.REWRITE),
121121

122122
ELIMINATE_LOGICAL_SELECT_HINT(RuleTypeClass.REWRITE),
123+
ELIMINATE_LOGICAL_PRE_AGG_ON_HINT(RuleTypeClass.REWRITE),
123124
ELIMINATE_ORDER_BY_CONSTANT(RuleTypeClass.REWRITE),
124125
ELIMINATE_ORDER_BY_UNDER_SUBQUERY(RuleTypeClass.REWRITE),
125126
ELIMINATE_ORDER_BY_UNDER_VIEW(RuleTypeClass.REWRITE),

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,8 @@
9595
import com.google.common.collect.Lists;
9696
import com.google.common.collect.Sets;
9797
import org.apache.commons.collections.CollectionUtils;
98+
import org.apache.logging.log4j.LogManager;
99+
import org.apache.logging.log4j.Logger;
98100

99101
import java.util.ArrayList;
100102
import java.util.List;
@@ -104,6 +106,7 @@
104106
* Rule to bind relations in query plan.
105107
*/
106108
public class BindRelation extends OneAnalysisRuleFactory {
109+
private static final Logger LOG = LogManager.getLogger(StatementContext.class);
107110

108111
public BindRelation() {}
109112

@@ -179,7 +182,8 @@ private LogicalPlan bind(CascadesContext cascadesContext, UnboundRelation unboun
179182
return getLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext);
180183
}
181184

182-
private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, List<String> qualifier) {
185+
private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, List<String> qualifier,
186+
CascadesContext cascadesContext) {
183187
LogicalOlapScan scan;
184188
List<Long> partIds = getPartitionIds(table, unboundRelation, qualifier);
185189
List<Long> tabletIds = unboundRelation.getTabletIds();
@@ -215,6 +219,9 @@ private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation,
215219
// This tabletIds is set manually, so need to set specifiedTabletIds
216220
scan = scan.withManuallySpecifiedTabletIds(tabletIds);
217221
}
222+
if (cascadesContext.getStatementContext().isHintForcePreAggOn()) {
223+
return scan.withPreAggStatus(PreAggStatus.on());
224+
}
218225
if (needGenerateLogicalAggForRandomDistAggTable(scan)) {
219226
// it's a random distribution agg table
220227
// add agg on olap scan
@@ -381,7 +388,7 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio
381388
switch (table.getType()) {
382389
case OLAP:
383390
case MATERIALIZED_VIEW:
384-
return makeOlapScan(table, unboundRelation, qualifierWithoutTableName);
391+
return makeOlapScan(table, unboundRelation, qualifierWithoutTableName, cascadesContext);
385392
case VIEW:
386393
View view = (View) table;
387394
isView = true;

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalPreAggOnHint.java‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.rules.analysis;
19+
20+
import org.apache.doris.nereids.rules.Rule;
21+
import org.apache.doris.nereids.rules.RuleType;
22+
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
23+
import org.apache.doris.nereids.trees.plans.Plan;
24+
import org.apache.doris.nereids.trees.plans.logical.LogicalPreAggOnHint;
25+
26+
/**
27+
* eliminate logical common hint and set them to cascade context
28+
*/
29+
public class EliminateLogicalPreAggOnHint extends OneRewriteRuleFactory {
30+
31+
@Override
32+
public Rule build() {
33+
return logicalPreAggOnHint().thenApply(ctx -> {
34+
LogicalPreAggOnHint<Plan> preAggHintPlan = ctx.root;
35+
ctx.statementContext.setHintForcePreAggOn(true);
36+
return preAggHintPlan.child();
37+
}).toRule(RuleType.ELIMINATE_LOGICAL_PRE_AGG_ON_HINT);
38+
}
39+
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ public enum PlanType {
6262
LOGICAL_APPLY,
6363
LOGICAL_ASSERT_NUM_ROWS,
6464
LOGICAL_CHECK_POLICY,
65+
LOGICAL_COMMON_HINT,
6566
LOGICAL_CTE,
6667
LOGICAL_CTE_ANCHOR,
6768
LOGICAL_CTE_PRODUCER,

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import org.apache.doris.nereids.rules.analysis.BindExpression;
4242
import org.apache.doris.nereids.rules.analysis.BindRelation;
4343
import org.apache.doris.nereids.rules.analysis.CheckPolicy;
44+
import org.apache.doris.nereids.rules.analysis.EliminateLogicalPreAggOnHint;
4445
import org.apache.doris.nereids.rules.analysis.EliminateLogicalSelectHint;
4546
import org.apache.doris.nereids.trees.expressions.Expression;
4647
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -310,7 +311,8 @@ public List<RewriteJob> getJobs() {
310311

311312
private static List<RewriteJob> buildAnalyzeViewJobsForStar() {
312313
return jobs(
313-
topDown(new EliminateLogicalSelectHint()),
314+
topDown(new EliminateLogicalSelectHint(),
315+
new EliminateLogicalPreAggOnHint()),
314316
topDown(new AnalyzeCTE()),
315317
bottomUp(
316318
new BindRelation(),

0 commit comments

Comments
 (0)