Skip to content

Commit 0b1d517

Browse files
Jibing-LiJibing-Li
authored
[improvement](statistics)Return -1 to neredis if report olap table row count for new table is not done for all tablets. (#40457) (#40838)
backport: #40457
1 parent 6ab8a7e commit 0b1d517

File tree

14 files changed

+169
-50
lines changed

14 files changed

+169
-50
lines changed

‎be/src/olap/tablet_manager.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1035,6 +1035,7 @@ void TabletManager::build_all_report_tablets_info(std::map<TTabletId, TTablet>*
10351035
t_tablet_stat.__set_row_count(tablet_info.row_count);
10361036
t_tablet_stat.__set_total_version_count(tablet_info.total_version_count);
10371037
t_tablet_stat.__set_visible_version_count(tablet_info.visible_version_count);
1038+
t_tablet_stat.__set_visible_version(tablet_info.version);
10381039
};
10391040
for_each_tablet(handler, filter_all_tablets);
10401041

‎fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java

Lines changed: 24 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,9 @@ public class ShowTableStatsStmt extends ShowStmt {
6363
new ImmutableList.Builder<String>()
6464
.add("table_name")
6565
.add("index_name")
66-
.add("row_count")
66+
.add("analyze_row_count")
67+
.add("report_row_count")
68+
.add("report_row_count_for_nereids")
6769
.build();
6870

6971
private final TableName tableName;
@@ -167,37 +169,33 @@ public long getTableId() {
167169
return tableId;
168170
}
169171

170-
public ShowResultSet constructResultSet(TableStatsMeta tableStatistic) {
172+
public ShowResultSet constructResultSet(TableStatsMeta tableStatistic, TableIf table) {
171173
if (indexName != null) {
172-
return constructIndexResultSet(tableStatistic);
174+
return constructIndexResultSet(tableStatistic, table);
173175
}
174-
return constructTableResultSet(tableStatistic);
176+
return constructTableResultSet(tableStatistic, table);
175177
}
176178

177179
public ShowResultSet constructEmptyResultSet() {
178180
return new ShowResultSet(getMetaData(), new ArrayList<>());
179181
}
180182

181-
public ShowResultSet constructResultSet(long rowCount) {
182-
List<List<String>> result = Lists.newArrayList();
183-
List<String> row = Lists.newArrayList();
184-
row.add("");
185-
row.add("");
186-
row.add(String.valueOf(rowCount));
187-
row.add("");
188-
row.add("");
189-
row.add("");
190-
row.add("");
191-
row.add("");
192-
result.add(row);
193-
return new ShowResultSet(getMetaData(), result);
194-
}
195-
196-
public ShowResultSet constructTableResultSet(TableStatsMeta tableStatistic) {
197-
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
183+
public ShowResultSet constructTableResultSet(TableStatsMeta tableStatistic, TableIf table) {
198184
if (tableStatistic == null) {
199-
return new ShowResultSet(getMetaData(), new ArrayList<>());
185+
List<List<String>> result = Lists.newArrayList();
186+
List<String> row = Lists.newArrayList();
187+
row.add("");
188+
row.add("");
189+
row.add(String.valueOf(table.getCachedRowCount()));
190+
row.add("");
191+
row.add("");
192+
row.add("");
193+
row.add("");
194+
row.add("");
195+
result.add(row);
196+
return new ShowResultSet(getMetaData(), result);
200197
}
198+
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
201199
List<List<String>> result = Lists.newArrayList();
202200
List<String> row = Lists.newArrayList();
203201
row.add(String.valueOf(tableStatistic.updatedRows));
@@ -216,7 +214,7 @@ public ShowResultSet constructTableResultSet(TableStatsMeta tableStatistic) {
216214
return new ShowResultSet(getMetaData(), result);
217215
}
218216

219-
public ShowResultSet constructIndexResultSet(TableStatsMeta tableStatistic) {
217+
public ShowResultSet constructIndexResultSet(TableStatsMeta tableStatistic, TableIf table) {
220218
List<List<String>> result = Lists.newArrayList();
221219
if (!(table instanceof OlapTable)) {
222220
return new ShowResultSet(getMetaData(), result);
@@ -226,14 +224,13 @@ public ShowResultSet constructIndexResultSet(TableStatsMeta tableStatistic) {
226224
if (indexId == null) {
227225
throw new RuntimeException(String.format("Index %s not exist.", indexName));
228226
}
229-
long rowCount = tableStatistic.getRowCount(olapTable.getIndexIdByName(indexName));
230-
if (rowCount == -1) {
231-
return new ShowResultSet(getMetaData(), result);
232-
}
227+
long rowCount = tableStatistic == null ? -1 : tableStatistic.getRowCount(olapTable.getIndexIdByName(indexName));
233228
List<String> row = Lists.newArrayList();
234229
row.add(table.getName());
235230
row.add(indexName);
236231
row.add(String.valueOf(rowCount));
232+
row.add(String.valueOf(olapTable.getRowCountForIndex(indexId, false)));
233+
row.add(String.valueOf(olapTable.getRowCountForIndex(indexId, true)));
237234
result.add(row);
238235
return new ShowResultSet(getMetaData(), result);
239236
}

‎fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,8 @@ public enum IndexExtState {
7373
@SerializedName(value = "rollupFinishedVersion")
7474
private long rollupFinishedVersion;
7575

76+
private boolean rowCountReported = false;
77+
7678
public MaterializedIndex() {
7779
this.state = IndexState.NORMAL;
7880
this.idToTablets = new HashMap<>();
@@ -206,6 +208,14 @@ public int getTabletOrderIdx(long tabletId) {
206208
return -1;
207209
}
208210

211+
public void setRowCountReported(boolean reported) {
212+
this.rowCountReported = reported;
213+
}
214+
215+
public boolean getRowCountReported() {
216+
return this.rowCountReported;
217+
}
218+
209219
@Override
210220
public void write(DataOutput out) throws IOException {
211221
super.write(out);

‎fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1396,18 +1396,17 @@ public List<Pair<String, String>> getColumnIndexPairs(Set<String> columns) {
13961396

13971397
@Override
13981398
public long fetchRowCount() {
1399-
long rowCount = 0;
1400-
for (Map.Entry<Long, Partition> entry : idToPartition.entrySet()) {
1401-
rowCount += entry.getValue().getBaseIndex().getRowCount();
1402-
}
1403-
return rowCount;
1399+
return getRowCountForIndex(baseIndexId, false);
14041400
}
14051401

1406-
public long getRowCountForIndex(long indexId) {
1402+
public long getRowCountForIndex(long indexId, boolean strict) {
14071403
long rowCount = 0;
14081404
for (Map.Entry<Long, Partition> entry : idToPartition.entrySet()) {
14091405
MaterializedIndex index = entry.getValue().getIndex(indexId);
1410-
rowCount += index == null ? 0 : index.getRowCount();
1406+
if (strict && !index.getRowCountReported()) {
1407+
return -1;
1408+
}
1409+
rowCount += (index == null || index.getRowCount() == -1) ? 0 : index.getRowCount();
14111410
}
14121411
return rowCount;
14131412
}

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,8 @@ public enum ReplicaStatus {
164164

165165
private long userDropTime = -1;
166166

167+
private long lastReportVersion = 0;
168+
167169
public Replica() {
168170
}
169171

@@ -811,4 +813,12 @@ public boolean isScheduleAvailable() {
811813
return Env.getCurrentSystemInfo().checkBackendScheduleAvailable(backendId)
812814
&& !isUserDrop();
813815
}
816+
817+
public void setLastReportVersion(long version) {
818+
this.lastReportVersion = version;
819+
}
820+
821+
public long getLastReportVersion() {
822+
return lastReportVersion;
823+
}
814824
}

‎fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,17 +118,41 @@ protected void runAfterCatalogReady() {
118118
long version = partition.getVisibleVersion();
119119
for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
120120
long indexRowCount = 0L;
121+
boolean indexReported = true;
121122
for (Tablet tablet : index.getTablets()) {
122123
long tabletRowCount = 0L;
124+
boolean tabletReported = false;
123125
for (Replica replica : tablet.getReplicas()) {
126+
LOG.debug("Table {} replica {} current version {}, report version {}",
127+
olapTable.getName(), replica.getId(),
128+
replica.getVersion(), replica.getLastReportVersion());
124129
if (replica.checkVersionCatchUp(version, false)
125-
&& replica.getRowCount() > tabletRowCount) {
130+
&& replica.getRowCount() >= tabletRowCount) {
131+
// 1. If replica version and reported replica version are all equal to
132+
// PARTITION_INIT_VERSION, set tabletReported to true, which indicates this
133+
// tablet is empty for sure when previous report.
134+
// 2. If last report version is larger than PARTITION_INIT_VERSION, set
135+
// tabletReported to true as well. That is, we only guarantee all replicas of
136+
// the tablet are reported for the init version.
137+
// e.g. When replica version is 2, but last reported version is 1,
138+
// tabletReported would be false.
139+
if (replica.getVersion() == Partition.PARTITION_INIT_VERSION
140+
&& replica.getLastReportVersion() == Partition.PARTITION_INIT_VERSION
141+
|| replica.getLastReportVersion() > Partition.PARTITION_INIT_VERSION) {
142+
tabletReported = true;
143+
}
126144
tabletRowCount = replica.getRowCount();
127145
}
128146
}
129147
indexRowCount += tabletRowCount;
148+
// Only when all tablets of this index are reported, we set indexReported to true.
149+
indexReported = indexReported && tabletReported;
130150
} // end for tablets
151+
index.setRowCountReported(indexReported);
131152
index.setRowCount(indexRowCount);
153+
LOG.debug("Table {} index {} all tablets reported[{}], row count {}",
154+
olapTable.getName(), olapTable.getIndexNameById(index.getId()),
155+
indexReported, indexRowCount);
132156
} // end for indices
133157
} // end for partitions
134158
if (LOG.isDebugEnabled()) {
@@ -157,6 +181,9 @@ private void updateTabletStat(Long beId, TTabletStatResult result) {
157181
replica.setTotalVersionCount(stat.getTotalVersionCount());
158182
replica.setVisibleVersionCount(stat.isSetVisibleVersionCount() ? stat.getVisibleVersionCount()
159183
: stat.getTotalVersionCount());
184+
// Older version BE doesn't set visible version. Set it to max for compatibility.
185+
replica.setLastReportVersion(stat.isSetVisibleVersion() ? stat.getVisibleVersion()
186+
: Long.MAX_VALUE);
160187
}
161188
}
162189
}

‎fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2520,19 +2520,12 @@ private void handleShowTableStats() {
25202520
if (tableStats == null) {
25212521
resultSet = showTableStatsStmt.constructEmptyResultSet();
25222522
} else {
2523-
resultSet = showTableStatsStmt.constructResultSet(tableStats);
2523+
resultSet = showTableStatsStmt.constructResultSet(tableStats, tableIf);
25242524
}
25252525
return;
25262526
}
25272527
TableStatsMeta tableStats = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(tableIf.getId());
2528-
/*
2529-
tableStats == null means it's not analyzed, in this case show the estimated row count.
2530-
*/
2531-
if (tableStats == null) {
2532-
resultSet = showTableStatsStmt.constructResultSet(tableIf.getCachedRowCount());
2533-
} else {
2534-
resultSet = showTableStatsStmt.constructResultSet(tableStats);
2535-
}
2528+
resultSet = showTableStatsStmt.constructResultSet(tableStats, tableIf);
25362529
}
25372530

25382531
private void handleShowColumnStats() throws AnalysisException {

‎fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -640,7 +640,7 @@ public void dropStats(DropStatsStmt dropStatsStmt) throws DdlException {
640640
if (tableStats == null) {
641641
return;
642642
}
643-
invalidateLocalStats(catalogId, dbId, tblId, cols, tableStats);
643+
invalidateLocalStats(catalogId, dbId, tblId, dropStatsStmt.isAllColumns() ? null : cols, tableStats);
644644
// Drop stats ddl is master only operation.
645645
invalidateRemoteStats(catalogId, dbId, tblId, cols, dropStatsStmt.isAllColumns());
646646
StatisticsRepository.dropStatisticsByColNames(catalogId, dbId, tblId, cols);
@@ -655,7 +655,7 @@ public void dropStats(TableIf table) throws DdlException {
655655
long dbId = table.getDatabase().getId();
656656
long tableId = table.getId();
657657
Set<String> cols = table.getSchemaAllIndexes(false).stream().map(Column::getName).collect(Collectors.toSet());
658-
invalidateLocalStats(catalogId, dbId, tableId, cols, tableStats);
658+
invalidateLocalStats(catalogId, dbId, tableId, null, tableStats);
659659
// Drop stats ddl is master only operation.
660660
invalidateRemoteStats(catalogId, dbId, tableId, cols, true);
661661
StatisticsRepository.dropStatisticsByColNames(catalogId, dbId, table.getId(), cols);
@@ -717,6 +717,8 @@ public void invalidateLocalStats(long catalogId, long dbId, long tableId,
717717
// To remove stale column name that is changed before.
718718
if (allColumn) {
719719
tableStats.removeAllColumn();
720+
tableStats.clearIndexesRowCount();
721+
removeTableStats(tableId);
720722
}
721723
tableStats.updatedTime = 0;
722724
tableStats.userInjected = false;

‎fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ protected void doSample() throws Exception {
102102
List<Long> tabletIds = pair.first;
103103
long totalRowCount = info.indexId == -1
104104
? tbl.getRowCount()
105-
: ((OlapTable) tbl).getRowCountForIndex(info.indexId);
105+
: ((OlapTable) tbl).getRowCountForIndex(info.indexId, false);
106106
double scaleFactor = (double) totalRowCount / (double) pair.second;
107107
// might happen if row count in fe metadata hasn't been updated yet
108108
if (Double.isInfinite(scaleFactor) || Double.isNaN(scaleFactor)) {

‎fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ public class TableStatsMeta implements Writable, GsonPostProcessable {
9595
public boolean userInjected;
9696

9797
@SerializedName("irc")
98-
public ConcurrentMap<Long, Long> indexesRowCount = new ConcurrentHashMap<>();
98+
private ConcurrentMap<Long, Long> indexesRowCount = new ConcurrentHashMap<>();
9999

100100
@VisibleForTesting
101101
public TableStatsMeta() {
@@ -212,6 +212,10 @@ public long getRowCount(long indexId) {
212212
return indexesRowCount.getOrDefault(indexId, -1L);
213213
}
214214

215+
public void clearIndexesRowCount() {
216+
indexesRowCount.clear();
217+
}
218+
215219
private void clearStaleIndexRowCount(OlapTable table) {
216220
Iterator<Long> iterator = indexesRowCount.keySet().iterator();
217221
List<Long> indexIds = table.getIndexIds();

0 commit comments

Comments
 (0)