Skip to content

Commit 00fca3e

Browse files
authored
[fix](inverted index) disable range query in StringTypeInvertedIndexReader (#38218)
## Proposed changes disable range query in StringTypeInvertedIndexReader
1 parent 1293912 commit 00fca3e

File tree

3 files changed

+61
-2
lines changed

3 files changed

+61
-2
lines changed

be/src/olap/rowset/segment_v2/segment_iterator.cpp

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -920,7 +920,8 @@ bool SegmentIterator::_check_apply_by_inverted_index(ColumnPredicate* pred, bool
920920
if (_opts.runtime_state && !_opts.runtime_state->query_options().enable_inverted_index_query) {
921921
return false;
922922
}
923-
if (_inverted_index_iterators[pred->column_id()] == nullptr) {
923+
auto pred_column_id = pred->column_id();
924+
if (_inverted_index_iterators[pred_column_id] == nullptr) {
924925
//this column without inverted index
925926
return false;
926927
}
@@ -935,13 +936,21 @@ bool SegmentIterator::_check_apply_by_inverted_index(ColumnPredicate* pred, bool
935936
return false;
936937
}
937938

939+
// UNTOKENIZED strings exceed ignore_above, they are written as null, causing range query errors
940+
if (PredicateTypeTraits::is_range(pred->type()) &&
941+
_inverted_index_iterators[pred_column_id] != nullptr &&
942+
_inverted_index_iterators[pred_column_id]->get_inverted_index_reader_type() ==
943+
InvertedIndexReaderType::STRING_TYPE) {
944+
return false;
945+
}
946+
938947
// Function filter no apply inverted index
939948
if (dynamic_cast<LikeColumnPredicate<TYPE_CHAR>*>(pred) != nullptr ||
940949
dynamic_cast<LikeColumnPredicate<TYPE_STRING>*>(pred) != nullptr) {
941950
return false;
942951
}
943952

944-
bool handle_by_fulltext = _column_has_fulltext_index(pred->column_id());
953+
bool handle_by_fulltext = _column_has_fulltext_index(pred_column_id);
945954
if (handle_by_fulltext) {
946955
// when predicate in compound condition which except leafNode of andNode,
947956
// only can apply match query for fulltext index,

regression-test/data/inverted_index_p0/test_ignore_above_in_index.out

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,6 @@
22
-- !sql --
33
3
44

5+
-- !sql --
6+
772
7+

regression-test/suites/inverted_index_p0/test_ignore_above_in_index.groovy

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,4 +39,51 @@ suite("test_ignore_above_in_index", "p0") {
3939
sql "insert into ${tableName} values (20, '1234567890');"
4040
sql "insert into ${tableName} values (20, '1234567890');"
4141
qt_sql "select count() from ${tableName} where c = '1234567890';"
42+
43+
def tableName2 = "test_ignore_above_in_index2"
44+
sql "DROP TABLE IF EXISTS ${tableName2}"
45+
sql """
46+
CREATE TABLE ${tableName2} (
47+
`@timestamp` int(11) NULL COMMENT "",
48+
`clientip` string NULL COMMENT "",
49+
`request` string NULL COMMENT "",
50+
`status` int NULL COMMENT "",
51+
`size` int NULL COMMENT "",
52+
INDEX clientip_idx (`clientip`) USING INVERTED PROPERTIES("ignore_above"="5") COMMENT '',
53+
INDEX request_idx (`request`) USING INVERTED PROPERTIES("parser" = "unicode", "support_phrase" = "true") COMMENT '',
54+
INDEX status_idx (`status`) USING INVERTED COMMENT '',
55+
INDEX size_idx (`size`) USING INVERTED COMMENT ''
56+
) ENGINE=OLAP
57+
DUPLICATE KEY(`@timestamp`)
58+
COMMENT "OLAP"
59+
DISTRIBUTED BY HASH(`@timestamp`) BUCKETS 1
60+
PROPERTIES (
61+
"replication_allocation" = "tag.location.default: 1"
62+
);
63+
"""
64+
65+
// load the json data
66+
streamLoad {
67+
table "${tableName2}"
68+
69+
set 'read_json_by_line', 'true'
70+
set 'format', 'json'
71+
file 'documents-1000.json' // import json file
72+
time 10000 // limit inflight 10s
73+
74+
// if declared a check callback, the default check condition will ignore.
75+
// So you must check all condition
76+
check { result, exception, startTime, endTime ->
77+
if (exception != null) {
78+
throw exception
79+
}
80+
log.info("Stream load result: ${result}".toString())
81+
def json = parseJson(result)
82+
assertEquals("success", json.Status.toLowerCase())
83+
assertEquals(json.NumberTotalRows, json.NumberLoadedRows + json.NumberUnselectedRows)
84+
assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
85+
}
86+
}
87+
88+
qt_sql "select count() from ${tableName2} where clientip > '17.0';"
4289
}

0 commit comments

Comments
 (0)