Skip to content

[opt](inverted index) set support_phrase default true if parser is set #37949

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jul 18, 2024
Merged

[opt](inverted index) set support_phrase default true if parser is set #37949

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,16 @@ public Index(long indexId, String indexName, List<String> columns,
this.comment = comment;
if (indexType == IndexDef.IndexType.INVERTED) {
if (this.properties != null && !this.properties.isEmpty()) {
String key = InvertedIndexUtil.INVERTED_INDEX_PARSER_LOWERCASE_KEY;
if (!properties.containsKey(key)) {
this.properties.put(key, "true");
if (this.properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY)) {
String lowerCaseKey = InvertedIndexUtil.INVERTED_INDEX_PARSER_LOWERCASE_KEY;
if (!properties.containsKey(lowerCaseKey)) {
this.properties.put(lowerCaseKey, "true");
}
String supportPhraseKey = InvertedIndexUtil
.INVERTED_INDEX_SUPPORT_PHRASE_KEY;
if (!properties.containsKey(supportPhraseKey)) {
this.properties.put(supportPhraseKey, "true");
}
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/test/java/org/apache/doris/common/proc/IndexesProcNodeTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ public void testFetchResult() throws AnalysisException {
Assert.assertEquals(procResult.getRows().get(1).get(5), "col_2");
Assert.assertEquals(procResult.getRows().get(1).get(11), "INVERTED");
Assert.assertEquals(procResult.getRows().get(1).get(12), "inverted index on col_2");
Assert.assertEquals(procResult.getRows().get(1).get(13), "(\"parser\" = \"unicode\", \"lower_case\" = \"true\")");
Assert.assertEquals(procResult.getRows().get(1).get(13), "(\"parser\" = \"unicode\", \"lower_case\" = \"true\", \"support_phrase\" = \"true\")");

Assert.assertEquals(procResult.getRows().get(2).get(0), "tbl_test_indexes_proc");
Assert.assertEquals(procResult.getRows().get(2).get(1), "3");
Expand Down
2 changes: 1 addition & 1 deletion regression-test/suites/backup_restore/test_backup_restore_inverted_index.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ suite("test_backup_restore_inverted_index", "backup_restore") {

def restore_index_comment = sql "SHOW CREATE TABLE ${dbName}.${tableName}"

assertTrue(restore_index_comment[0][1].contains("USING INVERTED PROPERTIES(\"parser\" = \"english\", \"lower_case\" = \"true\")"))
assertTrue(restore_index_comment[0][1].contains("USING INVERTED PROPERTIES(\"parser\" = \"english\", \"lower_case\" = \"true\", \"support_phrase\" = \"true\")"))

result = sql "SELECT id, TOKENIZE(comment,'\"parser\"=\"english\"') as token FROM ${dbName}.${tableName}"
assertEquals(result.size(), 2)
Expand Down
10 changes: 5 additions & 5 deletions regression-test/suites/index_p0/test_index_meta.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ suite("index_meta", "p0") {
assertEquals(show_result[1][4], "name")
assertEquals(show_result[1][10], "INVERTED")
assertEquals(show_result[1][11], "index for name")
assertEquals(show_result[1][12], "(\"parser\" = \"none\", \"lower_case\" = \"true\")")
assertEquals(show_result[1][12], "(\"parser\" = \"none\", \"lower_case\" = \"true\", \"support_phrase\" = \"true\")")

// add index on column description
sql "create index idx_desc on ${tableName}(description) USING INVERTED PROPERTIES(\"parser\"=\"standard\") COMMENT 'index for description';"
Expand All @@ -90,12 +90,12 @@ suite("index_meta", "p0") {
assertEquals(show_result[1][4], "name")
assertEquals(show_result[1][10], "INVERTED")
assertEquals(show_result[1][11], "index for name")
assertEquals(show_result[1][12], "(\"parser\" = \"none\", \"lower_case\" = \"true\")")
assertEquals(show_result[1][12], "(\"parser\" = \"none\", \"lower_case\" = \"true\", \"support_phrase\" = \"true\")")
assertEquals(show_result[2][2], "idx_desc")
assertEquals(show_result[2][4], "description")
assertEquals(show_result[2][10], "INVERTED")
assertEquals(show_result[2][11], "index for description")
assertEquals(show_result[2][12], "(\"parser\" = \"standard\", \"lower_case\" = \"true\")")
assertEquals(show_result[2][12], "(\"parser\" = \"standard\", \"lower_case\" = \"true\", \"support_phrase\" = \"true\")")

// drop index
// add index on column description
Expand All @@ -114,7 +114,7 @@ suite("index_meta", "p0") {
assertEquals(show_result[1][4], "description")
assertEquals(show_result[1][10], "INVERTED")
assertEquals(show_result[1][11], "index for description")
assertEquals(show_result[1][12], "(\"parser\" = \"standard\", \"lower_case\" = \"true\")")
assertEquals(show_result[1][12], "(\"parser\" = \"standard\", \"lower_case\" = \"true\", \"support_phrase\" = \"true\")")

// add index on column description
sql "create index idx_name on ${tableName}(name) USING INVERTED COMMENT 'new index for name';"
Expand All @@ -133,7 +133,7 @@ suite("index_meta", "p0") {
assertEquals(show_result[1][4], "description")
assertEquals(show_result[1][10], "INVERTED")
assertEquals(show_result[1][11], "index for description")
assertEquals(show_result[1][12], "(\"parser\" = \"standard\", \"lower_case\" = \"true\")")
assertEquals(show_result[1][12], "(\"parser\" = \"standard\", \"lower_case\" = \"true\", \"support_phrase\" = \"true\")")
assertEquals(show_result[2][2], "idx_name")
assertEquals(show_result[2][4], "name")
assertEquals(show_result[2][10], "INVERTED")
Expand Down