Skip to content

[fix](ES Catalog)Do not extract doc_values of field with ignore_above setting #40314

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Sep 5, 2024
Merged

[fix](ES Catalog)Do not extract doc_values of field with ignore_above setting #40314

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b833618
[fix](ES Catalog)Do not extract doc_values for field with ignore_abov…
qidaye Sep 3, 2024
7c1d3c7
fix
qidaye Sep 3, 2024
2d5e4b3
fix comment
qidaye Sep 4, 2024
6c3d999
add comments
qidaye Sep 4, 2024
f42f830
fix
qidaye Sep 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions docker/thirdparties/docker-compose/elasticsearch/scripts/data/data4_es6.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
{
"test1": "string_ignore_above_10",
"test2": "text_ignore_above_10",
"test3": 5.0,
"test4": "2022-08-08",
"test5": 3333.22,
"test6": "2022-08-08T12:10:10.151",
"c_bool": [true, false, true, true],
"c_byte": [1, -2, -3, 4],
"c_short": [128, 129, -129, -130],
"c_integer": [32768, 32769, -32769, -32770],
"c_long": [-1, 0, 1, 2],
"c_unsigned_long": [0, 1, 2, 3],
"c_float": [1.0, 1.1, 1.2, 1.3],
"c_half_float": [1, 2, 3, 4],
"c_double": [1, 2, 3, 4],
"c_scaled_float": [1, 2, 3, 4],
"c_date": ["2020-01-01", "2020-01-02"],
"c_datetime": ["2020-01-01 12:00:00", "2020-01-02 13:01:01"],
"c_keyword": ["a", "b", "c"],
"c_text": ["d", "e", "f"],
"c_ip": ["192.168.0.1", "127.0.0.1"],
"c_person": [
{"name": "Andy", "age": 18},
{"name": "Tim", "age": 28}
],
"message": "I'm not null or empty",
"c_user": [
{"first": "John", "last": "Smith"},
{"first": "Alice", "last": "White"}
]
}
35 changes: 33 additions & 2 deletions docker/thirdparties/docker-compose/elasticsearch/scripts/data/data5.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,35 @@
{
"field1": "value1",
"field2": "value2"
"test1": "string_ignore_above_10",
"test2": "text_ignore_above_10",
"test3": 6.0,
"test4": "2022-08-08",
"test5": "2022-08-11 12:10:10",
"test6": 1660191010000,
"test7": "2022-08-11 12:10:10",
"test8": "2022-08-11T12:10:10+09:00",
"test9": "4444.22",
"test10": "2022-08-08T12:10:10.151",
"c_bool": [true, false, true, true],
"c_byte": [1, -2, -3, 4],
"c_short": [128, 129, -129, -130],
"c_integer": [32768, 32769, -32769, -32770],
"c_long": [-1, 0, 1, 2],
"c_unsigned_long": [0, 1, 2, 3],
"c_float": [1.0, 1.1, 1.2, 1.3],
"c_half_float": [1, 2, 3, 4],
"c_double": [1, 2, 3, 4],
"c_scaled_float": [1, 2, 3, 4],
"c_date": ["2020-01-01", "2020-01-02"],
"c_datetime": ["2020-01-01 12:00:00", "2020-01-02 13:01:01"],
"c_keyword": ["a", "b", "c"],
"c_text": ["d", "e", "f"],
"c_ip": ["192.168.0.1", "127.0.0.1"],
"c_person": [
{"name": "Andy", "age": 18},
{"name": "Tim", "age": 28}
],
"c_user": [
{"first": "John", "last": "Smith"},
{"first": "Alice", "last": "White"}
]
}
4 changes: 4 additions & 0 deletions docker/thirdparties/docker-compose/elasticsearch/scripts/data/data6.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"field1": "value1",
"field2": "value2"
}
8 changes: 6 additions & 2 deletions docker/thirdparties/docker-compose/elasticsearch/scripts/es_init.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ curl "http://${ES_5_HOST}:9200/test1/doc/1" -H "Content-Type:application/json" -
curl "http://${ES_5_HOST}:9200/test1/doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2_es6.json'
# only difference between es5 and es6
curl "http://${ES_5_HOST}:9200/test1/doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3_es5.json'
curl "http://${ES_5_HOST}:9200/test1/doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4_es6.json'
# put data for test2_20220808
curl "http://${ES_5_HOST}:9200/test2_20220808/doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1_es6.json'
curl "http://${ES_5_HOST}:9200/test2_20220808/doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2_es6.json'
Expand Down Expand Up @@ -88,6 +89,7 @@ curl "http://${ES_6_HOST}:9200/test2_20220809" -H "Content-Type:application/json
curl "http://${ES_6_HOST}:9200/test1/doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1_es6.json'
curl "http://${ES_6_HOST}:9200/test1/doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2_es6.json'
curl "http://${ES_6_HOST}:9200/test1/doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3_es6.json'
curl "http://${ES_6_HOST}:9200/test1/doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4_es6.json'
# put data for test2_20220808
curl "http://${ES_6_HOST}:9200/test2_20220808/doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1_es6.json'
curl "http://${ES_6_HOST}:9200/test2_20220808/doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2_es6.json'
Expand Down Expand Up @@ -125,6 +127,7 @@ curl "http://${ES_7_HOST}:9200/test1/_doc/1" -H "Content-Type:application/json"
curl "http://${ES_7_HOST}:9200/test1/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json'
curl "http://${ES_7_HOST}:9200/test1/_doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json'
curl "http://${ES_7_HOST}:9200/test1/_doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json'
curl "http://${ES_7_HOST}:9200/test1/_doc/5" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data5.json'
# put data for test2_20220808
curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1.json'
curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json'
Expand All @@ -136,7 +139,7 @@ curl "http://${ES_7_HOST}:9200/test2_20220809/_doc/2" -H "Content-Type:applicati
curl "http://${ES_7_HOST}:9200/test2_20220809/_doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json'
curl "http://${ES_7_HOST}:9200/test2_20220809/_doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json'
# put data for test3_20231005
curl "http://${ES_7_HOST}:9200/test3_20231005/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data5.json'
curl "http://${ES_7_HOST}:9200/test3_20231005/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data6.json'

# put _meta for array
curl "http://${ES_7_HOST}:9200/test1/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json"
Expand Down Expand Up @@ -170,6 +173,7 @@ curl "http://${ES_8_HOST}:9200/test1/_doc/1" -H "Content-Type:application/json"
curl "http://${ES_8_HOST}:9200/test1/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json'
curl "http://${ES_8_HOST}:9200/test1/_doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json'
curl "http://${ES_8_HOST}:9200/test1/_doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json'
curl "http://${ES_8_HOST}:9200/test1/_doc/5" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data5.json'
# put data for test2_20220808
curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1.json'
curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json'
Expand All @@ -181,7 +185,7 @@ curl "http://${ES_8_HOST}:9200/test2_20220809/_doc/2" -H "Content-Type:applicati
curl "http://${ES_8_HOST}:9200/test2_20220809/_doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json'
curl "http://${ES_8_HOST}:9200/test2_20220809/_doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json'
# put data for test3_20231005
curl "http://${ES_8_HOST}:9200/test3_20231005/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data5.json'
curl "http://${ES_8_HOST}:9200/test3_20231005/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data6.json'

# put _meta for array
curl "http://${ES_8_HOST}:9200/test1/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json"
Expand Down
5 changes: 3 additions & 2 deletions docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_test1.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,15 @@
"doc": {
"properties": {
"test1": {
"type": "keyword"
"type": "keyword",
"ignore_above": 10
},
"test2": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
"ignore_above": 10
}
}
},
Expand Down
5 changes: 3 additions & 2 deletions docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test1.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,15 @@
"mappings": {
"properties": {
"test1": {
"type": "keyword"
"type": "keyword",
"ignore_above": 10
},
"test2": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
"ignore_above": 10
}
}
},
Expand Down
6 changes: 4 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@

import java.io.DataInput;
import java.io.IOException;
import java.util.Collections;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
Expand All @@ -52,7 +52,9 @@
@Getter
@Setter
public class EsTable extends Table implements GsonPostProcessable {
public static final Set<String> DEFAULT_DOCVALUE_DISABLED_FIELDS = new HashSet<>(Collections.singletonList("text"));
// refrence: https://www.elastic.co/guide/en/elasticsearch/reference/current/doc-values.html
public static final Set<String> DEFAULT_DOCVALUE_DISABLED_FIELDS =
new HashSet<>(Arrays.asList("text", "annotated_text"));

private static final Logger LOG = LogManager.getLogger(EsTable.class);
// Solr doc_values vs stored_fields performance-smackdown indicate:
Expand Down
9 changes: 9 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/es/MappingPhase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,10 @@ private static void resolveDocValuesFields(SearchContext searchContext, ObjectNo
if (docValue) {
docValueField = colName;
}
} else if (innerTypeObject.has("ignore_above")) {
// this field has `ignore_above` param
// Strings longer than the ignore_above setting will not be indexed or stored
// so we cannot rely on its doc_values
} else {
// a : {c : {}} -> a -> a.c
docValueField = colName + "." + fieldName;
Expand All @@ -146,6 +150,11 @@ private static void resolveDocValuesFields(SearchContext searchContext, ObjectNo
} else if (fieldType == null || "nested".equals(fieldType)) {
// The object field has no type, and nested not support doc value.
return;
} else if (fieldObject.has("ignore_above")) {
// this field has `ignore_above` param
// Strings longer than the ignore_above setting will not be indexed or stored
// so we cannot rely on its doc_values
return;
}
docValueField = colName;
}
Expand Down
Loading
Loading