Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

branch-3.0: [fix](ES catalog)Fix query long value exception with doc_value #46554 #46580

Merged
merged 1 commit into from
Jan 9, 2025
Merged

branch-3.0: [fix](ES catalog)Fix query long value exception with doc_value #46554 #46580

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions be/src/exec/es/es_scroll_parser.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -401,8 +401,7 @@ Status insert_int_value(const rapidjson::Value& col, PrimitiveType type,
};

if (pure_doc_value && col.IsArray() && !col.Empty()) {
if (col.IsNumber()) {
RETURN_ERROR_IF_COL_IS_NOT_NUMBER(col[0], type);
if (col[0].IsNumber()) {
T value = (T)(sizeof(T) < 8 ? col[0].GetInt() : col[0].GetInt64());
col_ptr->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&value)), 0);
return Status::OK();
Expand Down
1 change: 1 addition & 0 deletions docker/thirdparties/docker-compose/elasticsearch/scripts/data/data1_es6.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
"test4": "2022-08-08",
"test5": 12345,
"test6": "2022-08-08T12:10:10.151Z",
"test7": 123456789,
"c_bool": [true, false, true, true],
"c_byte": [1, -2, -3, 4],
"c_short": [128, 129, -129, -130],
Expand Down
1 change: 1 addition & 0 deletions docker/thirdparties/docker-compose/elasticsearch/scripts/data/data2_es6.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
"test4": "2022-08-08",
"test5": 2222.2,
"test6": "2022-08-08T12:10:10.151+08:00",
"test7": "123456789",
"c_bool": [true, false, true, true],
"c_byte": [1, -2, -3, 4],
"c_short": [128, 129, -129, -130],
Expand Down
1 change: 1 addition & 0 deletions docker/thirdparties/docker-compose/elasticsearch/scripts/data/data3_es6.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
"test4": "2022-08-08",
"test5": "3333.22",
"test6": "2022-08-08T12:10:10.151",
"test7": 1660104.44,
"c_bool": [true, false, true, true],
"c_byte": [1, -2, -3, 4],
"c_short": [128, 129, -129, -130],
Expand Down
1 change: 1 addition & 0 deletions docker/thirdparties/docker-compose/elasticsearch/scripts/data/data4_es6.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
"test4": "2022-08-08",
"test5": 3333.22,
"test6": "2022-08-08T12:10:10.151",
"test7": "1660104.44",
"c_bool": [true, false, true, true],
"c_byte": [1, -2, -3, 4],
"c_short": [128, 129, -129, -130],
Expand Down
3 changes: 3 additions & 0 deletions docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_test1.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@
"test6": {
"type": "date"
},
"test7": {
"type": "long"
},
"c_bool": {
"type": "boolean"
},
Expand Down
3 changes: 3 additions & 0 deletions docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_test2.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@
"test6": {
"type": "date"
},
"test7": {
"type": "long"
},
"c_bool": {
"type": "boolean"
},
Expand Down
312 changes: 232 additions & 80 deletions regression-test/data/external_table_p0/es/test_es_query.out
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion regression-test/data/external_table_p0/es/test_es_query_no_http_url.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
["2020-01-01", "2020-01-02"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770]

-- !sql61 --
[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N string1 text#1 3.14 2022-08-08T00:00 12345 2022-08-08T20:10:10
[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N string1 text#1 3.14 2022-08-08T00:00 12345 2022-08-08T20:10:10 123456789

-- !sql71 --
[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345
Expand Down
13 changes: 13 additions & 0 deletions regression-test/suites/external_table_p0/es/test_es_query.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
`test6` datetime NULL,
`test7` datetime NULL,
`test8` datetime NULL,
`test9` bigint NULL,
`c_byte` array<tinyint(4)> NULL,
`c_bool` array<boolean> NULL,
`c_integer` array<int(11)> NULL,
Expand Down Expand Up @@ -149,6 +150,7 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
`test6` datetimev2 NULL,
`test7` datetimev2 NULL,
`test8` datetimev2 NULL,
`test9` bigint NULL,
`c_byte` array<tinyint(4)> NULL,
`c_bool` array<boolean> NULL,
`c_integer` array<int(11)> NULL,
Expand Down Expand Up @@ -178,13 +180,15 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
order_qt_sql09 """select test1 from test_v1;"""
order_qt_sql10 """select test2 from test_v1;"""
order_qt_sql11 """select test6 from test_v1;"""
order_qt_sql12 """select test9 from test_v1;"""

order_qt_sql20 """select * from test_v2 where test2='text#1'"""
order_qt_sql21 """select * from test_v2 where esquery(test2, '{"match":{"test2":"text#1"}}')"""
order_qt_sql22 """select test4,test5,test6,test7,test8 from test_v2 order by test8"""
order_qt_sql23 """select * from test_v2 where esquery(c_long, '{"term":{"c_long":"-1"}}');"""
order_qt_sql24 """select c_person, c_user, json_extract(c_person, '\$.[0].name'), json_extract(c_user, '\$.[1].last') from test_v2;"""
order_qt_sql25 """select test6 from test_v2;"""
order_qt_sql26 """select test9 from test_v2;"""

sql """switch test_es_query_es5"""
order_qt_sql_5_02 """select * from test1 where test2='text#1'"""
Expand Down Expand Up @@ -216,6 +220,8 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
order_qt_sql_5_28 """select * from test3_20231005;"""
order_qt_sql_5_29 """select test1, test2 from test1 where test1 like 'string%';"""
order_qt_sql_5_30 """select test1, test2 from test1 where test2 like 'text%';"""
order_qt_sql_5_31 """select test7 from test1;"""
order_qt_sql_5_32 """select test7 from test2;"""

sql """switch test_es_query_es6"""
// order_qt_sql_6_01 """show tables"""
Expand Down Expand Up @@ -248,6 +254,9 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
order_qt_sql_6_28 """select * from test3_20231005;"""
order_qt_sql_6_29 """select test1, test2 from test1 where test1 like 'string%';"""
order_qt_sql_6_30 """select test1, test2 from test1 where test2 like 'text%';"""
order_qt_sql_6_31 """select test7 from test1;"""
order_qt_sql_6_32 """select test7 from test2;"""


List<List<String>> tables6N = sql """show tables"""
boolean notContainHide = true
Expand Down Expand Up @@ -305,6 +314,8 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
order_qt_sql_7_34 """select * from composite_type_array order by name;"""
order_qt_sql_7_35 """select test1, test2 from test1 where test1 like 'string%';"""
order_qt_sql_7_36 """select test1, test2 from test1 where test2 like 'text%';"""
order_qt_sql_7_37 """select test9 from test1;"""
order_qt_sql_7_38 """select test9 from test2;"""

List<List<String>> tables7N = sql """show tables"""
boolean notContainHide7 = true
Expand Down Expand Up @@ -362,6 +373,8 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
order_qt_sql_8_32 """select * from composite_type_array order by name;"""
order_qt_sql_8_33 """select test1, test2 from test1 where test1 like 'string%';"""
order_qt_sql_8_34 """select test1, test2 from test1 where test2 like 'text%';"""
order_qt_sql_8_35 """select test9 from test1;"""
order_qt_sql_8_36 """select test9 from test2;"""

}

Expand Down
Loading