Skip to content

Commit

Permalink
feat(draft): map data type (#3696)
Browse files Browse the repository at this point in the history
- deps: zetasql -> 0.3.3
- parser: map data type
- parser: create table with array/map data type
- parser & node: extract array/map value with []operator
- udf: map functions
- udf: map_keys function
- codegen: construct map data type
- codegen: access map value by key with []operator
- type system: NULL and VOID type
- refactor & IWYU

TODO
- []operator support for array type
- codegen: use vector type & SIMD instructions instead of array ?
  • Loading branch information
aceforeverd authored Jan 24, 2024
1 parent 3b473ab commit b2f85fb
Show file tree
Hide file tree
Showing 66 changed files with 1,605 additions and 534 deletions.
2 changes: 1 addition & 1 deletion cases/function/join/test_lastjoin_simple.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1067,4 +1067,4 @@ cases:
rows:
- [ "aa",2,131,1590738990000 ]
- [ "bb",21,NULL,NULL ]
- [ "dd", 41, NULL, NULL ]
- [ "dd", 41, NULL, NULL ]
2 changes: 0 additions & 2 deletions cases/plan/back_quote_identifier.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -131,12 +131,10 @@ cases:
| | +-node[kColumnDesc]
| | +-column_name: a-1
| | +-column_type: int32
| | +-NOT NULL: 0
| +-1:
| | +-node[kColumnDesc]
| | +-column_name: b-1
| | +-column_type: string
| | +-NOT NULL: 0
| +-2:
| +-node[kColumnIndex]
| +-keys: [a-1, b-1]
Expand Down
33 changes: 33 additions & 0 deletions cases/plan/const_query.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -39,3 +39,36 @@ cases:
mode: request-unsupport
sql: |
select int(NULL) as c1, bigint(NULL) as c2, float(NULL) as c3, double(NULL) as c4, timestamp(NULL) as c5, date(NULL) as c6, string(NULL) as c7;
- id: map_data_type
mode: request-unsupport
desc: access map value with []operator
sql: |
select map(1, 2)[1]
expect:
node_tree_str: |
+-node[kQuery]: kQuerySelect
+-distinct_opt: false
+-where_expr: null
+-group_expr_list: null
+-having_expr: null
+-order_expr_list: null
+-limit: null
+-select_list[list]:
| +-0:
| +-node[kResTarget]
| +-val:
| | map(1, 2)[1]
| +-name: <nil>
+-tableref_list: []
+-window_list: []
plan_tree_str: |
+-[kQueryPlan]
+-[kProjectPlan]
+-table: <nil>
+-project_list_vec[list]:
+-[kProjectList]
+-projects on table [list]:
+-[kProjectNode]
+-[0]map(1, 2)[1]: map(1, 2)[1]
null
113 changes: 56 additions & 57 deletions cases/plan/create.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -163,12 +163,10 @@ cases:
| | +-node[kColumnDesc]
| | +-column_name: a
| | +-column_type: int32
| | +-NOT NULL: 0
| +-1:
| | +-node[kColumnDesc]
| | +-column_name: b
| | +-column_type: string
| | +-NOT NULL: 0
| +-2:
| +-node[kColumnIndex]
| +-keys: [a, b]
Expand Down Expand Up @@ -218,12 +216,10 @@ cases:
| | +-node[kColumnDesc]
| | +-column_name: a
| | +-column_type: int16
| | +-NOT NULL: 0
| +-1:
| | +-node[kColumnDesc]
| | +-column_name: b
| | +-column_type: float
| | +-NOT NULL: 0
| +-2:
| +-node[kColumnIndex]
| +-keys: [a]
Expand Down Expand Up @@ -274,12 +270,10 @@ cases:
| | +-node[kColumnDesc]
| | +-column_name: a
| | +-column_type: int32
| | +-NOT NULL: 0
| +-1:
| | +-node[kColumnDesc]
| | +-column_name: b
| | +-column_type: timestamp
| | +-NOT NULL: 0
| +-2:
| +-node[kColumnIndex]
| +-keys: [a]
Expand Down Expand Up @@ -627,12 +621,10 @@ cases:
| | +-node[kColumnDesc]
| | +-column_name: a
| | +-column_type: int32
| | +-NOT NULL: 0
| +-1:
| | +-node[kColumnDesc]
| | +-column_name: b
| | +-column_type: timestamp
| | +-NOT NULL: 0
| +-2:
| +-node[kColumnIndex]
| +-keys: [a]
Expand Down Expand Up @@ -685,33 +677,27 @@ cases:
| +-0:
| | +-node[kColumnDesc]
| | +-column_name: column1
| | +-column_type: int32
| | +-NOT NULL: 1
| | +-column_type: int32 NOT NULL
| +-1:
| | +-node[kColumnDesc]
| | +-column_name: column2
| | +-column_type: int16
| | +-NOT NULL: 1
| | +-column_type: int16 NOT NULL
| +-2:
| | +-node[kColumnDesc]
| | +-column_name: column5
| | +-column_type: string
| | +-NOT NULL: 1
| | +-column_type: string NOT NULL
| +-3:
| | +-node[kColumnDesc]
| | +-column_name: column6
| | +-column_type: string
| | +-NOT NULL: 1
| | +-column_type: string NOT NULL
| +-4:
| | +-node[kColumnDesc]
| | +-column_name: std_ts
| | +-column_type: timestamp
| | +-NOT NULL: 1
| | +-column_type: timestamp NOT NULL
| +-5:
| | +-node[kColumnDesc]
| | +-column_name: std_date
| | +-column_type: date
| | +-NOT NULL: 1
| | +-column_type: date NOT NULL
| +-6:
| +-node[kColumnIndex]
| +-keys: [column2]
Expand Down Expand Up @@ -743,33 +729,27 @@ cases:
| +-0:
| | +-node[kColumnDesc]
| | +-column_name: column1
| | +-column_type: int32
| | +-NOT NULL: 1
| | +-column_type: int32 NOT NULL
| +-1:
| | +-node[kColumnDesc]
| | +-column_name: column2
| | +-column_type: int16
| | +-NOT NULL: 1
| | +-column_type: int16 NOT NULL
| +-2:
| | +-node[kColumnDesc]
| | +-column_name: column5
| | +-column_type: string
| | +-NOT NULL: 1
| | +-column_type: string NOT NULL
| +-3:
| | +-node[kColumnDesc]
| | +-column_name: column6
| | +-column_type: string
| | +-NOT NULL: 1
| | +-column_type: string NOT NULL
| +-4:
| | +-node[kColumnDesc]
| | +-column_name: std_ts
| | +-column_type: timestamp
| | +-NOT NULL: 1
| | +-column_type: timestamp NOT NULL
| +-5:
| | +-node[kColumnDesc]
| | +-column_name: std_date
| | +-column_type: date
| | +-NOT NULL: 1
| | +-column_type: date NOT NULL
| +-6:
| +-node[kColumnIndex]
| +-keys: [column2]
Expand All @@ -796,17 +776,11 @@ cases:
| +-0:
| | +-node[kColumnDesc]
| | +-column_name: column1
| | +-column_type: int32
| | +-NOT NULL: 0
| | +-default_value:
| | +-expr[primary]
| | +-value: 1
| | +-type: int32
| | +-column_type: int32 DEFAULT 1
| +-1:
| +-node[kColumnDesc]
| +-column_name: column2
| +-column_type: int32
| +-NOT NULL: 0
+-table_option_list: []
- id: 27
desc: Column default value with explicit type
Expand All @@ -824,20 +798,11 @@ cases:
| +-0:
| | +-node[kColumnDesc]
| | +-column_name: column1
| | +-column_type: string
| | +-NOT NULL: 0
| | +-default_value:
| | +-expr[cast]
| | +-cast_type: string
| | +-expr:
| | +-expr[primary]
| | +-value: 1
| | +-type: int32
| | +-column_type: string DEFAULT string(1)
| +-1:
| +-node[kColumnDesc]
| +-column_name: column3
| +-column_type: int32
| +-NOT NULL: 0
+-table_option_list: []
- id: 28
desc: Create table with database.table
Expand All @@ -856,12 +821,10 @@ cases:
| | +-node[kColumnDesc]
| | +-column_name: column1
| | +-column_type: string
| | +-NOT NULL: 0
| +-1:
| +-node[kColumnDesc]
| +-column_name: column3
| +-column_type: int32
| +-NOT NULL: 0
+-table_option_list: []
- id: 29
desc: create index with db name prefix
Expand Down Expand Up @@ -898,12 +861,10 @@ cases:
| | +-node[kColumnDesc]
| | +-column_name: column1
| | +-column_type: int32
| | +-NOT NULL: 0
| +-1:
| | +-node[kColumnDesc]
| | +-column_name: column2
| | +-column_type: timestamp
| | +-NOT NULL: 0
| +-2:
| +-node[kColumnIndex]
| +-keys: [column1]
Expand Down Expand Up @@ -934,12 +895,10 @@ cases:
| | +-node[kColumnDesc]
| | +-column_name: a
| | +-column_type: int32
| | +-NOT NULL: 0
| +-1:
| | +-node[kColumnDesc]
| | +-column_name: b
| | +-column_type: timestamp
| | +-NOT NULL: 0
| +-2:
| +-node[kColumnIndex]
| +-keys: [a]
Expand Down Expand Up @@ -1049,12 +1008,10 @@ cases:
| | +-node[kColumnDesc]
| | +-column_name: column1
| | +-column_type: int32
| | +-NOT NULL: 0
| +-1:
| | +-node[kColumnDesc]
| | +-column_name: column2
| | +-column_type: timestamp
| | +-NOT NULL: 0
| +-2:
| +-node[kColumnIndex]
| +-keys: [column1]
Expand All @@ -1068,3 +1025,45 @@ cases:
+-0:
+-node[kCompressType]
+-compress_type: snappy
- id: 35
desc: Create table with array & map type
sql: |
create table t1 (id int,
member ARRAY <STRING> NOT NULL,
attrs MAP <STRING, INT NOT NULL > NOT NULL);
expect:
node_tree_str: |
+-node[CREATE]
+-table: t1
+-IF NOT EXIST: 0
+-column_desc_list[list]:
| +-0:
| | +-node[kColumnDesc]
| | +-column_name: id
| | +-column_type: int32
| +-1:
| | +-node[kColumnDesc]
| | +-column_name: member
| | +-column_type: array<string> NOT NULL
| +-2:
| +-node[kColumnDesc]
| +-column_name: attrs
| +-column_type: map<string, int32 NOT NULL> NOT NULL
+-table_option_list: []
plan_tree_str: |
+-[kCreatePlan]
+-table: t1
+-column_desc_list[list]:
| +-0:
| | +-node[kColumnDesc]
| | +-column_name: id
| | +-column_type: int32
| +-1:
| | +-node[kColumnDesc]
| | +-column_name: member
| | +-column_type: array<string> NOT NULL
| +-2:
| +-node[kColumnDesc]
| +-column_name: attrs
| +-column_type: map<string, int32 NOT NULL> NOT NULL
+-table_option_list: []
1 change: 1 addition & 0 deletions cases/plan/simple_query.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -644,3 +644,4 @@ cases:
+-[kTablePlan]
+-table: t
+-alias: t1
34 changes: 34 additions & 0 deletions cases/query/udf_query.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -554,3 +554,37 @@ cases:
- c1 bool
data: |
true, false
# ================================================================
# Map data type
# ================================================================
- id: 13
mode: request-unsupport
sql: |
select
map(1, "2")[1] as e1,
map("abc", 100)["abc"] as e2,
map(1, "2", 3, "4")[5] as e3,
map("c", 99, "d", 101)["d"] as e4,
map(date("2012-12-12"), "e", date("2013-11-11"), "f", date("2014-10-10"), "g")[date("2013-11-11")] as e5,
map(timestamp(88), timestamp(1000), timestamp(99), timestamp(2000)) [timestamp(99)] as e6,
map('1', 2, '3', 4, '5', 6, '7', 8, '9', 10, '11', 12)['9'] as e7,
map('1', 2, '3', 4, '5', 6, '7', 8, '9', 10, '11', 12)['10'] as e8,
# first match on duplicate keys
map('1', 2, '1', 4, '1', 6, '7', 8, '9', 10, '11', 12)['1'] as e9,
map("c", 99, "d", NULL)["d"] as e10,
expect:
columns: ["e1 string", "e2 int", "e3 string", "e4 int", "e5 string", "e6 timestamp", "e7 int", "e8 int", "e9 int", "e10 int"]
data: |
2, 100, NULL, 101, f, 2000, 10, NULL, 2, NULL
- id: 14
mode: request-unsupport
sql: |
select
array_contains(map_keys(map(1, '2', 3, '4')), 1) as e1,
array_contains(map_keys(map('1', 2, '3', 4)), '2') as e2,
array_contains(map_keys(map(timestamp(88), timestamp(1000), timestamp(99), timestamp(2000))) , timestamp(99)) as e3,
expect:
columns: ["e1 bool", "e2 bool", "e3 bool"]
data: |
true, false, true
5 changes: 3 additions & 2 deletions hybridse/include/base/fe_status.h
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,12 @@

#ifndef HYBRIDSE_INCLUDE_BASE_FE_STATUS_H_
#define HYBRIDSE_INCLUDE_BASE_FE_STATUS_H_

#include <sstream>
#include <string>
#include <vector>
#include "glog/logging.h"

#include "proto/fe_common.pb.h"
#include "proto/fe_type.pb.h"

namespace hybridse {
namespace base {
Expand Down
1 change: 0 additions & 1 deletion hybridse/include/node/expr_node.h
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
#define HYBRIDSE_INCLUDE_NODE_EXPR_NODE_H_

#include <string>
#include <vector>

#include "base/fe_status.h"
#include "codec/fe_row_codec.h"
Expand Down
Loading

0 comments on commit b2f85fb

Please sign in to comment.