From e7204262962ad8dc72428c655c39f92ba6c9ccb2 Mon Sep 17 00:00:00 2001 From: marcus-snx Date: Wed, 18 Dec 2024 20:22:24 +0200 Subject: [PATCH] Store objects and arrays as string --- indexers/scripts/import_parquet.py | 8 +++++++- indexers/utils/clickhouse_schema.py | 9 +++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/indexers/scripts/import_parquet.py b/indexers/scripts/import_parquet.py index 9e7ba907..7808fdcf 100644 --- a/indexers/scripts/import_parquet.py +++ b/indexers/scripts/import_parquet.py @@ -41,7 +41,13 @@ def import_parquet_files(client, network_name: str, protocol_name: str): raise ValueError("Network and protocol must be provided") client = clickhouse_connect.get_client( - host="clickhouse", port=8123, username="default" + host="clickhouse", + port=8123, + username="default", + # settings={"allow_experimental_json_type": 1}, ) + db_name = f"raw_{network_name}" + client.command(f"CREATE DATABASE IF NOT EXISTS {db_name}") + import_parquet_files(client, network_name, protocol_name) diff --git a/indexers/utils/clickhouse_schema.py b/indexers/utils/clickhouse_schema.py index b1e7edd8..fee006eb 100644 --- a/indexers/utils/clickhouse_schema.py +++ b/indexers/utils/clickhouse_schema.py @@ -11,8 +11,6 @@ def to_snake(name): def map_to_clickhouse_type(sol_type): if sol_type in ["bytes32", "address", "string"]: return "String" - elif re.search(r"\(.*\)|\[\[$", sol_type): - return "JSON" elif re.match(r"uint\d+$", sol_type): bit_size = int(re.search(r"\d+", sol_type).group()) if bit_size <= 8: @@ -43,10 +41,13 @@ def map_to_clickhouse_type(sol_type): return "Int256" elif sol_type == "bool": return "Bool" + elif re.search(r"\(.*\)|\[\[$", sol_type): + return "String" elif sol_type.endswith("[]"): base_type = sol_type[:-2] - clickhouse_type = f"Array({map_to_clickhouse_type(base_type)})" - return clickhouse_type + # clickhouse_type = f"Array({map_to_clickhouse_type(base_type)})" + # return clickhouse_type + return "String" raise ValueError(f"Type {sol_type} not mapped")