From 6ad10f7ebbee2060902a3ec6925eaa4c3afafdb8 Mon Sep 17 00:00:00 2001 From: Troy Date: Thu, 22 Aug 2024 15:28:02 -0600 Subject: [PATCH 1/4] fix models --- .../core/fct_pool_pnl_hourly_eth_mainnet.sql | 61 ++++++++++++------- .../core/core_vault_debt_base_mainnet.sql | 52 ++++++++-------- .../core_vault_collateral_eth_mainnet.sql | 9 ++- .../core/core_vault_debt_eth_mainnet.sql | 9 ++- 4 files changed, 81 insertions(+), 50 deletions(-) diff --git a/transformers/synthetix/models/marts/eth/mainnet/core/fct_pool_pnl_hourly_eth_mainnet.sql b/transformers/synthetix/models/marts/eth/mainnet/core/fct_pool_pnl_hourly_eth_mainnet.sql index 9edd0694..e9330f41 100644 --- a/transformers/synthetix/models/marts/eth/mainnet/core/fct_pool_pnl_hourly_eth_mainnet.sql +++ b/transformers/synthetix/models/marts/eth/mainnet/core/fct_pool_pnl_hourly_eth_mainnet.sql @@ -4,6 +4,7 @@ ) }} WITH dim AS ( + SELECT generate_series(DATE_TRUNC('hour', MIN(t.ts)), DATE_TRUNC('hour', MAX(t.ts)), '1 hour' :: INTERVAL) AS ts, p.pool_id, @@ -22,9 +23,9 @@ WITH dim AS ( FROM {{ ref('fct_pool_debt_eth_mainnet') }} ) AS p -GROUP BY - p.pool_id, - p.collateral_type + GROUP BY + p.pool_id, + p.collateral_type ), issuance AS ( SELECT @@ -72,22 +73,14 @@ ffill AS ( dim.ts, dim.pool_id, dim.collateral_type, - coalesce( - last(debt) over ( - partition by dim.collateral_type, dim.pool_id - order by dim.ts - rows between unbounded preceding and current row - ), - 0 - ) as debt, - coalesce( - last(collateral_value) over ( - partition by dim.collateral_type, dim.pool_id - order by dim.ts - rows between unbounded preceding and current row - ), - 0 - ) as collateral_value + COALESCE(last(debt) over (PARTITION BY dim.collateral_type, dim.pool_id + ORDER BY + dim.ts rows BETWEEN unbounded preceding + AND CURRENT ROW), 0) AS debt, + COALESCE(last(collateral_value) over (PARTITION BY dim.collateral_type, dim.pool_id + ORDER BY + dim.ts rows BETWEEN unbounded preceding + AND CURRENT ROW), 0) AS collateral_value FROM dim LEFT JOIN debt @@ -121,6 +114,15 @@ hourly_rewards AS ( FROM {{ ref('fct_pool_rewards_hourly_eth_mainnet') }} ), +hourly_migration AS ( + SELECT + ts, + pool_id, + collateral_type, + hourly_debt_migrated + FROM + {{ ref('fct_core_migration_hourly_eth_mainnet') }} +), hourly_returns AS ( SELECT pnl.ts, @@ -132,9 +134,16 @@ hourly_returns AS ( iss.hourly_issuance, 0 ) hourly_issuance, + COALESCE( + migration.hourly_debt_migrated, + 0 + ) AS hourly_debt_migrated, pnl.hourly_pnl + COALESCE( iss.hourly_issuance, 0 + ) + COALESCE( + migration.hourly_debt_migrated, + 0 ) AS hourly_pnl, COALESCE( rewards.rewards_usd, @@ -149,11 +158,11 @@ hourly_returns AS ( END AS hourly_rewards_pct, CASE WHEN pnl.collateral_value = 0 THEN 0 - ELSE (COALESCE(iss.hourly_issuance, 0) + pnl.hourly_pnl) / pnl.collateral_value + ELSE (COALESCE(iss.hourly_issuance, 0) + pnl.hourly_pnl + COALESCE(migration.hourly_debt_migrated, 0)) / pnl.collateral_value END AS hourly_pnl_pct, CASE WHEN pnl.collateral_value = 0 THEN 0 - ELSE (COALESCE(rewards.rewards_usd, 0) + pnl.hourly_pnl + COALESCE(iss.hourly_issuance, 0)) / pnl.collateral_value + ELSE (COALESCE(rewards.rewards_usd, 0) + pnl.hourly_pnl + COALESCE(iss.hourly_issuance, 0) + COALESCE(migration.hourly_debt_migrated, 0)) / pnl.collateral_value END AS hourly_total_pct FROM hourly_pnl pnl @@ -169,8 +178,15 @@ hourly_returns AS ( ) = LOWER( iss.collateral_type ) + LEFT JOIN hourly_migration migration + ON pnl.ts = migration.ts + AND pnl.pool_id = migration.pool_id + AND LOWER( + pnl.collateral_type + ) = LOWER( + migration.collateral_type + ) ) - SELECT ts, pool_id, @@ -179,6 +195,7 @@ SELECT debt, hourly_issuance, hourly_pnl, + hourly_debt_migrated, rewards_usd, hourly_pnl_pct, hourly_rewards_pct, diff --git a/transformers/synthetix/models/raw/base/mainnet/core/core_vault_debt_base_mainnet.sql b/transformers/synthetix/models/raw/base/mainnet/core/core_vault_debt_base_mainnet.sql index d80c6bd8..49b5cce6 100644 --- a/transformers/synthetix/models/raw/base/mainnet/core/core_vault_debt_base_mainnet.sql +++ b/transformers/synthetix/models/raw/base/mainnet/core/core_vault_debt_base_mainnet.sql @@ -1,36 +1,40 @@ -with base as ( - select +WITH base AS ( + SELECT block_number, contract_address, chain_id, pool_id, collateral_type, - cast( - value_1 as numeric - ) as debt - from + CAST( + value_1 AS numeric + ) AS debt + FROM {{ source( 'raw_base_mainnet', "core_get_vault_debt" ) }} - where - value_1 is not null + WHERE + value_1 IS NOT NULL ) - -select - to_timestamp(blocks.timestamp) as ts, - cast( - blocks.block_number as integer - ) as block_number, +SELECT + TO_TIMESTAMP( + blocks.timestamp + ) AS ts, + CAST( + blocks.block_number AS INTEGER + ) AS block_number, base.contract_address, - cast( - base.pool_id as integer - ) as pool_id, - cast( - base.collateral_type as varchar - ) as collateral_type, - {{ convert_wei('base.debt') }} as debt -from + CAST( + base.pool_id AS INTEGER + ) AS pool_id, + CAST( + base.collateral_type AS VARCHAR + ) AS collateral_type, + {{ convert_wei('base.debt') }} AS debt +FROM base -inner join {{ source('raw_base_mainnet', 'blocks_parquet') }} as blocks - on base.block_number = blocks.block_number + INNER JOIN {{ source( + 'raw_base_mainnet', + 'blocks_parquet' + ) }} AS blocks + ON base.block_number = blocks.block_number diff --git a/transformers/synthetix/models/raw/eth/mainnet/core/core_vault_collateral_eth_mainnet.sql b/transformers/synthetix/models/raw/eth/mainnet/core/core_vault_collateral_eth_mainnet.sql index b1379093..8a13f0f3 100644 --- a/transformers/synthetix/models/raw/eth/mainnet/core/core_vault_collateral_eth_mainnet.sql +++ b/transformers/synthetix/models/raw/eth/mainnet/core/core_vault_collateral_eth_mainnet.sql @@ -20,7 +20,9 @@ WITH base AS ( amount IS NOT NULL ) SELECT - blocks.ts, + TO_TIMESTAMP( + blocks.timestamp + ) AS ts, base.block_number, base.contract_address, CAST( @@ -33,5 +35,8 @@ SELECT {{ convert_wei('base.collateral_value') }} AS collateral_value FROM base - JOIN {{ source('raw_eth_mainnet', 'blocks_parquet') }} AS blocks + JOIN {{ source( + 'raw_eth_mainnet', + 'blocks_parquet' + ) }} AS blocks ON base.block_number = blocks.block_number diff --git a/transformers/synthetix/models/raw/eth/mainnet/core/core_vault_debt_eth_mainnet.sql b/transformers/synthetix/models/raw/eth/mainnet/core/core_vault_debt_eth_mainnet.sql index b9fd9036..1e9f39a8 100644 --- a/transformers/synthetix/models/raw/eth/mainnet/core/core_vault_debt_eth_mainnet.sql +++ b/transformers/synthetix/models/raw/eth/mainnet/core/core_vault_debt_eth_mainnet.sql @@ -17,7 +17,9 @@ WITH base AS ( value_1 IS NOT NULL ) SELECT - blocks.ts, + TO_TIMESTAMP( + blocks.timestamp + ) AS ts, base.block_number, base.contract_address, CAST( @@ -29,5 +31,8 @@ SELECT {{ convert_wei('base.debt') }} AS debt FROM base - JOIN {{ source('raw_eth_mainnet', 'blocks_parquet') }} AS blocks + JOIN {{ source( + 'raw_eth_mainnet', + 'blocks_parquet' + ) }} AS blocks ON base.block_number = blocks.block_number From 61cada821bb65b97a93f48f9a0959750309e941a Mon Sep 17 00:00:00 2001 From: Troy Date: Thu, 22 Aug 2024 15:28:19 -0600 Subject: [PATCH 2/4] update example file --- .streamlit/secrets_example.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/.streamlit/secrets_example.toml b/.streamlit/secrets_example.toml index f88def57..9f2b47e8 100644 --- a/.streamlit/secrets_example.toml +++ b/.streamlit/secrets_example.toml @@ -4,6 +4,7 @@ DB_USER = 'analytics' DB_PASS = 'analytics' DB_HOST = 'db' DB_PORT = '5432' +DB_ENV = 'prod' [settings] SHOW_TESTNETS = 'false' From 6ea5104efe3d2e5478dff71b3c422a600c7ed1fb Mon Sep 17 00:00:00 2001 From: Troy Date: Thu, 22 Aug 2024 15:28:24 -0600 Subject: [PATCH 3/4] pin web3 version --- extractors/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extractors/requirements.txt b/extractors/requirements.txt index e30e5015..6d8b7f2e 100644 --- a/extractors/requirements.txt +++ b/extractors/requirements.txt @@ -5,4 +5,5 @@ duckdb==0.10.2 polars-lts-cpu==1.1.0 pandas numpy -synthetix==0.1.13 \ No newline at end of file +synthetix==0.1.13 +web3==6.20.2 \ No newline at end of file From 584c5372771c2b3d0b337e454a7018ea57233a14 Mon Sep 17 00:00:00 2001 From: Troy Date: Thu, 22 Aug 2024 15:36:33 -0600 Subject: [PATCH 4/4] add eth mainnet to dag --- scheduler/dags/v3_etl.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scheduler/dags/v3_etl.py b/scheduler/dags/v3_etl.py index 16eb70fe..47506f25 100644 --- a/scheduler/dags/v3_etl.py +++ b/scheduler/dags/v3_etl.py @@ -8,6 +8,7 @@ # environment variables WORKING_DIR = os.getenv("WORKING_DIR") NETWORK_RPCS = { + "eth_mainnet": "NETWORK_1_RPC", "base_mainnet": "NETWORK_8453_RPC", "base_sepolia": "NETWORK_84532_RPC", "arbitrum_mainnet": "NETWORK_42161_RPC", @@ -49,7 +50,7 @@ def create_docker_operator(dag, task_id, config_file, image, command, network_en ) -def create_dag(network, rpc_var, target='dev'): +def create_dag(network, rpc_var, target="dev"): version = f"{network}_{target}" dag = DAG( @@ -78,10 +79,10 @@ def create_dag(network, rpc_var, target='dev'): config_file=None, image="data-transformer", command=f"dbt test --target {target if network != 'optimism_mainnet' else target + '-op'} --select tag:{network} --profiles-dir profiles --profile synthetix", - network_env_var=rpc_var + network_env_var=rpc_var, ) - if target == 'prod': + if target == "prod": extract_task_id = f"extract_{version}" config_file = f"configs/{network}.yaml" extract_task = create_docker_operator( @@ -101,5 +102,5 @@ def create_dag(network, rpc_var, target='dev'): for network, rpc_var in NETWORK_RPCS.items(): - for target in ['dev', 'prod']: + for target in ["dev", "prod"]: globals()[f"v3_etl_{network}_{target}"] = create_dag(network, rpc_var, target)