Skip to content

Commit

Permalink
Add ClickHouse dbt adapter and update models
Browse files Browse the repository at this point in the history
  • Loading branch information
marcus-snx committed Oct 3, 2024
1 parent b76c4c9 commit 8971c82
Show file tree
Hide file tree
Showing 29 changed files with 273 additions and 235 deletions.
1 change: 1 addition & 0 deletions transformers/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,5 @@ pandas
pyarrow
psycopg2-binary
dbt-postgres
dbt-clickhouse
sqlfluff
7 changes: 7 additions & 0 deletions transformers/synthetix/macros/convert_case.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{% macro convert_case(field) %}
{% set re = modules.re %}
{% set field_snake = re.sub('(?<!^)(?=[A-Z])', '_', field) %}
{% set field_lower = field_snake | lower() %}
{{ field_lower }}
{% endmacro %}

8 changes: 2 additions & 6 deletions transformers/synthetix/macros/convert_hex.sql
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,9 @@
) %}
LEFT(
REGEXP_REPLACE(
encode(
DECODE(REPLACE({{ hex_column }}, '0x', ''), 'hex'),
'escape'
) :: text,
UNHEX(REPLACE({{ hex_column }}, '0x', '')),
'\\000',
'',
'g'
''
),
{{ max_length }}
)
Expand Down
2 changes: 1 addition & 1 deletion transformers/synthetix/macros/convert_wei.sql
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
{% macro convert_wei(column_name) %}
{{ column_name }} / 1e18
cast({{ column_name }} as UInt256) / 1e18
{% endmacro %}
16 changes: 11 additions & 5 deletions transformers/synthetix/macros/get_event_data.sql
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,17 @@
contract_name,
event_name
) %}
SELECT
*
FROM
{{ source(

{%- set relation = source(
'raw_' ~ chain ~ '_' ~ network,
contract_name ~ '_event_' ~ event_name
) }}
) -%}
{%- set columns = dbt_utils.get_filtered_columns_in_relation(relation) -%}

SELECT
{% for col in columns %}
{{ col }} as {{ convert_case(col) }}{% if not loop.last %},{% endif %}
{% endfor %}
FROM
{{ relation }}
{% endmacro %}
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,14 @@ with delegation_changes as (
pool_id,
collateral_type,
{{ convert_wei('amount') }}
- LAG({{ convert_wei('amount') }}, 1, 0) over (
- lagInFrame({{ convert_wei('amount') }}, 1, 0) over (
partition by
account_id,
pool_id,
collateral_type
order by
block_timestamp
rows between unbounded preceding and unbounded following
) as change_in_amount
from
{{ ref('core_delegation_updated_arbitrum_sepolia') }}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ avg_returns as (
collateral_type
order by
ts
range between interval '24 HOURS' preceding
range between 86400 preceding
and current row
) as avg_24h_pnl_pct,
AVG(
Expand All @@ -66,7 +66,7 @@ avg_returns as (
collateral_type
order by
ts
range between interval '7 DAYS' preceding
range between 86400 * 7 preceding
and current row
) as avg_7d_pnl_pct,
AVG(
Expand All @@ -77,7 +77,7 @@ avg_returns as (
collateral_type
order by
ts
range between interval '28 DAYS' preceding
range between 86400 * 28 preceding
and current row
) as avg_28d_pnl_pct,
AVG(
Expand All @@ -88,7 +88,7 @@ avg_returns as (
collateral_type
order by
ts
range between interval '24 HOURS' preceding
range between 86400 preceding
and current row
) as avg_24h_rewards_pct,
AVG(
Expand All @@ -99,7 +99,7 @@ avg_returns as (
collateral_type
order by
ts
range between interval '7 DAYS' preceding
range between 86400 * 7 preceding
and current row
) as avg_7d_rewards_pct,
AVG(
Expand All @@ -110,7 +110,7 @@ avg_returns as (
collateral_type
order by
ts
range between interval '28 DAYS' preceding
range between 86400 * 28 preceding
and current row
) as avg_28d_rewards_pct,
AVG(
Expand All @@ -121,7 +121,7 @@ avg_returns as (
collateral_type
order by
ts
range between interval '24 HOURS' preceding
range between 86400 preceding
and current row
) as avg_24h_total_pct,
AVG(
Expand All @@ -132,7 +132,7 @@ avg_returns as (
collateral_type
order by
ts
range between interval '7 DAYS' preceding
range between 86400 * 7 preceding
and current row
) as avg_7d_total_pct,
AVG(
Expand All @@ -143,7 +143,7 @@ avg_returns as (
collateral_type
order by
ts
range between interval '28 DAYS' preceding
range between 86400 * 28 preceding
and current row
) as avg_28d_total_pct
from
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ avg_returns as (
reward_token
order by
ts
range between interval '24 HOURS' preceding
range between 86400 preceding
and current row
) as avg_24h_rewards_pct,
AVG(
Expand All @@ -44,7 +44,7 @@ avg_returns as (
reward_token
order by
ts
range between interval '7 DAYS' preceding
range between 86400 * 7 preceding
and current row
) as avg_7d_rewards_pct,
AVG(
Expand All @@ -56,7 +56,7 @@ avg_returns as (
reward_token
order by
ts
range between interval '28 DAYS' preceding
range between 86400 * 28 preceding
and current row
) as avg_28d_rewards_pct
from
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,14 @@ with delegation_changes as (
pool_id,
collateral_type,
{{ convert_wei('amount') }}
- LAG({{ convert_wei('amount') }}, 1, 0) over (
- lagInFrame({{ convert_wei('amount') }}, 1, 0) over (
partition by
account_id,
pool_id,
collateral_type
order by
block_timestamp
rows between unbounded preceding and unbounded following
) as change_in_amount
from
{{ ref('core_delegation_updated_arbitrum_sepolia') }}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,15 @@ with dim as (
select
m.pool_id,
m.collateral_type,
generate_series(
date_trunc('hour', min(t.ts)),
date_trunc('hour', max(t.ts)),
'1 hour'::interval
arrayJoin(
arrayMap(
x -> toDateTime(x),
range(
toUInt32(date_trunc('hour', min(t.ts))),
toUInt32(date_trunc('hour', max(t.ts))),
3600
)
)
) as ts
from
(
Expand Down Expand Up @@ -86,17 +91,14 @@ issuance as (
date_trunc(
'hour',
ts
) as ts,
) as iss_ts,
pool_id,
collateral_type,
sum(amount) as hourly_issuance
from
filt_issuance
group by
date_trunc(
'hour',
ts
),
iss_ts,
pool_id,
collateral_type
)
Expand All @@ -119,4 +121,4 @@ left join issuance as i
) = lower(
i.collateral_type
)
and dim.ts = i.ts
and dim.ts = i.iss_ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,15 @@ with dim as (
select
p.pool_id,
p.collateral_type,
generate_series(
date_trunc('hour', min(t.ts)),
date_trunc('hour', max(t.ts)),
'1 hour'::INTERVAL
arrayJoin(
arrayMap(
x -> toDateTime(x),
range(
toUInt32(date_trunc('hour', min(t.ts))),
toUInt32(date_trunc('hour', max(t.ts))),
3600
)
)
) as ts
from
(
Expand Down Expand Up @@ -77,24 +82,24 @@ collateral as (
from
{{ ref('core_vault_collateral_arbitrum_sepolia') }}
where
pool_id = 1
pool_id = '1'
),

ffill as (
select
dim.ts,
dim.pool_id,
dim.collateral_type,
dim.ts as ts,
dim.pool_id as pool_id,
dim.collateral_type as collateral_type,
coalesce(
last(debt) over (
last_value(debt) over (
partition by dim.collateral_type, dim.pool_id
order by dim.ts
rows between unbounded preceding and current row
),
0
) as debt,
coalesce(
last(collateral_value) over (
last_value(collateral_value) over (
partition by dim.collateral_type, dim.pool_id
order by dim.ts
rows between unbounded preceding and current row
Expand Down Expand Up @@ -122,10 +127,11 @@ hourly_pnl as (
collateral_type,
collateral_value,
debt,
coalesce(lag(debt) over (
coalesce(lagInFrame(debt) over (
partition by pool_id, collateral_type
order by
ts
rows between unbounded preceding and unbounded following
) - debt, 0) as hourly_pnl
from
ffill
Expand All @@ -143,11 +149,11 @@ hourly_rewards as (

hourly_returns as (
select
pnl.ts,
pnl.pool_id,
pnl.collateral_type,
pnl.collateral_value,
pnl.debt,
pnl.ts as ts,
pnl.pool_id as pool_id,
pnl.collateral_type as collateral_type,
pnl.collateral_value as collateral_value,
pnl.debt as debt,
coalesce(
iss.hourly_issuance,
0
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,11 @@
with rewards_distributed as (
select
block_timestamp as ts,
CAST(
pool_id as INTEGER
) as pool_id,
pool_id,
collateral_type,
distributor,
{{ convert_wei('amount') }} as amount,
TO_TIMESTAMP("start") as ts_start,
FROM_UNIXTIME(CAST("start" AS INTEGER)) as ts_start,
"duration"
from
{{ ref('core_rewards_distributed_arbitrum_sepolia') }}
Expand Down
Loading

0 comments on commit 8971c82

Please sign in to comment.