Skip to content

Commit

Permalink
Query for all chains if no chain parameter present (#48)
Browse files Browse the repository at this point in the history 
* Query for all chains if no `chain` parameter present

Previously when not passing any `chain` parameter with the query, the
result would be the first row in the DB matching the expression. This
change now runs the query as a `UNION` between all the available chains
in the DB when no `chain` parameter is passed.

* Clean up `bun-test`
  • Loading branch information
@0237h
0237h authored Nov 3, 2023
1 parent 3636578 commit ffbffbe
Show file tree
Hide file tree
Showing 4 changed files with 61 additions and 54 deletions.
17 changes: 0 additions & 17 deletions .github/workflows/bun-test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ on:
jobs:
bun-test:
runs-on: ubuntu-latest
environment: dev-test
steps:
- name: Checkout
uses: actions/checkout@v3
Expand All @@ -21,23 +20,7 @@ jobs:
run: |
bun install
- name: 'Setup local Clickhouse DB'
uses: metrico/[email protected]

- name: 'Insert mock data into Clickhouse DB for testing'
run: |
curl https://clickhouse.com/ | sh
echo "CREATE TABLE IF NOT EXISTS block (block_id FixedString(64),block_number UInt32(),chain LowCardinality(String),timestamp DateTime64(3, 'UTC'),final_block Bool) ENGINE = ReplacingMergeTree PRIMARY KEY (block_id) ORDER BY (block_id, block_number, timestamp); INSERT INTO block (*) VALUES ('054d891671caf4978a4cfc3a13fe323d72cd805c215a14a16ea7417d5add3098', 18394250, 'mainnet', '2023-10-20 21:34:47.000', true) ,('167189cd0f77aa5320806a7150c73cbfc97dfd4418c089ecc33511fca48eb0d6', 18394255, 'testnet', '2023-10-20 21:35:47.000', true) ,('1df538c839f821a20b075dfb35ed6f266435190b99276a6651866407ecdd787e', 18394257, 'mainnet', '2023-10-20 21:36:11.000', false) ,('2b225df082189b69727ef443985edaae23c9edcb172fc3f1a37e17071cb2f827', 18394256, 'testnet', '2023-10-20 21:35:59.000', true) ,('34348c1ae22f33a404ec4ba5fbbe513d0da93172bce483f59ef749a465f50eaf', 18394249, 'mainnet', '2023-10-20 21:34:35.000', false);" > setup.sql
./clickhouse client --queries-file ./setup.sql
- name: 'Run test'
run: |
bun pretest
bun test
env:
HOSTNAME: ${{ vars.HOSTNAME }}
PORT: ${{ vars.PORT }}
DB_HOST: ${{ vars.DB_HOST }}
DB_NAME: ${{ secrets.DB_NAME }}
DB_USERNAME: ${{ secrets.DB_USERNAME }}
DB_PASSWORD: ''
2 changes: 1 addition & 1 deletion src/fetch/openapi.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ const TAGS = {
DOCS: "Documentation",
} as const;

const chains = await supportedChainsQuery();
export const chains = await supportedChainsQuery();
const block_example = (await makeQuery(await getBlock( new URLSearchParams({limit: "2"})))).data;

const timestampSchema: SchemaObject = { anyOf: [
Expand Down
7 changes: 7 additions & 0 deletions src/queries.spec.ts
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,19 @@
import { expect, test } from "bun:test";
import { getBlock, getChain } from "./queries.js";
import { chains } from './fetch/openapi.js';

test("getBlock", () => {
expect(getBlock(new URLSearchParams({ chain: "eth", block_number: "123" })))
.toBe(`SELECT * FROM blocks WHERE (chain == 'eth' AND block_number == '123') ORDER BY block_number DESC LIMIT 1`);

expect(getBlock(new URLSearchParams({ chain: "eth", greater_or_equals_by_timestamp: '1438270048', less_or_equals_by_timestamp: '1438270083', limit: '3' })))
.toBe(`SELECT * FROM blocks WHERE (toUnixTimestamp(timestamp) >= 1438270048 AND toUnixTimestamp(timestamp) <= 1438270083 AND chain == 'eth') ORDER BY block_number DESC LIMIT 3`);

// Check that if not chain parameter is passed, all chains are included in the selection
chains.forEach((chain) => {
expect(getBlock(new URLSearchParams({ block_number: "123" })))
.toContain(`SELECT * FROM blocks WHERE (chain == '${chain}' AND block_number == '123') ORDER BY block_number DESC LIMIT 1`);
});
});

test("getChain", () => {
Expand Down
89 changes: 53 additions & 36 deletions src/queries.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { DEFAULT_SORT_BY, config } from './config.js';
import { parseBlockId, parseLimit, parseTimestamp } from './utils.js';
import { chains } from './fetch/openapi.js';

export interface Block {
block_number: number;
Expand All @@ -11,44 +12,60 @@ export interface Block {
export function getBlock(searchParams: URLSearchParams) {
// TO-DO: Modulo block number (ex: search by every 1M blocks)

// SQL Query
let query = `SELECT * FROM ${config.table}`;
const where = [];

// Clickhouse Operators
// https://clickhouse.com/docs/en/sql-reference/operators
const operators = [
["greater_or_equals", ">="],
["greater", ">"],
["less_or_equals", "<="],
["less", "<"],
]
for ( const [key, operator] of operators ) {
const block_number = searchParams.get(`${key}_by_block_number`);
const timestamp = parseTimestamp(searchParams.get(`${key}_by_timestamp`));
if (block_number) where.push(`block_number ${operator} ${block_number}`);
if (timestamp) where.push(`toUnixTimestamp(timestamp) ${operator} ${timestamp}`);
}
let createBlockQuery = (searchParams: URLSearchParams) => {
// SQL Query
let query = `SELECT * FROM ${config.table}`;
const where = [];

// Clickhouse Operators
// https://clickhouse.com/docs/en/sql-reference/operators
const operators = [
["greater_or_equals", ">="],
["greater", ">"],
["less_or_equals", "<="],
["less", "<"],
]
for ( const [key, operator] of operators ) {
const block_number = searchParams.get(`${key}_by_block_number`);
const timestamp = parseTimestamp(searchParams.get(`${key}_by_timestamp`));
if (block_number) where.push(`block_number ${operator} ${block_number}`);
if (timestamp) where.push(`toUnixTimestamp(timestamp) ${operator} ${timestamp}`);
}

// equals
const chain = searchParams.get("chain");
const block_id = parseBlockId(searchParams.get("block_id"));
const block_number = searchParams.get('block_number');
const timestamp = parseTimestamp(searchParams.get('timestamp'));
if (chain) where.push(`chain == '${chain}'`);
if (block_id) where.push(`block_id == '${block_id}'`);
if (block_number) where.push(`block_number == '${block_number}'`);
if (timestamp) where.push(`toUnixTimestamp(timestamp) == ${timestamp}`);

// Join WHERE statements with AND
if ( where.length ) query += ` WHERE (${where.join(' AND ')})`;

// Sort and Limit
const limit = parseLimit(searchParams.get("limit"));
const sort_by = searchParams.get("sort_by");
query += ` ORDER BY block_number ${sort_by ?? DEFAULT_SORT_BY}`
query += ` LIMIT ${limit}`

return query;
};

// equals
const chain = searchParams.get("chain");
const block_id = parseBlockId(searchParams.get("block_id"));
const block_number = searchParams.get('block_number');
const timestamp = parseTimestamp(searchParams.get('timestamp'));
if (chain) where.push(`chain == '${chain}'`);
if (block_id) where.push(`block_id == '${block_id}'`);
if (block_number) where.push(`block_number == '${block_number}'`);
if (timestamp) where.push(`toUnixTimestamp(timestamp) == ${timestamp}`);

// Join WHERE statements with AND
if ( where.length ) query += ` WHERE (${where.join(' AND ')})`;

// Sort and Limit
const limit = parseLimit(searchParams.get("limit"));
const sort_by = searchParams.get("sort_by");
query += ` ORDER BY block_number ${sort_by ?? DEFAULT_SORT_BY}`
query += ` LIMIT ${limit}`
return query;

if (!chain) {
let queries = chains.map((chain) => {
searchParams.set('chain', chain);
return createBlockQuery(searchParams);
});

return queries.join(' UNION ALL ');
} else {
return createBlockQuery(searchParams);
}
}

export function getChain() {
Expand Down

0 comments on commit ffbffbe

Please sign in to comment.