From 30990d7376b7145e03486b9427c92837078b04aa Mon Sep 17 00:00:00 2001 From: Denis Date: Thu, 16 Nov 2023 13:49:35 -0500 Subject: [PATCH 1/2] Feature/subtivity (#51) * Added aggregation endpoints * Added /uaw endpoint * Added supported chains as store (periodically updated) * Added chain parameter verification * Simplified queries and tests * Added /uaw/history endpoint * fixed endpoint name and unix date to seconds * enum time range filter and results split by chain --------- Co-authored-by: Pelotfr --- README.md | 4 + src/clickhouse/stores.ts | 29 ++++++ src/config.ts | 1 + src/fetch/GET.ts | 9 +- src/fetch/aggregate.ts | 24 +++++ src/fetch/block.ts | 5 + src/fetch/chains.ts | 12 +-- src/fetch/history.ts | 25 +++++ src/fetch/openapi.ts | 202 +++++++++++++++++++++++++++++++++++++-- src/fetch/uaw.ts | 24 +++++ src/queries.spec.ts | 45 +++++++-- src/queries.ts | 123 +++++++++++++++++++++++- src/utils.spec.ts | 34 ++++++- src/utils.ts | 64 ++++++++++++- 14 files changed, 570 insertions(+), 31 deletions(-) create mode 100644 src/clickhouse/stores.ts create mode 100644 src/fetch/aggregate.ts create mode 100644 src/fetch/history.ts create mode 100644 src/fetch/uaw.ts diff --git a/README.md b/README.md index 90ddf16..0729e6c 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,10 @@ |-------------------------------------------|-----------------------| | GET `/chains` | Available `chains` | GET `/block` | Get block by `block_number`, `block_id` or `timestamp` +| GET `/trace_calls` | Get aggregate of trace_calls filtered by `chain`, `timestamp` or `block_number` +| GET `/transaction_traces` | Get aggregate of transaction_traces filtered by `chain`, `timestamp` or `block_number` +| GET `/uaw` | Get unique active wallets filtered by `chain` and `date` +| GET `/uaw/history` | Get daily unique active wallets for previous given number of days filtered by `chain` | GET `/health` | Health check | GET `/metrics` | Prometheus metrics | GET `/openapi` | [OpenAPI v3 JSON](https://spec.openapis.org/oas/v3.0.0) diff --git a/src/clickhouse/stores.ts b/src/clickhouse/stores.ts new file mode 100644 index 0000000..d1d9dbf --- /dev/null +++ b/src/clickhouse/stores.ts @@ -0,0 +1,29 @@ +import client from "./createClient.js"; + +class ClickhouseStore { + private ChainsPromise: Promise | undefined = undefined; + + constructor() { + // Fetch data initially + this.fetchData(); + + // Set up a timer to fetch data periodically (e.g., every 1 hour) + setInterval(() => { + this.fetchData(); + }, 10000); // 3600000 milliseconds = 1 hour + } + + private fetchData() { + this.ChainsPromise = client + .query({ query: "SELECT DISTINCT chain FROM blocks", format: "JSONEachRow" }) + .then((response) => response.json>()) + .then((chains) => chains.map(({ chain }) => chain)) + .catch(() => []); + } + + public get chains() { + return this.ChainsPromise; + } +} + +export const store = new ClickhouseStore(); diff --git a/src/config.ts b/src/config.ts index 0a8dd73..dd792df 100644 --- a/src/config.ts +++ b/src/config.ts @@ -15,6 +15,7 @@ export const DEFAULT_MAX_LIMIT = 500; export const DEFAULT_VERBOSE = false; export const APP_NAME = pkg.name; export const DEFAULT_SORT_BY = "DESC"; +export const DEFAULT_AGGREGATE_FUNCTION = "count"; // parse command line options const opts = program diff --git a/src/fetch/GET.ts b/src/fetch/GET.ts index f3c8e9e..ebb1e21 100644 --- a/src/fetch/GET.ts +++ b/src/fetch/GET.ts @@ -3,6 +3,9 @@ import openapi from "./openapi.js"; import health from "./health.js"; import chains from "./chains.js"; import block from "./block.js"; +import aggregate from "./aggregate.js"; +import uaw from "./uaw.js"; +import history from "./history.js"; import * as prometheus from "../prometheus.js"; import { logger } from "../logger.js"; import swaggerHtml from "../../swagger/index.html" @@ -17,8 +20,12 @@ export default async function (req: Request) { if ( pathname === "/health" ) return health(req); if ( pathname === "/metrics" ) return toText(await registry.metrics()); if ( pathname === "/openapi" ) return toJSON(openapi); - if ( pathname === "/chains" ) return chains(req); + if ( pathname === "/chains" ) return chains(); if ( pathname === "/block" ) return block(req); + if ( pathname === "/trace_calls" ) return aggregate(req, pathname); + if ( pathname === "/transaction_traces" ) return aggregate(req, pathname); + if ( pathname === "/uaw" ) return uaw(req); + if ( pathname === "/uaw/history" ) return history(req); logger.warn(`Not found: ${pathname}`); prometheus.request_error.inc({pathname, status: 404}); return NotFound; diff --git a/src/fetch/aggregate.ts b/src/fetch/aggregate.ts new file mode 100644 index 0000000..fc46947 --- /dev/null +++ b/src/fetch/aggregate.ts @@ -0,0 +1,24 @@ +import { makeQuery } from "../clickhouse/makeQuery.js"; +import { logger } from "../logger.js"; +import { getAggregate } from "../queries.js"; +import * as prometheus from "../prometheus.js"; +import { BadRequest, toJSON } from "./cors.js"; +import { verifyParameters } from "../utils.js"; + +export default async function (req: Request, pathname: string) { + const parametersResult = await verifyParameters(req); + if(parametersResult instanceof Response) { + return parametersResult; + } + try { + const { searchParams } = new URL(req.url); + logger.info({searchParams: Object.fromEntries(Array.from(searchParams))}); + const query = getAggregate(searchParams, pathname.replace("/", "")); + const response = await makeQuery(query) + return toJSON(response.data); + } catch (e: any) { + logger.error(e); + prometheus.request_error.inc({pathname: pathname, status: 400}); + return BadRequest + } +} \ No newline at end of file diff --git a/src/fetch/block.ts b/src/fetch/block.ts index 912b6fc..227b508 100644 --- a/src/fetch/block.ts +++ b/src/fetch/block.ts @@ -3,8 +3,13 @@ import { logger } from "../logger.js"; import { Block, getBlock } from "../queries.js"; import * as prometheus from "../prometheus.js"; import { BadRequest, toJSON } from "./cors.js"; +import { verifyParameters } from "../utils.js"; export default async function (req: Request) { + const parametersResult = await verifyParameters(req); + if(parametersResult instanceof Response) { + return parametersResult; + } try { const { searchParams } = new URL(req.url); logger.info({searchParams: Object.fromEntries(Array.from(searchParams))}); diff --git a/src/fetch/chains.ts b/src/fetch/chains.ts index 5a3001a..feb0d58 100644 --- a/src/fetch/chains.ts +++ b/src/fetch/chains.ts @@ -1,17 +1,11 @@ -import { makeQuery } from "../clickhouse/makeQuery.js"; import { logger } from "../logger.js"; +import { store } from "../clickhouse/stores.js"; import * as prometheus from "../prometheus.js"; -import { getChain } from "../queries.js"; import { BadRequest, toJSON } from "./cors.js"; -export async function supportedChainsQuery() { - const response = await makeQuery<{chain: string}>(getChain()); - return response.data.map((r) => r.chain); -} - -export default async function (req: Request) { +export default async function () { try { - const chains = await supportedChainsQuery(); + const chains = await store.chains; return toJSON(chains); } catch (e: any) { logger.error(e); diff --git a/src/fetch/history.ts b/src/fetch/history.ts new file mode 100644 index 0000000..ab4e791 --- /dev/null +++ b/src/fetch/history.ts @@ -0,0 +1,25 @@ +import { makeQuery } from "../clickhouse/makeQuery.js"; +import { logger } from "../logger.js"; +import { UAWHistory, getUAWHistory } from "../queries.js"; +import * as prometheus from "../prometheus.js"; +import { BadRequest, toJSON } from "./cors.js"; +import { parseUAWResponse, verifyParameters } from "../utils.js"; + +export default async function (req: Request) { + const parametersResult = await verifyParameters(req); + if(parametersResult instanceof Response) { + return parametersResult; + } + try { + const { searchParams } = new URL(req.url); + logger.info({searchParams: Object.fromEntries(Array.from(searchParams))}); + const query = getUAWHistory(searchParams); + const response = await makeQuery(query) + const formatted = parseUAWResponse(response.data); + return toJSON(formatted); + } catch (e: any) { + logger.error(e); + prometheus.request_error.inc({pathname: "/uaw/history", status: 400}); + return BadRequest + } +} \ No newline at end of file diff --git a/src/fetch/openapi.ts b/src/fetch/openapi.ts index dbdc721..e15ea86 100644 --- a/src/fetch/openapi.ts +++ b/src/fetch/openapi.ts @@ -2,10 +2,11 @@ import pkg from "../../package.json" assert { type: "json" }; import { OpenApiBuilder, SchemaObject, ExampleObject, ParameterObject } from "openapi3-ts/oas31"; import { config } from "../config.js"; -import { getBlock } from "../queries.js"; +import { store } from "../clickhouse/stores.js"; +import { getBlock, getAggregate, getUAWFromDate, getUAWHistory, UAWHistory } from "../queries.js"; import { registry } from "../prometheus.js"; import { makeQuery } from "../clickhouse/makeQuery.js"; -import { supportedChainsQuery } from "./chains.js"; +import { parseUAWResponse } from "../utils.js"; const TAGS = { MONITORING: "Monitoring", @@ -14,8 +15,11 @@ const TAGS = { DOCS: "Documentation", } as const; -const chains = await supportedChainsQuery(); const block_example = (await makeQuery(await getBlock( new URLSearchParams({limit: "2"})))).data; +const trace_calls_example = (await makeQuery(getAggregate( new URLSearchParams({aggregate_function: "count", chain: "wax"}), "trace_calls"))).data; +const transaction_traces_example = (await makeQuery(getAggregate( new URLSearchParams({aggregate_function: "count", chain: "wax"}), "transaction_traces"))).data; +const uaw_example = (await makeQuery(getUAWFromDate( new URLSearchParams({chain: "wax", date: "2023-09-06"})))).data; +const history_example = parseUAWResponse((await makeQuery(getUAWHistory( new URLSearchParams({chain: "eos", range: "7"})))).data); const timestampSchema: SchemaObject = { anyOf: [ {type: "number"}, @@ -28,6 +32,16 @@ const timestampExamples: ExampleObject = { date: { summary: `Full-date notation`, value: '2023-10-18' }, datetime: { summary: `Date-time notation`, value: '2023-10-18T00:00:00Z'}, } +const DateSchema: SchemaObject = { anyOf: [ + {type: "number"}, + {type: "string", format: "date"}, + ] +}; + +const DateExamples: ExampleObject = { + unix: { summary: `Unix Timestamp (seconds)`, value: 1693951200 }, + date: { summary: `Full-date notation`, value: '2023-09-06' }, +} export default new OpenApiBuilder() .addInfo({ @@ -47,8 +61,7 @@ export default new OpenApiBuilder() description: "Array of chains", content: { "application/json": { - schema: { type: "array" }, - example: chains, + schema: { enum: await store.chains }, } }, }, @@ -66,7 +79,7 @@ export default new OpenApiBuilder() in: "query", description: "Filter by chain", required: false, - schema: {enum: chains}, + schema: {enum: await store.chains}, }, { name: "block_number", @@ -137,6 +150,183 @@ export default new OpenApiBuilder() }, }, }) + .addPath("/trace_calls", { + get: { + tags: [TAGS.USAGE], + summary: "Get aggregate of trace_calls", + description: "Get aggregate of trace_calls filtered by `chain`, `timestamp` or `block_number`", + parameters: [ + { + name: "aggregate_function", + in: "query", + description: "Aggregate function", + required: false, + schema: {enum: ['count', 'min', 'max', 'sum', 'avg', 'median'] }, + }, + { + name: "chain", + in: "query", + description: "Filter by chain name", + required: false, + schema: {enum: await store.chains}, + }, + { + name: 'timestamp', + in: 'query', + description: 'Filter by exact timestamp', + required: false, + schema: timestampSchema, + examples: timestampExamples, + }, + { + name: "block_number", + description: "Filter by Block number (ex: 18399498)", + in: "query", + required: false, + schema: { type: "number" }, + }, + ...["greater_or_equals_by_timestamp", "greater_by_timestamp", "less_or_equals_by_timestamp", "less_by_timestamp"].map(name => { + return { + name, + in: "query", + description: "Filter " + name.replace(/_/g, " "), + required: false, + schema: timestampSchema, + examples: timestampExamples, + } as ParameterObject + }), + ...["greater_or_equals_by_block_number", "greater_by_block_number", "less_or_equals_by_block_number", "less_by_block_number"].map(name => { + return { + name, + in: "query", + description: "Filter " + name.replace(/_/g, " "), + required: false, + schema: { type: "number" }, + } as ParameterObject + }), + ], + responses: { + 200: { description: "Aggregate of sales", content: { "text/plain": { example: trace_calls_example} } }, + 400: { description: "Bad request", content: { "text/plain": { example: "Bad request", schema: { type: "string" } } }, }, + }, + }, + }) + .addPath("/transaction_traces", { + get: { + tags: [TAGS.USAGE], + summary: "Get aggregate of transaction_traces", + description: "Get aggregate of transaction_traces filtered by `chain`, `timestamp` or `block_number`", + parameters: [ + { + name: "aggregate_function", + in: "query", + description: "Aggregate function", + required: false, + schema: {enum: ['count', 'min', 'max', 'sum', 'avg', 'median'] }, + }, + { + name: "chain", + in: "query", + description: "Filter by chain name", + required: false, + schema: {enum: await store.chains}, + }, + { + name: 'timestamp', + in: 'query', + description: 'Filter by exact timestamp', + required: false, + schema: timestampSchema, + examples: timestampExamples, + }, + { + name: "block_number", + description: "Filter by Block number (ex: 18399498)", + in: "query", + required: false, + schema: { type: "number" }, + }, + ...["greater_or_equals_by_timestamp", "greater_by_timestamp", "less_or_equals_by_timestamp", "less_by_timestamp"].map(name => { + return { + name, + in: "query", + description: "Filter " + name.replace(/_/g, " "), + required: false, + schema: timestampSchema, + examples: timestampExamples, + } as ParameterObject + }), + ...["greater_or_equals_by_block_number", "greater_by_block_number", "less_or_equals_by_block_number", "less_by_block_number"].map(name => { + return { + name, + in: "query", + description: "Filter " + name.replace(/_/g, " "), + required: false, + schema: { type: "number" }, + } as ParameterObject + }), + ], + responses: { + 200: { description: "Aggregate of sales", content: { "text/plain": { example: transaction_traces_example} } }, + 400: { description: "Bad request", content: { "text/plain": { example: "Bad request", schema: { type: "string" } } }, }, + }, + }, + }) + .addPath("/uaw", { + get: { + tags: [TAGS.USAGE], + summary: "Get unique active wallets", + description: "Get unique active wallets filtered by `chain` and `date`", + parameters: [ + { + name: "chain", + in: "query", + description: "Filter by chain name", + required: false, + schema: {enum: await store.chains}, + }, + { + name: "date", + description: "Filter by date (ex: 2023-09-06)", + in: "query", + required: false, + schema: DateSchema, + examples: DateExamples, + }, + ], + responses: { + 200: { description: "Unique active wallets", content: { "text/plain": { example: uaw_example} } }, + 400: { description: "Bad request", content: { "text/plain": { example: "Bad request", schema: { type: "string" } } }, }, + }, + }, + }) + .addPath("/uaw/history", { + get: { + tags: [TAGS.USAGE], + summary: "Get daily unique active wallets", + description: "Get daily unique active wallets for given time range filtered by `chain`", + parameters: [ + { + name: "chain", + in: "query", + description: "Filter by chain name", + required: false, + schema: {enum: await store.chains}, + }, + { + name: "range", + in: "query", + description: "Time range to query (ex: 7d)", + required: false, + schema: { enum: ["24h", "7d", "30d", "90d", "1y", "all"] }, + } + ], + responses: { + 200: { description: "Daily active wallets", content: { "text/plain": { example: history_example} } }, + 400: { description: "Bad request", content: { "text/plain": { example: "Bad request", schema: { type: "string" } } }, }, + }, + }, + }) .addPath("/health", { get: { tags: [TAGS.HEALTH], diff --git a/src/fetch/uaw.ts b/src/fetch/uaw.ts new file mode 100644 index 0000000..102bf6e --- /dev/null +++ b/src/fetch/uaw.ts @@ -0,0 +1,24 @@ +import { makeQuery } from "../clickhouse/makeQuery.js"; +import { logger } from "../logger.js"; +import { getUAWFromDate } from "../queries.js"; +import * as prometheus from "../prometheus.js"; +import { BadRequest, toJSON } from "./cors.js"; +import { verifyParameters } from "../utils.js"; + +export default async function (req: Request) { + const parametersResult = await verifyParameters(req); + if(parametersResult instanceof Response) { + return parametersResult; + } + try { + const { searchParams } = new URL(req.url); + logger.info({searchParams: Object.fromEntries(Array.from(searchParams))}); + const query = getUAWFromDate(searchParams); + const response = await makeQuery(query) + return toJSON(response.data); + } catch (e: any) { + logger.error(e); + prometheus.request_error.inc({pathname: "/uaw", status: 400}); + return BadRequest + } +} \ No newline at end of file diff --git a/src/queries.spec.ts b/src/queries.spec.ts index 5b55e0c..5191049 100644 --- a/src/queries.spec.ts +++ b/src/queries.spec.ts @@ -1,9 +1,9 @@ import { expect, jest, mock, test } from "bun:test"; -import { createBlockQuery, getBlock, getChain } from "./queries.js"; -import { supportedChainsQuery } from "./fetch/chains.js"; +import { createBlockQuery, getBlock, getAggregate, getUAWFromDate, getUAWHistory } from "./queries.js"; +import { store } from "./clickhouse/stores.js"; // Mock supported chains data to prevent DB query -mock.module("./fetch/chains.ts", () => ({ supportedChainsQuery: jest.fn().mockResolvedValue(["eth", "polygon"]) })); +//mock.module("./fetch/chains.ts", () => ({ supportedChainsQuery: jest.fn().mockResolvedValue(["eth", "polygon"]) })); test("createBlockQuery", () => { expect(createBlockQuery(new URLSearchParams({ chain: "eth", block_number: "123" }))) @@ -18,13 +18,44 @@ test("getBlock", async () => { expect(getBlock(singleChainQuery)).resolves.toBe(createBlockQuery(singleChainQuery)); // Check that if no chain parameter is passed, all chains are included in the selection - let supportedChains = await supportedChainsQuery(); + let supportedChains = await store.chains; + if (!supportedChains) { + throw new Error("chains is null"); + } supportedChains.forEach((chain) => { expect(getBlock(new URLSearchParams({ block_number: "123" }))).resolves .toContain(`SELECT * FROM blocks WHERE (chain == '${chain}' AND block_number == '123') ORDER BY block_number DESC LIMIT 1`); }); }); -test("getChain", () => { - expect(getChain()).toBe(`SELECT DISTINCT chain FROM module_hashes`); -}); \ No newline at end of file +test("getAggregate", async () => { + const singleChainQuery = new URLSearchParams({ chain: "wax"}); + expect(getAggregate(singleChainQuery, "trace_calls")) + .toBe(`SELECT chain, count(trace_calls) FROM BlockStats WHERE (chain == 'wax') GROUP BY chain`); + + expect(getAggregate(new URLSearchParams(), "transaction_traces")) + .toBe(`SELECT chain, count(transaction_traces) FROM BlockStats GROUP BY chain`); +}); + +test("getUAWFromDate", async () => { + const singleChainQuery = new URLSearchParams({ chain: "wax", date: "2023-09-06" }); + expect(getUAWFromDate(singleChainQuery)) + .toBe(`SELECT chain, count(distinct uaw) FROM BlockStats ARRAY JOIN uaw WHERE (toUnixTimestamp(DATE(timestamp)) == toUnixTimestamp(DATE(1693958400)) AND chain == 'wax') GROUP BY chain`); + + expect(getUAWFromDate(new URLSearchParams({ date: "2023-09-06" }))) + .toBe(`SELECT chain, count(distinct uaw) FROM BlockStats ARRAY JOIN uaw WHERE (toUnixTimestamp(DATE(timestamp)) == toUnixTimestamp(DATE(1693958400))) GROUP BY chain`); +}); + +test("getUAWHistory", async () => { + const date_of_query = Math.floor(Number(new Date().setHours(0,0,0,0)) / 1000); + const datetime_of_query = Math.floor(Number(new Date()) / 1000); + + expect(getUAWHistory(new URLSearchParams({ chain: "eos", range: "7d" }))) + .toBe(`SELECT chain, toUnixTimestamp(DATE(timestamp)) as day, count(distinct uaw) as UAW FROM BlockStats ARRAY JOIN uaw WHERE (timestamp BETWEEN ${date_of_query} - 86400 * 7 AND ${date_of_query} AND chain == 'eos') GROUP BY chain, day ORDER BY day ASC`); + + expect(getUAWHistory(new URLSearchParams({ range: "24h" }))) + .toBe(`SELECT chain, toUnixTimestamp(DATE(timestamp)) as day, count(distinct uaw) as UAW FROM BlockStats ARRAY JOIN uaw WHERE (timestamp BETWEEN ${datetime_of_query} - 3600 * 24 AND ${datetime_of_query}) GROUP BY chain, day ORDER BY day ASC`); + + expect(getUAWHistory(new URLSearchParams({ range: "1y" }))) + .toBe(`SELECT chain, toUnixTimestamp(DATE(timestamp)) as day, count(distinct uaw) as UAW FROM BlockStats ARRAY JOIN uaw WHERE (timestamp BETWEEN ${date_of_query} - 31536000 * 1 AND ${date_of_query}) GROUP BY chain, day ORDER BY day ASC`); +}); diff --git a/src/queries.ts b/src/queries.ts index 6393b06..049ea9c 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -1,6 +1,6 @@ +import { store } from './clickhouse/stores.js'; import { config } from './config.js'; -import { parseBlockId, parseBlockNumber, parseChain, parseLimit, parseSortBy, parseTimestamp } from './utils.js'; -import { supportedChainsQuery } from './fetch/chains.js'; +import { parseBlockId, parseBlockNumber, parseChain, parseLimit, parseSortBy, parseTimestamp, parseAggregateFunction, parseHistoryRange} from './utils.js'; export interface Block { block_number: number; @@ -9,6 +9,12 @@ export interface Block { chain: string; } +export interface UAWHistory { + chain: string; + UAW: string; + day: number; +} + export function createBlockQuery (searchParams: URLSearchParams) { // SQL Query let query = `SELECT * FROM ${config.table}`; @@ -56,7 +62,10 @@ export async function getBlock(searchParams: URLSearchParams) { const chain = searchParams.get("chain"); if (!chain) { - const chains = await supportedChainsQuery(); + const chains = await store.chains; + if (!chains) { + throw new Error("chains is null"); + } let queries = chains.map((chain) => { searchParams.set('chain', chain); return createBlockQuery(searchParams); @@ -68,6 +77,110 @@ export async function getBlock(searchParams: URLSearchParams) { } } -export function getChain() { - return `SELECT DISTINCT chain FROM module_hashes`; +export function getAggregate(searchParams: URLSearchParams, aggregate_column: string) { + // SQL Query + let query = `SELECT chain,`; + + // Aggregate Function + const aggregate_function = parseAggregateFunction(searchParams.get("aggregate_function")); + + // Aggregate Column + if (aggregate_column == undefined) throw new Error("aggregate_column is undefined"); + else query += ` ${aggregate_function}(${aggregate_column})` + + query += ` FROM BlockStats`; + + const where = []; + // Clickhouse Operators + // https://clickhouse.com/docs/en/sql-reference/operators + const operators = [ + ["greater_or_equals", ">="], + ["greater", ">"], + ["less_or_equals", "<="], + ["less", "<"], + ] + for ( const [key, operator] of operators ) { + const block_number = parseBlockNumber(searchParams.get(`${key}_by_block_number`)); + const timestamp = parseTimestamp(searchParams.get(`${key}_by_timestamp`)); + if (block_number) where.push(`block_number ${operator} ${block_number}`); + if (timestamp) where.push(`toUnixTimestamp(timestamp) ${operator} ${timestamp}`); + } + + // equals + const block_number = parseBlockNumber(searchParams.get('block_number')); + if (block_number) where.push(`block_number == '${block_number}'`); + + const timestamp = parseTimestamp(searchParams.get('timestamp')); + if (timestamp) where.push(`toUnixTimestamp(timestamp) == ${timestamp}`); + + const chain = parseChain(searchParams.get('chain')); + if (chain) where.push(`chain == '${chain}'`); + + // Join WHERE statements with AND + if ( where.length ) query += ` WHERE (${where.join(' AND ')})`; + + // Group by chain + query += ` GROUP BY chain`; + + return query; } + +export function getUAWFromDate(searchParams: URLSearchParams) { + // SQL Query + let query = `SELECT chain, count(distinct uaw) FROM BlockStats ARRAY JOIN uaw`; + + const where = []; + + const date = parseTimestamp(searchParams.get('date')); + if (date) where.push(`toUnixTimestamp(DATE(timestamp)) == toUnixTimestamp(DATE(${date}))`); + + const chain = parseChain(searchParams.get('chain')); + if (chain) where.push(`chain == '${chain}'`); + + // Join WHERE statements with AND + if ( where.length ) query += ` WHERE (${where.join(' AND ')})`; + + // Group by chain + query += ` GROUP BY chain`; + + return query; +} + +export function getUAWHistory(searchParams: URLSearchParams) { + // SQL Query + let query = `SELECT chain, toUnixTimestamp(DATE(timestamp)) as day, count(distinct uaw) as UAW FROM BlockStats ARRAY JOIN uaw`; + + const where = []; + + const datetime_of_query = Math.floor(Number(new Date()) / 1000); + const date_of_query = Math.floor(Number(new Date().setHours(0,0,0,0)) / 1000); + + //const test = 1694296800; + + const range = parseHistoryRange(searchParams.get('range')); + + if (range?.includes('h')) { + const hours = parseInt(range); + if (hours) where.push(`timestamp BETWEEN ${datetime_of_query} - 3600 * ${hours} AND ${datetime_of_query}`); + } + + if (range?.includes('d')) { + const days = parseInt(range); + if (days) where.push(`timestamp BETWEEN ${date_of_query} - 86400 * ${days} AND ${date_of_query}`); + } + + if(range?.includes('y')) { + const years = parseInt(range); + if (years) where.push(`timestamp BETWEEN ${date_of_query} - 31536000 * ${years} AND ${date_of_query}`); + } + + const chain = parseChain(searchParams.get('chain')); + if (chain) where.push(`chain == '${chain}'`); + + // Join WHERE statements with AND + if ( where.length ) query += ` WHERE (${where.join(' AND ')})`; + + query += ` GROUP BY chain, day ORDER BY day ASC`; + + return query; +} \ No newline at end of file diff --git a/src/utils.spec.ts b/src/utils.spec.ts index a69cac9..35b8b90 100644 --- a/src/utils.spec.ts +++ b/src/utils.spec.ts @@ -1,6 +1,8 @@ import { expect, test } from "bun:test"; -import { parseBlockId, parseBlockNumber, parseChain, parseLimit, parseSortBy, parseTimestamp } from "./utils.js"; +import { parseBlockId, parseBlockNumber, parseChain, parseLimit, parseSortBy, + parseTimestamp, parseAggregateColumn, parseAggregateFunction, parseHistoryRange } from "./utils.js"; import { DEFAULT_MAX_LIMIT, DEFAULT_SORT_BY } from "./config.js"; +import { parse } from "querystring"; test("parseBlockId", () => { expect(parseBlockId()).toBeUndefined(); @@ -65,3 +67,33 @@ test("parseTimestamp", () => { expect(() => parseTimestamp(10)).toThrow("Invalid timestamp"); expect(() => parseTimestamp("10")).toThrow("Invalid timestamp"); }); + +test("parseAggregateFunction", () => { + expect(parseAggregateFunction()).toBe("count"); + expect(parseAggregateFunction(null)).toBe("count"); + expect(parseAggregateFunction("invalid")).toBe("count"); + expect(parseAggregateFunction("count")).toBe("count"); + expect(parseAggregateFunction("sum")).toBe("sum"); + expect(parseAggregateFunction("avg")).toBe("avg"); + expect(parseAggregateFunction("min")).toBe("min"); + expect(parseAggregateFunction("max")).toBe("max"); +}); + +test("parseAggregateColumn", () => { + expect(parseAggregateColumn()).toBeUndefined(); + expect(parseAggregateColumn(null)).toBeUndefined(); + expect(parseAggregateColumn("invalid")).toBeUndefined(); + expect(parseAggregateColumn("transaction_traces")).toBe("transaction_traces"); + expect(parseAggregateColumn("trace_calls")).toBe("trace_calls"); + expect(parseAggregateColumn("total_uaw")).toBe("total_uaw"); +}); + +test("parseHistoryRange", () => { + expect(parseHistoryRange()).toBe("7d"); + expect(parseHistoryRange(null)).toBe("7d"); + expect(parseHistoryRange("invalid")).toBe("7d"); + expect(parseHistoryRange("24h")).toBe("24h"); + expect(parseHistoryRange("7d")).toBe("7d"); + expect(parseHistoryRange("1y")).toBe("1y"); + expect(parseHistoryRange("all")).toBe("all"); +}); \ No newline at end of file diff --git a/src/utils.ts b/src/utils.ts index 2e8ccbf..aa0158a 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,5 +1,16 @@ -import { z } from 'zod'; -import { DEFAULT_SORT_BY, config } from "./config.js"; +import { string, z } from 'zod'; +import { DEFAULT_SORT_BY, DEFAULT_AGGREGATE_FUNCTION, config } from "./config.js"; +import { logger } from './logger.js'; +import { store } from "./clickhouse/stores.js"; +import { toText } from './fetch/cors.js'; +import { UAWHistory } from './queries.js'; + +export interface FormattedUAWHistory { + [chain: string]: { + UAW: number[]; + day: number[]; + }; +} export function parseBlockId(block_id?: string|null) { // Match against hexadecimal string (with or without '0x' prefix) @@ -68,3 +79,52 @@ export function parseTimestamp(timestamp?: string|null|number) { } return undefined; } + +export function parseAggregateFunction(aggregate_function?: string|null) { + if (!z.enum(["min", "max", "avg", "sum", "count", "median"]).safeParse(aggregate_function).success) { + logger.info("Aggregate function not supported, using default"); + return DEFAULT_AGGREGATE_FUNCTION; + } + + return aggregate_function; +} + +export function parseAggregateColumn(aggregate_column?: string|null) { + if (!z.enum(["transaction_traces", "trace_calls", "total_uaw"]).safeParse(aggregate_column).success) { + return undefined; + } + return aggregate_column; +} + +export async function verifyParameters(req: Request) { + const url = new URL(req.url); + const chain = url.searchParams.get("chain"); + + if(chain && (parseChain(chain) == undefined)) { + return toText("Invalid chain name: " + chain, 400); + } + else if (chain && !(await store.chains)?.includes(chain)) { + return toText("Chain not found: " + chain, 404); + } +} + +export function parseUAWResponse(data: UAWHistory[]) { + return data.reduce((formattedData, item) => { + const { chain, UAW, day } = item; + + formattedData[chain] = formattedData[chain] || { UAW: [], day: [] }; + + formattedData[chain].UAW.push(parseInt(UAW, 10)); + formattedData[chain].day.push(day); + + return formattedData; + }, {} as FormattedUAWHistory); +} + +export function parseHistoryRange(range?: string|null) { + if (!z.enum(["24h", "7d", "30d", "90d", "1y", "all"]).safeParse(range).success) { + return "7d"; + } + + return range; +} \ No newline at end of file From ac5cebe6eb71ef56e81d2cb676ef194a39a27f30 Mon Sep 17 00:00:00 2001 From: Denis Carriere Date: Thu, 16 Nov 2023 13:51:05 -0500 Subject: [PATCH 2/2] update version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5f1ebe6..c77576d 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "substreams-clock-api", "description": "Block & Timestamps API", - "version": "1.2.1", + "version": "1.4.0", "homepage": "https://github.com/pinax-network/substreams-clock-api", "license": "MIT", "type": "module",