diff --git a/buffer.db b/buffer.db new file mode 100644 index 0000000..326fa9b Binary files /dev/null and b/buffer.db differ diff --git a/package.json b/package.json index 5612dd7..d679163 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "antelope-token-api", - "description": "Token prices from the Antelope blockchains", + "description": "Transfers and account balances from the Antelope blockchains", "version": "0.1.0", "homepage": "https://github.com/pinax-network/antelope-token-api", "license": "MIT", diff --git a/src/clickhouse/createClient.ts b/src/clickhouse/createClient.ts index e566dfe..c6311da 100644 --- a/src/clickhouse/createClient.ts +++ b/src/clickhouse/createClient.ts @@ -1,3 +1,4 @@ +// from: https://github.com/pinax-network/substreams-clock-api/blob/main/src/clickhouse/createClient.ts import { createClient } from "@clickhouse/client-web"; import { ping } from "./ping.js"; import { APP_NAME, config } from "../config.js"; diff --git a/src/clickhouse/makeQuery.ts b/src/clickhouse/makeQuery.ts index c61c0c9..a3a9c23 100644 --- a/src/clickhouse/makeQuery.ts +++ b/src/clickhouse/makeQuery.ts @@ -1,3 +1,4 @@ +// from: https://github.com/pinax-network/substreams-clock-api/blob/main/src/clickhouse/makeQuery.ts import { logger } from "../logger.js"; import * as prometheus from "../prometheus.js"; import client from "./createClient.js"; @@ -18,12 +19,19 @@ export interface Query { } export async function makeQuery(query: string) { - const response = await client.query({ query }) - const data: Query = await response.json(); - prometheus.query.inc(); - prometheus.bytes_read.inc(data.statistics.bytes_read); - prometheus.rows_read.inc(data.statistics.rows_read); - prometheus.elapsed.inc(data.statistics.elapsed); - logger.info({ query, statistics: data.statistics, rows: data.rows }); - return data; -} + try { + const response = await client.query({ query }) + const data: Query = await response.json(); + prometheus.query.inc(); + prometheus.bytes_read.inc(data.statistics.bytes_read); + prometheus.rows_read.inc(data.statistics.rows_read); + prometheus.elapsed.inc(data.statistics.elapsed); + logger.info({ query, statistics: data.statistics, rows: data.rows }); + return data; + } catch (e: any) { + + console.error(e.message) + return { data: [] } + } + +} \ No newline at end of file diff --git a/src/clickhouse/ping.ts b/src/clickhouse/ping.ts index 5646c6e..afdb5d9 100644 --- a/src/clickhouse/ping.ts +++ b/src/clickhouse/ping.ts @@ -1,3 +1,4 @@ +// from: https://github.com/pinax-network/substreams-clock-api/blob/main/src/clickhouse/ping.ts import { PingResult } from "@clickhouse/client-web"; import client from "./createClient.js"; diff --git a/src/config.ts b/src/config.ts index 0a8dd73..61f5471 100644 --- a/src/config.ts +++ b/src/config.ts @@ -8,13 +8,12 @@ export const DEFAULT_PORT = "8080"; export const DEFAULT_HOSTNAME = "localhost"; export const DEFAULT_HOST = "http://localhost:8123"; export const DEFAULT_DATABASE = "default"; -export const DEFAULT_TABLE = "blocks"; export const DEFAULT_USERNAME = "default"; export const DEFAULT_PASSWORD = ""; -export const DEFAULT_MAX_LIMIT = 500; +export const DEFAULT_MAX_LIMIT = 10000; export const DEFAULT_VERBOSE = false; -export const APP_NAME = pkg.name; export const DEFAULT_SORT_BY = "DESC"; +export const APP_NAME = pkg.name; // parse command line options const opts = program @@ -29,7 +28,6 @@ const opts = program .addOption(new Option("--username ", "Database user").env("USERNAME").default(DEFAULT_USERNAME)) .addOption(new Option("--password ", "Password associated with the specified username").env("PASSWORD").default(DEFAULT_PASSWORD)) .addOption(new Option("--database ", "The database to use inside ClickHouse").env("DATABASE").default(DEFAULT_DATABASE)) - .addOption(new Option("--table ", "Clickhouse table name").env("TABLE").default(DEFAULT_TABLE)) .addOption(new Option("--max-limit ", "Maximum LIMIT queries").env("MAX_LIMIT").default(DEFAULT_MAX_LIMIT)) .parse() .opts(); @@ -38,10 +36,9 @@ export const config = z.object({ port: z.string(), hostname: z.string(), host: z.string(), - table: z.string(), database: z.string(), username: z.string(), password: z.string(), maxLimit: z.coerce.number(), verbose: z.coerce.boolean(), -}).parse(opts); +}).parse(opts); \ No newline at end of file diff --git a/src/fetch/GET.ts b/src/fetch/GET.ts index 2b75de3..07e61ee 100644 --- a/src/fetch/GET.ts +++ b/src/fetch/GET.ts @@ -2,23 +2,27 @@ import { registry } from "../prometheus.js"; import openapi from "./openapi.js"; import health from "./health.js"; import chains from "./chains.js"; -import block from "./block.js"; +import balance from "./balance.js"; +import supply from "./supply.js"; import * as prometheus from "../prometheus.js"; import { logger } from "../logger.js"; import swaggerHtml from "../../swagger/index.html" import swaggerFavicon from "../../swagger/favicon.png" +import transfers from "./transfers.js"; export default async function (req: Request) { - const { pathname} = new URL(req.url); - prometheus.request.inc({pathname}); - if ( pathname === "/" ) return new Response(Bun.file(swaggerHtml)); - if ( pathname === "/favicon.png" ) return new Response(Bun.file(swaggerFavicon)); - if ( pathname === "/health" ) return health(req); - if ( pathname === "/metrics" ) return new Response(await registry.metrics(), {headers: {"Content-Type": registry.contentType}}); - if ( pathname === "/openapi" ) return new Response(openapi, {headers: {"Content-Type": "application/json"}}); - if ( pathname === "/chains" ) return chains(req); - //if ( pathname === "/block" ) return block(req); + const { pathname } = new URL(req.url); + prometheus.request.inc({ pathname }); + if (pathname === "/") return new Response(Bun.file(swaggerHtml)); + if (pathname === "/favicon.png") return new Response(Bun.file(swaggerFavicon)); + if (pathname === "/health") return health(req); + if (pathname === "/metrics") return new Response(await registry.metrics(), { headers: { "Content-Type": registry.contentType } }); + if (pathname === "/openapi") return new Response(openapi, { headers: { "Content-Type": "application/json" } }); + if (pathname === "/chains") return chains(req); + if (pathname === "/supply") return supply(req); + if (pathname === "/balance") return balance(req); + if (pathname === "/transfers") return transfers(req); logger.warn(`Not found: ${pathname}`); - prometheus.request_error.inc({pathname, status: 404}); + prometheus.request_error.inc({ pathname, status: 404 }); return new Response("Not found", { status: 404 }); -} +} \ No newline at end of file diff --git a/src/fetch/balance.ts b/src/fetch/balance.ts new file mode 100644 index 0000000..cfc0741 --- /dev/null +++ b/src/fetch/balance.ts @@ -0,0 +1,32 @@ +// from: https://github.com/pinax-network/substreams-clock-api/blob/main/src/fetch/block.ts +import { makeQuery } from "../clickhouse/makeQuery.js"; +import { logger } from "../logger.js"; +import { getBalanceChanges } from "../queries.js"; +import * as prometheus from "../prometheus.js"; +import { toJSON } from "./utils.js"; + + +function verifyParams(searchParams: URLSearchParams) { + const chain = searchParams.get("chain"); + const owner = searchParams.get("owner"); + const contract = searchParams.get("contract"); + + if (!chain) throw new Error("chain is required"); + if (!owner && !contract) throw new Error("owner or contract is required"); +} + +export default async function (req: Request) { + try { + const { searchParams } = new URL(req.url); + logger.info({ searchParams: Object.fromEntries(Array.from(searchParams)) }); + //Verify required params + verifyParams(searchParams); + const query = getBalanceChanges(searchParams); + const response = await makeQuery(query) + return toJSON(response.data); + } catch (e: any) { + logger.error(e); + prometheus.request_error.inc({ pathname: "/balance", status: 400 }); + return new Response(e.message, { status: 400 }); + } +} \ No newline at end of file diff --git a/src/fetch/chains.ts b/src/fetch/chains.ts index d8c192d..0f61ee8 100644 --- a/src/fetch/chains.ts +++ b/src/fetch/chains.ts @@ -1,20 +1,22 @@ +// from: https://github.com/pinax-network/substreams-clock-api/blob/main/src/fetch/chains.ts import { makeQuery } from "../clickhouse/makeQuery.js"; import { logger } from "../logger.js"; import * as prometheus from "../prometheus.js"; import { getChain } from "../queries.js"; +import { toJSON } from "./utils.js"; export async function supportedChainsQuery() { - const response = await makeQuery<{chain: string}>(getChain()); + const response = await makeQuery<{ chain: string }>(getChain()); return response.data.map((r) => r.chain); } -export default async function (req: Request) { +export default async function (_req: Request) { try { const chains = await supportedChainsQuery(); - return new Response(JSON.stringify(chains), { headers: { "Content-Type": "application/json" } }); + return toJSON(chains); } catch (e: any) { logger.error(e); - prometheus.request_error.inc({pathname: "/chains", status: 400}); + prometheus.request_error.inc({ pathname: "/chains", status: 400 }); return new Response(e.message, { status: 400 }); } } \ No newline at end of file diff --git a/src/fetch/health.ts b/src/fetch/health.ts index a0b6684..30b2cf6 100644 --- a/src/fetch/health.ts +++ b/src/fetch/health.ts @@ -1,8 +1,9 @@ +// from: https://github.com/pinax-network/substreams-clock-api/blob/main/src/fetch/health.ts import client from "../clickhouse/createClient.js"; import { logger } from "../logger.js"; import * as prometheus from "../prometheus.js"; -export default async function (req: Request) { +export default async function (_req: Request) { try { const response = await client.ping(); if (response.success === false) throw new Error(response.error.message); diff --git a/src/fetch/openapi.ts b/src/fetch/openapi.ts index dbdc721..dcb02d0 100644 --- a/src/fetch/openapi.ts +++ b/src/fetch/openapi.ts @@ -1,12 +1,12 @@ +// from: https://github.com/pinax-network/substreams-clock-api/blob/main/src/fetch/openapi.ts import pkg from "../../package.json" assert { type: "json" }; import { OpenApiBuilder, SchemaObject, ExampleObject, ParameterObject } from "openapi3-ts/oas31"; import { config } from "../config.js"; -import { getBlock } from "../queries.js"; import { registry } from "../prometheus.js"; -import { makeQuery } from "../clickhouse/makeQuery.js"; import { supportedChainsQuery } from "./chains.js"; - +import { makeQuery } from "../clickhouse/makeQuery.js"; +import { getBalanceChanges, getTotalSupply, getTransfers } from "../queries.js"; const TAGS = { MONITORING: "Monitoring", HEALTH: "Health", @@ -14,27 +14,99 @@ const TAGS = { DOCS: "Documentation", } as const; +const timestampExamplesArrayFilter = ["greater_or_equals_by_timestamp", "greater_by_timestamp", "less_or_equals_by_timestamp", "less_by_timestamp"]; +const blockExamplesArrayFilter = ["greater_or_equals_by_block", "greater_by_block", "less_or_equals_by_block", "less_by_block"]; +const amountExamplesArrayFilter = ["amount_greater_or_equals", "amount_greater", "amount_less_or_equals", "amount_less"]; + const chains = await supportedChainsQuery(); -const block_example = (await makeQuery(await getBlock( new URLSearchParams({limit: "2"})))).data; +const supply_example = (await makeQuery(getTotalSupply(new URLSearchParams({ limit: "2" }), true))).data; +const balance_example = (await makeQuery(getBalanceChanges(new URLSearchParams({ limit: "2" }), true))).data; +const transfers_example = (await makeQuery(getTransfers(new URLSearchParams({ limit: "5" }), true))).data; -const timestampSchema: SchemaObject = { anyOf: [ - {type: "number"}, - {type: "string", format: "date"}, - {type: "string", format: "date-time"} +const timestampSchema: SchemaObject = { + anyOf: [ + { type: "number" }, + { type: "string", format: "date" }, + { type: "string", format: "date-time" } ] }; const timestampExamples: ExampleObject = { unix: { summary: `Unix Timestamp (seconds)` }, date: { summary: `Full-date notation`, value: '2023-10-18' }, - datetime: { summary: `Date-time notation`, value: '2023-10-18T00:00:00Z'}, + datetime: { summary: `Date-time notation`, value: '2023-10-18T00:00:00Z' }, +} + +const parameterChain: ParameterObject = { + name: "chain", + in: "query", + description: "Filter by chain", + required: false, + schema: { enum: chains }, +} +const parameterString = (name: string = "address", required = false) => ({ + name, + in: "query", + description: `Filter by ${name}`, + required, + schema: { type: "string" }, +} as ParameterObject); + + +const parameterLimit: ParameterObject = { + name: "limit", + in: "query", + description: "Used to specify the number of records to return.", + required: false, + schema: { type: "number", maximum: config.maxLimit, minimum: 1 }, +} + +const parameterOffset: ParameterObject = { + name: "offset", + in: "query", + description: "Used to offset data. Combined with limit can be used for pagination.", + required: false, + schema: { type: "number", minimum: 1 }, } + +const timestampFilter = timestampExamplesArrayFilter.map(name => { + return { + name, + in: "query", + description: "Filter " + name.replace(/_/g, " "), + required: false, + schema: timestampSchema, + examples: timestampExamples, + } as ParameterObject +}) + +const blockFilter = blockExamplesArrayFilter.map(name => { + return { + name, + in: "query", + description: "Filter " + name.replace(/_/g, " "), + required: false, + schema: { type: "number" }, + } as ParameterObject +}) + +const amountFilter = amountExamplesArrayFilter.map(name => { + return { + name, + in: "query", + description: "Filter " + name.replace(/_/g, " "), + required: false, + schema: { type: "number" }, + } as ParameterObject +}) + + export default new OpenApiBuilder() .addInfo({ title: pkg.name, version: pkg.version, description: pkg.description, - license: {name: pkg.license}, + license: { name: pkg.license, url: `${pkg.homepage}/blob/main/LICENSE` }, }) .addExternalDocs({ url: pkg.homepage, description: "Extra documentation" }) .addSecurityScheme("auth-key", { type: "http", scheme: "bearer" }) @@ -55,84 +127,62 @@ export default new OpenApiBuilder() }, }, }) - .addPath("/block", { + .addPath("/supply", { get: { tags: [TAGS.USAGE], - summary: "Get block", - description: "Get block by `block_number`, `block_id` or `timestamp`", + summary: "Antelope Tokens total supply", parameters: [ - { - name: "chain", - in: "query", - description: "Filter by chain", - required: false, - schema: {enum: chains}, - }, - { - name: "block_number", - description: "Filter by Block number (ex: 18399498)", - in: "query", - required: false, - schema: { type: "number" }, - }, - { - name: "block_id", - in: "query", - description: "Filter by Block hash ID (ex: 00fef8cf2a2c73266f7c0b71fb5762f9a36419e51a7c05b0e82f9e3bacb859bc)", - required: false, - schema: { type: "string" }, - }, - { - name: 'timestamp', - in: 'query', - description: 'Filter by exact timestamp', - required: false, - schema: timestampSchema, - examples: timestampExamples, - }, - { - name: "final_block", - description: "If true, only returns final blocks", - in: "query", - required: false, - schema: { type: "boolean" }, - }, - { - name: "sort_by", - in: "query", - description: "Sort by `block_number`", - required: false, - schema: {enum: ['ASC', 'DESC'] }, - }, - ...["greater_or_equals_by_timestamp", "greater_by_timestamp", "less_or_equals_by_timestamp", "less_by_timestamp"].map(name => { - return { - name, - in: "query", - description: "Filter " + name.replace(/_/g, " "), - required: false, - schema: timestampSchema, - examples: timestampExamples, - } as ParameterObject - }), - ...["greater_or_equals_by_block_number", "greater_by_block_number", "less_or_equals_by_block_number", "less_by_block_number"].map(name => { - return { - name, - in: "query", - description: "Filter " + name.replace(/_/g, " "), - required: false, - schema: { type: "number" }, - } as ParameterObject - }), - { - name: "limit", - in: "query", - description: "Used to specify the number of records to return.", - required: false, - schema: { type: "number", maximum: config.maxLimit, minimum: 1 }, - }, + parameterChain, + parameterString("address"), + parameterString("name"), + parameterString("symbol"), + ...timestampFilter, + ...blockFilter, + parameterLimit, + parameterOffset, + ], + responses: { + 200: { description: "Array of supply", content: { "application/json": { example: supply_example, schema: { type: "array" } } } }, + 400: { description: "Bad request" }, + }, + }, + }) + .addPath("/balance", { + get: { + tags: [TAGS.USAGE], + summary: "Antelope Tokens balance changes", + parameters: [ + parameterChain, + parameterString("owner"), + parameterString("contract"), + ...timestampFilter, + ...blockFilter, + parameterLimit, + parameterOffset, + ], + responses: { + 200: { description: "Array of balance changes", content: { "application/json": { example: balance_example, schema: { type: "array" } } } }, + 400: { description: "Bad request" }, + }, + }, + }).addPath("/transfers", { + get: { + tags: [TAGS.USAGE], + summary: "Antelope Tokens Transfers", + parameters: [ + parameterChain, + parameterString("contract"), + parameterString("from"), + parameterString("to"), + parameterString("transaction_id"), + ...amountFilter, + ...timestampFilter, + ...blockFilter, + parameterLimit, + parameterOffset, ], responses: { - 200: { description: "Array of blocks", content: { "application/json": { example: block_example, schema: { type: "array" } } } }, + 200: { description: "Array of supply", content: { "application/json": { example: transfers_example, schema: { type: "array" } } } }, 400: { description: "Bad request" }, }, }, @@ -141,21 +191,21 @@ export default new OpenApiBuilder() get: { tags: [TAGS.HEALTH], summary: "Performs health checks and checks if the database is accessible", - responses: {200: { description: "OK", content: { "text/plain": {example: "OK"}} } }, + responses: { 200: { description: "OK", content: { "text/plain": { example: "OK" } } } }, }, }) .addPath("/metrics", { get: { tags: [TAGS.MONITORING], summary: "Prometheus metrics", - responses: {200: { description: "Prometheus metrics", content: { "text/plain": { example: await registry.metrics(), schema: { type: "string" } } }}}, + responses: { 200: { description: "Prometheus metrics", content: { "text/plain": { example: await registry.metrics(), schema: { type: "string" } } } } }, }, }) .addPath("/openapi", { get: { tags: [TAGS.DOCS], summary: "OpenAPI specification", - responses: {200: {description: "OpenAPI JSON Specification", content: { "application/json": { schema: { type: "string" } } } }}, + responses: { 200: { description: "OpenAPI JSON Specification", content: { "application/json": { schema: { type: "string" } } } } }, }, }) .getSpecAsJson(); \ No newline at end of file diff --git a/src/fetch/block.ts b/src/fetch/supply.ts similarity index 53% rename from src/fetch/block.ts rename to src/fetch/supply.ts index c58dd4a..78e45bc 100644 --- a/src/fetch/block.ts +++ b/src/fetch/supply.ts @@ -1,18 +1,20 @@ +// from: https://github.com/pinax-network/substreams-clock-api/blob/main/src/fetch/block.ts import { makeQuery } from "../clickhouse/makeQuery.js"; import { logger } from "../logger.js"; -import { Block, getBlock } from "../queries.js"; +import { getTotalSupply } from "../queries.js"; import * as prometheus from "../prometheus.js"; +import { toJSON } from "./utils.js"; export default async function (req: Request) { try { const { searchParams } = new URL(req.url); logger.info({searchParams: Object.fromEntries(Array.from(searchParams))}); - const query = await getBlock(searchParams); - const response = await makeQuery(query) - return new Response(JSON.stringify(response.data), { headers: { "Content-Type": "application/json" } }); + const query = getTotalSupply(searchParams); + const response = await makeQuery(query) + return toJSON(response.data); } catch (e: any) { logger.error(e); - prometheus.request_error.inc({pathname: "/block", status: 400}); + prometheus.request_error.inc({pathname: "/supply", status: 400}); return new Response(e.message, { status: 400 }); } } \ No newline at end of file diff --git a/src/fetch/transfers.ts b/src/fetch/transfers.ts new file mode 100644 index 0000000..949823d --- /dev/null +++ b/src/fetch/transfers.ts @@ -0,0 +1,20 @@ +// from: https://github.com/pinax-network/substreams-clock-api/blob/main/src/fetch/block.ts +import { makeQuery } from "../clickhouse/makeQuery.js"; +import { logger } from "../logger.js"; +import { getTransfers } from "../queries.js"; +import * as prometheus from "../prometheus.js"; +import { toJSON } from "./utils.js"; + +export default async function (req: Request) { + try { + const { searchParams } = new URL(req.url); + logger.info({ searchParams: Object.fromEntries(Array.from(searchParams)) }); + const query = getTransfers(searchParams); + const response = await makeQuery(query) + return toJSON(response.data); + } catch (e: any) { + logger.error(e); + prometheus.request_error.inc({ pathname: "/transfers", status: 400 }); + return new Response(e.message, { status: 400 }); + } +} \ No newline at end of file diff --git a/src/fetch/utils.ts b/src/fetch/utils.ts new file mode 100644 index 0000000..be854ec --- /dev/null +++ b/src/fetch/utils.ts @@ -0,0 +1,3 @@ +export function toJSON(data: any, status: number = 200) { + return new Response(JSON.stringify(data), { status, headers: { "Content-Type": "application/json" } }); +} \ No newline at end of file diff --git a/src/prometheus.ts b/src/prometheus.ts index 0235628..0b8c2a5 100644 --- a/src/prometheus.ts +++ b/src/prometheus.ts @@ -37,4 +37,3 @@ export const query = registerCounter('query', 'Clickhouse DB queries made'); export const bytes_read = registerCounter('bytes_read', 'Clickhouse DB Statistics bytes read'); export const rows_read = registerCounter('rows_read', 'Clickhouse DB Statistics rows read'); export const elapsed = registerCounter('elapsed', 'Clickhouse DB Statistics query elapsed time'); - diff --git a/src/queries.spec.ts b/src/queries.spec.ts index 55a290e..682d592 100644 --- a/src/queries.spec.ts +++ b/src/queries.spec.ts @@ -1,14 +1,262 @@ +// from: https://github.com/pinax-network/substreams-clock-api/blob/main/src/queries.spec.ts + import { expect, test } from "bun:test"; -import { getBlock, getChain } from "./queries.js"; +import { + getChain, + getTotalSupply, + getBalanceChanges, + addTimestampBlockFilter, + getTransfers, + addAmountFilter, +} from "./queries.js"; + +const chain = "eth"; +const address = "dac17f958d2ee523a2206206994597c13d831ec7"; +const limit = "1"; +const symbol = "usdt"; +const name = "tether usd"; +const greater_or_equals_by_timestamp = "1697587200"; +const less_or_equals_by_timestamp = "1697587100"; +const transaction_id = + "ab3612eed62a184eed2ae86bcad766183019cf40f82e5316f4d7c4e61f4baa44"; +const SQLTestQuery = new URLSearchParams({ + chain, + address, + symbol, + greater_or_equals_by_timestamp, + less_or_equals_by_timestamp, + name, + limit, +}); + +function formatSQL(query: string) { + return query.replace(/\s+/g, ""); +} + +//Timestamp and Block Filter +test("addTimestampBlockFilter", () => { + let where: any[] = []; + const searchParams = new URLSearchParams({ + address: address, + greater_or_equals_by_timestamp: "1697587200", + less_or_equals_by_timestamp: "1697587100", + greater_or_equals_by_block: "123", + less_or_equals_by_block: "123", + }); + addTimestampBlockFilter(searchParams, where); + expect(where).toContain("block_number >= 123"); + expect(where).toContain("block_number <= 123"); + expect(where).toContain("toUnixTimestamp(timestamp) >= 1697587200"); + expect(where).toContain("toUnixTimestamp(timestamp) <= 1697587100"); +}); + +//Test Amount Filter +test("addAmountFilter", () => { + let where: any[] = []; + const searchParams = new URLSearchParams({ + address: address, + amount_greater_or_equals: "123123", + amount_less_or_equals: "123123", + amount_greater: "2323", + amount_less: "2332", + }); + addAmountFilter(searchParams, where); + expect(where).toContain("amount >= 123123"); + expect(where).toContain("amount <= 123123"); + expect(where).toContain("amount > 2323"); + expect(where).toContain("amount < 2332"); +}); + + +// Test TotalSupply +test("getTotalSupply", () => { + const parameters = new URLSearchParams({ chain, address }); + expect(formatSQL(getTotalSupply(parameters))).toContain( + formatSQL(`SELECT + TotalSupply.address as address, + TotalSupply.supply as supply, + TotalSupply.block_number, + TotalSupply.chain as chain, + Contracts.name as name, + Contracts.symbol as symbol, + Contracts.decimals as decimals, + TotalSupply.timestamp`) + ); + expect(formatSQL(getTotalSupply(parameters))).toContain( + formatSQL(`FROM TotalSupply`) + ); + + + expect(formatSQL(getTotalSupply(parameters))).toContain( + formatSQL(`LEFT JOIN Contracts ON Contracts.address = TotalSupply.address`) + ); + + expect(formatSQL(getTotalSupply(parameters))).toContain( + formatSQL( + `WHERE(TotalSupply.chain == '${chain}' AND TotalSupply.address == '${address}')` + ) + ); + + expect(formatSQL(getTotalSupply(parameters))).toContain( + formatSQL(`ORDER BY block_number DESC`) + ); + + expect(formatSQL(getTotalSupply(parameters))).toContain(formatSQL(`LIMIT 1`)); +}); + +test("getTotalSupply with options", () => { + const parameters = new URLSearchParams({ + chain, + address, + symbol, + greater_or_equals_by_timestamp, + less_or_equals_by_timestamp, + name, + limit, + }); + expect(formatSQL(getTotalSupply(parameters))).toContain( + formatSQL( + `WHERE(TotalSupply.chain == '${chain}' AND TotalSupply.address == '${address}' AND toUnixTimestamp(timestamp) >= ${greater_or_equals_by_timestamp} AND toUnixTimestamp(timestamp) <= ${less_or_equals_by_timestamp} AND LOWER(symbol) == '${symbol}' AND LOWER(name) == '${name}')` + ) + ); +}); + +// Test Balance Change +test("getBalanceChange", () => { + const parameters1 = new URLSearchParams({ chain, owner: address, contract: address }); + const parameters2 = new URLSearchParams({ chain, owner: address }); + const parameters3 = new URLSearchParams({ chain, contract: address }); + + expect(formatSQL(getBalanceChanges(parameters1))).toContain( + formatSQL(`SELECT + contract as contract, + owner as owner, + newBalance as balance, + toDateTime(timestamp) as timestamp, + transaction as transaction_id, + chain as chain, + block_number`) + ); + expect(formatSQL(getBalanceChanges(parameters2))).toContain( + formatSQL(`SELECT + owner, + contract, + toDateTime(last_value(timestamp)) AS timestamp, + last_value(newBalance) AS balance`) + ); + + expect(formatSQL(getBalanceChanges(parameters3))).toContain( + formatSQL(`SELECT + owner, + contract, + toDateTime(last_value(timestamp)) as timestamp, + last_value(newBalance) as balance`) + ); + + expect(formatSQL(getBalanceChanges(parameters1))).toContain( + formatSQL(`FROM mv_balance_changes_owner`) + ); + + expect(formatSQL(getBalanceChanges(parameters2))).toContain( + formatSQL(`FROM mv_balance_changes_contract`) + ); -test.skip("getBlock", () => { - expect(getBlock(new URLSearchParams({ chain: "eth", block_number: "123" }))) - .toBe(`SELECT * FROM blocks WHERE (chain == 'eth' AND block_number == '123') ORDER BY block_number DESC LIMIT 1`); + expect(formatSQL(getBalanceChanges(parameters3))).toContain( + formatSQL(`FROM mv_balance_changes_contract`) + ); - expect(getBlock(new URLSearchParams({ chain: "eth", greater_or_equals_by_timestamp: '1438270048', less_or_equals_by_timestamp: '1438270083', limit: '3' }))) - .toBe(`SELECT * FROM blocks WHERE (toUnixTimestamp(timestamp) >= 1438270048 AND toUnixTimestamp(timestamp) <= 1438270083 AND chain == 'eth') ORDER BY block_number DESC LIMIT 3`); + + expect(formatSQL(getBalanceChanges(parameters1))).toContain( + formatSQL(`WHERE(chain == '${chain}' AND owner == '${address}' AND contract == '${address}')`) + ); + + expect(formatSQL(getBalanceChanges(parameters1))).toContain( + formatSQL(`ORDER BY timestamp DESC`) + ); + + expect(formatSQL(getBalanceChanges(parameters2))).toContain( + formatSQL(`GROUP BY (contract, owner) ORDER BY timestamp DESC`) + ); + + expect(formatSQL(getBalanceChanges(parameters3))).toContain( + formatSQL(`GROUP BY (contract, owner) ORDER BY timestamp DESC`) + ); + + + expect(formatSQL(getBalanceChanges(parameters1))).toContain( + formatSQL(`LIMIT 1`) + ); +}); + +test("getBalanceChanges with options", () => { + const parameters = new URLSearchParams({ + chain, + owner: address, + transaction_id, + greater_or_equals_by_timestamp, + less_or_equals_by_timestamp, + limit, + }); + expect(formatSQL(getBalanceChanges(parameters))).toContain( + formatSQL( + `WHERE(chain == '${chain}' AND owner == '${address}' AND toUnixTimestamp(timestamp) >= ${greater_or_equals_by_timestamp} AND toUnixTimestamp(timestamp) <= ${less_or_equals_by_timestamp})` + ) + ); +}); + +// Test getTransfers + +test("getTransfers", () => { + const parameters1 = new URLSearchParams({ chain, contract: address }); + const parameters2 = new URLSearchParams({ chain, from: address }); + const parameters3 = new URLSearchParams({ chain, to: address }); + const parameters4 = new URLSearchParams({ chain, from: address, to: address }); + const parameters5 = new URLSearchParams({ chain, contract: address, from: address, to: address, transaction_id }); + expect(formatSQL(getTransfers(parameters1))).toContain( + formatSQL(`SELECT + address, + from, + to, + value as amount, + transaction as transaction_id, + block_number, + timestamp, + chain`) + + ); + expect(formatSQL(getTransfers(parameters1))).toContain( + formatSQL(`FROM mv_transfers_contract`) + ); + + expect(formatSQL(getTransfers(parameters2))).toContain( + formatSQL(`FROM mv_transfers_from`) + ); + + expect(formatSQL(getTransfers(parameters3))).toContain( + formatSQL(`FROM mv_transfers_to`) + ); + + expect(formatSQL(getTransfers(parameters4))).toContain( + formatSQL(`FROM mv_transfers_from`) + ); + + + + + + expect(formatSQL(getTransfers(parameters5))).toContain( + formatSQL(`WHERE(chain == '${chain}' AND address == '${address}' AND from == '${address}' AND to == '${address}' AND transaction == '${transaction_id}')`) + ); + + expect(formatSQL(getTransfers(parameters5))).toContain( + formatSQL(`ORDER BY timestamp DESC`) + ); + + expect(formatSQL(getTransfers(parameters5))).toContain( + formatSQL(`LIMIT 100`) + ); }); test("getChain", () => { expect(getChain()).toBe(`SELECT DISTINCT chain FROM module_hashes`); -}); \ No newline at end of file +}); diff --git a/src/queries.ts b/src/queries.ts index 1bcb53a..78d72fe 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -1,56 +1,222 @@ -import { DEFAULT_SORT_BY, config } from './config.js'; -import { parseBlockId, parseLimit, parseTimestamp } from './utils.js'; +import { DEFAULT_SORT_BY } from "./config.js"; +import { getAddress, parseLimit, parseTimestamp } from "./utils.js"; -export interface Block { - block_number: number; - block_id: string; - timestamp: string; - chain: string; -} - -export function getBlock(searchParams: URLSearchParams) { - // TO-DO: Modulo block number (ex: search by every 1M blocks) - - // SQL Query - let query = `SELECT * FROM ${config.table}`; - const where = []; - - // Clickhouse Operators - // https://clickhouse.com/docs/en/sql-reference/operators +export function addTimestampBlockFilter(searchParams: URLSearchParams, where: any[]) { const operators = [ ["greater_or_equals", ">="], ["greater", ">"], ["less_or_equals", "<="], ["less", "<"], ] - for ( const [key, operator] of operators ) { - const block_number = searchParams.get(`${key}_by_block_number`); + for (const [key, operator] of operators) { + const block_number = searchParams.get(`${key}_by_block`); const timestamp = parseTimestamp(searchParams.get(`${key}_by_timestamp`)); if (block_number) where.push(`block_number ${operator} ${block_number}`); if (timestamp) where.push(`toUnixTimestamp(timestamp) ${operator} ${timestamp}`); } +} + +export function addAmountFilter(searchParams: URLSearchParams, where: any[]) { + const operators = [ + ["greater_or_equals", ">="], + ["greater", ">"], + ["less_or_equals", "<="], + ["less", "<"], + ] + for (const [key, operator] of operators) { + const amount = searchParams.get(`amount_${key}`); + if (amount) where.push(`amount ${operator} ${amount}`); + } +} + +function balance_changes_owner_contract_query(table: string) { + let query = `SELECT + contract as contract, + owner as owner, + newBalance as balance, + toDateTime(timestamp) as timestamp, + transaction as transaction_id, + chain as chain, + block_number`; + + query += ` FROM ${table}` + return query; +} + +function balance_changes_owner_query(table: string) { + let query = `SELECT + owner, + contract, + toDateTime(last_value(timestamp)) AS timestamp, + last_value(newBalance) AS balance`; + + query += ` FROM ${table}` + return query; +} + +function balance_changes_contract_query(table: string) { + let query = `SELECT + owner, + contract, + toDateTime(last_value(timestamp)) as timestamp, + last_value(newBalance) as balance`; + + query += ` FROM ${table}` + return query; +} + +export function getTotalSupply(searchParams: URLSearchParams, example?: boolean) { + // Params + const address = getAddress(searchParams, "address", false)?.toLowerCase(); + const chain = searchParams.get("chain"); + const symbol = searchParams.get("symbol")?.toLowerCase(); + const name = searchParams.get("name")?.toLowerCase(); + + // Query + const table = 'stats' + let query = `SELECT + * + FROM ${table} `; + + // JOIN Contracts table + //query += ` LEFT JOIN Contracts ON ${contractTable}.address = ${table}.address`; + if (!example) { + // WHERE statements + const where = []; + + // equals + if (chain) where.push(`chain == '${chain}'`); + if (address) where.push(`contract == '${address}'`); + + // timestamp and block filter + addTimestampBlockFilter(searchParams, where); + + if (symbol) where.push(`symcode == '${symbol}'`); + if (name) where.push(`issuer == '${name}'`); + + // Join WHERE statements with AND + if (where.length) query += ` WHERE(${where.join(' AND ')})`; + + // Sort and Limit + const sort_by = searchParams.get("sort_by"); + query += ` ORDER BY block_number ${sort_by ?? DEFAULT_SORT_BY} ` + } + const limit = parseLimit(searchParams.get("limit")); + query += ` LIMIT ${limit} ` + const offset = searchParams.get("offset"); + if (offset) query += ` OFFSET ${offset} ` + return query; +} - // equals +export function getBalanceChanges(searchParams: URLSearchParams, example?: boolean) { const chain = searchParams.get("chain"); - const block_id = parseBlockId(searchParams.get("block_id")); - const block_number = searchParams.get('block_number'); - const timestamp = parseTimestamp(searchParams.get('timestamp')); - if (chain) where.push(`chain == '${chain}'`); - if (block_id) where.push(`block_id == '${block_id}'`); - if (block_number) where.push(`block_number == '${block_number}'`); - if (timestamp) where.push(`toUnixTimestamp(timestamp) == ${timestamp}`); - - // Join WHERE statements with AND - if ( where.length ) query += ` WHERE (${where.join(' AND ')})`; - - // Sort and Limit + const contract = getAddress(searchParams, "contract", false)?.toLowerCase(); + const owner = getAddress(searchParams, "owner", false)?.toLowerCase(); + + let table; + let contractTable; + let mvOwnerTable = "mv_balance_changes_owner" + let mvContractTable = "mv_balance_changes_contract" + let query = ""; + + // SQL Query + table = 'accounts' + + /*if (contract && owner) query += balance_changes_owner_contract_query(mvOwnerTable); + else if (!contract && owner) query += balance_changes_owner_query(mvContractTable); + else if (contract && !owner) query += balance_changes_contract_query(mvContractTable); + else */query += `SELECT * FROM ${table}` + + if (!example) { + // WHERE statements + const where = []; + + // equals + + if (chain) where.push(`chain == '${chain}'`); + if (owner) where.push(`account == '${owner}'`); + if (contract) where.push(`contract == '${contract}'`); + + // timestamp and block filter + addTimestampBlockFilter(searchParams, where); + + // Join WHERE statements with AND + if (where.length) query += ` WHERE(${where.join(' AND ')})`; + + //add ORDER BY and GROUP BY + if (contract && owner) query += ` ORDER BY timestamp DESC` + if (!contract && owner) query += `GROUP BY (contract, owner) ORDER BY timestamp DESC` + if (contract && !owner) query += `GROUP BY (contract, owner) ORDER BY timestamp DESC` + } + + //ADD limit const limit = parseLimit(searchParams.get("limit")); - const sort_by = searchParams.get("sort_by"); - query += ` ORDER BY block_number ${sort_by ?? DEFAULT_SORT_BY}` - query += ` LIMIT ${limit}` + query += ` LIMIT ${limit} ` + const offset = searchParams.get("offset"); + if (offset) query += ` OFFSET ${offset} ` + return query; +} + +export function getTransfers(searchParams: URLSearchParams, example?: boolean) { + + const contract = getAddress(searchParams, "contract", false)?.toLowerCase(); + const from = getAddress(searchParams, "from", false)?.toLowerCase(); + const to = getAddress(searchParams, "to", false)?.toLowerCase(); + const chain = searchParams.get("chain"); + const transaction_id = searchParams.get("transaction_id")?.toLowerCase(); + + // SQL Query + let table = "transfers" + let mvFromTable = "mv_transfers_from" + let mvToTable = "mv_transfers_to" + let mvContractTable = "mv_transfers_contract" + + let query = `SELECT + from, + to, + value as amount, + symcode, + trx_id as transaction_id, + block_number, + timestamp, + chain` + + /*if (contract) query += ` FROM ${mvContractTable}` + else if (!contract && from && !to) query += ` FROM ${mvFromTable}` + else if (!contract && !from && to) query += ` FROM ${mvToTable}` + else if (!contract && from && to) query += ` FROM ${mvFromTable}` + else */query += ` FROM ${table}` + + if (!example) { + // WHERE statements + const where = []; + + // equals + if (chain) where.push(`chain == '${chain}'`); + if (contract) where.push(`contract == '${contract}'`); + if (from) where.push(`from == '${from}'`); + if (to) where.push(`to == '${to}'`); + if (transaction_id) where.push(`transaction == '${transaction_id}'`); + + //add amount filter + addAmountFilter(searchParams, where); + // timestamp and block filter + addTimestampBlockFilter(searchParams, where); + + // Join WHERE statements with AND + if (where.length) query += ` WHERE(${where.join(' AND ')})`; + //add ORDER BY and GROUP BY + query += ` ORDER BY timestamp DESC` + } + + //ADD limit + const limit = parseLimit(searchParams.get("limit"), 100); + query += ` LIMIT ${limit} ` + const offset = searchParams.get("offset"); + if (offset) query += ` OFFSET ${offset} ` return query; } export function getChain() { return `SELECT DISTINCT chain FROM module_hashes`; -} +} \ No newline at end of file diff --git a/src/utils.spec.ts b/src/utils.spec.ts index c74ddc3..99b8e6d 100644 --- a/src/utils.spec.ts +++ b/src/utils.spec.ts @@ -1,41 +1,32 @@ import { expect, test } from "bun:test"; -import { parseBlockId, parseLimit, parseTimestamp } from "./utils.js"; -import { DEFAULT_MAX_LIMIT } from "./config.js"; +import { formatAddress, checkValidAddress, getAddress, parseBlockId, parseTimestamp } from "./utils.js"; +const address = "0xdac17f958d2ee523a2206206994597c13d831ec7"; +test("formatAddress", () => { + expect(formatAddress(address)).toBe("dac17f958d2ee523a2206206994597c13d831ec7") +}); -test("parseBlockId", () => { - expect(parseBlockId()).toBeUndefined(); - expect(parseBlockId(null)).toBeUndefined(); - expect(parseBlockId("00fef8cf2a2c73266f7c0b71fb5762f9a36419e51a7c05b0e82f9e3bacb859bc")).toBe("00fef8cf2a2c73266f7c0b71fb5762f9a36419e51a7c05b0e82f9e3bacb859bc"); - expect(parseBlockId("0x00fef8cf2a2c73266f7c0b71fb5762f9a36419e51a7c05b0e82f9e3bacb859bc")).toBe("00fef8cf2a2c73266f7c0b71fb5762f9a36419e51a7c05b0e82f9e3bacb859bc"); + +test("checkValidAddress", () => { + checkValidAddress(address) + expect(() => checkValidAddress(address)).not.toThrow(); + expect(() => checkValidAddress("foobar")).toThrow("Invalid address"); }); -test("parseLimit", () => { - expect(parseLimit()).toBe(1); - expect(parseLimit(null)).toBe(1); - expect(parseLimit("10")).toBe(10); - expect(parseLimit(10)).toBe(10); - expect(parseLimit(999999)).toBe(DEFAULT_MAX_LIMIT); +test("parseBlockId", () => { + expect(parseBlockId("0x123") as string).toBe("123"); }); test("parseTimestamp", () => { - const seconds = 1672531200; - expect(parseTimestamp()).toBeUndefined(); + expect(parseTimestamp("1697587100")).toBe(1697587100); + expect(parseTimestamp("1697587100000")).toBe(1697587100); + expect(parseTimestamp("awdawd")).toBeNaN(); expect(parseTimestamp(null)).toBeUndefined(); - expect(parseTimestamp(1672531200000)).toBe(seconds); // Milliseconds (13 digits) => Seconds (10 digits) - expect(parseTimestamp("1672531200")).toBe(seconds); - expect(parseTimestamp(1672531200000)).toBe(seconds); - expect(parseTimestamp("2023-01-01T00:00:00.000Z")).toBe(seconds); - expect(parseTimestamp("2023-01-01T00:00:00.000")).toBe(seconds); - expect(parseTimestamp("2023-01-01 00:00:00")).toBe(seconds); // Datetime - expect(parseTimestamp("2023-01-01T00:00:00Z")).toBe(seconds); // ISO - expect(parseTimestamp("2023-01-01T00:00:00")).toBe(seconds); - expect(parseTimestamp("2023-01-01")).toBe(seconds); - expect(parseTimestamp("2023-01")).toBe(seconds); - expect(parseTimestamp(Number(new Date("2023")))).toBe(seconds); - - // errors - expect(() => parseTimestamp(10)).toThrow("Invalid timestamp"); - expect(() => parseTimestamp("10")).toThrow("Invalid timestamp"); }); +test("getAddress", () => { + expect(() => getAddress(new URLSearchParams({ address: address }), "address", false)).not.toThrow(); + expect(() => getAddress(new URLSearchParams({ address: address }), "address", true)).not.toThrow(); + expect(() => getAddress(new URLSearchParams({ address: "" }), "address", true)).toThrow("Missing [address] parameter"); + expect(() => getAddress(new URLSearchParams({ address: "foobar" }), "address")).toThrow("Invalid address"); +}); diff --git a/src/utils.ts b/src/utils.ts index 0d1d106..56207bb 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,21 +1,44 @@ import { config } from "./config.js"; -export function parseLimit(limit?: string|null|number) { - let value = 1; // default 1 +export function getAddress(searchParams: URLSearchParams, key: string, required: boolean = false) { + const address = formatAddress(searchParams.get(key)); + if (required && !address) throw new Error(`Missing [${key}] parameter`); + if (address) checkValidAddress(address); + return address; +} + +export function formatAddress(address: string | null) { + if (!address) return undefined; + if (address.startsWith("0x")) { + // Remove the "0x" prefix and return the address + return address.slice(2); + } + // If it doesn't start with "0x", return the address as is + return address; +} + +export function checkValidAddress(address?: string) { + // todo +} + +export function parseLimit(limit?: string | null | number, defaultLimit?: number) { + let value = 1 // default 1 + if (defaultLimit) + value = defaultLimit; if (limit) { if (typeof limit === "string") value = parseInt(limit); if (typeof limit === "number") value = limit; } // limit must be between 1 and maxLimit - if ( value > config.maxLimit ) value = config.maxLimit; + if (value > config.maxLimit) value = config.maxLimit; return value; } -export function parseBlockId(block_id?: string|null) { +export function parseBlockId(block_id?: string | null) { return block_id ? block_id.replace("0x", "") : undefined; } -export function parseTimestamp(timestamp?: string|null|number) { +export function parseTimestamp(timestamp?: string | null | number) { if (timestamp !== undefined && timestamp !== null) { if (typeof timestamp === "string") { if (/^[0-9]+$/.test(timestamp)) { @@ -27,10 +50,10 @@ export function parseTimestamp(timestamp?: string|null|number) { } if (typeof timestamp === "number") { const length = timestamp.toString().length; - if ( length === 10 ) return timestamp; // seconds - if ( length === 13 ) return Math.floor(timestamp / 1000); // convert milliseconds to seconds + if (length === 10) return timestamp; // seconds + if (length === 13) return Math.floor(timestamp / 1000); // convert milliseconds to seconds throw new Error("Invalid timestamp"); } } return undefined; -} +} \ No newline at end of file