Feature/subtivity (#52)

* Added aggregation endpoints * Added /uaw endpoint * Added supported chains as store (periodically updated) * Added chain parameter verification * Simplified queries and tests * Added /uaw/history endpoint * fixed endpoint name and unix date to seconds * enum time range filter and results split by chain * Renamed /uaw/history into /uaw and removed previous /uaw endpoint * Refactored parameter verification and defaults * Added multi select + history logic to agg endpoints * added normalized data format and grouped similar queries in one * Removed multi-select for aggregate functions, defined new standard response format * Updated docs and endpoints to use new standard response and simpler queries * Changed from blocks to module_hashes for faster result * Default agg to sum and simplified checks * moved uaw request handling into aggregate, added some comments * Fixed description * Revert "Merge branch 'main' into feature/subtivity" This reverts commit e18d74e, reversing changes made to 5eab96a. --------- Co-authored-by: Pelotfr <[email protected]> Co-authored-by: Paul Meignan <[email protected]>
pinax-network · Nov 22, 2023 · ff26fe1 · ff26fe1
1 parent ac5cebe
commit ff26fe1
Show file tree

Hide file tree

Showing 13 changed files with 115 additions and 321 deletions.
diff --git a/README.md b/README.md
@@ -10,10 +10,9 @@
 |-------------------------------------------|-----------------------|
 | GET `/chains`                             | Available `chains`
 | GET `/block`                              | Get block by `block_number`, `block_id` or `timestamp`
-| GET `/trace_calls`                        | Get aggregate of trace_calls filtered by `chain`, `timestamp` or `block_number`
-| GET `/transaction_traces`                 | Get aggregate of transaction_traces filtered by `chain`, `timestamp` or `block_number`
-| GET `/uaw`                                | Get unique active wallets filtered by `chain` and `date`
-| GET `/uaw/history`                        | Get daily unique active wallets for previous given number of days filtered by `chain`
+| GET `/trace_calls`                        | Get aggregate of trace_calls for given time range filtered by `chain`
+| GET `/transaction_traces`                 | Get aggregate of transaction_traces for given time range filtered by `chain`
+| GET `/uaw`                                | Get daily unique active wallets for given time range filtered by `chain`
 | GET `/health`                             | Health check
 | GET `/metrics`                            | Prometheus metrics
 | GET `/openapi`                            | [OpenAPI v3 JSON](https://spec.openapis.org/oas/v3.0.0)

diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "substreams-clock-api",
   "description": "Block & Timestamps API",
-  "version": "1.4.0",
+  "version": "1.4.1",
   "homepage": "https://github.com/pinax-network/substreams-clock-api",
   "license": "MIT",
   "type": "module",

diff --git a/src/clickhouse/stores.ts b/src/clickhouse/stores.ts
@@ -7,15 +7,15 @@ class ClickhouseStore {
     // Fetch data initially
     this.fetchData();
 
-    // Set up a timer to fetch data periodically (e.g., every 1 hour)
+    // Fetch periodically
     setInterval(() => {
       this.fetchData();
-    }, 10000); // 3600000 milliseconds = 1 hour
+    }, 10000); // in milliseconds
   }
 
   private fetchData() {
     this.ChainsPromise = client
-      .query({ query: "SELECT DISTINCT chain FROM blocks", format: "JSONEachRow" })
+      .query({ query: "SELECT DISTINCT chain FROM module_hashes", format: "JSONEachRow" })
       .then((response) => response.json<Array<{ chain: string }>>())
       .then((chains) => chains.map(({ chain }) => chain))
       .catch(() => []);

diff --git a/src/config.ts b/src/config.ts
@@ -15,7 +15,7 @@ export const DEFAULT_MAX_LIMIT = 500;
 export const DEFAULT_VERBOSE = false;
 export const APP_NAME = pkg.name;
 export const DEFAULT_SORT_BY = "DESC";
-export const DEFAULT_AGGREGATE_FUNCTION = "count";
+export const DEFAULT_AGGREGATE_FUNCTION = "sum";
 
 // parse command line options
 const opts = program

diff --git a/src/fetch/GET.ts b/src/fetch/GET.ts
@@ -4,8 +4,6 @@ import health from "./health.js";
 import chains from "./chains.js";
 import block from "./block.js";
 import aggregate from "./aggregate.js";
-import uaw from "./uaw.js";
-import history from "./history.js";
 import * as prometheus from "../prometheus.js";
 import { logger } from "../logger.js";
 import swaggerHtml from "../../swagger/index.html"
@@ -24,8 +22,7 @@ export default async function (req: Request) {
     if ( pathname === "/block" ) return block(req);
     if ( pathname === "/trace_calls" ) return aggregate(req, pathname);
     if ( pathname === "/transaction_traces" ) return aggregate(req, pathname);
-    if ( pathname === "/uaw" ) return uaw(req);
-    if ( pathname === "/uaw/history" ) return history(req);
+    if ( pathname === "/uaw" ) return aggregate(req, pathname);
     logger.warn(`Not found: ${pathname}`);
     prometheus.request_error.inc({pathname, status: 404});
     return NotFound;

diff --git a/src/fetch/aggregate.ts b/src/fetch/aggregate.ts
@@ -1,21 +1,26 @@
 import { makeQuery } from "../clickhouse/makeQuery.js";
 import { logger } from "../logger.js";
-import { getAggregate } from "../queries.js";
+import { NormalizedHistoryData, getAggregate } from "../queries.js";
 import * as prometheus from "../prometheus.js";
 import { BadRequest, toJSON } from "./cors.js";
-import { verifyParameters } from "../utils.js";
+import { parseNormalized, verifyParameters } from "../utils.js";
 
+// endpoint for aggregates (trace_calls, transaction_traces, uaw)
 export default async function (req: Request, pathname: string) {
+  // verify some crucial parameters beforehand
   const parametersResult = await verifyParameters(req);
   if(parametersResult instanceof Response) {
     return parametersResult;
   }
   try {
     const { searchParams } = new URL(req.url);
     logger.info({searchParams: Object.fromEntries(Array.from(searchParams))});
+    // creates the query for requested aggregate column based on pathname
     const query = getAggregate(searchParams, pathname.replace("/", ""));
-    const response = await makeQuery<number>(query)
-    return toJSON(response.data);
+    const response = await makeQuery<NormalizedHistoryData>(query)
+    // formats the response into daily intervals
+    const formatted = parseNormalized(response.data, 86400);
+    return toJSON(formatted);
   } catch (e: any) {
     logger.error(e);
     prometheus.request_error.inc({pathname: pathname, status: 400});

diff --git a/src/fetch/history.ts b/src/fetch/history.ts
diff --git a/src/fetch/openapi.ts b/src/fetch/openapi.ts
@@ -3,10 +3,10 @@ import pkg from "../../package.json" assert { type: "json" };
 import { OpenApiBuilder, SchemaObject, ExampleObject, ParameterObject } from "openapi3-ts/oas31";
 import { config } from "../config.js";
 import { store } from "../clickhouse/stores.js";
-import { getBlock, getAggregate, getUAWFromDate, getUAWHistory, UAWHistory } from "../queries.js";
+import { getBlock, getAggregate, NormalizedHistoryData } from "../queries.js";
 import { registry } from "../prometheus.js";
 import { makeQuery } from "../clickhouse/makeQuery.js";
-import { parseUAWResponse } from "../utils.js";
+import { parseNormalized } from "../utils.js";
 
 const TAGS = {
   MONITORING: "Monitoring",
@@ -16,10 +16,9 @@ const TAGS = {
 } as const;
 
 const block_example = (await makeQuery(await getBlock( new URLSearchParams({limit: "2"})))).data;
-const trace_calls_example = (await makeQuery(getAggregate( new URLSearchParams({aggregate_function: "count", chain: "wax"}), "trace_calls"))).data;
-const transaction_traces_example = (await makeQuery(getAggregate( new URLSearchParams({aggregate_function: "count", chain: "wax"}), "transaction_traces"))).data;
-const uaw_example = (await makeQuery(getUAWFromDate( new URLSearchParams({chain: "wax", date: "2023-09-06"})))).data;
-const history_example = parseUAWResponse((await makeQuery<UAWHistory>(getUAWHistory( new URLSearchParams({chain: "eos", range: "7"})))).data);
+const trace_calls_example = parseNormalized((await makeQuery<NormalizedHistoryData>(getAggregate( new URLSearchParams({aggregate_function: "count", chain: "wax"}), "trace_calls"))).data, 86400);
+const transaction_traces_example = parseNormalized((await makeQuery<NormalizedHistoryData>(getAggregate( new URLSearchParams({aggregate_function: "count", chain: "wax"}), "transaction_traces"))).data, 86400);
+const uaw_example = parseNormalized((await makeQuery<NormalizedHistoryData>(getAggregate( new URLSearchParams({chain: "eos", range: "24h"}), "uaw"))).data, 86400);
 
 const timestampSchema: SchemaObject = { anyOf: [
     {type: "number"},
@@ -154,7 +153,7 @@ export default new OpenApiBuilder()
     get: {
       tags: [TAGS.USAGE],
       summary: "Get aggregate of trace_calls",
-      description: "Get aggregate of trace_calls filtered by `chain`, `timestamp` or `block_number`",
+      description: "Get aggregate of trace_calls for given time range filtered by `chain`",
       parameters: [
         {
           name: "aggregate_function",
@@ -171,39 +170,12 @@ export default new OpenApiBuilder()
           schema: {enum: await store.chains},
         },
         {
-          name: 'timestamp',
-          in: 'query',
-          description: 'Filter by exact timestamp',
-          required: false,
-          schema: timestampSchema,
-          examples: timestampExamples,
-        },
-        {
-          name: "block_number",
-          description: "Filter by Block number (ex: 18399498)",
+          name: "range",
           in: "query",
+          description: "Time range to query (ex: 24h)",
           required: false,
-          schema: { type: "number" },
-        },
-        ...["greater_or_equals_by_timestamp", "greater_by_timestamp", "less_or_equals_by_timestamp", "less_by_timestamp"].map(name => {
-          return {
-            name,
-            in: "query",
-            description: "Filter " + name.replace(/_/g, " "),
-            required: false,
-            schema: timestampSchema,
-            examples: timestampExamples,
-          } as ParameterObject
-        }),
-        ...["greater_or_equals_by_block_number", "greater_by_block_number", "less_or_equals_by_block_number", "less_by_block_number"].map(name => {
-          return {
-            name,
-            in: "query",
-            description: "Filter " + name.replace(/_/g, " "),
-            required: false,
-            schema: { type: "number" },
-          } as ParameterObject
-        }),
+          schema: { enum: ["24h", "7d", "30d", "90d", "1y", "all"] },
+        }
       ],
       responses: {
         200: { description: "Aggregate of sales", content: { "text/plain": { example: trace_calls_example} } },
@@ -215,7 +187,7 @@ export default new OpenApiBuilder()
     get: {
       tags: [TAGS.USAGE],
       summary: "Get aggregate of transaction_traces",
-      description: "Get aggregate of transaction_traces filtered by `chain`, `timestamp` or `block_number`",
+      description: "Get aggregate of transaction_traces for given time range filtered by `chain`",
       parameters: [
         {
           name: "aggregate_function",
@@ -232,39 +204,12 @@ export default new OpenApiBuilder()
           schema: {enum: await store.chains},
         },
         {
-          name: 'timestamp',
-          in: 'query',
-          description: 'Filter by exact timestamp',
-          required: false,
-          schema: timestampSchema,
-          examples: timestampExamples,
-        },
-        {
-          name: "block_number",
-          description: "Filter by Block number (ex: 18399498)",
+          name: "range",
           in: "query",
+          description: "Time range to query (ex: 24h)",
           required: false,
-          schema: { type: "number" },
-        },
-        ...["greater_or_equals_by_timestamp", "greater_by_timestamp", "less_or_equals_by_timestamp", "less_by_timestamp"].map(name => {
-          return {
-            name,
-            in: "query",
-            description: "Filter " + name.replace(/_/g, " "),
-            required: false,
-            schema: timestampSchema,
-            examples: timestampExamples,
-          } as ParameterObject
-        }),
-        ...["greater_or_equals_by_block_number", "greater_by_block_number", "less_or_equals_by_block_number", "less_by_block_number"].map(name => {
-          return {
-            name,
-            in: "query",
-            description: "Filter " + name.replace(/_/g, " "),
-            required: false,
-            schema: { type: "number" },
-          } as ParameterObject
-        }),
+          schema: { enum: ["24h", "7d", "30d", "90d", "1y", "all"] },
+        }
       ],
       responses: {
         200: { description: "Aggregate of sales", content: { "text/plain": { example: transaction_traces_example} } },
@@ -273,34 +218,6 @@ export default new OpenApiBuilder()
     },
   })
   .addPath("/uaw", {
-    get: {
-      tags: [TAGS.USAGE],
-      summary: "Get unique active wallets",
-      description: "Get unique active wallets filtered by `chain` and `date`",
-      parameters: [
-        {
-          name: "chain",
-          in: "query",
-          description: "Filter by chain name",
-          required: false,
-          schema: {enum: await store.chains},
-        },
-        {
-          name: "date",
-          description: "Filter by date (ex: 2023-09-06)",
-          in: "query",
-          required: false,
-          schema: DateSchema,
-          examples: DateExamples,
-        },
-      ],
-      responses: {
-        200: { description: "Unique active wallets", content: { "text/plain": { example: uaw_example} } },
-        400: { description: "Bad request", content: { "text/plain": { example: "Bad request", schema: { type: "string" } } }, },
-      },
-    },
-  })
-  .addPath("/uaw/history", {
     get: {
       tags: [TAGS.USAGE],
       summary: "Get daily unique active wallets",
@@ -316,13 +233,13 @@ export default new OpenApiBuilder()
         {
           name: "range",
           in: "query",
-          description: "Time range to query (ex: 7d)",
+          description: "Time range to query (ex: 24h)",
           required: false,
           schema: { enum: ["24h", "7d", "30d", "90d", "1y", "all"] },
         }
       ],
       responses: {
-        200: { description: "Daily active wallets", content: { "text/plain": { example: history_example} } },
+        200: { description: "Daily active wallets", content: { "text/plain": { example: uaw_example} } },
         400: { description: "Bad request", content: { "text/plain": { example: "Bad request", schema: { type: "string" } } }, },
       },
     },

diff --git a/src/fetch/uaw.ts b/src/fetch/uaw.ts
diff --git a/src/queries.spec.ts b/src/queries.spec.ts
@@ -1,5 +1,5 @@
 import { expect, jest, mock, test } from "bun:test";
-import { createBlockQuery, getBlock, getAggregate, getUAWFromDate, getUAWHistory } from "./queries.js";
+import { createBlockQuery, getBlock, getAggregate } from "./queries.js";
 import { store } from "./clickhouse/stores.js";
 
 // Mock supported chains data to prevent DB query
@@ -29,33 +29,11 @@ test("getBlock", async () => {
 });
 
 test("getAggregate", async () => {
-    const singleChainQuery = new URLSearchParams({ chain: "wax"});
-    expect(getAggregate(singleChainQuery, "trace_calls"))
-        .toBe(`SELECT chain, count(trace_calls) FROM BlockStats WHERE (chain == 'wax') GROUP BY chain`);
-
-    expect(getAggregate(new URLSearchParams(), "transaction_traces"))
-        .toBe(`SELECT chain, count(transaction_traces) FROM BlockStats GROUP BY chain`);
-});
-
-test("getUAWFromDate", async () => {
-    const singleChainQuery = new URLSearchParams({ chain: "wax", date: "2023-09-06" });
-    expect(getUAWFromDate(singleChainQuery))
-        .toBe(`SELECT chain, count(distinct uaw) FROM BlockStats ARRAY JOIN uaw WHERE (toUnixTimestamp(DATE(timestamp)) == toUnixTimestamp(DATE(1693958400)) AND chain == 'wax') GROUP BY chain`);
-
-    expect(getUAWFromDate(new URLSearchParams({ date: "2023-09-06" })))
-        .toBe(`SELECT chain, count(distinct uaw) FROM BlockStats ARRAY JOIN uaw WHERE (toUnixTimestamp(DATE(timestamp)) == toUnixTimestamp(DATE(1693958400))) GROUP BY chain`);
-});
-
-test("getUAWHistory", async () => {
     const date_of_query = Math.floor(Number(new Date().setHours(0,0,0,0)) / 1000);
     const datetime_of_query = Math.floor(Number(new Date()) / 1000);
+    expect(getAggregate(new URLSearchParams({ chain: "wax" }), "trace_calls"))
+        .toBe(`SELECT chain, toUnixTimestamp(DATE(timestamp)) as timestamp, sum(trace_calls) as value FROM BlockStats WHERE (timestamp BETWEEN ${datetime_of_query} - 3600 * 24 AND ${datetime_of_query} AND chain == 'wax') GROUP BY chain, timestamp ORDER BY timestamp ASC`);
 
-    expect(getUAWHistory(new URLSearchParams({ chain: "eos", range: "7d" })))
-        .toBe(`SELECT chain, toUnixTimestamp(DATE(timestamp)) as day, count(distinct uaw) as UAW  FROM BlockStats ARRAY JOIN uaw WHERE (timestamp BETWEEN ${date_of_query} - 86400 * 7 AND ${date_of_query} AND chain == 'eos') GROUP BY chain, day ORDER BY day ASC`);
-
-    expect(getUAWHistory(new URLSearchParams({ range: "24h" })))
-        .toBe(`SELECT chain, toUnixTimestamp(DATE(timestamp)) as day, count(distinct uaw) as UAW  FROM BlockStats ARRAY JOIN uaw WHERE (timestamp BETWEEN ${datetime_of_query} - 3600 * 24 AND ${datetime_of_query}) GROUP BY chain, day ORDER BY day ASC`);
-
-    expect(getUAWHistory(new URLSearchParams({ range: "1y" })))
-        .toBe(`SELECT chain, toUnixTimestamp(DATE(timestamp)) as day, count(distinct uaw) as UAW  FROM BlockStats ARRAY JOIN uaw WHERE (timestamp BETWEEN ${date_of_query} - 31536000 * 1 AND ${date_of_query}) GROUP BY chain, day ORDER BY day ASC`);
+    expect(getAggregate(new URLSearchParams({ range: "7d"}), "transaction_traces"))
+        .toBe(`SELECT chain, toUnixTimestamp(DATE(timestamp)) as timestamp, sum(transaction_traces) as value FROM BlockStats WHERE (timestamp BETWEEN ${date_of_query} - 86400 * 7 AND ${date_of_query}) GROUP BY chain, timestamp ORDER BY timestamp ASC`);
 });