Merge latest changes (#17)

* Code refactoring and comments (#16) * Fix typo and console highlights in README * Fix wrong environment variables names
pinax-network · Oct 17, 2023 · dd69ea2 · dd69ea2
1 parent bb6a1fc
commit dd69ea2
Show file tree

Hide file tree

Showing 11 changed files with 196 additions and 97 deletions.
diff --git a/.github/workflows/bun-build.yml b/.github/workflows/bun-build.yml
@@ -14,8 +14,13 @@ jobs:
       - uses: oven-sh/setup-bun@v1
         with:
           bun-version: latest
-      - run: bun install
-      - run: bun run build
+
+      - name: 'Install Dependencies'
+        run: bun install
+
+      - name: 'Build app'
+        run: bun run build
+
       - uses: softprops/action-gh-release@v1
         with:
           files: |

diff --git a/.github/workflows/bun-test.yml b/.github/workflows/bun-test.yml
@@ -38,5 +38,5 @@ jobs:
           PORT: ${{ vars.PORT }}
           DB_HOST: ${{ vars.DB_HOST }}
           DB_NAME: ${{ secrets.DB_NAME }}
-          DB_USERNAME: ${{ secrets.USERNAME }}
+          DB_USERNAME: ${{ secrets.DB_USERNAME }}
           DB_PASSWORD: ''
diff --git a/README.md b/README.md
@@ -1,5 +1,93 @@
-# substreams-clock-api
+# [`Substreams`](https://substreams.streamingfast.io/) Clock API
 
 [![.github/workflows/bun-test.yml](https://github.com/pinax-network/substreams-clock-api/actions/workflows/bun-test.yml/badge.svg)](https://github.com/pinax-network/substreams-clock-api/actions/workflows/bun-test.yml)
 
-> Timestamps <> Block numbers conversion for your favorite chains
+> Convert Timestamps <> Block numbers and query latest blocks height for your favorite chains !
+
+## REST API
+
+| Pathname                                  | Description           |
+|-------------------------------------------|-----------------------|
+| GET `/`                                   | Banner
+| GET `/chains`                             | Returns all available `chains`
+| GET `/health`                             | Health check
+| ~~GET `/metrics`~~ (SOON)                 | ~~Prometheus metrics~~
+| GET `/openapi`                            | [OpenAPI v3 JSON](https://spec.openapis.org/oas/v3.0.0)
+| GET `/swagger`                            | [Swagger UI](https://swagger.io/resources/open-api/)
+| GET `/{chain}/current`                   | Latest block number on the chain
+| GET `/{chain}/final`                      | Latest finalized block number on the chain
+| GET `/{chain}/timestamp?block_number=`    | Timestamp query from a block number or array (comma-separated)
+| GET `/{chain}/blocknum?timestamp=`        | Block number query from a timestamp or array (comma-separated)
+
+## Requirements
+
+- [Clickhouse](clickhouse.com/)
+
+Additionnaly to pull data directly from a substream:
+- [Substreams Sink Clickhouse](https://github.com/pinax-network/substreams-sink-clickhouse/)
+
+## Quickstart
+
+```console
+$ bun install
+$ bun dev
+```
+
+## [`Bun` Binary Releases](https://github.com/pinax-network/substreams-sink-websockets/releases)
+
+> Linux Only
+
+```console
+$ wget https://github.com/pinax-network/substreams-clock-api/releases/download/v0.2.0/substreams-clock-api
+$ chmod +x ./substreams-clock-api
+```
+
+## `.env` Environment variables
+
+```env
+# Optional
+PORT=8080
+HOSTNAME=localhost
+DB_HOST=http://localhost:8123
+DB_NAME=demo
+DB_USERNAME=default
+DB_PASSWORD=
+```
+
+## Help
+
+```console
+$ ./substreams-clock-api --help
+Usage: substreams-clock-api [options]
+
+Timestamps <> Block numbers conversion for your favorite chains
+
+Options:
+  --port <int>                     Server listen on HTTP port (default: "8080", env: PORT)
+  --hostname <string>              Server listen on HTTP hostname (default: "localhost", env: HOST)
+  --db-host <string>               Clickhouse DB HTTP hostname (default: "http://localhost:8123", env: dbHost)
+  --name <string>                  Clickhouse DB table name (default: "demo", env: DB_NAME)
+  --username <string>              Clickhouse DB username (default: "default", env: DB_USERNAME)
+  --password <string>              Clickhouse DB password (default: "", env: DB_PASSWORD)
+  --max-elements-queried <string>  Maximum number of query elements when using arrays as parameters (default: 10, env: MAX_ELEMENTS_QUERIED)
+  --verbose <boolean>              Enable verbose logging (default: false, env: VERBOSE)
+  -V, --version                    output the version number
+  -h, --help                       display help for command
+```
+
+## Docker environment (SOON)
+
+<!-- Pull from GitHub Container registry
+```bash
+docker pull ghcr.io/pinax-network/substreams-sink-websockets:latest
+```
+
+Build from source
+```bash
+docker build -t substreams-sink-websockets .
+```
+
+Run with `.env` file
+```bash
+docker run -it --rm --env-file .env ghcr.io/pinax-network/substreams-sink-websockets
+``` -->
diff --git a/src/banner.ts b/src/banner.ts
@@ -21,8 +21,8 @@ export function banner()  {
         /health
         /openapi
         /swagger
-        /{chain}/timestamp?block_number=<positive integer>
-        /{chain}/blocknum?timestamp=<UNIX or date>
+        /{chain}/timestamp?block_number=<positive integer or comma-separated>
+        /{chain}/blocknum?timestamp=<UNIX or date or comma-separated>
         /{chain}/current
         /{chain}/final
 `

diff --git a/src/config.ts b/src/config.ts
@@ -21,7 +21,7 @@ const CommanderSchema = z.object({
     name: z.string().default(DEFAULT_DB_NAME),
     username: z.string().default(DEFAULT_DB_USERNAME),
     password: z.string().default(DEFAULT_DB_PASSWORD),
-    maxElementsQueried: z.coerce.number().default(DEFAULT_MAX_ELEMENTS_QUERIES).describe(
+    maxElementsQueried: z.coerce.number().gte(2).default(DEFAULT_MAX_ELEMENTS_QUERIES).describe(
         'Maximum number of query elements when using arrays as parameters'
     )
 });
@@ -37,12 +37,13 @@ const opts = new Command()
     .showHelpAfterError()
     .addOption(new Option("--port <int>", "Server listen on HTTP port").default(DEFAULT_PORT).env("PORT"))
     .addOption(new Option("--hostname <string>", "Server listen on HTTP hostname").default(DEFAULT_HOSTNAME).env("HOST"))
-    .addOption(new Option("--db-host <string>", "Clickhouse DB HTTP hostname").default(DEFAULT_DB_HOST).env("dbHost"))
+    .addOption(new Option("--db-host <string>", "Clickhouse DB HTTP hostname").default(DEFAULT_DB_HOST).env("DB_HOST"))
     .addOption(new Option("--name <string>", "Clickhouse DB table name").default(DEFAULT_DB_NAME).env("DB_NAME"))
     .addOption(new Option("--username <string>", "Clickhouse DB username").default(DEFAULT_DB_USERNAME).env("DB_USERNAME"))
     .addOption(new Option("--password <string>", "Clickhouse DB password").default(DEFAULT_DB_PASSWORD).env("DB_PASSWORD"))
-    .addOption(new Option("--max-elements-queried <string>", "Maximum number of query elements when using arrays as parameters")
-        .default(DEFAULT_MAX_ELEMENTS_QUERIES).env("MAX_ELEMENTS_QUERIED"))
+    .addOption(new Option("--max-elements-queried <string>",
+        "Maximum number of query elements when using arrays as parameters (warning: setting a very high number can allow for intensive DB workload)"
+        ).default(DEFAULT_MAX_ELEMENTS_QUERIES).env("MAX_ELEMENTS_QUERIED"))
     .addOption(new Option("--verbose <boolean>", "Enable verbose logging").default(DEFAULT_VERBOSE).env("VERBOSE")) // TODO: Use verbose logging
     .version(pkg.version)
     .parse(process.argv).opts();

diff --git a/src/index.ts b/src/index.ts
@@ -4,25 +4,26 @@ import { serveStatic } from 'hono/bun'
 import { HTTPException } from 'hono/http-exception';
 import { logger } from 'hono/logger';
 
-import pkg from "../package.json";
 import * as routes from './routes';
+import config from "./config";
+import pkg from "../package.json";
 import {
     type BlockchainSchema, type BlocknumSchema, type TimestampSchema,
     type BlocktimeQueryResponseSchema, type SingleBlocknumQueryResponseSchema, type SupportedChainsQueryResponseSchema
 } from './schemas';
-import config from "./config";
 import { banner } from "./banner";
 import { supportedChainsQuery, timestampQuery, blocknumQuery, currentBlocknumQuery, finalBlocknumQuery } from "./queries";
 
-// Export a function to be able to create App in tests as default export is different for setting Bun port/hostname
+// Export app as a function to be able to create it in tests as well.
+// Default export is different for setting Bun port/hostname than running tests.
 // See (https://hono.dev/getting-started/bun#change-port-number) vs. (https://hono.dev/getting-started/bun#_3-hello-world)
 export function generateApp() {
     const app = new OpenAPIHono();
 
     if ( config.NODE_ENV !== "production" )
-        app.use('*', logger());
+        app.use('*', logger()); // TODO: Custom logger based on config.verbose
 
-    app.use('/swagger/*', serveStatic({ root: './' }))
+    app.use('/swagger/*', serveStatic({ root: './' }));
 
     app.doc('/openapi', {
         openapi: '3.0.0',

diff --git a/src/queries.ts b/src/queries.ts
@@ -11,7 +11,7 @@ type JSONObjectEachRow = {
     }
 };
 
-async function makeQuery(query: string, format: string = 'JSONObjectEachRow') {
+async function makeQuery(query: string, format: string = 'JSONObjectEachRow'): Promise<any> {
     const response = await fetch(`${config.dbHost}/?default_format=${format}`, {
         method: "POST",
         body: query,
@@ -36,10 +36,7 @@ async function makeQuery(query: string, format: string = 'JSONObjectEachRow') {
     return json;
 }
 
-export async function timestampQuery(blockchain: string, blocknum: number | number[]) { // TODO: Merge `timestampQuery` / `blocknumQuery`
-    const query = `SELECT (blockchain, blocknum, timestamp) FROM ${config.name} WHERE (blockchain == '${blockchain}') AND (blocknum IN (${blocknum.toString()}))`;
-    const json = await makeQuery(query);
-
+function parseBlockTimeQueryResponse(json: JSONObjectEachRow): BlocktimeQueryResponsesSchema {
     return BlocktimeQueryResponsesSchema.parse(
         Object.values(json as JSONObjectEachRow).map((r: {
             [key: string]: Array<string>
@@ -52,23 +49,21 @@ export async function timestampQuery(blockchain: string, blocknum: number | numb
     }));
 }
 
-export async function blocknumQuery(blockchain: string, timestamp: Date | Date[]) {
+export async function timestampQuery(blockchain: string, blocknum: number | number[]): Promise<BlocktimeQueryResponsesSchema> {
+    const query = `SELECT (blockchain, blocknum, timestamp) FROM ${config.name} WHERE (blockchain == '${blockchain}') AND (blocknum IN (${blocknum.toString()}))`;
+    const json = await makeQuery(query);
+
+    return parseBlockTimeQueryResponse(json);
+}
+
+export async function blocknumQuery(blockchain: string, timestamp: Date | Date[]): Promise<BlocktimeQueryResponsesSchema> {
     timestamp = Array.isArray(timestamp) ? timestamp : [timestamp];
     const query = `SELECT (blockchain, blocknum, timestamp) FROM ${config.name} WHERE (blockchain == '${blockchain}') AND (timestamp IN (${
-        timestamp.map((t) => '\'' + t.toISOString().replace('T', ' ').substring(0, 19) + '\'').toString()
+        timestamp.map((t) => '\'' + t.toISOString().replace('T', ' ').substring(0, 19) + '\'').toString() // Format dates to find them in DB (mock data)
     }))`; // TODO: Find closest instead of matching timestamp or another route ?
     const json = await makeQuery(query);
 
-    return BlocktimeQueryResponsesSchema.parse(
-        Object.values(json as JSONObjectEachRow).map((r: {
-            [key: string]: Array<string>
-        }) => {
-        return BlocktimeQueryResponseSchema.parse({
-            blockchain: Object.values(r)[0][0],
-            block_number: Object.values(r)[0][1],
-            timestamp: Object.values(r)[0][2]
-        });
-    }));
+    return parseBlockTimeQueryResponse(json)
 }
 
 export async function currentBlocknumQuery(blockchain: string) {
@@ -95,7 +90,8 @@ export async function finalBlocknumQuery(blockchain: string) {
 
 export async function supportedChainsQuery() {
     const query = `SELECT DISTINCT blockchain FROM ${config.name}`;
-    // Required for returning a const value in order to make z.enum() work in the schema definitions
+
+    // Required format for returning a const value in order to make z.enum() work in the schema definitions
     const json = await makeQuery(query, 'JSONColumns');
 
     return json.blockchain

diff --git a/src/routes.ts b/src/routes.ts
@@ -1,6 +1,43 @@
 import { createRoute } from '@hono/zod-openapi';
 import * as schemas from './schemas';
 
+export const indexRoute = createRoute({
+    method: 'get',
+    path: '/',
+    responses: {
+        200: {
+            description: 'Index page banner.',
+        },
+    },
+});
+
+export const healthCheckRoute = createRoute({
+    method: 'get',
+    path: '/health',
+    responses: {
+        200: {
+            description: 'Health check service.',
+        },
+    },
+});
+
+export const supportedChainsRoute = createRoute({
+    method: 'get',
+    path: '/chains',
+    responses: {
+        200: {
+            content: {
+                'application/json': {
+                    schema: schemas.SupportedChainsQueryResponseSchema,
+                },
+            },
+            description: 'Fetch supported chains from the Clickhouse DB.',
+        },
+    },
+});
+
+// Note: OpenAPI and SwaggerUI routes are created directly in `index.ts`
+
 export const blocknumQueryRoute = createRoute({
     method: 'get',
     path: '/{chain}/blocknum',
@@ -73,39 +110,4 @@ export const finalBlocknumQueryRoute = createRoute({
             description: 'Retrieve the latest final block number on the blockchain.',
         },
     },
-});
-
-export const indexRoute = createRoute({
-    method: 'get',
-    path: '/',
-    responses: {
-        200: {
-            description: 'Index page banner.',
-        },
-    },
-});
-
-export const healthCheckRoute = createRoute({
-    method: 'get',
-    path: '/health',
-    responses: {
-        200: {
-            description: 'Health check service.',
-        },
-    },
-});
-
-export const supportedChainsRoute = createRoute({
-    method: 'get',
-    path: '/chains',
-    responses: {
-        200: {
-            content: {
-                'application/json': {
-                    schema: schemas.SupportedChainsQueryResponseSchema,
-                },
-            },
-            description: 'Fetch supported chains from the Clickhouse DB.',
-        },
-    },
-});
+});