From 78b6bbf0947a70bbf02c1fa39acea4dcdc6da098 Mon Sep 17 00:00:00 2001 From: Etienne Donneger Date: Fri, 21 Jun 2024 13:02:21 -0400 Subject: [PATCH] Add support for `block_range` parameter Includes a refactor of the parameters type coercion to be run only once when starting the app. --- index.ts | 64 ++++-------------------------------- src/types/api.ts | 84 +++++++++++++++++++++++++++++++++++++++++++++--- src/usage.ts | 23 ++++++++++--- 3 files changed, 105 insertions(+), 66 deletions(-) diff --git a/index.ts b/index.ts index 0c42e8c..d083749 100644 --- a/index.ts +++ b/index.ts @@ -2,13 +2,14 @@ import client from './src/clickhouse/client.js'; import openapi from "./tsp-output/@typespec/openapi3/openapi.json"; import { Hono } from "hono"; -import { ZodBigInt, ZodBoolean, ZodDate, ZodDefault, ZodNumber, ZodOptional, ZodTypeAny, ZodUndefined, ZodUnion, z } from "zod"; +import { z } from 'zod'; import { EndpointByMethod } from './src/types/zod.gen.js'; -import { APP_VERSION, config } from "./src/config.js"; +import { APP_VERSION } from "./src/config.js"; import { logger } from './src/logger.js'; import * as prometheus from './src/prometheus.js'; import { makeUsageQuery } from "./src/usage.js"; import { APIErrorResponse } from "./src/utils.js"; +import { fixEndpointParametersCoercion } from "./src/types/api.js"; import type { Context } from "hono"; import type { EndpointParameters, EndpointReturnTypes, UsageEndpoints } from "./src/types/api.js"; @@ -61,68 +62,15 @@ function AntelopeTokenAPI() { "/metrics", async (_) => new Response(await prometheus.registry.metrics(), { headers: { "Content-Type": prometheus.registry.contentType } }) ); + + // Call once + fixEndpointParametersCoercion(); const createUsageEndpoint = (endpoint: UsageEndpoints) => app.get( // Hono using different syntax than OpenAPI for path parameters // `/{path_param}` (OpenAPI) VS `/:path_param` (Hono) endpoint.replace(/{([^}]+)}/, ":$1"), async (ctx: Context) => { - // Add type coercion for query and path parameters since the codegen doesn't coerce types natively - const endpoint_parameters = Object.values(EndpointByMethod["get"][endpoint].parameters.shape).map(p => p.shape); - endpoint_parameters.forEach( - // `p` can be query or path parameters - (p) => Object.keys(p).forEach( - (key, _) => { - let zod_type = p[key] as ZodTypeAny; - let underlying_zod_type: ZodTypeAny; - let isOptional = false; - - // Strip default layer for value - if (zod_type instanceof ZodDefault) { - zod_type = zod_type.removeDefault(); - } - - // Detect the underlying type from the codegen - if (zod_type instanceof ZodUnion) { - underlying_zod_type = zod_type.options[0]; - isOptional = zod_type.options.some((o: ZodTypeAny) => o instanceof ZodUndefined); - } else if (zod_type instanceof ZodOptional) { - underlying_zod_type = zod_type.unwrap(); - isOptional = true; - } else { - underlying_zod_type = zod_type; - } - - // Query and path user input parameters come as strings and we need to coerce them to the right type using Zod - if (underlying_zod_type instanceof ZodNumber) { - p[key] = z.coerce.number(); - if (key === "limit") - p[key] = p[key].max(config.maxLimit); - } else if (underlying_zod_type instanceof ZodBoolean) { - p[key] = z.coerce.boolean(); - } else if (underlying_zod_type instanceof ZodBigInt) { - p[key] = z.coerce.bigint(); - } else if (underlying_zod_type instanceof ZodDate) { - p[key] = z.coerce.date(); - // Any other type will be coerced as string value directly - } else { - p[key] = z.string(); - } - - if (isOptional) - p[key] = p[key].optional(); - - // Mark parameters with default values explicitly as a workaround - // See https://github.com/astahmer/typed-openapi/issues/34 - if (key == "limit") - p[key] = p[key].default(10); - else if (key == "page") - p[key] = p[key].default(1); - - } - ) - ); - const result = EndpointByMethod["get"][endpoint].parameters.safeParse({ query: ctx.req.query(), path: ctx.req.param() diff --git a/src/types/api.ts b/src/types/api.ts index 905fe3e..6a05ce2 100644 --- a/src/types/api.ts +++ b/src/types/api.ts @@ -1,6 +1,7 @@ -import z from "zod"; +import { ZodArray, ZodBigInt, ZodBoolean, ZodDate, ZodDefault, ZodNumber, ZodOptional, ZodType, ZodTypeAny, ZodUndefined, ZodUnion, z } from "zod"; -import type { GetEndpoints } from './zod.gen.js'; +import { EndpointByMethod, type GetEndpoints } from './zod.gen.js'; +import { config } from "../config.js"; export type EndpointReturnTypes = E extends UsageEndpoints ? UsageResponse["data"] : z.infer; export type EndpointParameters = z.infer; @@ -9,10 +10,85 @@ export type EndpointParameters = z.infer; export type UsageResponse = z.infer; -export type ValidUserParams = EndpointParameters extends { path: unknown; } ? +export type ValidUserParams = EndpointParameters extends { path: unknown; } ? // Combine path and query parameters only if path exists to prevent "never" on intersection Extract, { query: unknown; }>["query"] & Extract, { path: unknown; }>["path"] : Extract, { query: unknown; }>["query"]; +export type AdditionalQueryParams = { offset?: number; min_block?: number; max_block?: number;} // Allow any valid parameters from the endpoint to be used as SQL query parameters with the addition of the `OFFSET` for pagination -export type ValidQueryParams = ValidUserParams & { offset?: number; }; \ No newline at end of file +export type ValidQueryParams = ValidUserParams & AdditionalQueryParams; + +export function fixEndpointParametersCoercion() { + // Add type coercion for query and path parameters since the codegen doesn't coerce types natively + for (const endpoint in EndpointByMethod["get"]) { + if (EndpointByMethod["get"][endpoint as UsageEndpoints].parameters.shape) { + Object.values(EndpointByMethod["get"][endpoint as UsageEndpoints].parameters.shape).map(p => p.shape).forEach( + // `p` can be query or path parameters + (p) => Object.keys(p).forEach( + (key, _) => { + let zod_type = p[key] as ZodTypeAny; + let underlying_zod_type: ZodTypeAny; + let isOptional = false; + + // Strip default layer for value + if (zod_type instanceof ZodDefault) { + zod_type = zod_type.removeDefault(); + } + + // Detect the underlying type from the codegen + if (zod_type instanceof ZodUnion) { + underlying_zod_type = zod_type.options[0]; + isOptional = zod_type.options.some((o: ZodTypeAny) => o instanceof ZodUndefined); + } else if (zod_type instanceof ZodOptional) { + underlying_zod_type = zod_type.unwrap(); + isOptional = true; + } else { + underlying_zod_type = zod_type; + } + + const coercePrimitive = (zod_type: ZodType) => { + if (zod_type instanceof ZodNumber) { + return z.coerce.number(); + } else if (zod_type instanceof ZodBoolean) { + return z.coerce.boolean(); + } else if (zod_type instanceof ZodBigInt) { + return z.coerce.bigint(); + } else if (zod_type instanceof ZodDate) { + return z.coerce.date(); + // Any other type will be coerced as string value directly + } else { + return z.string(); + } + }; + + if (underlying_zod_type instanceof ZodArray && underlying_zod_type.element instanceof ZodNumber) { + // Special case for `block_range` coercion, input is expected to be one or two values separated by comma + p[key] = z.preprocess( + (x) => String(x).split(','), + z.coerce.number().positive().array().min(1).max(2) + ); + } else { + p[key] = coercePrimitive(underlying_zod_type); + } + + if (key === "limit") + p[key] = p[key].max(config.maxLimit); + + // Need to mark optional before adding defaults + if (isOptional) + p[key] = p[key].optional(); + + // Mark parameters with default values explicitly as a workaround + // See https://github.com/astahmer/typed-openapi/issues/34 + if (key === "limit") + p[key] = p[key].default(10); + else if (key === "page") + p[key] = p[key].default(1); + + } + ) + ); + } + } +} \ No newline at end of file diff --git a/src/usage.ts b/src/usage.ts index b263d27..d57ba8c 100644 --- a/src/usage.ts +++ b/src/usage.ts @@ -2,7 +2,7 @@ import { makeQuery } from "./clickhouse/makeQuery.js"; import { APIErrorResponse } from "./utils.js"; import type { Context } from "hono"; -import type { EndpointReturnTypes, UsageEndpoints, UsageResponse, ValidUserParams } from "./types/api.js"; +import type { AdditionalQueryParams, EndpointReturnTypes, UsageEndpoints, UsageResponse, ValidUserParams } from "./types/api.js"; export async function makeUsageQuery(ctx: Context, endpoint: UsageEndpoints, user_params: ValidUserParams) { type EndpointElementReturnType = EndpointReturnTypes[number]; @@ -16,7 +16,8 @@ export async function makeUsageQuery(ctx: Context, endpoint: UsageEndpoints, use page = 1; let filters = ""; - for (const k of Object.keys(query_params).filter(k => k !== "limit")) // Don't add `limit` to WHERE clause + // Don't add `limit` and `block_range` to WHERE clause + for (const k of Object.keys(query_params).filter(k => k !== "limit" && k !== "block_range")) filters += ` (${k} = {${k}: String}) AND`; filters = filters.substring(0, filters.lastIndexOf(' ')); // Remove last item ` AND` @@ -24,6 +25,7 @@ export async function makeUsageQuery(ctx: Context, endpoint: UsageEndpoints, use filters = `WHERE ${filters}` let query = ""; + let additional_query_params: AdditionalQueryParams = {}; if (endpoint == "/balance" || endpoint == "/supply") { // Need to narrow the type of `query_params` explicitly to access properties based on endpoint value // See https://github.com/microsoft/TypeScript/issues/33014 @@ -47,7 +49,19 @@ export async function makeUsageQuery(ctx: Context, endpoint: UsageEndpoints, use query += `SELECT * FROM `; const q = query_params as ValidUserParams; - if (q.from) { + if (q.block_range) { + query += `transfers_block_num`; + console.log(q.block_range); + if (q.block_range[0] && q.block_range[1]) { + filters += "AND (block_num >= {min_block: int} AND block_num <= {max_block: int})" + // Use Min/Max to account for any ordering of parameters + additional_query_params.min_block = Math.min(q.block_range[0], q.block_range[1]); + additional_query_params.max_block = Math.max(q.block_range[0], q.block_range[1]); + } else if (q.block_range[0]) { + filters += "AND (block_num >= {min_block: int})" + additional_query_params.min_block = q.block_range[0]; + } + } else if (q.from) { // Find all incoming and outgoing transfers from single account if (q.to && q.to === q.from) filters = filters.replace( @@ -79,8 +93,9 @@ export async function makeUsageQuery(ctx: Context, endpoint: UsageEndpoints, use query += " OFFSET {offset: int}"; let query_results; + additional_query_params.offset = query_params.limit * (page - 1); try { - query_results = await makeQuery(query, { ...query_params, offset: query_params.limit * (page - 1) }); + query_results = await makeQuery(query, { ...query_params, ...additional_query_params }); } catch (err) { return APIErrorResponse(ctx, 500, "bad_database_response", err); }