Skip to content

Commit

Permalink
Add support for block_range parameter
Browse files Browse the repository at this point in the history 
Includes a refactor of the parameters type coercion to be run only
once when starting the app.
  • Loading branch information
@0237h
0237h committed Jun 21, 2024
1 parent a2415a3 commit 78b6bbf
Show file tree
Hide file tree
Showing 3 changed files with 105 additions and 66 deletions.
64 changes: 6 additions & 58 deletions index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,14 @@ import client from './src/clickhouse/client.js';
import openapi from "./tsp-output/@typespec/openapi3/openapi.json";

import { Hono } from "hono";
import { ZodBigInt, ZodBoolean, ZodDate, ZodDefault, ZodNumber, ZodOptional, ZodTypeAny, ZodUndefined, ZodUnion, z } from "zod";
import { z } from 'zod';
import { EndpointByMethod } from './src/types/zod.gen.js';
import { APP_VERSION, config } from "./src/config.js";
import { APP_VERSION } from "./src/config.js";
import { logger } from './src/logger.js';
import * as prometheus from './src/prometheus.js';
import { makeUsageQuery } from "./src/usage.js";
import { APIErrorResponse } from "./src/utils.js";
import { fixEndpointParametersCoercion } from "./src/types/api.js";

import type { Context } from "hono";
import type { EndpointParameters, EndpointReturnTypes, UsageEndpoints } from "./src/types/api.js";
Expand Down Expand Up @@ -61,68 +62,15 @@ function AntelopeTokenAPI() {
"/metrics",
async (_) => new Response(await prometheus.registry.metrics(), { headers: { "Content-Type": prometheus.registry.contentType } })
);

// Call once
fixEndpointParametersCoercion();

const createUsageEndpoint = (endpoint: UsageEndpoints) => app.get(
// Hono using different syntax than OpenAPI for path parameters
// `/{path_param}` (OpenAPI) VS `/:path_param` (Hono)
endpoint.replace(/{([^}]+)}/, ":$1"),
async (ctx: Context) => {
// Add type coercion for query and path parameters since the codegen doesn't coerce types natively
const endpoint_parameters = Object.values(EndpointByMethod["get"][endpoint].parameters.shape).map(p => p.shape);
endpoint_parameters.forEach(
// `p` can be query or path parameters
(p) => Object.keys(p).forEach(
(key, _) => {
let zod_type = p[key] as ZodTypeAny;
let underlying_zod_type: ZodTypeAny;
let isOptional = false;

// Strip default layer for value
if (zod_type instanceof ZodDefault) {
zod_type = zod_type.removeDefault();
}

// Detect the underlying type from the codegen
if (zod_type instanceof ZodUnion) {
underlying_zod_type = zod_type.options[0];
isOptional = zod_type.options.some((o: ZodTypeAny) => o instanceof ZodUndefined);
} else if (zod_type instanceof ZodOptional) {
underlying_zod_type = zod_type.unwrap();
isOptional = true;
} else {
underlying_zod_type = zod_type;
}

// Query and path user input parameters come as strings and we need to coerce them to the right type using Zod
if (underlying_zod_type instanceof ZodNumber) {
p[key] = z.coerce.number();
if (key === "limit")
p[key] = p[key].max(config.maxLimit);
} else if (underlying_zod_type instanceof ZodBoolean) {
p[key] = z.coerce.boolean();
} else if (underlying_zod_type instanceof ZodBigInt) {
p[key] = z.coerce.bigint();
} else if (underlying_zod_type instanceof ZodDate) {
p[key] = z.coerce.date();
// Any other type will be coerced as string value directly
} else {
p[key] = z.string();
}

if (isOptional)
p[key] = p[key].optional();

// Mark parameters with default values explicitly as a workaround
// See https://github.com/astahmer/typed-openapi/issues/34
if (key == "limit")
p[key] = p[key].default(10);
else if (key == "page")
p[key] = p[key].default(1);

}
)
);

const result = EndpointByMethod["get"][endpoint].parameters.safeParse({
query: ctx.req.query(),
path: ctx.req.param()
Expand Down
84 changes: 80 additions & 4 deletions src/types/api.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import z from "zod";
import { ZodArray, ZodBigInt, ZodBoolean, ZodDate, ZodDefault, ZodNumber, ZodOptional, ZodType, ZodTypeAny, ZodUndefined, ZodUnion, z } from "zod";

import type { GetEndpoints } from './zod.gen.js';
import { EndpointByMethod, type GetEndpoints } from './zod.gen.js';
import { config } from "../config.js";

export type EndpointReturnTypes<E extends keyof GetEndpoints> = E extends UsageEndpoints ? UsageResponse["data"] : z.infer<GetEndpoints[E]["response"]>;
export type EndpointParameters<E extends keyof GetEndpoints> = z.infer<GetEndpoints[E]["parameters"]>;
Expand All @@ -9,10 +10,85 @@ export type EndpointParameters<E extends keyof GetEndpoints> = z.infer<GetEndpoi
export type UsageEndpoints = Exclude<keyof GetEndpoints, "/health" | "/metrics" | "/version" | "/openapi">;
export type UsageResponse = z.infer<GetEndpoints[UsageEndpoints]["response"]>;

export type ValidUserParams<E extends UsageEndpoints> = EndpointParameters<E> extends { path: unknown; } ?
export type ValidUserParams<E extends UsageEndpoints> = EndpointParameters<E> extends { path: unknown; } ?
// Combine path and query parameters only if path exists to prevent "never" on intersection
Extract<EndpointParameters<E>, { query: unknown; }>["query"] & Extract<EndpointParameters<E>, { path: unknown; }>["path"]
:
Extract<EndpointParameters<E>, { query: unknown; }>["query"];
export type AdditionalQueryParams = { offset?: number; min_block?: number; max_block?: number;}
// Allow any valid parameters from the endpoint to be used as SQL query parameters with the addition of the `OFFSET` for pagination
export type ValidQueryParams = ValidUserParams<UsageEndpoints> & { offset?: number; };
export type ValidQueryParams = ValidUserParams<UsageEndpoints> & AdditionalQueryParams;

export function fixEndpointParametersCoercion() {
// Add type coercion for query and path parameters since the codegen doesn't coerce types natively
for (const endpoint in EndpointByMethod["get"]) {
if (EndpointByMethod["get"][endpoint as UsageEndpoints].parameters.shape) {
Object.values(EndpointByMethod["get"][endpoint as UsageEndpoints].parameters.shape).map(p => p.shape).forEach(
// `p` can be query or path parameters
(p) => Object.keys(p).forEach(
(key, _) => {
let zod_type = p[key] as ZodTypeAny;
let underlying_zod_type: ZodTypeAny;
let isOptional = false;

// Strip default layer for value
if (zod_type instanceof ZodDefault) {
zod_type = zod_type.removeDefault();
}

// Detect the underlying type from the codegen
if (zod_type instanceof ZodUnion) {
underlying_zod_type = zod_type.options[0];
isOptional = zod_type.options.some((o: ZodTypeAny) => o instanceof ZodUndefined);
} else if (zod_type instanceof ZodOptional) {
underlying_zod_type = zod_type.unwrap();
isOptional = true;
} else {
underlying_zod_type = zod_type;
}

const coercePrimitive = (zod_type: ZodType) => {
if (zod_type instanceof ZodNumber) {
return z.coerce.number();
} else if (zod_type instanceof ZodBoolean) {
return z.coerce.boolean();
} else if (zod_type instanceof ZodBigInt) {
return z.coerce.bigint();
} else if (zod_type instanceof ZodDate) {
return z.coerce.date();
// Any other type will be coerced as string value directly
} else {
return z.string();
}
};

if (underlying_zod_type instanceof ZodArray && underlying_zod_type.element instanceof ZodNumber) {
// Special case for `block_range` coercion, input is expected to be one or two values separated by comma
p[key] = z.preprocess(
(x) => String(x).split(','),
z.coerce.number().positive().array().min(1).max(2)
);
} else {
p[key] = coercePrimitive(underlying_zod_type);
}

if (key === "limit")
p[key] = p[key].max(config.maxLimit);

// Need to mark optional before adding defaults
if (isOptional)
p[key] = p[key].optional();

// Mark parameters with default values explicitly as a workaround
// See https://github.com/astahmer/typed-openapi/issues/34
if (key === "limit")
p[key] = p[key].default(10);
else if (key === "page")
p[key] = p[key].default(1);

}
)
);
}
}
}
23 changes: 19 additions & 4 deletions src/usage.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { makeQuery } from "./clickhouse/makeQuery.js";
import { APIErrorResponse } from "./utils.js";

import type { Context } from "hono";
import type { EndpointReturnTypes, UsageEndpoints, UsageResponse, ValidUserParams } from "./types/api.js";
import type { AdditionalQueryParams, EndpointReturnTypes, UsageEndpoints, UsageResponse, ValidUserParams } from "./types/api.js";

export async function makeUsageQuery(ctx: Context, endpoint: UsageEndpoints, user_params: ValidUserParams<typeof endpoint>) {
type EndpointElementReturnType = EndpointReturnTypes<typeof endpoint>[number];
Expand All @@ -16,14 +16,16 @@ export async function makeUsageQuery(ctx: Context, endpoint: UsageEndpoints, use
page = 1;

let filters = "";
for (const k of Object.keys(query_params).filter(k => k !== "limit")) // Don't add `limit` to WHERE clause
// Don't add `limit` and `block_range` to WHERE clause
for (const k of Object.keys(query_params).filter(k => k !== "limit" && k !== "block_range"))
filters += ` (${k} = {${k}: String}) AND`;
filters = filters.substring(0, filters.lastIndexOf(' ')); // Remove last item ` AND`

if (filters.length)
filters = `WHERE ${filters}`

let query = "";
let additional_query_params: AdditionalQueryParams = {};
if (endpoint == "/balance" || endpoint == "/supply") {
// Need to narrow the type of `query_params` explicitly to access properties based on endpoint value
// See https://github.com/microsoft/TypeScript/issues/33014
Expand All @@ -47,7 +49,19 @@ export async function makeUsageQuery(ctx: Context, endpoint: UsageEndpoints, use
query += `SELECT * FROM `;

const q = query_params as ValidUserParams<typeof endpoint>;
if (q.from) {
if (q.block_range) {
query += `transfers_block_num`;
console.log(q.block_range);
if (q.block_range[0] && q.block_range[1]) {
filters += "AND (block_num >= {min_block: int} AND block_num <= {max_block: int})"
// Use Min/Max to account for any ordering of parameters
additional_query_params.min_block = Math.min(q.block_range[0], q.block_range[1]);
additional_query_params.max_block = Math.max(q.block_range[0], q.block_range[1]);
} else if (q.block_range[0]) {
filters += "AND (block_num >= {min_block: int})"
additional_query_params.min_block = q.block_range[0];
}
} else if (q.from) {
// Find all incoming and outgoing transfers from single account
if (q.to && q.to === q.from)
filters = filters.replace(
Expand Down Expand Up @@ -79,8 +93,9 @@ export async function makeUsageQuery(ctx: Context, endpoint: UsageEndpoints, use
query += " OFFSET {offset: int}";

let query_results;
additional_query_params.offset = query_params.limit * (page - 1);
try {
query_results = await makeQuery<EndpointElementReturnType>(query, { ...query_params, offset: query_params.limit * (page - 1) });
query_results = await makeQuery<EndpointElementReturnType>(query, { ...query_params, ...additional_query_params });
} catch (err) {
return APIErrorResponse(ctx, 500, "bad_database_response", err);
}
Expand Down

0 comments on commit 78b6bbf

Please sign in to comment.