From cc231184cbfc467d13484fecf611b61c795b6278 Mon Sep 17 00:00:00 2001 From: Nikita Klimin Date: Tue, 23 Jul 2024 11:49:24 +0300 Subject: [PATCH] feat: add s3 ops metrics (#317) * feat: add s3 ops metrics * changes * fix: add event emitter and change metric to Counter --- .dockerignore | 2 + .gitignore | 1 + ...egraph-compatebility_2024-07-11-14-20.json | 10 + ...egraph-compatebility_2024-07-11-14-20.json | 20 + ...-feat-add-s3-metrics_2024-07-22-11-40.json | 10 + ...-feat-add-s3-metrics_2024-07-22-11-40.json | 10 + ...-feat-add-s3-metrics_2024-07-22-11-40.json | 10 + common/config/rush/pnpm-lock.yaml | 24 +- graphql/graphql-server/src/main.ts | 3 + graphql/graphql-server/src/server.ts | 22 +- graphql/graphql-server/src/typeorm.ts | 3 +- graphql/openreader/package.json | 6 +- graphql/openreader/src/context.ts | 4 +- graphql/openreader/src/db.ts | 6 +- graphql/openreader/src/dialect.ts | 2 - graphql/openreader/src/dialect/common.ts | 49 + graphql/openreader/src/dialect/index.ts | 20 + .../src/{ => dialect}/opencrud/orderBy.ts | 25 +- .../src/{ => dialect}/opencrud/schema.ts | 85 +- .../src/{ => dialect}/opencrud/tree.ts | 13 +- .../src/{ => dialect}/opencrud/where.ts | 20 +- .../openreader/src/dialect/thegraph/locale.ts | 284 ++++++ .../src/dialect/thegraph/orderBy.ts | 75 ++ .../openreader/src/dialect/thegraph/schema.ts | 484 ++++++++++ .../openreader/src/dialect/thegraph/tree.ts | 162 ++++ .../openreader/src/dialect/thegraph/where.ts | 184 ++++ graphql/openreader/src/ir/args.ts | 2 + graphql/openreader/src/main.ts | 6 +- graphql/openreader/src/model.schema.ts | 41 +- graphql/openreader/src/model.ts | 2 + graphql/openreader/src/server.ts | 40 +- graphql/openreader/src/sql/cursor.ts | 8 +- graphql/openreader/src/sql/printer.ts | 40 +- graphql/openreader/src/sql/query.ts | 10 +- graphql/openreader/src/sql/util.ts | 4 +- graphql/openreader/src/test/basic.test.ts | 852 ++++++++++++------ graphql/openreader/src/test/isNull.test.ts | 133 ++- graphql/openreader/src/test/limits.test.ts | 303 +++++-- graphql/openreader/src/test/lookup.test.ts | 273 ++++-- graphql/openreader/src/test/setup.ts | 4 +- graphql/openreader/src/test/where.test.ts | 343 ++++--- util/util-internal-dump-cli/src/dumper.ts | 12 +- util/util-internal-dump-cli/src/prometheus.ts | 20 +- util/util-internal-fs/src/factory.ts | 10 +- util/util-internal-fs/src/s3.ts | 12 +- util/util-internal-ingest-cli/package.json | 4 +- util/util-internal-ingest-cli/src/ingest.ts | 26 +- .../src/prometheus.ts | 29 + 48 files changed, 2915 insertions(+), 793 deletions(-) create mode 100644 common/changes/@subsquid/graphql-server/thegraph-compatebility_2024-07-11-14-20.json create mode 100644 common/changes/@subsquid/openreader/thegraph-compatebility_2024-07-11-14-20.json create mode 100644 common/changes/@subsquid/util-internal-dump-cli/octo-gone-feat-add-s3-metrics_2024-07-22-11-40.json create mode 100644 common/changes/@subsquid/util-internal-fs/octo-gone-feat-add-s3-metrics_2024-07-22-11-40.json create mode 100644 common/changes/@subsquid/util-internal-ingest-cli/octo-gone-feat-add-s3-metrics_2024-07-22-11-40.json delete mode 100644 graphql/openreader/src/dialect.ts create mode 100644 graphql/openreader/src/dialect/common.ts create mode 100644 graphql/openreader/src/dialect/index.ts rename graphql/openreader/src/{ => dialect}/opencrud/orderBy.ts (80%) rename graphql/openreader/src/{ => dialect}/opencrud/schema.ts (91%) rename graphql/openreader/src/{ => dialect}/opencrud/tree.ts (95%) rename graphql/openreader/src/{ => dialect}/opencrud/where.ts (90%) create mode 100644 graphql/openreader/src/dialect/thegraph/locale.ts create mode 100644 graphql/openreader/src/dialect/thegraph/orderBy.ts create mode 100644 graphql/openreader/src/dialect/thegraph/schema.ts create mode 100644 graphql/openreader/src/dialect/thegraph/tree.ts create mode 100644 graphql/openreader/src/dialect/thegraph/where.ts create mode 100644 util/util-internal-ingest-cli/src/prometheus.ts diff --git a/.dockerignore b/.dockerignore index e47a865a3..85b889157 100644 --- a/.dockerignore +++ b/.dockerignore @@ -32,3 +32,5 @@ docker-compose.yml /ops/docker-publish.sh **/.DS_Store + +*.temp \ No newline at end of file diff --git a/.gitignore b/.gitignore index 8868156ed..fef86899d 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ common/autoinstallers/*/.npmrc # IDE .idea +.vscode # Built js libs /*/*/lib diff --git a/common/changes/@subsquid/graphql-server/thegraph-compatebility_2024-07-11-14-20.json b/common/changes/@subsquid/graphql-server/thegraph-compatebility_2024-07-11-14-20.json new file mode 100644 index 000000000..55ac86f93 --- /dev/null +++ b/common/changes/@subsquid/graphql-server/thegraph-compatebility_2024-07-11-14-20.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@subsquid/graphql-server", + "comment": "add dialect system support", + "type": "minor" + } + ], + "packageName": "@subsquid/graphql-server" +} \ No newline at end of file diff --git a/common/changes/@subsquid/openreader/thegraph-compatebility_2024-07-11-14-20.json b/common/changes/@subsquid/openreader/thegraph-compatebility_2024-07-11-14-20.json new file mode 100644 index 000000000..dc8ef8bc0 --- /dev/null +++ b/common/changes/@subsquid/openreader/thegraph-compatebility_2024-07-11-14-20.json @@ -0,0 +1,20 @@ +{ + "changes": [ + { + "packageName": "@subsquid/openreader", + "comment": "introduce dialect system", + "type": "major" + }, + { + "packageName": "@subsquid/openreader", + "comment": "remove `ByUniqueInput` queries", + "type": "major" + }, + { + "packageName": "@subsquid/openreader", + "comment": "add `thegraph` dialect support", + "type": "minor" + } + ], + "packageName": "@subsquid/openreader" +} \ No newline at end of file diff --git a/common/changes/@subsquid/util-internal-dump-cli/octo-gone-feat-add-s3-metrics_2024-07-22-11-40.json b/common/changes/@subsquid/util-internal-dump-cli/octo-gone-feat-add-s3-metrics_2024-07-22-11-40.json new file mode 100644 index 000000000..73f42eaf7 --- /dev/null +++ b/common/changes/@subsquid/util-internal-dump-cli/octo-gone-feat-add-s3-metrics_2024-07-22-11-40.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@subsquid/util-internal-dump-cli", + "comment": "add prometheus metrics for S3 file system handler", + "type": "patch" + } + ], + "packageName": "@subsquid/util-internal-dump-cli" +} \ No newline at end of file diff --git a/common/changes/@subsquid/util-internal-fs/octo-gone-feat-add-s3-metrics_2024-07-22-11-40.json b/common/changes/@subsquid/util-internal-fs/octo-gone-feat-add-s3-metrics_2024-07-22-11-40.json new file mode 100644 index 000000000..4f57e76bf --- /dev/null +++ b/common/changes/@subsquid/util-internal-fs/octo-gone-feat-add-s3-metrics_2024-07-22-11-40.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@subsquid/util-internal-fs", + "comment": "add metrics for S3 file system handler", + "type": "patch" + } + ], + "packageName": "@subsquid/util-internal-fs" +} \ No newline at end of file diff --git a/common/changes/@subsquid/util-internal-ingest-cli/octo-gone-feat-add-s3-metrics_2024-07-22-11-40.json b/common/changes/@subsquid/util-internal-ingest-cli/octo-gone-feat-add-s3-metrics_2024-07-22-11-40.json new file mode 100644 index 000000000..02005b7fd --- /dev/null +++ b/common/changes/@subsquid/util-internal-ingest-cli/octo-gone-feat-add-s3-metrics_2024-07-22-11-40.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@subsquid/util-internal-ingest-cli", + "comment": "add prometheus metrics", + "type": "minor" + } + ], + "packageName": "@subsquid/util-internal-ingest-cli" +} \ No newline at end of file diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index fd4802e4e..6ecfae756 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -2741,6 +2741,10 @@ packages: pg-types: 4.0.2 dev: false + /@types/pluralize@0.0.33: + resolution: {integrity: sha512-JOqsl+ZoCpP4e8TDke9W79FDcSgPAR0l6pixx2JHkhnRjvShyYiAYw2LVsnA7K08Y6DeOnaU6ujmENO4os/cYg==} + dev: false + /@types/qs@6.9.14: resolution: {integrity: sha512-5khscbd3SwWMhFqylJBLQ0zIu7c1K6Vz0uBIt915BI3zV0q1nfjRQD3RqSBcPaO6PHEF4ov/t9y89fSiyThlPA==} dev: false @@ -5398,6 +5402,11 @@ packages: pathe: 1.1.2 dev: false + /pluralize@8.0.0: + resolution: {integrity: sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==} + engines: {node: '>=4'} + dev: false + /possible-typed-array-names@1.0.0: resolution: {integrity: sha512-d7Uw+eZoloe0EHDIYoe+bQ5WXnGMOpmiZFTuMWCwpjzzkL2nTjcKiAk4hh8TjnGye2TwWOk3UXucZ+3rbmBa8Q==} engines: {node: '>= 0.4'} @@ -6662,7 +6671,7 @@ packages: dev: false file:projects/astar-erc20.tgz(supports-color@8.1.1)(ts-node@10.9.2): - resolution: {integrity: sha512-+NndRNTiqeADahJ7MnhjM4sv8dNWjkutfX2jSedZwaWcdfu2yuWFTZ19FksmiOjSNS0oEH1q58L4FSkLtMx/Hw==, tarball: file:projects/astar-erc20.tgz} + resolution: {integrity: sha512-zD/CYvTv02jjAzFp2EfCEARshN2qyWZtO+gPx7/IlgQkzph0JbMbpQRtn2rbOpv6jcAg/wG+2HRVazRxghS5fg==, tarball: file:projects/astar-erc20.tgz} id: file:projects/astar-erc20.tgz name: '@rush-temp/astar-erc20' version: 0.0.0 @@ -6799,7 +6808,7 @@ packages: dev: false file:projects/erc20-transfers.tgz(supports-color@8.1.1)(ts-node@10.9.2): - resolution: {integrity: sha512-JBoemaBay82HLCQAvkaSebJXSH/Ek7PBvOg4+FUn77kHUKavaATl60EEnsgDERGitBiftBUD2K4TUeU7hgwJtA==, tarball: file:projects/erc20-transfers.tgz} + resolution: {integrity: sha512-W9AXPSgGr0Ln1FX88xqytYQdYJ2SqrNDNgjCMIRcIRqd3fyij8HJWP5wm6Yd3BvjWoUJzXbUrWzk5w+TM8vmGw==, tarball: file:projects/erc20-transfers.tgz} id: file:projects/erc20-transfers.tgz name: '@rush-temp/erc20-transfers' version: 0.0.0 @@ -7135,7 +7144,7 @@ packages: dev: false file:projects/openreader.tgz(supports-color@8.1.1): - resolution: {integrity: sha512-1yTnDvAUuWzyPC2bH2BZUtYhRMgK7Yh1MDnWwEyBjEhkhrJp/oqZ+8LbN8GpWREaQusvqQn+0U5PLiWqSjfNrw==, tarball: file:projects/openreader.tgz} + resolution: {integrity: sha512-vS4NkBdy9ZWY3/fRkEwmDEH+cu61QDbjVs22Vzct75k49TvsXVxIyTvY2Lw4Y1vWBwNLj8ddji+gDw2CncWTKg==, tarball: file:projects/openreader.tgz} id: file:projects/openreader.tgz name: '@rush-temp/openreader' version: 0.0.0 @@ -7144,9 +7153,11 @@ packages: '@subsquid/graphiql-console': 0.3.0 '@types/deep-equal': 1.0.4 '@types/express': 4.17.21 + '@types/inflected': 2.1.3 '@types/mocha': 10.0.6 '@types/node': 18.19.31 '@types/pg': 8.11.5 + '@types/pluralize': 0.0.33 '@types/ws': 8.5.10 apollo-server-core: 3.13.0(graphql@15.8.0) apollo-server-express: 3.13.0(express@4.19.2)(graphql@15.8.0) @@ -7158,8 +7169,10 @@ packages: graphql: 15.8.0 graphql-parse-resolve-info: 4.14.0(graphql@15.8.0)(supports-color@8.1.1) graphql-ws: 5.16.0(graphql@15.8.0) + inflected: 2.1.0 mocha: 10.4.0 pg: 8.11.5 + pluralize: 8.0.0 typescript: 5.3.3 ws: 8.16.0(bufferutil@4.0.8)(utf-8-validate@5.0.10) transitivePeerDependencies: @@ -7290,7 +7303,7 @@ packages: dev: false file:projects/solana-example.tgz(supports-color@8.1.1)(ts-node@10.9.2): - resolution: {integrity: sha512-Ns7M8TC//cSLfJybpLFsop3TVKD33gDaCcfjs6VUh/UDsMpEPOmxyGEXmYzujPpmpc/7GD7iBXJ52kZ/po/fHg==, tarball: file:projects/solana-example.tgz} + resolution: {integrity: sha512-Nj4TlOCzEKhmatFmf28UjvnZ2E/tO+P5/ey7Bp9zrqZ4TKnz0SZge5bnkB/NPWNYzVh91vrvjP2qmJs/Br4oRQ==, tarball: file:projects/solana-example.tgz} id: file:projects/solana-example.tgz name: '@rush-temp/solana-example' version: 0.0.0 @@ -7725,12 +7738,13 @@ packages: dev: false file:projects/util-internal-ingest-cli.tgz: - resolution: {integrity: sha512-aUnjqkCZ1K036D1WsGDalurA8VO5LOChQxgqrTXNHZwIsNU92wERfGdHQ8IkfXDQ/n7TcL/MRd1fNjS6HzftWg==, tarball: file:projects/util-internal-ingest-cli.tgz} + resolution: {integrity: sha512-hF3MhECI+li0SPLDDdrnzoKUFnIcyO2gsIKEIkAkoZaqF/GJ1S2AIhqvIAR/aSU+bUJMWbzekN6pt/4MqDunlA==, tarball: file:projects/util-internal-ingest-cli.tgz} name: '@rush-temp/util-internal-ingest-cli' version: 0.0.0 dependencies: '@types/node': 18.19.31 commander: 11.1.0 + prom-client: 14.2.0 typescript: 5.3.3 dev: false diff --git a/graphql/graphql-server/src/main.ts b/graphql/graphql-server/src/main.ts index 0b84ab45f..784f03ee2 100644 --- a/graphql/graphql-server/src/main.ts +++ b/graphql/graphql-server/src/main.ts @@ -6,6 +6,7 @@ import {registerTsNodeIfRequired} from '@subsquid/util-internal-ts-node' import assert from 'assert' import {Command, Option} from 'commander' import {DumbInMemoryCacheOptions, DumbRedisCacheOptions, Server} from './server' +import {Dialect} from '@subsquid/openreader/lib/dialect' const LOG = createLogger('sqd:graphql-server') @@ -26,6 +27,7 @@ runProgram(async () => { program.option('--dumb-cache-max-age ', 'cache-control max-age in milliseconds', nat, 5000) program.option('--dumb-cache-ttl ', 'in-memory cached item TTL in milliseconds', nat, 5000) program.option('--dumb-cache-size ', 'max in-memory cache size in megabytes', nat, 50) + program.addOption(new Option('--dialect ').choices(Object.values(Dialect))) let opts = program.parse().opts() as { maxRequestSize: number @@ -41,6 +43,7 @@ runProgram(async () => { subscriptionPollInterval: number subscriptionMaxResponseSize?: number tsNode?: boolean + dialect?: Dialect } await registerTsNodeIfRequired() diff --git a/graphql/graphql-server/src/server.ts b/graphql/graphql-server/src/server.ts index bf4dc0a27..02f21601c 100644 --- a/graphql/graphql-server/src/server.ts +++ b/graphql/graphql-server/src/server.ts @@ -3,10 +3,9 @@ import {InMemoryLRUCache} from '@apollo/utils.keyvaluecache' import {mergeSchemas} from '@graphql-tools/schema' import {Logger} from '@subsquid/logger' import {Context, OpenreaderContext} from '@subsquid/openreader/lib/context' -import {PoolOpenreaderContext} from '@subsquid/openreader/lib/db' -import {Dialect} from '@subsquid/openreader/lib/dialect' +import {DbType, PoolOpenreaderContext} from '@subsquid/openreader/lib/db' +import {Dialect, getSchemaBuilder} from '@subsquid/openreader/lib/dialect' import type {Model} from '@subsquid/openreader/lib/model' -import {SchemaBuilder} from '@subsquid/openreader/lib/opencrud/schema' import {addServerCleanup, Dispose, runApollo} from '@subsquid/openreader/lib/server' import {loadModel, resolveGraphqlSchema} from '@subsquid/openreader/lib/tools' import {ResponseSizeLimit} from '@subsquid/openreader/lib/util/limit' @@ -38,6 +37,7 @@ export interface ServerOptions { subscriptionPollInterval?: number subscriptionMaxResponseNodes?: number dumbCache?: DumbRedisCacheOptions | DumbInMemoryCacheOptions + dialect?: Dialect } @@ -103,8 +103,14 @@ export class Server { @def private async schema(): Promise { + const schemaBuilder = await getSchemaBuilder({ + model: this.model(), + subscriptions: this.options.subscriptions, + dialect: this.options.dialect, + }) + let schemas = [ - new SchemaBuilder({model: this.model(), subscriptions: this.options.subscriptions}).build() + schemaBuilder.build(), ] if (this.options.squidStatus !== false) { @@ -185,14 +191,14 @@ export class Server { @def private async context(): Promise<() => Context> { - let dialect = this.dialect() + let dbType = this.dbType() let createOpenreader: () => OpenreaderContext if (await this.customResolvers()) { let con = await this.createTypeormConnection({sqlStatementTimeout: this.options.sqlStatementTimeout}) this.disposals.push(() => con.destroy()) createOpenreader = () => { return new TypeormOpenreaderContext( - dialect, + dbType, con, con, this.options.subscriptionPollInterval, @@ -204,7 +210,7 @@ export class Server { this.disposals.push(() => pool.end()) createOpenreader = () => { return new PoolOpenreaderContext( - dialect, + dbType, pool, pool, this.options.subscriptionPollInterval, @@ -261,7 +267,7 @@ export class Server { return envNat('GQL_DB_CONNECTION_POOL_SIZE') || 5 } - private dialect(): Dialect { + private dbType(): DbType { let type = process.env.DB_TYPE if (!type) return 'postgres' switch(type) { diff --git a/graphql/graphql-server/src/typeorm.ts b/graphql/graphql-server/src/typeorm.ts index 4ed42e8ae..35c56a890 100644 --- a/graphql/graphql-server/src/typeorm.ts +++ b/graphql/graphql-server/src/typeorm.ts @@ -1,5 +1,6 @@ import type {Logger} from '@subsquid/logger' import type {OpenreaderContext} from '@subsquid/openreader/lib/context' +import {DbType} from '@subsquid/openreader/lib/db' import type {Dialect} from '@subsquid/openreader/lib/dialect' import type {Query} from '@subsquid/openreader/lib/sql/query' import {Subscription} from '@subsquid/openreader/lib/subscription' @@ -19,7 +20,7 @@ export class TypeormOpenreaderContext implements OpenreaderContext { private queryCounter = 0 constructor( - public readonly dialect: Dialect, + public readonly dbType: DbType, private connection: DataSource, subscriptionConnection?: DataSource, private subscriptionPollInterval: number = 1000, diff --git a/graphql/openreader/package.json b/graphql/openreader/package.json index 89bf7da39..6d2004948 100644 --- a/graphql/openreader/package.json +++ b/graphql/openreader/package.json @@ -42,7 +42,8 @@ "graphql-parse-resolve-info": "^4.14.0", "graphql-ws": "^5.14.2", "pg": "^8.11.3", - "ws": "^8.14.2" + "ws": "^8.14.2", + "inflected": "^2.1.0" }, "peerDependencies": { "@subsquid/big-decimal": "^1.0.0" @@ -63,6 +64,7 @@ "expect": "^29.7.0", "gql-test-client": "^0.0.0", "mocha": "^10.2.0", - "typescript": "~5.3.2" + "typescript": "~5.3.2", + "@types/inflected": "^2.1.3" } } diff --git a/graphql/openreader/src/context.ts b/graphql/openreader/src/context.ts index 882a2709b..027f28e62 100644 --- a/graphql/openreader/src/context.ts +++ b/graphql/openreader/src/context.ts @@ -1,5 +1,5 @@ import type {Logger} from '@subsquid/logger' -import type {Dialect} from './dialect' +import type {DbType} from './db' import type {Query} from './sql/query' import type {Limit} from './util/limit' @@ -11,7 +11,7 @@ export interface Context { export interface OpenreaderContext { id: number - dialect: Dialect + dbType: DbType executeQuery(query: Query): Promise subscription(query: Query): AsyncIterable responseSizeLimit?: Limit diff --git a/graphql/openreader/src/db.ts b/graphql/openreader/src/db.ts index bf109a31e..1ce248d4b 100644 --- a/graphql/openreader/src/db.ts +++ b/graphql/openreader/src/db.ts @@ -3,7 +3,6 @@ import {addErrorContext} from '@subsquid/util-internal' import type {ClientBase, Pool} from 'pg' import {QueryResult} from 'pg' import {OpenreaderContext} from './context' -import {Dialect} from './dialect' import {Query} from './sql/query' import {Subscription} from './subscription' import {LazyTransaction} from './util/lazy-transaction' @@ -12,6 +11,9 @@ import {LazyTransaction} from './util/lazy-transaction' let CTX_COUNTER = 0 +export type DbType = 'postgres' | 'cockroach' + + export class PoolOpenreaderContext implements OpenreaderContext { public id = (CTX_COUNTER = (CTX_COUNTER + 1) % Number.MAX_SAFE_INTEGER) public log?: Logger @@ -20,7 +22,7 @@ export class PoolOpenreaderContext implements OpenreaderContext { private queryCounter = 0 constructor( - public readonly dialect: Dialect, + public readonly dbType: DbType, pool: Pool, subscriptionPool?: Pool, private subscriptionPollInterval: number = 1000, diff --git a/graphql/openreader/src/dialect.ts b/graphql/openreader/src/dialect.ts deleted file mode 100644 index 98e88d16b..000000000 --- a/graphql/openreader/src/dialect.ts +++ /dev/null @@ -1,2 +0,0 @@ - -export type Dialect = 'postgres' | 'cockroach' diff --git a/graphql/openreader/src/dialect/common.ts b/graphql/openreader/src/dialect/common.ts new file mode 100644 index 000000000..7e15131c5 --- /dev/null +++ b/graphql/openreader/src/dialect/common.ts @@ -0,0 +1,49 @@ +import {GraphQLFieldConfigMap, GraphQLSchema} from 'graphql' +import {Model} from '../model' +import {Context} from '../context' +import {OrderBy, Where} from '../ir/args' +import assert from 'assert' + +export enum Dialect { + OpenCrud = 'opencrud', + TheGraph = 'thegraph', +} + +export interface SchemaBuilder { + build(): GraphQLSchema +} + +export interface SchemaOptions { + model: Model + subscriptions?: boolean +} + +export type GqlFieldMap = GraphQLFieldConfigMap + +export function mergeOrderBy(list: OrderBy[]): OrderBy { + let result: OrderBy = {} + list.forEach((item) => { + for (let key in item) { + let current = result[key] + if (current == null) { + result[key] = item[key] + } else if (typeof current != 'string') { + let it = item[key] + assert(typeof it == 'object') + result[key] = mergeOrderBy([current, it]) + } + } + }) + return result +} + +export function toCondition(op: 'AND' | 'OR', operands: Where[]): Where | undefined { + switch(operands.length) { + case 0: + return undefined + case 1: + return operands[0] + default: + return {op, args: operands} + } +} diff --git a/graphql/openreader/src/dialect/index.ts b/graphql/openreader/src/dialect/index.ts new file mode 100644 index 000000000..5e661aa93 --- /dev/null +++ b/graphql/openreader/src/dialect/index.ts @@ -0,0 +1,20 @@ +import {unexpectedCase} from '@subsquid/util-internal' +import {Dialect, SchemaBuilder, SchemaOptions} from './common' + +export * from './common' + +export async function getSchemaBuilder(options: SchemaOptions & {dialect?: Dialect}): Promise { + switch (options.dialect) { + case undefined: + case Dialect.OpenCrud: { + const {SchemaBuilder} = await import('./opencrud/schema') + return new SchemaBuilder(options) + } + case Dialect.TheGraph: { + const {SchemaBuilder} = await import('./thegraph/schema') + return new SchemaBuilder(options) + } + default: + throw unexpectedCase(options.dialect) + } +} diff --git a/graphql/openreader/src/opencrud/orderBy.ts b/graphql/openreader/src/dialect/opencrud/orderBy.ts similarity index 80% rename from graphql/openreader/src/opencrud/orderBy.ts rename to graphql/openreader/src/dialect/opencrud/orderBy.ts index dc88ba088..dd9cc7805 100644 --- a/graphql/openreader/src/opencrud/orderBy.ts +++ b/graphql/openreader/src/dialect/opencrud/orderBy.ts @@ -1,7 +1,8 @@ import assert from "assert" -import type { Model } from "../model" -import { getUniversalProperties } from '../model.tools' -import { OrderBy } from "../ir/args" +import type { Model } from "../../model" +import { getUniversalProperties } from '../../model.tools' +import { OrderBy } from "../../ir/args" +import {mergeOrderBy} from '../common' /** @@ -80,21 +81,3 @@ export function parseOrderBy(model: Model, typeName: string, input: OpenCrudOrde }) ) } - - -export function mergeOrderBy(list: OrderBy[]): OrderBy { - let result: OrderBy = {} - list.forEach(item => { - for (let key in item) { - let current = result[key] - if (current == null) { - result[key] = item[key] - } else if (typeof current != 'string') { - let it = item[key] - assert(typeof it == 'object') - result[key] = mergeOrderBy([current, it]) - } - } - }) - return result -} diff --git a/graphql/openreader/src/opencrud/schema.ts b/graphql/openreader/src/dialect/opencrud/schema.ts similarity index 91% rename from graphql/openreader/src/opencrud/schema.ts rename to graphql/openreader/src/dialect/opencrud/schema.ts index 44635d449..5f03f1398 100644 --- a/graphql/openreader/src/opencrud/schema.ts +++ b/graphql/openreader/src/dialect/opencrud/schema.ts @@ -27,29 +27,21 @@ import { GraphQLFieldConfigMap, GraphQLInputFieldConfigMap } from 'graphql/type/definition' -import {Context} from '../context' -import {decodeRelayConnectionCursor, RelayConnectionRequest} from '../ir/connection' -import {AnyFields} from '../ir/fields' -import {getConnectionSize, getListSize, getObjectSize} from '../limit.size' -import {Entity, Interface, JsonObject, Model, Prop} from '../model' -import {getObject, getUniversalProperties} from '../model.tools' -import {customScalars} from '../scalars' -import {ConnectionQuery, CountQuery, EntityByIdQuery, ListQuery, Query} from '../sql/query' -import {Limit} from '../util/limit' -import {getResolveTree, getTreeRequest, hasTreeRequest, simplifyResolveTree} from '../util/resolve-tree' -import {ensureArray, identity} from '../util/util' +import {Context} from '../../context' +import {decodeRelayConnectionCursor, RelayConnectionRequest} from '../../ir/connection' +import {AnyFields} from '../../ir/fields' +import {getConnectionSize, getListSize, getObjectSize} from '../../limit.size' +import {Entity, Interface, JsonObject, Model, Prop} from '../../model' +import {getEntity, getObject, getUniversalProperties} from '../../model.tools' +import {customScalars} from '../../scalars' +import {ConnectionQuery, CountQuery, EntityByIdQuery, ListQuery, Query} from '../../sql/query' +import {Limit} from '../../util/limit' +import {getResolveTree, getTreeRequest, hasTreeRequest, simplifyResolveTree} from '../../util/resolve-tree' +import {ensureArray, identity} from '../../util/util' import {getOrderByMapping, parseOrderBy} from './orderBy' import {parseAnyTree, parseObjectTree, parseSqlArguments} from './tree' import {parseWhere} from './where' - - -type GqlFieldMap = GraphQLFieldConfigMap - - -export interface SchemaOptions { - model: Model - subscriptions?: boolean -} +import {GqlFieldMap, SchemaOptions} from '../common' export class SchemaBuilder { @@ -377,7 +369,6 @@ export class SchemaBuilder { case "entity": this.installListQuery(name, query, subscription) this.installEntityById(name, query, subscription) - this.installEntityByUniqueInput(name, query) this.installRelayConnection(name, query) break case 'interface': @@ -403,7 +394,9 @@ export class SchemaBuilder { private installListQuery(typeName: string, query: GqlFieldMap, subscription: GqlFieldMap): void { let model = this.model - let queryName = toPlural(toCamelCase(typeName)) + + let entity = model[typeName] + let queryName = entity.kind === 'entity' && entity.listQueryName || this.normalizeEntityName(typeName).plural let outputType = new GraphQLNonNull(new GraphQLList(new GraphQLNonNull(this.get(typeName)))) let argsType = this.listArguments(typeName) @@ -414,7 +407,7 @@ export class SchemaBuilder { limit?.check(() => getListSize(model, typeName, fields, args.limit, args.where) + 1) return new ListQuery( model, - context.openreader.dialect, + context.openreader.dbType, typeName, fields, args @@ -443,7 +436,9 @@ export class SchemaBuilder { private installEntityById(entityName: string, query: GqlFieldMap, subscription: GqlFieldMap): void { let model = this.model - let queryName = `${toCamelCase(entityName)}ById` + + let entity = model[entityName] + let queryName = (entity.kind === 'entity' && entity.queryName) || `${this.normalizeEntityName(entityName).singular}ById` let argsType = { id: {type: new GraphQLNonNull(GraphQLString)} } @@ -454,7 +449,7 @@ export class SchemaBuilder { limit?.check(() => getObjectSize(model, fields) + 1) return new EntityByIdQuery( model, - context.openreader.dialect, + context.openreader.dbType, entityName, fields, tree.args.id as string @@ -481,39 +476,12 @@ export class SchemaBuilder { } } - private installEntityByUniqueInput(entityName: string, query: GqlFieldMap): void { - let model = this.model - - query[`${toCamelCase(entityName)}ByUniqueInput`] = { - deprecationReason: `Use ${toCamelCase(entityName)}ById`, - type: this.get(entityName), - args: { - where: {type: this.whereIdInput()} - }, - async resolve(source, args, context, info) { - let tree = getResolveTree(info) - let fields = parseObjectTree(model, entityName, info.schema, tree) - context.openreader.responseSizeLimit?.check(() => getObjectSize(model, fields) + 1) - let query = new ListQuery( - model, - context.openreader.dialect, - entityName, - fields, - {where: {op: 'eq', field: 'id', value: args.where.id}} - ) - let result = await context.openreader.executeQuery(query) - assert(result.length < 2) - return result[0] - } - } - } - private installRelayConnection(typeName: string, query: GqlFieldMap): void { let model = this.model let outputType = toPlural(typeName) + 'Connection' let edgeType = `${typeName}Edge` - query[`${toPlural(toCamelCase(typeName))}Connection`] = { + query[`${this.normalizeEntityName(typeName).plural}Connection`] = { type: new GraphQLNonNull(new GraphQLObjectType({ name: outputType, fields: { @@ -582,7 +550,7 @@ export class SchemaBuilder { let result = await context.openreader.executeQuery(new ConnectionQuery( model, - context.openreader.dialect, + context.openreader.dbType, typeName, req )) @@ -590,7 +558,7 @@ export class SchemaBuilder { if (req.totalCount && result.totalCount == null) { result.totalCount = await context.openreader.executeQuery(new CountQuery( model, - context.openreader.dialect, + context.openreader.dbType, typeName, req.where )) @@ -627,6 +595,13 @@ export class SchemaBuilder { }) ) } + + private normalizeEntityName(typeName: string) { + let singular = toCamelCase(typeName) + let plural = toPlural(singular) + + return {singular, plural} + } } diff --git a/graphql/openreader/src/opencrud/tree.ts b/graphql/openreader/src/dialect/opencrud/tree.ts similarity index 95% rename from graphql/openreader/src/opencrud/tree.ts rename to graphql/openreader/src/dialect/opencrud/tree.ts index bcbe62860..c13142aa5 100644 --- a/graphql/openreader/src/opencrud/tree.ts +++ b/graphql/openreader/src/dialect/opencrud/tree.ts @@ -2,12 +2,12 @@ import {unexpectedCase} from '@subsquid/util-internal' import assert from 'assert' import {GraphQLSchema} from 'graphql' import {ResolveTree} from 'graphql-parse-resolve-info' -import {SqlArguments} from '../ir/args' -import {AnyFields, FieldRequest, FieldsByEntity, OpaqueRequest} from '../ir/fields' -import {Model} from '../model' -import {getQueryableEntities} from '../model.tools' -import {simplifyResolveTree} from '../util/resolve-tree' -import {ensureArray} from '../util/util' +import {SqlArguments} from '../../ir/args' +import {AnyFields, FieldRequest, FieldsByEntity, OpaqueRequest} from '../../ir/fields' +import {Model} from '../../model' +import {getQueryableEntities} from '../../model.tools' +import {simplifyResolveTree} from '../../util/resolve-tree' +import {ensureArray} from '../../util/util' import {parseOrderBy} from './orderBy' import {parseWhere} from './where' @@ -18,7 +18,6 @@ export function parseObjectTree( schema: GraphQLSchema, tree: ResolveTree ): FieldRequest[] { - let requests: FieldRequest[] = [] let requestedScalars: Record = {} let object = model[typeName] diff --git a/graphql/openreader/src/opencrud/where.ts b/graphql/openreader/src/dialect/opencrud/where.ts similarity index 90% rename from graphql/openreader/src/opencrud/where.ts rename to graphql/openreader/src/dialect/opencrud/where.ts index d6d804acb..248efb25d 100644 --- a/graphql/openreader/src/opencrud/where.ts +++ b/graphql/openreader/src/dialect/opencrud/where.ts @@ -1,7 +1,8 @@ import {unexpectedCase} from "@subsquid/util-internal" import assert from "assert" -import {Where} from "../ir/args" -import {ensureArray} from "../util/util" +import {Where} from "../../ir/args" +import {ensureArray} from "../../util/util" +import {toCondition} from '../common' export function parseWhere(whereArg?: any): Where | undefined { @@ -86,19 +87,6 @@ export function parseWhere(whereArg?: any): Where | undefined { } } - -function toCondition(op: 'AND' | 'OR', operands: Where[]): Where | undefined { - switch(operands.length) { - case 0: - return undefined - case 1: - return operands[0] - default: - return {op, args: operands} - } -} - - export function parseWhereKey(key: string): {op: Where['op'], field: string} { let m = WHERE_KEY_REGEX.exec(key) if (m) { @@ -138,4 +126,4 @@ const WHERE_KEY_REGEX = (() => { "not_in", ] return new RegExp(`^([^_]*)_(${ops.join('|')})$`) -})() +})() \ No newline at end of file diff --git a/graphql/openreader/src/dialect/thegraph/locale.ts b/graphql/openreader/src/dialect/thegraph/locale.ts new file mode 100644 index 000000000..b8d37f849 --- /dev/null +++ b/graphql/openreader/src/dialect/thegraph/locale.ts @@ -0,0 +1,284 @@ +import {inflections, pluralize} from 'inflected' + +const THEGRAPH_LOCALE = 'thegraph' + +// ref https://github.com/whatisinternet/Inflector/blob/master/src/string/pluralize/mod.rs +inflections(THEGRAPH_LOCALE, (inflector) => { + inflector.plural(/(\w*)$/, '$1s') + inflector.plural(/(\w*)s$/, '$1s') + inflector.plural(/(\w*([^aeiou]ese))$/, '$1') + inflector.plural(/(\w*(ax|test))is$/, '$1es') + inflector.plural(/(\w*(alias|[^aou]us|tlas|gas|ris))$/, '$1es') + inflector.plural(/(\w*(e[mn]u))s?$/, '$1s') + inflector.plural(/(\w*([^l]ias|[aeiou]las|[emjzr]as|[iu]am))$/, '$1') + inflector.plural( + /(\w*(alumn|syllab|octop|vir|radi|nucle|fung|cact|stimul|termin|bacill|foc|uter|loc|strat))(?:us|i)$/, + '$1i' + ) + inflector.plural(/(\w*(alumn|alg|vertebr))(?:a|ae)$/, '$1ae') + inflector.plural(/(\w*(seraph|cherub))(?:im)?$/, '$1im') + inflector.plural(/(\w*(her|at|gr))o$/, '$1oes') + inflector.plural( + /(\w*(agend|addend|millenni|dat|extrem|bacteri|desiderat|strat|candelabr|errat|ov|symposi|curricul|automat|quor))(?:a|um)$/, + '$1a' + ) + inflector.plural( + /(\w*(apheli|hyperbat|periheli|asyndet|noumen|phenomen|criteri|organ|prolegomen|hedr|automat))(?:a|on)$/, + '$1a' + ) + inflector.plural(/(\w*)sis$/, '$1ses') + inflector.plural(/(\w*(kni|wi|li))fe$/, '$1ves') + inflector.plural(/(\w*(ar|l|ea|eo|oa|hoo))f$/, '$1ves') + inflector.plural(/(\w*([^aeiouy]|qu))y$/, '$1ies') + inflector.plural(/(\w*([^ch][ieo][ln]))ey$/, '$1ies') + inflector.plural(/(\w*(x|ch|ss|sh|zz)es)$/, '$1') + inflector.plural(/(\w*(x|ch|ss|sh|zz))$/, '$1es') + inflector.plural(/(\w*(matr|cod|mur|sil|vert|ind|append))(?:ix|ex)$/, '$1ices') + inflector.plural(/(\w*(m|l)(?:ice|ouse))$/, '$1ice') + inflector.plural(/(\w*(pe)(?:rson|ople))$/, '$1ople') + inflector.plural(/(\w*(child))(?:ren)?$/, '$1ren') + inflector.plural(/(\w*eaux)$/, '$1') + + inflector.irregular('ox', 'oxes') + inflector.irregular('man', 'men') + inflector.irregular('woman', 'women') + inflector.irregular('die', 'dice') + inflector.irregular('yes', 'yeses') + inflector.irregular('foot', 'feet') + inflector.irregular('eave', 'eaves') + inflector.irregular('goose', 'geese') + inflector.irregular('tooth', 'teeth') + inflector.irregular('quiz', 'quizzes') + + inflector.uncountable( + 'accommodation', + 'adulthood', + 'advertising', + 'advice', + 'aggression', + 'aid', + 'air', + 'aircraft', + 'alcohol', + 'anger', + 'applause', + 'arithmetic', + 'assistance', + 'athletics', + + 'bacon', + 'baggage', + 'beef', + 'biology', + 'blood', + 'botany', + 'bread', + 'butter', + + 'carbon', + 'cardboard', + 'cash', + 'chalk', + 'chaos', + 'chess', + 'crossroads', + 'countryside', + + 'dancing', + 'deer', + 'dignity', + 'dirt', + 'dust', + + 'economics', + 'education', + 'electricity', + 'engineering', + 'enjoyment', + 'envy', + 'equipment', + 'ethics', + 'evidence', + 'evolution', + + 'fame', + 'fiction', + 'flour', + 'flu', + 'food', + 'fuel', + 'fun', + 'furniture', + + 'gallows', + 'garbage', + 'garlic', + 'genetics', + 'gold', + 'golf', + 'gossip', + 'grammar', + 'gratitude', + 'grief', + 'guilt', + 'gymnastics', + + 'happiness', + 'hardware', + 'harm', + 'hate', + 'hatred', + 'health', + 'heat', + 'help', + 'homework', + 'honesty', + 'honey', + 'hospitality', + 'housework', + 'humour', + 'hunger', + 'hydrogen', + + 'ice', + 'importance', + 'inflation', + 'information', + 'innocence', + 'iron', + 'irony', + + 'jam', + 'jewelry', + 'judo', + + 'karate', + 'knowledge', + + 'lack', + 'laughter', + 'lava', + 'leather', + 'leisure', + 'lightning', + 'linguine', + 'linguini', + 'linguistics', + 'literature', + 'litter', + 'livestock', + 'logic', + 'loneliness', + 'luck', + 'luggage', + + 'macaroni', + 'machinery', + 'magic', + 'management', + 'mankind', + 'marble', + 'mathematics', + 'mayonnaise', + 'measles', + 'methane', + 'milk', + 'money', + 'mud', + 'music', + 'mumps', + + 'nature', + 'news', + 'nitrogen', + 'nonsense', + 'nurture', + 'nutrition', + + 'obedience', + 'obesity', + 'oxygen', + + 'pasta', + 'patience', + 'physics', + 'poetry', + 'pollution', + 'poverty', + 'pride', + 'psychology', + 'publicity', + 'punctuation', + + 'quartz', + + 'racism', + 'relaxation', + 'reliability', + 'research', + 'respect', + 'revenge', + 'rice', + 'rubbish', + 'rum', + + 'safety', + 'scenery', + 'seafood', + 'seaside', + 'series', + 'shame', + 'sheep', + 'shopping', + 'sleep', + 'smoke', + 'smoking', + 'snow', + 'soap', + 'software', + 'soil', + 'spaghetti', + 'species', + 'steam', + 'stuff', + 'stupidity', + 'sunshine', + 'symmetry', + + 'tennis', + 'thirst', + 'thunder', + 'timber', + 'traffic', + 'transportation', + 'trust', + + 'underwear', + 'unemployment', + 'unity', + + 'validity', + 'veal', + 'vegetation', + 'vegetarianism', + 'vengeance', + 'violence', + 'vitality', + + 'warmth', + 'wealth', + 'weather', + 'welfare', + 'wheat', + 'wildlife', + 'wisdom', + 'yoga', + + 'zinc', + 'zoology' + ) +}) + +export function toPlural(value: string) { + return pluralize(value, THEGRAPH_LOCALE) +} \ No newline at end of file diff --git a/graphql/openreader/src/dialect/thegraph/orderBy.ts b/graphql/openreader/src/dialect/thegraph/orderBy.ts new file mode 100644 index 000000000..99583de72 --- /dev/null +++ b/graphql/openreader/src/dialect/thegraph/orderBy.ts @@ -0,0 +1,75 @@ +import assert from 'assert' +import type {Model} from '../../model' +import {getUniversalProperties} from '../../model.tools' +import {OrderBy, SortOrder} from '../../ir/args' +import {mergeOrderBy} from '../common' + +export type TheGraphOrderByValue = string + +export type TheGraph_OrderBy_List = ReadonlySet + +const MAPPING_CACHE = new WeakMap>() + +export function getOrderByList(model: Model, typeName: string): TheGraph_OrderBy_List { + let cache = MAPPING_CACHE.get(model) + if (cache == null) { + cache = {} + MAPPING_CACHE.set(model, cache) + } + if (cache[typeName]) return cache[typeName] + return (cache[typeName] = buildOrderByList(model, typeName, 2)) +} + +function buildOrderByList(model: Model, typeName: string, depth: number): TheGraph_OrderBy_List { + if (depth <= 0) return new Set() + let properties = getUniversalProperties(model, typeName) + let m = new Set() + for (let key in properties) { + let propType = properties[key].type + switch (propType.kind) { + case 'scalar': + case 'enum': + if (propType.name != 'JSON') { + m.add(key) + } + break + case 'object': + case 'union': + for (let name of buildOrderByList(model, propType.name, depth - 1)) { + m.add(key + '__' + name) + } + break + case 'fk': + case 'lookup': + m.add(key) + for (let name of buildOrderByList(model, propType.entity, depth - 1)) { + m.add(key + '__' + name) + } + break + } + } + return m +} + +export const ORDER_DIRECTIONS: Record = { + asc: 'ASC', + asc_nulls_first: 'ASC NULLS FIRST', + asc_nulls_last: 'ASC NULLS LAST', + desc: 'DESC', + desc_nulls_first: 'DESC NULLS FIRST', + desc_nulls_last: 'DESC NULLS LAST', +} + +export function parseOrderBy(model: Model, typeName: string, input: {orderBy: string; direction?: string}): OrderBy { + let list = getOrderByList(model, typeName) + assert(list.has(input.orderBy)) + + const sortOrder = input.direction ? ORDER_DIRECTIONS[input.direction] : ORDER_DIRECTIONS['asc'] + assert(sortOrder) + + const keys = input.orderBy.split('__').reverse() + const res = keys.reduce((res: OrderBy | null, key) => ({[key]: res ?? sortOrder}), null) + assert(res) + + return res +} diff --git a/graphql/openreader/src/dialect/thegraph/schema.ts b/graphql/openreader/src/dialect/thegraph/schema.ts new file mode 100644 index 000000000..481ea373f --- /dev/null +++ b/graphql/openreader/src/dialect/thegraph/schema.ts @@ -0,0 +1,484 @@ +import {def, unexpectedCase} from '@subsquid/util-internal' +import {toCamelCase} from '@subsquid/util-naming' +import assert from 'assert' +import { + GraphQLBoolean, + GraphQLEnumType, + GraphQLFieldConfig, + GraphQLFloat, + GraphQLInputObjectType, + GraphQLInputType, + GraphQLInt, + GraphQLInterfaceType, + GraphQLList, + GraphQLNonNull, + GraphQLObjectType, + GraphQLOutputType, + GraphQLResolveInfo, + GraphQLScalarType, + GraphQLSchema, + GraphQLString, + GraphQLUnionType, +} from 'graphql' +import { + GraphQLEnumValueConfigMap, + GraphQLFieldConfigArgumentMap, + GraphQLFieldConfigMap, + GraphQLInputFieldConfigMap, +} from 'graphql/type/definition' +import {Context} from '../../context' +import {getListSize, getObjectSize} from '../../limit.size' +import {Entity, Interface, JsonObject, Model, Prop} from '../../model' +import {getEntity, getObject, getUniversalProperties} from '../../model.tools' +import {customScalars} from '../../scalars' +import {EntityByIdQuery, ListQuery} from '../../sql/query' +import {Limit} from '../../util/limit' +import {getResolveTree} from '../../util/resolve-tree' +import {identity} from '../../util/util' +import {getOrderByList, ORDER_DIRECTIONS} from './orderBy' +import {parseAnyTree, parseObjectTree, parseSqlArguments} from './tree' +import {GqlFieldMap, SchemaOptions} from '../common' +import {toPlural} from './locale' + +export class SchemaBuilder { + private model: Model + private types = new Map() + private where = new Map() + private orderBy = new Map() + + constructor(private options: SchemaOptions) { + this.model = options.model + } + + private get(name: string): GraphQLOutputType + private get(name: string, kind: Type): T + private get(name: string, kind?: Type): GraphQLOutputType { + switch (name) { + case 'ID': + case 'String': + return GraphQLString + case 'Int': + return GraphQLInt + case 'Boolean': + return GraphQLBoolean + case 'Float': + return GraphQLFloat + case 'DateTime': + return customScalars.DateTime + case 'BigInt': + return customScalars.BigInt + case 'BigDecimal': + return customScalars.BigDecimal + case 'Bytes': + return customScalars.Bytes + case 'JSON': + return customScalars.JSON + } + + let type = this.types.get(name) + if (type == null) { + type = this.buildType(name) + this.types.set(name, type) + } + if (kind) { + assert(type instanceof kind) + } + return type + } + + private buildType(name: string): GraphQLOutputType { + const item = this.model[name] + switch (item.kind) { + case 'entity': + case 'object': + return new GraphQLObjectType({ + name, + description: item.description, + interfaces: () => item.interfaces?.map((name) => this.get(name, GraphQLInterfaceType)), + fields: () => this.buildObjectFields(item), + }) + case 'interface': + return new GraphQLInterfaceType({ + name, + description: item.description, + fields: () => this.buildObjectFields(item), + resolveType: item.queryable ? (value: any) => value._isTypeOf : undefined, + }) + case 'enum': + return new GraphQLEnumType({ + name, + description: item.description, + values: Object.keys(item.values).reduce((values, variant) => { + values[variant] = {} + return values + }, {} as GraphQLEnumValueConfigMap), + }) + case 'union': + return new GraphQLUnionType({ + name, + description: item.description, + types: () => item.variants.map((variant) => this.get(variant, GraphQLObjectType)), + resolveType(value: any) { + return value.isTypeOf + }, + }) + default: + throw unexpectedCase() + } + } + + private buildObjectFields(object: Entity | JsonObject | Interface): GraphQLFieldConfigMap { + let fields: GraphQLFieldConfigMap = {} + for (let key in object.properties) { + let prop = object.properties[key] + let field: GraphQLFieldConfig = { + description: prop.description, + type: this.getPropType(prop), + } + if (prop.type.kind == 'list-lookup') { + field.args = this.listArguments(prop.type.entity) + } + if (object.kind == 'entity' || object.kind == 'object') { + switch (prop.type.kind) { + case 'object': + case 'union': + case 'fk': + case 'lookup': + case 'list-lookup': + field.resolve = (source, args, context, info) => source[info.path.key] + break + } + } + fields[key] = field + } + return fields + } + + private getPropType(prop: Prop): GraphQLOutputType { + let type: GraphQLOutputType + switch (prop.type.kind) { + case 'list': + type = new GraphQLList(this.getPropType(prop.type.item)) + break + case 'fk': + type = this.get(prop.type.entity) + break + case 'lookup': + return this.get(prop.type.entity) + case 'list-lookup': + return new GraphQLNonNull(new GraphQLList(new GraphQLNonNull(this.get(prop.type.entity)))) + default: + type = this.get(prop.type.name) + } + if (!prop.nullable) { + type = new GraphQLNonNull(type) + } + return type + } + + private listArguments(typeName: string): GraphQLFieldConfigArgumentMap { + return { + where: {type: this.getWhere(typeName)}, + orderBy: {type: this.getOrderBy(typeName)}, + orderDirection: {type: this.getOrderDirection()}, + skip: {type: GraphQLInt}, + first: {type: GraphQLInt}, + } + } + + private getWhere(typeName: string): GraphQLInputType { + let where = this.where.get(typeName) + if (where) return where + + let object = this.model[typeName] + let properties = getUniversalProperties(this.model, typeName) + + where = new GraphQLInputObjectType({ + name: `${typeName}_filter`, + fields: () => { + let fields: GraphQLInputFieldConfigMap = {} + + for (let key in properties) { + this.buildPropWhereFilters(key, properties[key], fields) + } + + if (object.kind == 'entity' || object.kind == 'interface') { + let whereList = new GraphQLList(new GraphQLNonNull(this.getWhere(typeName))) + fields['and'] = { + type: whereList, + } + fields['or'] = { + type: whereList, + } + } + + return fields + }, + }) + + this.where.set(typeName, where) + return where + } + + private buildPropWhereFilters(key: string, prop: Prop, fields: GraphQLInputFieldConfigMap): void { + switch (prop.type.kind) { + case 'scalar': { + let type = this.get(prop.type.name, GraphQLScalarType) + let listType = new GraphQLList(new GraphQLNonNull(type)) + + fields[`${key}_is_null`] = {type: GraphQLBoolean} + fields[`${key}`] = {type} + fields[`${key}_not`] = {type} + + switch (prop.type.name) { + case 'ID': + case 'String': + case 'Int': + case 'Float': + case 'DateTime': + case 'BigInt': + case 'BigDecimal': + fields[`${key}_gt`] = {type} + fields[`${key}_gte`] = {type} + fields[`${key}_lt`] = {type} + fields[`${key}_lte`] = {type} + fields[`${key}_in`] = {type: listType} + fields[`${key}_not_in`] = {type: listType} + break + case 'JSON': + fields[`${key}_json_contains`] = {type} + fields[`${key}_json_has_key`] = {type} + break + } + + if (prop.type.name == 'ID' || prop.type.name == 'String') { + fields[`${key}_contains`] = {type} + fields[`${key}_not_contains`] = {type} + fields[`${key}_contains_nocase`] = {type} + fields[`${key}_not_contains_nocase`] = {type} + fields[`${key}_starts_with`] = {type} + fields[`${key}_starts_with_nocase`] = {type} + fields[`${key}_not_starts_with`] = {type} + fields[`${key}_not_starts_with_nocase`] = {type} + fields[`${key}_ends_with`] = {type} + fields[`${key}_ends_with_nocase`] = {type} + fields[`${key}_not_ends_with`] = {type} + fields[`${key}_not_ends_with_nocase`] = {type} + } + + break + } + case 'enum': { + let type = this.get(prop.type.name, GraphQLEnumType) + let listType = new GraphQLList(new GraphQLNonNull(type)) + fields[`${key}_is_null`] = {type: GraphQLBoolean} + fields[`${key}`] = {type} + fields[`${key}_not`] = {type} + fields[`${key}_in`] = {type: listType} + fields[`${key}_not_in`] = {type: listType} + break + } + case 'list': + fields[`${key}_isNull`] = {type: GraphQLBoolean} + if (prop.type.item.type.kind == 'scalar' || prop.type.item.type.kind == 'enum') { + let item = this.getPropType(prop.type.item) + let list = new GraphQLList(item) + fields[`${key}_contains_all`] = {type: list} + fields[`${key}_contains_any`] = {type: list} + fields[`${key}_contains_none`] = {type: list} + } + break + case 'object': + fields[`${key}_is_null`] = {type: GraphQLBoolean} + if (this.hasFilters(getObject(this.model, prop.type.name))) { + fields[`${key}_`] = {type: this.getWhere(prop.type.name)} + } + break + case 'union': + fields[`${key}_is_null`] = {type: GraphQLBoolean} + fields[key] = {type: this.getWhere(prop.type.name)} + break + case 'fk': + case 'lookup': + fields[`${key}_is_null`] = {type: GraphQLBoolean} + fields[`${key}_`] = {type: this.getWhere(prop.type.entity)} + break + case 'list-lookup': { + let where = this.getWhere(prop.type.entity) + fields[`${key}_every`] = {type: where} + fields[`${key}_some`] = {type: where} + fields[`${key}_none`] = {type: where} + break + } + } + } + + private hasFilters(obj: JsonObject): boolean { + for (let key in obj.properties) { + let propType = obj.properties[key].type + switch (propType.kind) { + case 'scalar': + case 'enum': + case 'union': + return true + case 'object': { + let ref = getObject(this.model, propType.name) + if (ref !== obj && this.hasFilters(ref)) { + return true + } + } + } + } + return false + } + + private getOrderBy(typeName: string): GraphQLInputType { + let orderBy = this.orderBy.get(typeName) + if (orderBy) return orderBy + + let values: GraphQLEnumValueConfigMap = {} + for (let variant of getOrderByList(this.model, typeName)) { + values[variant] = {} + } + + orderBy = new GraphQLEnumType({ + name: `${typeName}_orderBy`, + values, + }) + this.orderBy.set(typeName, orderBy) + return orderBy + } + + @def + private getOrderDirection(): GraphQLInputType { + let values: GraphQLEnumValueConfigMap = {} + for (let variant of Object.keys(ORDER_DIRECTIONS)) { + values[variant] = {} + } + + return new GraphQLEnumType({ + name: `OrderDirection`, + values, + }) + } + + @def + build(): GraphQLSchema { + let query: GqlFieldMap = {} + let subscription: GqlFieldMap = {} + + for (let name in this.model) { + let item = this.model[name] + switch (item.kind) { + case 'entity': + this.installEntityQuery(name, query, subscription) + this.installListQuery(name, query, subscription) + break + case 'interface': + if (item.queryable) { + this.installListQuery(name, query, subscription) + } + break + } + } + + return new GraphQLSchema({ + query: new GraphQLObjectType({ + name: 'Query', + fields: query, + }), + subscription: this.options.subscriptions + ? new GraphQLObjectType({ + name: 'Subscription', + fields: subscription, + }) + : undefined, + }) + } + + private installListQuery(typeName: string, query: GqlFieldMap, subscription: GqlFieldMap): void { + let model = this.model + + let entity = model[typeName] + let queryName = (entity.kind === 'entity' && entity.listQueryName) || this.normalizeQueryName(typeName).plural + let outputType = new GraphQLNonNull(new GraphQLList(new GraphQLNonNull(this.get(typeName)))) + let argsType = this.listArguments(typeName) + + function createQuery(context: Context, info: GraphQLResolveInfo, limit?: Limit) { + let tree = getResolveTree(info) + let args = parseSqlArguments(model, typeName, tree.args) + let fields = parseAnyTree(model, typeName, info.schema, tree) + limit?.check(() => getListSize(model, typeName, fields, args.limit, args.where) + 1) + return new ListQuery(model, context.openreader.dbType, typeName, fields, args) + } + + query[queryName] = { + type: outputType, + args: argsType, + resolve(source, args, context, info) { + let q = createQuery(context, info, context.openreader.responseSizeLimit) + return context.openreader.executeQuery(q) + }, + } + + subscription[queryName] = { + type: outputType, + args: argsType, + resolve: identity, + subscribe(source, args, context, info) { + let q = createQuery(context, info, context.openreader.subscriptionResponseSizeLimit) + return context.openreader.subscription(q) + }, + } + } + + private installEntityQuery(entityName: string, query: GqlFieldMap, subscription: GqlFieldMap): void { + let model = this.model + + let entity = model[entityName] + let queryName = (entity.kind === 'entity' && entity.queryName) || this.normalizeQueryName(entityName).singular + let argsType = { + id: {type: new GraphQLNonNull(GraphQLString)}, + } + + function createQuery(context: Context, info: GraphQLResolveInfo, limit?: Limit) { + let tree = getResolveTree(info) + let fields = parseObjectTree(model, entityName, info.schema, tree) + limit?.check(() => getObjectSize(model, fields) + 1) + return new EntityByIdQuery(model, context.openreader.dbType, entityName, fields, tree.args.id as string) + } + + query[queryName] = { + type: this.get(entityName), + args: argsType, + async resolve(source, args, context, info) { + let q = createQuery(context, info, context.openreader.responseSizeLimit) + return context.openreader.executeQuery(q) + }, + } + + subscription[queryName] = { + type: this.get(entityName), + args: argsType, + resolve: identity, + subscribe(source, args, context, info) { + let q = createQuery(context, info, context.openreader.subscriptionResponseSizeLimit) + return context.openreader.subscription(q) + }, + } + } + + private normalizeQueryName(typeName: string) { + let singular = toCamelCase(typeName) + let plural = toPlural(singular) + if (singular === plural) { + plural += '_collection' + } + + return {singular, plural} + } +} + +interface Type { + new (...args: any[]): T +} diff --git a/graphql/openreader/src/dialect/thegraph/tree.ts b/graphql/openreader/src/dialect/thegraph/tree.ts new file mode 100644 index 000000000..6167bc390 --- /dev/null +++ b/graphql/openreader/src/dialect/thegraph/tree.ts @@ -0,0 +1,162 @@ +import {unexpectedCase} from '@subsquid/util-internal' +import assert from 'assert' +import {GraphQLSchema} from 'graphql' +import {ResolveTree} from 'graphql-parse-resolve-info' +import {SqlArguments} from '../../ir/args' +import {AnyFields, FieldRequest, FieldsByEntity, OpaqueRequest} from '../../ir/fields' +import {Model} from '../../model' +import {getQueryableEntities} from '../../model.tools' +import {simplifyResolveTree} from '../../util/resolve-tree' +import {parseWhere} from './where' +import {parseOrderBy} from './orderBy' + +export function parseObjectTree( + model: Model, + typeName: string, + schema: GraphQLSchema, + tree: ResolveTree +): FieldRequest[] { + let requests: FieldRequest[] = [] + let requestedScalars: Record = {} + let object = model[typeName] + assert(object.kind == 'entity' || object.kind == 'object') + + let fields = simplifyResolveTree(schema, tree, typeName).fields + for (let alias in fields) { + let f = fields[alias] + let prop = object.properties[f.name] + switch (prop.type.kind) { + case 'scalar': + case 'enum': + case 'list': + if (requestedScalars[f.name] == null) { + requestedScalars[f.name] = true + requests.push({ + field: f.name, + aliases: [f.name], + kind: prop.type.kind, + type: prop.type, + prop, + index: 0, + } as OpaqueRequest) + } + break + case 'object': + requests.push({ + field: f.name, + aliases: [f.alias], + kind: prop.type.kind, + type: prop.type, + prop, + index: 0, + children: parseObjectTree(model, prop.type.name, schema, f), + }) + break + case 'union': { + let union = model[prop.type.name] + assert(union.kind == 'union') + let children: FieldRequest[] = [] + for (let variant of union.variants) { + for (let req of parseObjectTree(model, variant, schema, f)) { + req.ifType = variant + children.push(req) + } + } + requests.push({ + field: f.name, + aliases: [f.alias], + kind: prop.type.kind, + type: prop.type, + prop, + index: 0, + children, + }) + break + } + case 'fk': + requests.push({ + field: f.name, + aliases: [f.alias], + kind: prop.type.kind, + type: prop.type, + prop, + index: 0, + children: parseObjectTree(model, prop.type.entity, schema, f), + }) + break + case 'lookup': + requests.push({ + field: f.name, + aliases: [f.alias], + kind: prop.type.kind, + type: prop.type, + prop, + index: 0, + children: parseObjectTree(model, prop.type.entity, schema, f), + }) + break + case 'list-lookup': + requests.push({ + field: f.name, + aliases: [f.alias], + kind: prop.type.kind, + type: prop.type, + prop, + index: 0, + args: parseSqlArguments(model, prop.type.entity, f.args), + children: parseObjectTree(model, prop.type.entity, schema, f), + }) + break + default: + throw unexpectedCase() + } + } + + return requests +} + +export function parseSqlArguments(model: Model, typeName: string, gqlArgs: any): SqlArguments { + let args: SqlArguments = {} + + let where = parseWhere(gqlArgs.where) + if (where) { + args.where = where + } + + if (gqlArgs.orderBy) { + args.orderBy = parseOrderBy(model, typeName, {orderBy: gqlArgs.orderBy, direction: gqlArgs.orderDirection}) + } + + if (gqlArgs.skip) { + assert(typeof gqlArgs.skip == 'number') + args.offset = gqlArgs.skip + } + + if (gqlArgs.first != null) { + assert(typeof gqlArgs.first == 'number') + args.limit = gqlArgs.first + } + + return args +} + +export function parseQueryableTree( + model: Model, + queryableName: string, + schema: GraphQLSchema, + tree: ResolveTree +): FieldsByEntity { + let fields: FieldsByEntity = {} + for (let entity of getQueryableEntities(model, queryableName)) { + fields[entity] = parseObjectTree(model, entity, schema, tree) + } + return fields +} + +export function parseAnyTree(model: Model, typeName: string, schema: GraphQLSchema, tree: ResolveTree): AnyFields { + if (model[typeName].kind == 'interface') { + return parseQueryableTree(model, typeName, schema, tree) + } else { + return parseObjectTree(model, typeName, schema, tree) + } +} diff --git a/graphql/openreader/src/dialect/thegraph/where.ts b/graphql/openreader/src/dialect/thegraph/where.ts new file mode 100644 index 000000000..18a8c3969 --- /dev/null +++ b/graphql/openreader/src/dialect/thegraph/where.ts @@ -0,0 +1,184 @@ +import {unexpectedCase} from '@subsquid/util-internal' +import assert from 'assert' +import {Where} from '../../ir/args' +import {ensureArray} from '../../util/util' +import {toCondition} from '../common' + +export function parseWhere(whereArg?: any): Where | undefined { + if (whereArg == null) return undefined + let {and, or, ...fields} = whereArg + let conj: Where[] = [] + + for (let key in fields) { + let arg = fields[key] + let {field, op} = parseWhereKey(key) + switch (op) { + case 'SELF': + conj.push({op: 'eq', field, value: arg}) + break + case '_': { + let where = parseWhere(arg) + where && conj.push({op: 'REF', field, where}) + break + } + case '_every': { + let where = parseWhere(arg) + where && conj.push({op: 'every', field, where}) + break + } + case '_some': + conj.push({op: 'some', field, where: parseWhere(arg)}) + break + case '_none': + conj.push({op: 'none', field, where: parseWhere(arg)}) + break + case '_in': + conj.push({op: 'in', field, values: ensureArray(arg)}) + break + case '_not_in': + conj.push({op: 'not_in', field, values: ensureArray(arg)}) + break + case '_not': + conj.push({op: 'not_eq', field, value: arg}) + break + case '_gt': + conj.push({op: 'gt', field, value: arg}) + break + case '_gte': + conj.push({op: 'gte', field, value: arg}) + break + case '_lt': + conj.push({op: 'lt', field, value: arg}) + break + case '_lte': + conj.push({op: 'lte', field, value: arg}) + break + case '_contains': + conj.push({op: 'contains', field, value: arg}) + break + case '_not_contains': + conj.push({op: 'not_contains', field, value: arg}) + break + case '_contains_nocase': + conj.push({op: 'containsInsensitive', field, value: arg}) + break + case '_not_contains_nocase': + conj.push({op: 'not_containsInsensitive', field, value: arg}) + break + case '_starts_with': + conj.push({op: 'startsWith', field, value: arg}) + break + case '_starts_with_nocase': + conj.push({op: 'startsWithInsensitive', field, value: arg}) + break + case '_not_starts_with': + conj.push({op: 'not_startsWith', field, value: arg}) + break + case '_not_starts_with_nocase': + conj.push({op: 'not_startsWithInsensitive', field, value: arg}) + break + case '_ends_with': + conj.push({op: 'endsWith', field, value: arg}) + break + case '_ends_with_nocase': + conj.push({op: 'endsWithInsensitive', field, value: arg}) + break + case '_not_ends_with': + conj.push({op: 'not_endsWith', field, value: arg}) + break + case '_not_ends_with_nocase': + conj.push({op: 'not_endsWithInsensitive', field, value: arg}) + break + case '_json_has_key': + conj.push({op: 'jsonHasKey', field, value: arg}) + break + case '_json_contains': + conj.push({op: 'jsonContains', field, value: arg}) + break + case '_contains_none': + conj.push({op: 'containsNone', field, value: ensureArray(arg)}) + break + case '_contains_all': + conj.push({op: 'containsAll', field, value: ensureArray(arg)}) + break + case '_contains_any': + conj.push({op: 'containsAny', field, value: ensureArray(arg)}) + break + case '_is_null': + assert(typeof arg == 'boolean') + conj.push({op: 'isNull', field, yes: arg}) + break + default: + throw unexpectedCase(op) + } + } + + if (and) { + for (let arg of ensureArray(and)) { + let where = parseWhere(arg) + if (where) { + conj.push(where) + } + } + } + + let conjunction = toCondition('AND', conj) + if (or) { + let disjunctions: Where[] = [] + if (conjunction) { + disjunctions.push(conjunction) + } + for (let arg of ensureArray(or)) { + let where = parseWhere(arg) + if (where) { + disjunctions.push(where) + } + } + return toCondition('OR', disjunctions) + } else { + return conjunction + } +} + +export function parseWhereKey(key: string): {op: (typeof WHERE_OPS)[number] | 'SELF'; field: string} { + let m = WHERE_KEY_REGEX.exec(key) + if (m) { + return {op: m[2] as (typeof WHERE_OPS)[number], field: m[1]} + } else { + return {op: 'SELF', field: key} + } +} + +const WHERE_OPS = [ + '_', + '_not', + '_gt', + '_gte', + '_lt', + '_lte', + '_in', + '_not_in', + '_contains', + '_contains_nocase', + '_not_contains', + '_not_contains_nocase', + '_starts_with', + '_starts_with_nocase', + '_not_starts_with', + '_not_starts_with_nocase', + '_ends_with', + '_ends_with_nocase', + '_not_ends_with', + '_not_ends_with_nocase', + '_contains_all', + '_contains_any', + '_contains_none', + '_json_contains', + '_json_has_key', + '_is_null', + '_some', + '_every', + '_none', +] as const + +const WHERE_KEY_REGEX = new RegExp(`^([^_]*)(${WHERE_OPS.join('|')})$`) diff --git a/graphql/openreader/src/ir/args.ts b/graphql/openreader/src/ir/args.ts index ec72a75c8..0dc844b37 100644 --- a/graphql/openreader/src/ir/args.ts +++ b/graphql/openreader/src/ir/args.ts @@ -66,7 +66,9 @@ export type BinaryOp = 'contains' | 'not_contains' | 'containsInsensitive' | 'not_containsInsensitive' | 'startsWith' | 'not_startsWith' | + 'startsWithInsensitive' | 'not_startsWithInsensitive' | 'endsWith' | 'not_endsWith' | + 'endsWithInsensitive' | 'not_endsWithInsensitive' | 'containsAll' | 'containsAny' | 'containsNone' | diff --git a/graphql/openreader/src/main.ts b/graphql/openreader/src/main.ts index 0ec950217..c6fe194cd 100644 --- a/graphql/openreader/src/main.ts +++ b/graphql/openreader/src/main.ts @@ -4,7 +4,7 @@ import {nat, Url} from '@subsquid/util-internal-commander' import {waitForInterruption} from '@subsquid/util-internal-http-server' import {Command, Option} from 'commander' import {Pool} from 'pg' -import {Dialect} from './dialect' +import {DbType} from './db' import {serve} from './server' import {loadModel} from './tools' @@ -36,7 +36,7 @@ GraphQL server for postgres-compatible databases let opts = program.parse().opts() as { schema: string dbUrl: string - dbType: Dialect + dbType: DbType port: number maxRequestSize: number maxRootFields?: number @@ -56,7 +56,7 @@ GraphQL server for postgres-compatible databases let server = await serve({ model, - dialect: opts.dbType, + dbType: opts.dbType, connection, port: opts.port, log: LOG, diff --git a/graphql/openreader/src/model.schema.ts b/graphql/openreader/src/model.schema.ts index 3e76f4225..6747d439e 100644 --- a/graphql/openreader/src/model.schema.ts +++ b/graphql/openreader/src/model.schema.ts @@ -24,7 +24,7 @@ import {customScalars} from './scalars' const baseSchema = buildASTSchema(parse(` - directive @entity on OBJECT + directive @entity(queryName: String listQueryName: String) on OBJECT directive @query on INTERFACE directive @derivedFrom(field: String!) on FIELD_DEFINITION directive @unique on FIELD_DEFINITION @@ -80,10 +80,18 @@ function addEntityOrJsonObjectOrInterface(model: Model, type: GraphQLObjectType let indexes: Index[] = type instanceof GraphQLObjectType ? checkEntityIndexes(type) : [] let cardinality = checkEntityCardinality(type) let description = type.description || undefined - - switch(kind) { + + switch (kind) { case 'entity': - model[type.name] = {kind, properties, description, interfaces, indexes, ...cardinality} + model[type.name] = { + kind, + properties, + description, + interfaces, + indexes, + ...cardinality, + ...handleEntityDirective(model, type), + } break case 'object': model[type.name] = {kind, properties, description, interfaces} @@ -528,4 +536,29 @@ function unsupportedFieldTypeError(propName: string): Error { } +function handleEntityDirective(model: Model, type: GraphQLObjectType | GraphQLInterfaceType) { + let directive = type.astNode?.directives?.find(d => d.name.value == 'entity') + if (directive == null) return + + let queryNameArg = directive.arguments?.find(d => d.name.value === 'queryName') + let queryName: string | undefined + if (queryNameArg != null) { + assert(queryNameArg?.value.kind == 'StringValue') + queryName = queryNameArg.value.value + } + + let listQueryNameArg = directive.arguments?.find(d => d.name.value === 'listQueryName') + let listQueryName: string | undefined + if (listQueryNameArg != null) { + assert(listQueryNameArg?.value.kind == 'StringValue') + listQueryName = listQueryNameArg.value.value + } + + return { + queryName, + listQueryName, + } +} + + export class SchemaError extends Error {} diff --git a/graphql/openreader/src/model.ts b/graphql/openreader/src/model.ts index 456d0b522..dc074b49e 100644 --- a/graphql/openreader/src/model.ts +++ b/graphql/openreader/src/model.ts @@ -10,6 +10,8 @@ export interface Entity extends TypeMeta { interfaces?: Name[] indexes?: Index[] cardinality?: number + queryName?: string + listQueryName?: string } diff --git a/graphql/openreader/src/server.ts b/graphql/openreader/src/server.ts index cce2cc6ed..c2fce148c 100644 --- a/graphql/openreader/src/server.ts +++ b/graphql/openreader/src/server.ts @@ -12,17 +12,18 @@ import type {Pool} from 'pg' import {WebSocketServer} from 'ws' import {Context, OpenreaderContext} from './context' import {PoolOpenreaderContext} from './db' -import type {Dialect} from './dialect' +import type {DbType} from './db' import type {Model} from './model' -import {SchemaBuilder} from './opencrud/schema' import {openreaderExecute, openreaderSubscribe} from './util/execute' import {ResponseSizeLimit} from './util/limit' +import {Dialect, getSchemaBuilder} from './dialect' export interface ServerOptions { port: number | string model: Model connection: Pool + dbType?: DbType dialect?: Dialect graphiqlConsole?: boolean log?: Logger @@ -32,7 +33,7 @@ export interface ServerOptions { subscriptions?: boolean subscriptionPollInterval?: number subscriptionConnection?: Pool - subscriptionMaxResponseNodes?: number, + subscriptionMaxResponseNodes?: number cache?: KeyValueCache } @@ -43,16 +44,17 @@ export async function serve(options: ServerOptions): Promise { subscriptionPollInterval, maxResponseNodes, subscriptionMaxResponseNodes, - log + log, } = options - let dialect = options.dialect ?? 'postgres' + let dbType = options.dbType ?? 'postgres' - let schema = new SchemaBuilder(options).build() + let schemaBuilder = await getSchemaBuilder(options) + let schema = schemaBuilder.build() let context = () => { let openreader: OpenreaderContext = new PoolOpenreaderContext( - dialect, + dbType, connection, subscriptionConnection, subscriptionPollInterval, @@ -69,23 +71,23 @@ export async function serve(options: ServerOptions): Promise { } return { - openreader + openreader, } } let disposals: Dispose[] = [] return addServerCleanup(disposals, runApollo({ - port: options.port, - schema, - context, - disposals, - subscriptions: options.subscriptions, - log: options.log, - graphiqlConsole: options.graphiqlConsole, - maxRequestSizeBytes: options.maxRequestSizeBytes, - maxRootFields: options.maxRootFields, - cache: options.cache, + port: options.port, + schema, + context, + disposals, + subscriptions: options.subscriptions, + log: options.log, + graphiqlConsole: options.graphiqlConsole, + maxRequestSizeBytes: options.maxRequestSizeBytes, + maxRootFields: options.maxRootFields, + cache: options.cache, }), options.log) } @@ -117,7 +119,7 @@ export async function runApollo(options: ApolloOptions): Promise openreaderExecute(args, { maxRootFields: maxRootFields - }) + }) if (options.subscriptions) { let wsServer = new WebSocketServer({ diff --git a/graphql/openreader/src/sql/cursor.ts b/graphql/openreader/src/sql/cursor.ts index 28748d579..9f59a2401 100644 --- a/graphql/openreader/src/sql/cursor.ts +++ b/graphql/openreader/src/sql/cursor.ts @@ -1,7 +1,7 @@ import {assertNotNull, unexpectedCase} from "@subsquid/util-internal" import {toSnakeCase} from "@subsquid/util-naming" import assert from "assert" -import {Dialect} from "../dialect" +import {DbType} from "../db" import {Entity, JsonObject, Model, ObjectPropType, Prop, UnionPropType} from "../model" import {getEntity, getFtsQuery, getObject, getUnionProps} from "../model.tools" import {toColumn, toFkColumn, toTable} from "../util/util" @@ -10,7 +10,7 @@ import {AliasSet, escapeIdentifier, JoinSet} from "./util" export interface CursorCtx { model: Model - dialect: Dialect + dialect: DbType aliases: AliasSet join: JoinSet } @@ -62,7 +62,7 @@ export class EntityCursor implements Cursor { } prop(field: string): Prop { - return assertNotNull(this.entity.properties[field]) + return assertNotNull(this.entity.properties[field], `property ${field} is missing`) } output(field: string): string { @@ -209,7 +209,7 @@ export class ObjectCursor implements Cursor { } prop(field: string): Prop { - return assertNotNull(this.object.properties[field]) + return assertNotNull(this.object.properties[field], `property ${field} is missing`) } output(field: string): string { diff --git a/graphql/openreader/src/sql/printer.ts b/graphql/openreader/src/sql/printer.ts index c4721fbc4..0b7dc43e8 100644 --- a/graphql/openreader/src/sql/printer.ts +++ b/graphql/openreader/src/sql/printer.ts @@ -1,6 +1,6 @@ import {unexpectedCase} from '@subsquid/util-internal' import assert from 'assert' -import {Dialect} from '../dialect' +import {DbType} from '../db' import {OrderBy, SortOrder, SqlArguments, Where} from '../ir/args' import {FieldRequest, FieldsByEntity} from '../ir/fields' import {Model} from '../model' @@ -19,7 +19,7 @@ export class EntitySqlPrinter { constructor( private model: Model, - private dialect: Dialect, + private dialect: DbType, public readonly entityName: string, private params: unknown[], private args: SqlArguments = {}, @@ -92,6 +92,10 @@ export class EntitySqlPrinter { } private populateWhere(cursor: Cursor, where: Where, exps: string[]): void { + function printStr(value: string) { + return !!where.op?.endsWith?.('Insensitive') ? `lower(${value})` : value + } + switch(where.op) { case "AND": for (let cond of where.args) { @@ -178,47 +182,46 @@ export class EntitySqlPrinter { break } case "startsWith": + case "startsWithInsensitive": if (this.dialect == "cockroach") { let f = cursor.native(where.field) let p = this.param(where.value) + "::text" - exps.push(`${f} >= ${p}`) - exps.push(`left(${f}, length(${p})) = ${p}`) + exps.push(`${printStr(f)} >= ${printStr(p)}`) + exps.push(`left(${printStr(f)}, length(${p})) = ${printStr(p)}`) } else { - exps.push(`starts_with(${cursor.native(where.field)}, ${this.param(where.value)})`) + exps.push(`starts_with(${printStr(cursor.native(where.field))}, ${printStr(this.param(where.value))})`) } break case "not_startsWith": + case "not_startsWithInsensitive": if (this.dialect == "cockroach") { let f = cursor.native(where.field) let p = this.param(where.value) + "::text" - exps.push(`(${f} < ${p} OR left(${f}, length(${p})) != ${p})`) + exps.push(`(${printStr(f)} < ${printStr(p)} OR left(${printStr(f)}, length(${printStr(p)})) != ${printStr(p)})`) } else { - exps.push(`NOT starts_with(${cursor.native(where.field)}, ${this.param(where.value)})`) + exps.push(`NOT starts_with(${printStr(cursor.native(where.field))}, ${printStr(this.param(where.value))})`) } break - case "endsWith": { + case "endsWith": + case "not_startsWithInsensitive": { let f = cursor.native(where.field) let p = this.param(where.value) + "::text" - exps.push(`right(${f}, length(${p})) = ${p}`) + exps.push(`right(${printStr(f)}, length(${p})) = ${printStr(p)}`) break } case "not_endsWith": { let f = cursor.native(where.field) let p = this.param(where.value) + "::text" - exps.push(`right(${f}, length(${p})) != ${p}`) + exps.push(`right(${printStr(f)}, length(${p})) != ${printStr(p)}`) break } case "contains": - exps.push(`position(${this.param(where.value)} in ${cursor.native(where.field)}) > 0`) - break - case "not_contains": - exps.push(`position(${this.param(where.value)} in ${cursor.native(where.field)}) = 0`) - break case "containsInsensitive": - exps.push(`position(lower(${this.param(where.value)}) in lower(${cursor.native(where.field)})) > 0`) + exps.push(`position(${printStr(this.param(where.value))} in ${printStr(cursor.native(where.field))}) > 0`) break + case "not_contains": case "not_containsInsensitive": - exps.push(`position(lower(${this.param(where.value)}) in lower(${cursor.native(where.field)})) = 0`) + exps.push(`position(${printStr(this.param(where.value))} in ${printStr(cursor.native(where.field))}) = 0`) break case "every": { let rel = cursor.prop(where.field) @@ -360,7 +363,7 @@ export class QueryableSqlPrinter { constructor( private model: Model, - private dialect: Dialect, + private dialect: DbType, private queryableName: string, private params: unknown[], private args: SqlArguments = {}, @@ -446,3 +449,4 @@ export class QueryableSqlPrinter { return sql } } + diff --git a/graphql/openreader/src/sql/query.ts b/graphql/openreader/src/sql/query.ts index 9ec560c8c..0d30a4a57 100644 --- a/graphql/openreader/src/sql/query.ts +++ b/graphql/openreader/src/sql/query.ts @@ -1,6 +1,6 @@ import {assertNotNull} from '@subsquid/util-internal' import assert from 'assert' -import type {Dialect} from '../dialect' +import type {DbType} from '../db' import type {SqlArguments, Where} from '../ir/args' import { decodeRelayConnectionCursor, @@ -30,7 +30,7 @@ export class ListQuery implements Query { constructor( model: Model, - dialect: Dialect, + dialect: DbType, typeName: string, private fields: AnyFields, args: SqlArguments @@ -60,7 +60,7 @@ export class EntityByIdQuery { constructor( model: Model, - dialect: Dialect, + dialect: DbType, entityName: string, private fields: FieldRequest[], id: string @@ -88,7 +88,7 @@ export class CountQuery implements Query { constructor( model: Model, - dialect: Dialect, + dialect: DbType, typeName: string, where?: Where ) { @@ -114,7 +114,7 @@ export class ConnectionQuery implements Query { constructor( model: Model, - dialect: Dialect, + dialect: DbType, typeName: string, req: RelayConnectionRequest ) { diff --git a/graphql/openreader/src/sql/util.ts b/graphql/openreader/src/sql/util.ts index 8bd0fe026..cfbc31bc7 100644 --- a/graphql/openreader/src/sql/util.ts +++ b/graphql/openreader/src/sql/util.ts @@ -1,7 +1,7 @@ -import type {Dialect} from "../dialect" +import type {DbType} from "../db" -export function escapeIdentifier(dialect: Dialect, name: string): string { +export function escapeIdentifier(dialect: DbType, name: string): string { return `"${name.replace(/"/g, '""')}"` } diff --git a/graphql/openreader/src/test/basic.test.ts b/graphql/openreader/src/test/basic.test.ts index 124469857..9fb29cf6d 100644 --- a/graphql/openreader/src/test/basic.test.ts +++ b/graphql/openreader/src/test/basic.test.ts @@ -1,3 +1,4 @@ +import {Dialect} from '../dialect' import {useDatabase, useServer} from "./setup" @@ -16,324 +17,611 @@ describe('basic tests', function() { `insert into historical_balance (id, account_id, balance) values ('3-1', '3', 300)`, ]) - const client = useServer(` - interface HasBalance { - balance: Int! - } - - type Account implements HasBalance @entity { - id: ID! - wallet: String! - balance: Int! - history: [HistoricalBalance!] @derivedFrom(field: "account") - } - - "Historical record of account balance" - type HistoricalBalance implements HasBalance @entity { - "Unique identifier" - id: ID! - - "Related account" - account: Account! - - "Balance" - balance: Int! - } - `) - - it('can fetch all accounts', function() { - return client.test( - `query { - accounts(orderBy: id_ASC) { - id - wallet - balance - history(orderBy: id_ASC) { balance } - } - }`, - { - accounts: [ - {id: '1', wallet: 'a', balance: 100, history: [{balance: 20}, {balance: 80}]}, - {id: '2', wallet: 'b', balance: 200, history: [{balance: 50}, {balance: 90}, {balance: 60}]}, - {id: '3', wallet: 'c', balance: 300, history: [{balance: 300}]}, - ] + describe('opencrud', function() { + const client = useServer(` + interface HasBalance { + balance: Int! } - ) - }) - - it('supports filtering by id', function () { - return client.test( - `query { - accounts(where: {id_eq: "3"}) { - id - wallet - } - }`, - { - accounts: [{id: '3', wallet: 'c'}] + + type Account implements HasBalance @entity { + id: ID! + wallet: String! + balance: Int! + history: [HistoricalBalance!] @derivedFrom(field: "account") } - ) - }) - - it('supports by id query', function () { - return client.test( - `query { - a3: accountById(id: "3") { - id - wallet - } - nonexistent: accountById(id: "foo") { - id - wallet - } - }`, - { - a3: {id: '3', wallet: 'c'}, - nonexistent: null + + "Historical record of account balance" + type HistoricalBalance implements HasBalance @entity { + "Unique identifier" + id: ID! + + "Related account" + account: Account! + + "Balance" + balance: Int! } - ) - }) - - it('supports by unique input query', function () { - return client.test( - `query { - a2: accountByUniqueInput(where: {id: "2"}) { - id - wallet + `) + + it('can fetch all accounts', function() { + return client.test( + `query { + accounts(orderBy: id_ASC) { + id + wallet + balance + history(orderBy: id_ASC) { balance } + } + }`, + { + accounts: [ + {id: '1', wallet: 'a', balance: 100, history: [{balance: 20}, {balance: 80}]}, + {id: '2', wallet: 'b', balance: 200, history: [{balance: 50}, {balance: 90}, {balance: 60}]}, + {id: '3', wallet: 'c', balance: 300, history: [{balance: 300}]}, + ] } - nonexistent: accountByUniqueInput(where: {id: "foo"}) { - id - wallet + ) + }) + + it('supports filtering by id', function () { + return client.test( + `query { + accounts(where: {id_eq: "3"}) { + id + wallet + } + }`, + { + accounts: [{id: '3', wallet: 'c'}] } - }`, - { - a2: {id: '2', wallet: 'b'}, - nonexistent: null - } - ) - }) - - it('can fetch deep relations', function () { - return client.test( - `query { - accounts(where: {id_eq: "3"}) { - id - history { + ) + }) + + it('supports by id query', function () { + return client.test( + `query { + a3: accountById(id: "3") { id - account { - wallet - history { - balance - account { - id + wallet + } + nonexistent: accountById(id: "foo") { + id + wallet + } + }`, + { + a3: {id: '3', wallet: 'c'}, + nonexistent: null + } + ) + }) + + it('can fetch deep relations', function () { + return client.test( + `query { + accounts(where: {id_eq: "3"}) { + id + history { + id + account { + wallet + history { + balance + account { + id + } } } } } + }`, + { + accounts: [{ + id: '3', + history: [{ + id: '3-1', + account: { + wallet: 'c', + history: [{ + balance: 300, + account: { + id: '3' + } + }] + } + }] + }] } - }`, - { - accounts: [{ - id: '3', - history: [{ - id: '3-1', - account: { - wallet: 'c', - history: [{ - balance: 300, - account: { - id: '3' - } - }] + ) + }) + + it('supports *_some filter', function () { + return client.test( + `query { + accounts(where: {history_some: {balance_lt: 50}}) { + id + } + }`, + { + accounts: [{id: '1'}] + } + ) + }) + + it('supports *_every filter', function () { + return client.test( + `query { + accounts(where: {history_every: {balance_gt: 20}}) { + wallet + } + }`, + { + accounts: [{wallet: 'b'}, {wallet: 'c'}] + } + ) + }) + + it('supports *_none filter', function () { + return client.test( + `query { + accounts(where: {history_none: {balance_lt: 60}}) { + wallet + } + }`, + { + accounts: [{wallet: 'c'}] + } + ) + }) + + it('supports gql aliases', function () { + return client.test( + `query { + accounts(where: {id_eq: "1"}) { + balance + bag: wallet + purse: wallet + payment1: history(where: {id_eq: "1-1"}) { + balance } + payment2: history(where: {id_eq: "1-2"}) { + balance + } + } + }`, + { + accounts: [{ + balance: 100, + bag: 'a', + purse: 'a', + payment1: [{balance: 20}], + payment2: [{balance: 80}] }] - }] - } - ) - }) - - it('supports *_some filter', function () { - return client.test( - `query { - accounts(where: {history_some: {balance_lt: 50}}) { - id } - }`, - { - accounts: [{id: '1'}] - } - ) - }) - - it('supports *_every filter', function () { - return client.test( - `query { - accounts(where: {history_every: {balance_gt: 20}}) { - wallet + ) + }) + + it('supports gql fragments', function () { + return client.test( + `query { + accounts(where: {id_eq: "1"}) { + ...accountFields + history { + ...historicalBalance + } + } } - }`, - { - accounts: [{wallet: 'b'}, {wallet: 'c'}] - } - ) - }) - - it('supports *_none filter', function () { - return client.test( - `query { - accounts(where: {history_none: {balance_lt: 60}}) { + + fragment accountFields on Account { + id wallet } - }`, - { - accounts: [{wallet: 'c'}] - } - ) - }) - - it('supports gql aliases', function () { - return client.test( - `query { - accounts(where: {id_eq: "1"}) { + + fragment historicalBalance on HistoricalBalance { balance - bag: wallet - purse: wallet - payment1: history(where: {id_eq: "1-1"}) { + }`, + { + accounts: [{ + id: '1', + wallet: 'a', + history: [{balance: 20}, {balance: 80}] + }] + } + ) + }) + + it('supports gql fragments on interfaces', function () { + return client.test( + `query { + accounts(where: {id_eq: "1"}) { + ...balance + history { + ...balance + } + } + } + + fragment balance on HasBalance { + ... on Account { + accountBalance: balance + } + ... on HistoricalBalance { + payment: balance + } + }`, + { + accounts: [{ + accountBalance: 100, + history: [{payment: 20}, {payment: 80}] + }] + } + ) + }) + + it('supports sorting', function () { + return client.test( + `query { + historicalBalances(orderBy: balance_ASC) { balance } - payment2: history(where: {id_eq: "1-2"}) { + }`, + { + historicalBalances: [ + {balance: 20}, + {balance: 50}, + {balance: 60}, + {balance: 80}, + {balance: 90}, + {balance: 300} + ] + } + ) + }) + + it('supports sorting by referenced entity field', function () { + return client.test( + `query { + historicalBalances(orderBy: [account_wallet_ASC, balance_DESC]) { balance } + }`, + { + historicalBalances: [ + {balance: 80}, + {balance: 20}, + {balance: 90}, + {balance: 60}, + {balance: 50}, + {balance: 300} + ] } - }`, - { - accounts: [{ - balance: 100, - bag: 'a', - purse: 'a', - payment1: [{balance: 20}], - payment2: [{balance: 80}] - }] - } - ) - }) - - it('supports gql fragments', function () { - return client.test( - `query { - accounts(where: {id_eq: "1"}) { - ...accountFields - history { - ...historicalBalance + ) + }) + + it('supports descriptions', function () { + return client.test(` + query { + HistoricalBalance: __type(name: "HistoricalBalance") { + description + fields { + description + } } } + `, { + HistoricalBalance: { + description: 'Historical record of account balance', + fields: [ + {description: 'Unique identifier'}, + {description: 'Related account'}, + {description: 'Balance'}, + ] + } + }) + }) + }) + + describe('thegraph', function() { + const client = useServer(` + interface HasBalance { + balance: Int! } - - fragment accountFields on Account { - id - wallet + + type Account implements HasBalance @entity { + id: ID! + wallet: String! + balance: Int! + history: [HistoricalBalance!] @derivedFrom(field: "account") } - - fragment historicalBalance on HistoricalBalance { - balance - }`, - { - accounts: [{ - id: '1', - wallet: 'a', - history: [{balance: 20}, {balance: 80}] - }] + + "Historical record of account balance" + type HistoricalBalance implements HasBalance @entity { + "Unique identifier" + id: ID! + + "Related account" + account: Account! + + "Balance" + balance: Int! } - ) - }) - - it('supports gql fragments on interfaces', function () { - return client.test( - `query { - accounts(where: {id_eq: "1"}) { - ...balance - history { - ...balance + `, {dialect: Dialect.TheGraph}) + + it('can fetch all accounts', function() { + return client.test( + `query { + accounts(orderBy: id, orderDirection: asc) { + id + wallet + balance + history(orderBy: id, orderDirection: asc) { balance } } + }`, + { + accounts: [ + {id: '1', wallet: 'a', balance: 100, history: [{balance: 20}, {balance: 80}]}, + {id: '2', wallet: 'b', balance: 200, history: [{balance: 50}, {balance: 90}, {balance: 60}]}, + {id: '3', wallet: 'c', balance: 300, history: [{balance: 300}]}, + ] } - } - - fragment balance on HasBalance { - ... on Account { - accountBalance: balance - } - ... on HistoricalBalance { - payment: balance - } - }`, - { - accounts: [{ - accountBalance: 100, - history: [{payment: 20}, {payment: 80}] - }] - } - ) - }) - - it('supports sorting', function () { - return client.test( - `query { - historicalBalances(orderBy: balance_ASC) { - balance + ) + }) + + it('supports filtering by id', function () { + return client.test( + `query { + accounts(where: {id: "3"}) { + id + wallet + } + }`, + { + accounts: [{id: '3', wallet: 'c'}] } - }`, - { - historicalBalances: [ - {balance: 20}, - {balance: 50}, - {balance: 60}, - {balance: 80}, - {balance: 90}, - {balance: 300} - ] - } - ) - }) - - it('supports sorting by referenced entity field', function () { - return client.test( - `query { - historicalBalances(orderBy: [account_wallet_ASC, balance_DESC]) { + ) + }) + + it('supports by id query', function () { + return client.test( + `query { + a3: account(id: "3") { + id + wallet + } + nonexistent: account(id: "foo") { + id + wallet + } + }`, + { + a3: {id: '3', wallet: 'c'}, + nonexistent: null + } + ) + }) + + it('can fetch deep relations', function () { + return client.test( + `query { + accounts(where: {id: "3"}) { + id + history { + id + account { + wallet + history { + balance + account { + id + } + } + } + } + } + }`, + { + accounts: [{ + id: '3', + history: [{ + id: '3-1', + account: { + wallet: 'c', + history: [{ + balance: 300, + account: { + id: '3' + } + }] + } + }] + }] + } + ) + }) + + it('supports *_some filter', function () { + return client.test( + `query { + accounts(where: {history_some: {balance_lt: 50}}) { + id + } + }`, + { + accounts: [{id: '1'}] + } + ) + }) + + it('supports *_every filter', function () { + return client.test( + `query { + accounts(where: {history_every: {balance_gt: 20}}) { + wallet + } + }`, + { + accounts: [{wallet: 'b'}, {wallet: 'c'}] + } + ) + }) + + it('supports *_none filter', function () { + return client.test( + `query { + accounts(where: {history_none: {balance_lt: 60}}) { + wallet + } + }`, + { + accounts: [{wallet: 'c'}] + } + ) + }) + + it('supports gql aliases', function () { + return client.test( + `query { + accounts(where: {id: "1"}) { + balance + bag: wallet + purse: wallet + payment1: history(where: {id: "1-1"}) { + balance + } + payment2: history(where: {id: "1-2"}) { + balance + } + } + }`, + { + accounts: [{ + balance: 100, + bag: 'a', + purse: 'a', + payment1: [{balance: 20}], + payment2: [{balance: 80}] + }] + } + ) + }) + + it('supports gql fragments', function () { + return client.test( + `query { + accounts(where: {id: "1"}) { + ...accountFields + history { + ...historicalBalance + } + } + } + + fragment accountFields on Account { + id + wallet + } + + fragment historicalBalance on HistoricalBalance { balance + }`, + { + accounts: [{ + id: '1', + wallet: 'a', + history: [{balance: 20}, {balance: 80}] + }] } - }`, - { - historicalBalances: [ - {balance: 80}, - {balance: 20}, - {balance: 90}, - {balance: 60}, - {balance: 50}, - {balance: 300} - ] - } - ) - }) - - it('supports descriptions', function () { - return client.test(` - query { - HistoricalBalance: __type(name: "HistoricalBalance") { - description - fields { + ) + }) + + it('supports gql fragments on interfaces', function () { + return client.test( + `query { + accounts(where: {id: "1"}) { + ...balance + history { + ...balance + } + } + } + + fragment balance on HasBalance { + ... on Account { + accountBalance: balance + } + ... on HistoricalBalance { + payment: balance + } + }`, + { + accounts: [{ + accountBalance: 100, + history: [{payment: 20}, {payment: 80}] + }] + } + ) + }) + + it('supports sorting', function () { + return client.test( + `query { + historicalBalances(orderBy: balance, orderDirection: asc) { + balance + } + }`, + { + historicalBalances: [ + {balance: 20}, + {balance: 50}, + {balance: 60}, + {balance: 80}, + {balance: 90}, + {balance: 300} + ] + } + ) + }) + + it('supports sorting by referenced entity field', function () { + return client.test( + `query { + historicalBalances(orderBy: account__wallet, orderDirection: asc) { + balance + } + }`, + { + historicalBalances: [ + {balance: 20}, + {balance: 80}, + {balance: 50}, + {balance: 90}, + {balance: 60}, + {balance: 300} + ] + } + ) + }) + + it('supports descriptions', function () { + return client.test(` + query { + HistoricalBalance: __type(name: "HistoricalBalance") { description + fields { + description + } } } - } - `, { - HistoricalBalance: { - description: 'Historical record of account balance', - fields: [ - {description: 'Unique identifier'}, - {description: 'Related account'}, - {description: 'Balance'}, - ] - } + `, { + HistoricalBalance: { + description: 'Historical record of account balance', + fields: [ + {description: 'Unique identifier'}, + {description: 'Related account'}, + {description: 'Balance'}, + ] + } + }) }) }) }) diff --git a/graphql/openreader/src/test/isNull.test.ts b/graphql/openreader/src/test/isNull.test.ts index 40e70d3f6..861a31f4e 100644 --- a/graphql/openreader/src/test/isNull.test.ts +++ b/graphql/openreader/src/test/isNull.test.ts @@ -1,3 +1,4 @@ +import {Dialect} from '../dialect' import {useDatabase, useServer} from "./setup" @@ -12,7 +13,7 @@ describe('isNull operator', function() { `insert into entity (id, scalar, json, meta_id) values ('4', 'foo', '{}', '1')`, ]) - const client = useServer(` + const schema = ` type Meta @entity { id: ID! } @@ -27,53 +28,109 @@ describe('isNull operator', function() { type JsonObject { a: Int } - `) + ` - it("on scalar", function() { - return client.test(` - query { - entities(where: {scalar_isNull: true}) { - id + describe(Dialect.OpenCrud, function() { + const client = useServer(schema) + + it("on scalar", function() { + return client.test(` + query { + entities(where: {scalar_isNull: true}) { + id + } } - } - `, { - entities: [{id: '1'}] + `, { + entities: [{id: '1'}] + }) }) - }) - - it("on json", function() { - return client.test(` - query { - entities(where: {json_isNull: true}) { - id + + it("on json", function() { + return client.test(` + query { + entities(where: {json_isNull: true}) { + id + } } - } - `, { - entities: [{id: '2'}] + `, { + entities: [{id: '2'}] + }) }) - }) - - it("on nested json prop", function() { - return client.test(` - query { - entities(where: {json: {a_isNull: true}} orderBy: id_ASC) { - id + + it("on nested json prop", function() { + return client.test(` + query { + entities(where: {json: {a_isNull: true}} orderBy: id_ASC) { + id + } + } + `, { + entities: [{id: '2'}, {id: '4'}] + }) + }) + + it("on fk", function() { + return client.test(` + query { + entities(where: {meta_isNull: true}) { + id + } } - } - `, { - entities: [{id: '2'}, {id: '4'}] + `, { + entities: [{id: '3'}] + }) }) }) - it("on fk", function() { - return client.test(` - query { - entities(where: {meta_isNull: true}) { - id + describe(Dialect.TheGraph, function() { + const client = useServer(schema, {dialect: Dialect.TheGraph}) + + it("on scalar", function() { + return client.test(` + query { + entities(where: {scalar_is_null: true}) { + id + } + } + `, { + entities: [{id: '1'}] + }) + }) + + it("on json", function() { + return client.test(` + query { + entities(where: {json_is_null: true}) { + id + } + } + `, { + entities: [{id: '2'}] + }) + }) + + it("on nested json prop", function() { + return client.test(` + query { + entities(where: {json_: {a_is_null: true}} orderBy: id, orderDirection: asc) { + id + } + } + `, { + entities: [{id: '2'}, {id: '4'}] + }) + }) + + it("on fk", function() { + return client.test(` + query { + entities(where: {meta_is_null: true}) { + id + } } - } - `, { - entities: [{id: '3'}] + `, { + entities: [{id: '3'}] + }) }) }) }) diff --git a/graphql/openreader/src/test/limits.test.ts b/graphql/openreader/src/test/limits.test.ts index 7fb252820..eec6b0e05 100644 --- a/graphql/openreader/src/test/limits.test.ts +++ b/graphql/openreader/src/test/limits.test.ts @@ -1,5 +1,6 @@ import expect from 'expect' import {useDatabase, useServer} from './setup' +import {Dialect} from '../dialect' describe('response size limits', function() { @@ -12,7 +13,7 @@ describe('response size limits', function() { `create table item3 (id text primary key, order_id text, name text)`, ]) - const client = useServer(` + let schema = ` type Order1 @entity { id: ID! items: [Item1!]! @derivedFrom(field: "order") @@ -45,119 +46,239 @@ describe('response size limits', function() { order: Order3! name: String @byteWeight(value: 10.0) } - `, { + ` + let opts = { maxResponseNodes: 50, maxRootFields: 3 - }) + } + + describe(Dialect.OpenCrud, function() { + const client = useServer(schema, opts) - it('unlimited requests fail', async function() { - let result = await client.query(` - query { - order1s { - id - } - } - `) - expect(result).toMatchObject({ - data: null, - errors: [ - expect.objectContaining({message: 'response might exceed the size limit', path: ['order1s']}) - ] + it('unlimited requests fail', async function() { + let result = await client.query(` + query { + order1s { + id + } + } + `) + expect(result).toMatchObject({ + data: null, + errors: [ + expect.objectContaining({message: 'response might exceed the size limit', path: ['order1s']}) + ] + }) }) - }) - it('limited requests work', function() { - return client.test(` - query { - order1s(limit: 10) { - items(limit: 2) { + it('limited requests work', function() { + return client.test(` + query { + order1s(limit: 10) { + items(limit: 2) { + id + } + } + } + `, { + order1s: [] + }) + }) + + it('entity level cardinalities are respected', function() { + return client.test(` + query { + order2s { id } } - } - `, { - order1s: [] + `, { + order2s: [] + }) }) - }) - it('entity level cardinalities are respected', function() { - return client.test(` - query { - order2s { - id + it('item cardinalities are respected', function() { + return client.test(` + query { + order3s(limit: 1) { + items { id } + } } - } - `, { - order2s: [] + `, { + order3s: [] + }) }) - }) - it('item cardinalities are respected', function() { - return client.test(` - query { - order3s(limit: 1) { - items { id } + it('@byteWeight annotations are respected', async function() { + let result = await client.query(` + query { + order3s(limit: 1) { + items(limit: 8) { name } + } + } + `) + expect(result).toEqual({ + data: null, + errors: [ + expect.objectContaining({ + message: 'response might exceed the size limit', + path: ['order3s'] + }) + ] + }) + await client.test(` + query { + order3s(limit: 1) { + items(limit: 4) { name } + } } - } - `, { - order3s: [] + `, { + order3s: [] + }) }) - }) - it('@byteWeight annotations are respected', async function() { - let result = await client.query(` - query { - order3s(limit: 1) { - items(limit: 8) { name } - } - } - `) - expect(result).toEqual({ - data: null, - errors: [ - expect.objectContaining({ - message: 'response might exceed the size limit', - path: ['order3s'] - }) - ] + it('id_in conditions are understood', function() { + return client.test(` + query { + order1s(where: {id_in: ["1", "2", "3"]}) { + id + } + } + `, { + order1s: [] + }) }) - await client.test(` - query { - order3s(limit: 1) { - items(limit: 4) { name } - } - } - `, { - order3s: [] + + it('root query fields limit', async function() { + return client.httpErrorTest(` + query { + a: order1ById(id: "1") { id } + b: order1ById(id: "1") { id } + c: order1ById(id: "1") { id } + d: order1ById(id: "1") { id } + } + `, { + errors: [ + expect.objectContaining({ + message: 'only 3 root fields allowed, but got 4' + }) + ] + }) }) }) - it('id_in conditions are understood', function() { - return client.test(` - query { - order1s(where: {id_in: ["1", "2", "3"]}) { - id + describe(Dialect.TheGraph, function() { + const client = useServer(schema, {...opts, dialect: Dialect.TheGraph}) + + it('unlimited requests fail', async function() { + let result = await client.query(` + query { + order1s { + id + } } - } - `, { - order1s: [] + `) + expect(result).toMatchObject({ + data: null, + errors: [ + expect.objectContaining({message: 'response might exceed the size limit', path: ['order1s']}) + ] + }) }) - }) - it('root query fields limit', async function() { - return client.httpErrorTest(` - query { - a: order1ById(id: "1") { id } - b: order1ById(id: "1") { id } - c: order1ById(id: "1") { id } - d: order1ById(id: "1") { id } - } - `, { - errors: [ - expect.objectContaining({ - message: 'only 3 root fields allowed, but got 4' - }) - ] + it('limited requests work', function() { + return client.test(` + query { + order1s(first: 10) { + items(first: 2) { + id + } + } + } + `, { + order1s: [] + }) + }) + + it('entity level cardinalities are respected', function() { + return client.test(` + query { + order2s { + id + } + } + `, { + order2s: [] + }) + }) + + it('item cardinalities are respected', function() { + return client.test(` + query { + order3s(first: 1) { + items { id } + } + } + `, { + order3s: [] + }) + }) + + it('@byteWeight annotations are respected', async function() { + let result = await client.query(` + query { + order3s(first: 1) { + items(first: 8) { name } + } + } + `) + expect(result).toEqual({ + data: null, + errors: [ + expect.objectContaining({ + message: 'response might exceed the size limit', + path: ['order3s'] + }) + ] + }) + await client.test(` + query { + order3s(first: 1) { + items(first: 4) { name } + } + } + `, { + order3s: [] + }) + }) + + it('id_in conditions are understood', function() { + return client.test(` + query { + order1s(where: {id_in: ["1", "2", "3"]}) { + id + } + } + `, { + order1s: [] + }) + }) + + it('root query fields limit', async function() { + return client.httpErrorTest(` + query { + a: order1(id: "1") { id } + b: order1(id: "1") { id } + c: order1(id: "1") { id } + d: order1(id: "1") { id } + } + `, { + errors: [ + expect.objectContaining({ + message: 'only 3 root fields allowed, but got 4' + }) + ] + }) }) }) }) diff --git a/graphql/openreader/src/test/lookup.test.ts b/graphql/openreader/src/test/lookup.test.ts index c6539e51e..6334d82f1 100644 --- a/graphql/openreader/src/test/lookup.test.ts +++ b/graphql/openreader/src/test/lookup.test.ts @@ -1,3 +1,4 @@ +import {Dialect} from '../dialect' import {isCockroach, useDatabase, useServer} from "./setup" describe('lookup test', function () { @@ -13,105 +14,211 @@ describe('lookup test', function () { `insert into issue_cancellation (id, issue_id, height) values ('3', '3', 10)`, ]) - const client = useServer(` - type Issue @entity { - id: ID! - payment: IssuePayment @derivedFrom(field: "issue") - cancellation: IssueCancellation @derivedFrom(field: "issue") - } - - type IssuePayment @entity { - id: ID! - issue: Issue! @unique - amount: Int! - } - - type IssueCancellation @entity { - id: ID! - issue: Issue! @unique - height: Int! - } - `) - - it('fetches correctly', function () { - return client.test(` - query { - issues(orderBy: [id_ASC]) { - id - payment { - amount - } - cancellation { - height - issue { - cancellation { - id + describe('opencrud', function () { + const client = useServer(` + type Issue @entity { + id: ID! + payment: IssuePayment @derivedFrom(field: "issue") + cancellation: IssueCancellation @derivedFrom(field: "issue") + } + + type IssuePayment @entity { + id: ID! + issue: Issue! @unique + amount: Int! + } + + type IssueCancellation @entity { + id: ID! + issue: Issue! @unique + height: Int! + } + `) + + it('fetches correctly', function () { + return client.test(` + query { + issues(orderBy: [id_ASC]) { + id + payment { + amount + } + cancellation { + height + issue { + cancellation { + id + } } } } } - } - `, { - issues: [ - { - id: '1', - payment: {amount: 2}, - cancellation: null - }, - { - id: '2', - payment: {amount: 1}, - cancellation: null - }, - { - id: '3', - payment: null, - cancellation: { - height: 10, - issue: { - cancellation: { - id: '3' + `, { + issues: [ + { + id: '1', + payment: {amount: 2}, + cancellation: null + }, + { + id: '2', + payment: {amount: 1}, + cancellation: null + }, + { + id: '3', + payment: null, + cancellation: { + height: 10, + issue: { + cancellation: { + id: '3' + } } } } + ] + }) + }) + + it('supports sorting on lookup fields', function () { + return client.test(` + query { + issues(orderBy: [payment_amount_ASC]) { + id + } } - ] + `, { + issues: isCockroach() + ? [ + {id: '3'}, + {id: '2'}, + {id: '1'} + ] + : [ + {id: '2'}, + {id: '1'}, + {id: '3'} + ] + }) }) - }) - - it('supports sorting on lookup fields', function () { - return client.test(` - query { - issues(orderBy: [payment_amount_ASC]) { - id + + it('supports where conditions', function () { + return client.test(` + query { + issues(where: {payment: {amount_gt: 1}}) { + id + } } - } - `, { - issues: isCockroach() - ? [ - {id: '3'}, - {id: '2'}, + `, { + issues: [ {id: '1'} ] - : [ - {id: '2'}, - {id: '1'}, - {id: '3'} - ] + }) }) }) - it('supports where conditions', function () { - return client.test(` - query { - issues(where: {payment: {amount_gt: 1}}) { - id - } + describe('thegraph', function () { + const client = useServer(` + type Issue @entity { + id: ID! + payment: IssuePayment @derivedFrom(field: "issue") + cancellation: IssueCancellation @derivedFrom(field: "issue") + } + + type IssuePayment @entity { + id: ID! + issue: Issue! @unique + amount: Int! + } + + type IssueCancellation @entity { + id: ID! + issue: Issue! @unique + height: Int! } - `, { - issues: [ - {id: '1'} - ] + `, {dialect: Dialect.TheGraph}) + + it('fetches correctly', function () { + return client.test(` + query { + issues(orderBy: id, orderDirection: asc) { + id + payment { + amount + } + cancellation { + height + issue { + cancellation { + id + } + } + } + } + } + `, { + issues: [ + { + id: '1', + payment: {amount: 2}, + cancellation: null + }, + { + id: '2', + payment: {amount: 1}, + cancellation: null + }, + { + id: '3', + payment: null, + cancellation: { + height: 10, + issue: { + cancellation: { + id: '3' + } + } + } + } + ] + }) + }) + + it('supports sorting on lookup fields', function () { + return client.test(` + query { + issues(orderBy: payment__amount, orderDirection: asc) { + id + } + } + `, { + issues: isCockroach() + ? [ + {id: '3'}, + {id: '2'}, + {id: '1'} + ] + : [ + {id: '2'}, + {id: '1'}, + {id: '3'} + ] + }) + }) + + it('supports where conditions', function () { + return client.test(` + query { + issues(where: {payment_: {amount_gt: 1}}) { + id + } + } + `, { + issues: [ + {id: '1'} + ] + }) }) }) }) diff --git a/graphql/openreader/src/test/setup.ts b/graphql/openreader/src/test/setup.ts index 84ae3e261..6f0d8449d 100644 --- a/graphql/openreader/src/test/setup.ts +++ b/graphql/openreader/src/test/setup.ts @@ -69,12 +69,12 @@ export function useServer(schema: string, options?: Partial): Cli connection: db, model: buildModel(buildSchema(parse(schema))), port: 0, - dialect: isCockroach() ? 'cockroach' : 'postgres', + dbType: isCockroach() ? 'cockroach' : 'postgres', subscriptions: true, subscriptionPollInterval: 500, maxRootFields: 10, // log: createLogger('sqd:openreader'), - ...options + ...options, }) client.endpoint = `http://localhost:${info.port}/graphql` }) diff --git a/graphql/openreader/src/test/where.test.ts b/graphql/openreader/src/test/where.test.ts index adb21185f..db58a863f 100644 --- a/graphql/openreader/src/test/where.test.ts +++ b/graphql/openreader/src/test/where.test.ts @@ -1,3 +1,4 @@ +import {Dialect} from '../dialect' import {useDatabase, useServer} from "./setup" describe('AND, OR on entity filters', function () { @@ -11,125 +12,251 @@ describe('AND, OR on entity filters', function () { `insert into item (id, a, b) values ('6', 5, 6)`, ]) - const client = useServer(` - type Item @entity { - id: ID! - a: Int - b: Int - } - `) - - it('{c, and: {c}}', function () { - return client.test(` - query { - items(where: {a_eq: 1, AND: {b_eq: 1}} orderBy: id_ASC) { id } + describe('opencrud', function() { + const client = useServer(` + type Item @entity { + id: ID! + a: Int + b: Int } - `, { - items: [ - {id: '1'} - ] + `) + + it('{c, and: {c}}', function () { + return client.test(` + query { + items(where: {a_eq: 1, AND: {b_eq: 1}} orderBy: id_ASC) { id } + } + `, { + items: [ + {id: '1'} + ] + }) }) - }) - - it('{and: {and: {c}, c}}', function () { - return client.test(` - query { - items(where: {AND: {b_eq: 2, AND: {a_eq: 3}}} orderBy: id_ASC) { id } - } - `, { - items: [ - {id: '3'} - ] + + it('{and: {and: {c}, c}}', function () { + return client.test(` + query { + items(where: {AND: {b_eq: 2, AND: {a_eq: 3}}} orderBy: id_ASC) { id } + } + `, { + items: [ + {id: '3'} + ] + }) }) - }) - - it('{and: [{c}, {c}]}', function () { - return client.test(` - query { - items(where: {AND: [{a_eq: 2}, {b_eq: 2}]} orderBy: id_ASC) { id } - } - `, { - items: [ - {id: '2'} - ] + + it('{and: [{c}, {c}]}', function () { + return client.test(` + query { + items(where: {AND: [{a_eq: 2}, {b_eq: 2}]} orderBy: id_ASC) { id } + } + `, { + items: [ + {id: '2'} + ] + }) }) - }) - - it('{c, {or: {c}}}', function () { - return client.test(` - query { - items(where: {a_eq: 1, OR: {a_eq: 2}} orderBy: id_ASC) { id } - } - `, { - items: [ - {id: '1'}, - {id: '2'} - ] + + it('{c, {or: {c}}}', function () { + return client.test(` + query { + items(where: {a_eq: 1, OR: {a_eq: 2}} orderBy: id_ASC) { id } + } + `, { + items: [ + {id: '1'}, + {id: '2'} + ] + }) }) - }) - - it('{or: [{c}, {c}]}', function () { - return client.test(` - query { - items(where: {OR: [{a_eq: 2}, {a_eq: 3}]} orderBy: id_ASC) { id } - } - `, { - items: [ - {id: '2'}, - {id: '3'} - ] + + it('{or: [{c}, {c}]}', function () { + return client.test(` + query { + items(where: {OR: [{a_eq: 2}, {a_eq: 3}]} orderBy: id_ASC) { id } + } + `, { + items: [ + {id: '2'}, + {id: '3'} + ] + }) }) - }) - - it('{or: {or: {c}, c}}', function () { - return client.test(` - query { - items(where: {OR: {a_eq: 1, OR: {b_eq: 2}}} orderBy: id_ASC) { id } - } - `, { - items: [ - {id: '1'}, - {id: '2'}, - {id: '3'} - ] + + it('{or: {or: {c}, c}}', function () { + return client.test(` + query { + items(where: {OR: {a_eq: 1, OR: {b_eq: 2}}} orderBy: id_ASC) { id } + } + `, { + items: [ + {id: '1'}, + {id: '2'}, + {id: '3'} + ] + }) }) - }) - - it('{and: [{or: {c}, c}, {or: {c}, c}]}', function () { - return client.test(` - query { - items(where: {AND: [{OR: {a_eq: 5}, a_eq: 4}, {OR: {b_eq: 2}, b_eq: 4}]} orderBy: id_ASC) { id } - } - `, { - items: [ - {id: '4'}, - {id: '5'} - ] + + it('{and: [{or: {c}, c}, {or: {c}, c}]}', function () { + return client.test(` + query { + items(where: {AND: [{OR: {a_eq: 5}, a_eq: 4}, {OR: {b_eq: 2}, b_eq: 4}]} orderBy: id_ASC) { id } + } + `, { + items: [ + {id: '4'}, + {id: '5'} + ] + }) }) - }) - - it('{c, and: {c}, or: {c}}', function () { - return client.test(` - query { - items(where: { a_eq: 4, AND: {b_eq: 4}, OR: {b_eq: 6} } orderBy: id_ASC) { id } - } - `, { - items: [ - {id: '4'}, - {id: '6'} - ] + + it('{c, and: {c}, or: {c}}', function () { + return client.test(` + query { + items(where: { a_eq: 4, AND: {b_eq: 4}, OR: {b_eq: 6} } orderBy: id_ASC) { id } + } + `, { + items: [ + {id: '4'}, + {id: '6'} + ] + }) + }) + + it('handles empty wheres', function () { + return client.test(` + query { + items(where: { a_eq: 4, AND: { OR: {}, AND: {} }, OR: { OR: {AND: {} } } } orderBy: id_ASC) { id } + } + `, { + items: [ + {id: '4'} + ] + }) }) }) - - it('handles empty wheres', function () { - return client.test(` - query { - items(where: { a_eq: 4, AND: { OR: {}, AND: {} }, OR: { OR: {AND: {} } } } orderBy: id_ASC) { id } + + describe('thegraph', function() { + const client = useServer(` + type Item @entity { + id: ID! + a: Int + b: Int } - `, { - items: [ - {id: '4'} - ] + `, {dialect: Dialect.TheGraph}) + + it('{c, and: {c}}', function () { + return client.test(` + query { + items(where: {a: 1, and: {b: 1}}, orderBy: id, orderDirection: asc) { id } + } + `, { + items: [ + {id: '1'} + ] + }) + }) + + it('{and: {and: {c}, c}}', function () { + return client.test(` + query { + items(where: {and: {b: 2, and: {a: 3}}}, orderBy: id, orderDirection: asc) { id } + } + `, { + items: [ + {id: '3'} + ] + }) + }) + + it('{and: [{c}, {c}]}', function () { + return client.test(` + query { + items(where: {and: [{a: 2}, {b: 2}]}, orderBy: id, orderDirection: asc) { id } + } + `, { + items: [ + {id: '2'} + ] + }) + }) + + it('{c, {or: {c}}}', function () { + return client.test(` + query { + items(where: {a: 1, or: {a: 2}}, orderBy: id, orderDirection: asc) { id } + } + `, { + items: [ + {id: '1'}, + {id: '2'} + ] + }) + }) + + it('{or: [{c}, {c}]}', function () { + return client.test(` + query { + items(where: {or: [{a: 2}, {a: 3}]}, orderBy: id, orderDirection: asc) { id } + } + `, { + items: [ + {id: '2'}, + {id: '3'} + ] + }) + }) + + it('{or: {or: {c}, c}}', function () { + return client.test(` + query { + items(where: {or: {a: 1, or: {b: 2}}}, orderBy: id, orderDirection: asc) { id } + } + `, { + items: [ + {id: '1'}, + {id: '2'}, + {id: '3'} + ] + }) + }) + + it('{and: [{or: {c}, c}, {or: {c}, c}]}', function () { + return client.test(` + query { + items(where: {and: [{or: {a: 5}, a: 4}, {or: {b: 2}, b: 4}]}, orderBy: id, orderDirection: asc) { id } + } + `, { + items: [ + {id: '4'}, + {id: '5'} + ] + }) + }) + + it('{c, and: {c}, or: {c}}', function () { + return client.test(` + query { + items(where: { a: 4, and: {b: 4}, or: {b: 6} }, orderBy: id, orderDirection: asc) { id } + } + `, { + items: [ + {id: '4'}, + {id: '6'} + ] + }) + }) + + it('handles empty wheres', function () { + return client.test(` + query { + items(where: { a: 4, and: { or: {}, and: {} }, or: { or: {and: {} } } }, orderBy: id, orderDirection: asc) { id } + } + `, { + items: [ + {id: '4'} + ] + }) }) }) }) diff --git a/util/util-internal-dump-cli/src/dumper.ts b/util/util-internal-dump-cli/src/dumper.ts index 5a704d4dc..fab11b50c 100644 --- a/util/util-internal-dump-cli/src/dumper.ts +++ b/util/util-internal-dump-cli/src/dumper.ts @@ -8,6 +8,7 @@ import {createFs, Fs} from '@subsquid/util-internal-fs' import {assertRange, printRange, Range, rangeEnd} from '@subsquid/util-internal-range' import {Command} from 'commander' import {PrometheusServer} from './prometheus' +import {EventEmitter} from 'events' export interface DumperOptions { @@ -93,7 +94,7 @@ export abstract class Dumper this.getFinalizedHeight(), this.rpc(), this.log().child('prometheus') ) + this.eventEmitter().on('S3FsOperation', (op: string) => server.incS3Requests(op)) + return server } private async *ingest(from?: number, prevHash?: string): AsyncIterable { diff --git a/util/util-internal-dump-cli/src/prometheus.ts b/util/util-internal-dump-cli/src/prometheus.ts index 7d12b8c64..ef19aa755 100644 --- a/util/util-internal-dump-cli/src/prometheus.ts +++ b/util/util-internal-dump-cli/src/prometheus.ts @@ -1,7 +1,7 @@ import {Logger} from '@subsquid/logger' import {RpcClient} from '@subsquid/rpc-client' import {createPrometheusServer, ListeningServer} from '@subsquid/util-internal-prometheus-server' -import {collectDefaultMetrics, Gauge, Registry} from 'prom-client' +import {collectDefaultMetrics, Gauge, Counter, Registry} from 'prom-client' export class PrometheusServer { @@ -9,6 +9,7 @@ export class PrometheusServer { private chainHeightGauge: Gauge private lastWrittenBlockGauge: Gauge private rpcRequestsGauge: Gauge + private s3RequestsCounter: Counter constructor( private port: number, @@ -30,13 +31,13 @@ export class PrometheusServer { } this.set(chainHeight) } - }); + }) this.lastWrittenBlockGauge = new Gauge({ name: 'sqd_dump_last_written_block', help: 'Last saved block', registers: [this.registry] - }); + }) this.rpcRequestsGauge = new Gauge({ name: 'sqd_rpc_request_count', @@ -56,7 +57,14 @@ export class PrometheusServer { kind: 'failure' }, metrics.connectionErrors) } - }); + }) + + this.s3RequestsCounter = new Counter({ + name: 'sqd_s3_request_count', + help: 'Number of s3 requests made', + labelNames: ['kind'], + registers: [this.registry], + }) collectDefaultMetrics({register: this.registry}) } @@ -65,6 +73,10 @@ export class PrometheusServer { this.lastWrittenBlockGauge.set(block) } + incS3Requests(kind: string, value?: number) { + this.s3RequestsCounter.inc({kind}, value) + } + serve(): Promise { return createPrometheusServer(this.registry, this.port) } diff --git a/util/util-internal-fs/src/factory.ts b/util/util-internal-fs/src/factory.ts index a4204d443..38a339a38 100644 --- a/util/util-internal-fs/src/factory.ts +++ b/util/util-internal-fs/src/factory.ts @@ -2,14 +2,15 @@ import {S3Client} from '@aws-sdk/client-s3' import {Fs} from './interface' import {LocalFs} from './local' import {S3Fs} from './s3' +import {EventEmitter} from 'events' -export function createFs(url: string): Fs { +export function createFs(url: string, eventEmitter?: EventEmitter): Fs { if (url.includes('://')) { let protocol = new URL(url).protocol switch(protocol) { case 's3:': - return createS3Fs(url.slice('s3://'.length)) + return createS3Fs(url.slice('s3://'.length), eventEmitter) default: throw new Error(`Unsupported protocol: ${protocol}`) } @@ -19,12 +20,13 @@ export function createFs(url: string): Fs { } -function createS3Fs(root: string): S3Fs { +function createS3Fs(root: string, eventEmitter?: EventEmitter): S3Fs { let client = new S3Client({ endpoint: process.env.AWS_S3_ENDPOINT }) return new S3Fs({ root, - client + client, + eventEmitter }) } diff --git a/util/util-internal-fs/src/s3.ts b/util/util-internal-fs/src/s3.ts index b087e08f0..6d6bb0b7e 100644 --- a/util/util-internal-fs/src/s3.ts +++ b/util/util-internal-fs/src/s3.ts @@ -10,21 +10,25 @@ import assert from 'assert' import {Readable} from 'stream' import Upath from 'upath' import {Fs} from './interface' +import {EventEmitter} from 'events' export interface S3FsOptions { root: string client: S3Client + eventEmitter?: EventEmitter } export class S3Fs implements Fs { public readonly client: S3Client private root: string + private eventEmitter?: EventEmitter constructor(options: S3FsOptions) { this.client = options.client this.root = Upath.normalizeTrim(options.root) + this.eventEmitter = options.eventEmitter splitPath(this.root) } @@ -52,7 +56,8 @@ export class S3Fs implements Fs { cd(...path: string[]): S3Fs { return new S3Fs({ client: this.client, - root: this.resolve(path) + root: this.resolve(path), + eventEmitter: this.eventEmitter }) } @@ -74,6 +79,7 @@ export class S3Fs implements Fs { ContinuationToken }) ) + this.eventEmitter?.emit('S3FsOperation', 'ListObjectsV2') // process folder names if (res.CommonPrefixes) { @@ -116,6 +122,7 @@ export class S3Fs implements Fs { Key, Body: content })) + this.eventEmitter?.emit('S3FsOperation', 'PutObject') } async delete(path: string): Promise { @@ -129,6 +136,7 @@ export class S3Fs implements Fs { ContinuationToken }) ) + this.eventEmitter?.emit('S3FsOperation', 'ListObjectsV2') if (list.Contents) { let Objects: ObjectIdentifier[] = [] @@ -144,6 +152,7 @@ export class S3Fs implements Fs { Objects } })) + this.eventEmitter?.emit('S3FsOperation', 'DeleteObjects') } if (list.IsTruncated) { @@ -160,6 +169,7 @@ export class S3Fs implements Fs { Bucket, Key })) + this.eventEmitter?.emit('S3FsOperation', 'GetObject') assert(res.Body instanceof Readable) return res.Body } diff --git a/util/util-internal-ingest-cli/package.json b/util/util-internal-ingest-cli/package.json index 33059ddbc..a65b02e07 100644 --- a/util/util-internal-ingest-cli/package.json +++ b/util/util-internal-ingest-cli/package.json @@ -22,9 +22,11 @@ "@subsquid/util-internal-archive-layout": "^0.4.0", "@subsquid/util-internal-commander": "^1.4.0", "@subsquid/util-internal-fs": "^0.1.2", + "@subsquid/util-internal-prometheus-server": "^1.3.0", "@subsquid/util-internal-http-server": "^2.0.0", "@subsquid/util-internal-range": "^0.3.0", - "commander": "^11.1.0" + "commander": "^11.1.0", + "prom-client": "^14.2.0" }, "devDependencies": { "@types/node": "^18.18.14", diff --git a/util/util-internal-ingest-cli/src/ingest.ts b/util/util-internal-ingest-cli/src/ingest.ts index 1efa70d85..c25d90558 100644 --- a/util/util-internal-ingest-cli/src/ingest.ts +++ b/util/util-internal-ingest-cli/src/ingest.ts @@ -8,6 +8,8 @@ import {HttpApp, HttpContext, HttpError, waitForInterruption} from '@subsquid/ut import {assertRange, isRange, Range} from '@subsquid/util-internal-range' import {Command} from 'commander' import {Writable} from 'stream' +import {PrometheusServer} from './prometheus' +import {EventEmitter} from 'events' export interface IngestOptions { @@ -19,6 +21,7 @@ export interface IngestOptions { endpointCapacity?: number endpointRateLimit?: number endpointMaxBatchCallSize?: number + metrics?: number } @@ -55,6 +58,7 @@ export class Ingest { program.option('--first-block ', 'Height of the first block to ingest', nat) program.option('--last-block ', 'Height of the last block to ingest', nat) program.option('--service ', 'Run as HTTP data service', nat) + program.option('--metrics ', 'Enable prometheus metrics server', nat) return program } @@ -95,10 +99,24 @@ export class Ingest { @def protected archive(): ArchiveLayout { let url = assertNotNull(this.options().rawArchive, 'archive is not specified') - let fs = createFs(url) + let fs = createFs(url, this.eventEmitter()) return new ArchiveLayout(fs) } + @def + protected eventEmitter(): EventEmitter { + return new EventEmitter() + } + + @def + protected prometheus() { + let server = new PrometheusServer( + this.options().metrics ?? 0, + ) + this.eventEmitter().on('S3FsOperation', (op: string) => server.incS3Requests(op)) + return server + } + private async ingest(range: Range, writable: Writable): Promise { for await (let blocks of this.getBlocks(range)) { await waitDrain(writable) @@ -113,6 +131,7 @@ export class Ingest { let log = this.log().child('service') let app = new HttpApp() let self = this + let prometheus = this.prometheus() app.setMaxRequestBody(1024) app.setLogger(log) @@ -138,6 +157,11 @@ export class Ingest { } }) + if (this.options().metrics != null) { + let server = await prometheus.serve() + this.log().info(`prometheus metrics are available on port ${server.port}`) + } + let server = await app.listen(port) log.info( `Data service is listening on port ${server.port}. ` + diff --git a/util/util-internal-ingest-cli/src/prometheus.ts b/util/util-internal-ingest-cli/src/prometheus.ts new file mode 100644 index 000000000..118bca38e --- /dev/null +++ b/util/util-internal-ingest-cli/src/prometheus.ts @@ -0,0 +1,29 @@ +import {createPrometheusServer, ListeningServer} from '@subsquid/util-internal-prometheus-server' +import {collectDefaultMetrics, Counter, Registry} from 'prom-client' + + +export class PrometheusServer { + private registry = new Registry() + private s3RequestsCounter: Counter + + constructor( + private port: number, + ) { + this.s3RequestsCounter = new Counter({ + name: 'sqd_s3_request_count', + help: 'Number of s3 requests made', + labelNames: ['kind'], + registers: [this.registry], + }) + + collectDefaultMetrics({register: this.registry}) + } + + incS3Requests(kind: string, value?: number) { + this.s3RequestsCounter.inc({kind}, value) + } + + serve(): Promise { + return createPrometheusServer(this.registry, this.port) + } +}