feat: add s3 ops metrics (#317)

* feat: add s3 ops metrics

* changes

* fix: add event emitter and change metric to Counter

.dockerignore

@@ -32,3 +32,5 @@ docker-compose.yml
 /ops/docker-publish.sh
 
 **/.DS_Store
+
+*.temp

.gitignore

@@ -19,6 +19,7 @@ common/autoinstallers/*/.npmrc
 
 # IDE
 .idea
+.vscode
 
 # Built js libs
 /*/*/lib

common/changes/@subsquid/graphql-server/thegraph-compatebility_2024-07-11-14-20.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@subsquid/graphql-server",
+      "comment": "add dialect system support",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@subsquid/graphql-server"
+}

common/changes/@subsquid/openreader/thegraph-compatebility_2024-07-11-14-20.json

@@ -0,0 +1,20 @@
+{
+  "changes": [
+    {
+      "packageName": "@subsquid/openreader",
+      "comment": "introduce dialect system",
+      "type": "major"
+    },
+    {
+      "packageName": "@subsquid/openreader",
+      "comment": "remove `ByUniqueInput` queries",
+      "type": "major"
+    },
+    {
+      "packageName": "@subsquid/openreader",
+      "comment": "add `thegraph` dialect support",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@subsquid/openreader"
+}

common/changes/@subsquid/util-internal-dump-cli/octo-gone-feat-add-s3-metrics_2024-07-22-11-40.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@subsquid/util-internal-dump-cli",
+      "comment": "add prometheus metrics for S3 file system handler",
+      "type": "patch"
+    }
+  ],
+  "packageName": "@subsquid/util-internal-dump-cli"
+}

common/changes/@subsquid/util-internal-fs/octo-gone-feat-add-s3-metrics_2024-07-22-11-40.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@subsquid/util-internal-fs",
+      "comment": "add metrics for S3 file system handler",
+      "type": "patch"
+    }
+  ],
+  "packageName": "@subsquid/util-internal-fs"
+}

common/changes/@subsquid/util-internal-ingest-cli/octo-gone-feat-add-s3-metrics_2024-07-22-11-40.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@subsquid/util-internal-ingest-cli",
+      "comment": "add prometheus metrics",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@subsquid/util-internal-ingest-cli"
+}

graphql/graphql-server/src/main.ts

@@ -6,6 +6,7 @@ import {registerTsNodeIfRequired} from '@subsquid/util-internal-ts-node'
 import assert from 'assert'
 import {Command, Option} from 'commander'
 import {DumbInMemoryCacheOptions, DumbRedisCacheOptions, Server} from './server'
+import {Dialect} from '@subsquid/openreader/lib/dialect'
 
 
 const LOG = createLogger('sqd:graphql-server')
@@ -26,6 +27,7 @@ runProgram(async () => {
     program.option('--dumb-cache-max-age <ms>', 'cache-control max-age in milliseconds', nat, 5000)
     program.option('--dumb-cache-ttl <ms>', 'in-memory cached item TTL in milliseconds', nat, 5000)
     program.option('--dumb-cache-size <mb>', 'max in-memory cache size in megabytes', nat, 50)
+    program.addOption(new Option('--dialect <type>').choices(Object.values(Dialect)))
 
     let opts = program.parse().opts() as {
         maxRequestSize: number
@@ -41,6 +43,7 @@ runProgram(async () => {
         subscriptionPollInterval: number
         subscriptionMaxResponseSize?: number
         tsNode?: boolean
+        dialect?: Dialect
     }
 
     await registerTsNodeIfRequired()

graphql/graphql-server/src/server.ts

@@ -3,10 +3,9 @@ import {InMemoryLRUCache} from '@apollo/utils.keyvaluecache'
 import {mergeSchemas} from '@graphql-tools/schema'
 import {Logger} from '@subsquid/logger'
 import {Context, OpenreaderContext} from '@subsquid/openreader/lib/context'
-import {PoolOpenreaderContext} from '@subsquid/openreader/lib/db'
-import {Dialect} from '@subsquid/openreader/lib/dialect'
+import {DbType, PoolOpenreaderContext} from '@subsquid/openreader/lib/db'
+import {Dialect, getSchemaBuilder} from '@subsquid/openreader/lib/dialect'
 import type {Model} from '@subsquid/openreader/lib/model'
-import {SchemaBuilder} from '@subsquid/openreader/lib/opencrud/schema'
 import {addServerCleanup, Dispose, runApollo} from '@subsquid/openreader/lib/server'
 import {loadModel, resolveGraphqlSchema} from '@subsquid/openreader/lib/tools'
 import {ResponseSizeLimit} from '@subsquid/openreader/lib/util/limit'
@@ -38,6 +37,7 @@ export interface ServerOptions {
     subscriptionPollInterval?: number
     subscriptionMaxResponseNodes?: number
     dumbCache?: DumbRedisCacheOptions | DumbInMemoryCacheOptions
+    dialect?: Dialect
 }
 
 
@@ -103,8 +103,14 @@ export class Server {
 
     @def
     private async schema(): Promise<GraphQLSchema> {
+        const schemaBuilder = await getSchemaBuilder({
+            model: this.model(),
+            subscriptions: this.options.subscriptions,
+            dialect: this.options.dialect,
+        })
+
         let schemas = [
-            new SchemaBuilder({model: this.model(), subscriptions: this.options.subscriptions}).build()
+            schemaBuilder.build(),
         ]
 
         if (this.options.squidStatus !== false) {
@@ -185,14 +191,14 @@ export class Server {
 
     @def
     private async context(): Promise<() => Context> {
-        let dialect = this.dialect()
+        let dbType = this.dbType()
         let createOpenreader: () => OpenreaderContext
         if (await this.customResolvers()) {
             let con = await this.createTypeormConnection({sqlStatementTimeout: this.options.sqlStatementTimeout})
             this.disposals.push(() => con.destroy())
             createOpenreader = () => {
                 return new TypeormOpenreaderContext(
-                    dialect,
+                    dbType,
                     con,
                     con,
                     this.options.subscriptionPollInterval,
@@ -204,7 +210,7 @@ export class Server {
             this.disposals.push(() => pool.end())
             createOpenreader = () => {
                 return new PoolOpenreaderContext(
-                    dialect,
+                    dbType,
                     pool,
                     pool,
                     this.options.subscriptionPollInterval,
@@ -261,7 +267,7 @@ export class Server {
         return envNat('GQL_DB_CONNECTION_POOL_SIZE') || 5
     }
 
-    private dialect(): Dialect {
+    private dbType(): DbType {
         let type = process.env.DB_TYPE
         if (!type) return 'postgres'
         switch(type) {

graphql/graphql-server/src/typeorm.ts

@@ -1,5 +1,6 @@
 import type {Logger} from '@subsquid/logger'
 import type {OpenreaderContext} from '@subsquid/openreader/lib/context'
+import {DbType} from '@subsquid/openreader/lib/db'
 import type {Dialect} from '@subsquid/openreader/lib/dialect'
 import type {Query} from '@subsquid/openreader/lib/sql/query'
 import {Subscription} from '@subsquid/openreader/lib/subscription'
@@ -19,7 +20,7 @@ export class TypeormOpenreaderContext implements OpenreaderContext {
     private queryCounter = 0
 
     constructor(
-        public readonly dialect: Dialect,
+        public readonly dbType: DbType,
         private connection: DataSource,
         subscriptionConnection?: DataSource,
         private subscriptionPollInterval: number = 1000,

graphql/openreader/package.json

@@ -42,7 +42,8 @@
     "graphql-parse-resolve-info": "^4.14.0",
     "graphql-ws": "^5.14.2",
     "pg": "^8.11.3",
-    "ws": "^8.14.2"
+    "ws": "^8.14.2",
+    "inflected": "^2.1.0"
   },
   "peerDependencies": {
     "@subsquid/big-decimal": "^1.0.0"
@@ -63,6 +64,7 @@
     "expect": "^29.7.0",
     "gql-test-client": "^0.0.0",
     "mocha": "^10.2.0",
-    "typescript": "~5.3.2"
+    "typescript": "~5.3.2",
+    "@types/inflected": "^2.1.3"
   }
 }

graphql/openreader/src/context.ts

@@ -1,5 +1,5 @@
 import type {Logger} from '@subsquid/logger'
-import type {Dialect} from './dialect'
+import type {DbType} from './db'
 import type {Query} from './sql/query'
 import type {Limit} from './util/limit'
 
@@ -11,7 +11,7 @@ export interface Context {
 
 export interface OpenreaderContext {
     id: number
-    dialect: Dialect
+    dbType: DbType
     executeQuery<T>(query: Query<T>): Promise<T>
     subscription<T>(query: Query<T>): AsyncIterable<T>
     responseSizeLimit?: Limit

graphql/openreader/src/db.ts

@@ -3,7 +3,6 @@ import {addErrorContext} from '@subsquid/util-internal'
 import type {ClientBase, Pool} from 'pg'
 import {QueryResult} from 'pg'
 import {OpenreaderContext} from './context'
-import {Dialect} from './dialect'
 import {Query} from './sql/query'
 import {Subscription} from './subscription'
 import {LazyTransaction} from './util/lazy-transaction'
@@ -12,6 +11,9 @@ import {LazyTransaction} from './util/lazy-transaction'
 let CTX_COUNTER = 0
 
 
+export type DbType = 'postgres' | 'cockroach'
+
+
 export class PoolOpenreaderContext implements OpenreaderContext {
     public id = (CTX_COUNTER = (CTX_COUNTER + 1) % Number.MAX_SAFE_INTEGER)
     public log?: Logger
@@ -20,7 +22,7 @@ export class PoolOpenreaderContext implements OpenreaderContext {
     private queryCounter = 0
 
     constructor(
-        public readonly dialect: Dialect,
+        public readonly dbType: DbType,
         pool: Pool,
         subscriptionPool?: Pool,
         private subscriptionPollInterval: number = 1000,