From 041dd04737495d3aac51ef73d8dd94aa8e2c61b0 Mon Sep 17 00:00:00 2001 From: Charles Morin Date: Wed, 6 Sep 2023 14:20:57 -0400 Subject: [PATCH] Added trace_id to prometheus metrics and logs --- package-lock.json | 226 +++++++++++----------------------------------- package.json | 10 +- src/prometheus.ts | 11 ++- 3 files changed, 68 insertions(+), 179 deletions(-) diff --git a/package-lock.json b/package-lock.json index 57d8852..4f8d988 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,13 +9,13 @@ "version": "0.9.5", "license": "MIT OR Apache-2.0", "dependencies": { - "@substreams/core": "^0.1.18", - "@substreams/manifest": "^0.0.8", - "@substreams/node": "^0.2.0", + "@substreams/core": "^0.1.19", + "@substreams/manifest": "^0.0.9", + "@substreams/node": "^0.2.1", "commander": "latest", "dotenv": "latest", - "node-fetch": "^3.3.2", - "prom-client": "^14.2.0", + "node-fetch": "latest", + "prom-client": "latest", "tslog": "latest" }, "devDependencies": { @@ -25,9 +25,9 @@ } }, "node_modules/@bufbuild/buf": { - "version": "1.23.1", - "resolved": "https://registry.npmjs.org/@bufbuild/buf/-/buf-1.23.1.tgz", - "integrity": "sha512-jSm3lvXVYLQA7TXy173SubUakLLDkNuKZ5mdyua6kRKWHgxXOykMeFdf1GkT0s4zg0WGNHCW0g5NrJxdz1fQgQ==", + "version": "1.26.1", + "resolved": "https://registry.npmjs.org/@bufbuild/buf/-/buf-1.26.1.tgz", + "integrity": "sha512-NyYx4T//3ndtFYV3BfqX9Xrm1NZEx3eChXniAKc/osCVViFooC5nuLQUbyqglMonH0w39RohiURMXN+e/oEB4g==", "hasInstallScript": true, "bin": { "buf": "bin/buf", @@ -38,18 +38,18 @@ "node": ">=12" }, "optionalDependencies": { - "@bufbuild/buf-darwin-arm64": "1.23.1", - "@bufbuild/buf-darwin-x64": "1.23.1", - "@bufbuild/buf-linux-aarch64": "1.23.1", - "@bufbuild/buf-linux-x64": "1.23.1", - "@bufbuild/buf-win32-arm64": "1.23.1", - "@bufbuild/buf-win32-x64": "1.23.1" + "@bufbuild/buf-darwin-arm64": "1.26.1", + "@bufbuild/buf-darwin-x64": "1.26.1", + "@bufbuild/buf-linux-aarch64": "1.26.1", + "@bufbuild/buf-linux-x64": "1.26.1", + "@bufbuild/buf-win32-arm64": "1.26.1", + "@bufbuild/buf-win32-x64": "1.26.1" } }, "node_modules/@bufbuild/buf-darwin-arm64": { - "version": "1.23.1", - "resolved": "https://registry.npmjs.org/@bufbuild/buf-darwin-arm64/-/buf-darwin-arm64-1.23.1.tgz", - "integrity": "sha512-klt7SNqzJAuenunjPm9LLlk0eajXCXh/xJxDFJqaAUK8qQPOTAVV7YvkE2QE0PbjyT9NUgS65OHn/T25t+B7mw==", + "version": "1.26.1", + "resolved": "https://registry.npmjs.org/@bufbuild/buf-darwin-arm64/-/buf-darwin-arm64-1.26.1.tgz", + "integrity": "sha512-nmyWiT/59RFja0ZuXFxjNGoAMDPTApU66CZUUevkFVWbNB9nzeQDjx2vsJyACY64k5fTgZiaelSiyppwObQknw==", "cpu": [ "arm64" ], @@ -62,9 +62,9 @@ } }, "node_modules/@bufbuild/buf-darwin-x64": { - "version": "1.23.1", - "resolved": "https://registry.npmjs.org/@bufbuild/buf-darwin-x64/-/buf-darwin-x64-1.23.1.tgz", - "integrity": "sha512-mO+CMKDZCEdMWudfYSxVCjAc6EfdzP5o2gvFyfVBULKiBfOiYZeE67vbV5HuGzIjALKP+ZiWaYnxKeCxEDuJJg==", + "version": "1.26.1", + "resolved": "https://registry.npmjs.org/@bufbuild/buf-darwin-x64/-/buf-darwin-x64-1.26.1.tgz", + "integrity": "sha512-jl5WmUv30OW8JiRLid9+mVx1XVH0XttpUfkQfmqDFdUHGfdy4XWYK8kr84YyWu0SiMTIt1mPXkqG5UM3x+tdIQ==", "cpu": [ "x64" ], @@ -77,9 +77,9 @@ } }, "node_modules/@bufbuild/buf-linux-aarch64": { - "version": "1.23.1", - "resolved": "https://registry.npmjs.org/@bufbuild/buf-linux-aarch64/-/buf-linux-aarch64-1.23.1.tgz", - "integrity": "sha512-Sel6ZbnGtvC095KDZHqNC+wErZ0rhrqmjjkVc5fdllZA6of0sEk49KrL2UC9Z1AYKbmnAs4fH1bfvyJOXjqwxg==", + "version": "1.26.1", + "resolved": "https://registry.npmjs.org/@bufbuild/buf-linux-aarch64/-/buf-linux-aarch64-1.26.1.tgz", + "integrity": "sha512-EedR2KDW/yDIxQKWuq1Y/g7IuwTgvelqylGVO7muMxt2JWShobyUaU6GIU8JB4yhIbqRQYCL2KqBsvDJbJtCUw==", "cpu": [ "arm64" ], @@ -92,9 +92,9 @@ } }, "node_modules/@bufbuild/buf-linux-x64": { - "version": "1.23.1", - "resolved": "https://registry.npmjs.org/@bufbuild/buf-linux-x64/-/buf-linux-x64-1.23.1.tgz", - "integrity": "sha512-QdkdD2OZiw4pPjoXsK9UT6IzWPaqARftmdWFO+I04WfXklcUBohco1GHFhu3zG9hb8vS8KI6ONN8w8Vt4dWOJQ==", + "version": "1.26.1", + "resolved": "https://registry.npmjs.org/@bufbuild/buf-linux-x64/-/buf-linux-x64-1.26.1.tgz", + "integrity": "sha512-5iFL+MmWqR4cBLVNpgsjRecdHgcTxFaIkVYlQV9q8acbaJn5rgOIjUr1tzcBao9YsL3rdBhHvKkgnQ9gi1IiTw==", "cpu": [ "x64" ], @@ -107,9 +107,9 @@ } }, "node_modules/@bufbuild/buf-win32-arm64": { - "version": "1.23.1", - "resolved": "https://registry.npmjs.org/@bufbuild/buf-win32-arm64/-/buf-win32-arm64-1.23.1.tgz", - "integrity": "sha512-qfMWnBjorZFrvKzRmxzN690LhD602eseRFnSS3Z9miWF36FexI+CHdywRwF7VyMaKaRAQISS5cARB8Sw5mT2Zw==", + "version": "1.26.1", + "resolved": "https://registry.npmjs.org/@bufbuild/buf-win32-arm64/-/buf-win32-arm64-1.26.1.tgz", + "integrity": "sha512-/ayymSD12gBetN98ErkH0CBGRLTmtYAp4fmbPuvq8zuJcL0eiAnK6d7ZFjTc+vDMuKY/aelQN7dj9WhzdYAQSQ==", "cpu": [ "arm64" ], @@ -122,9 +122,9 @@ } }, "node_modules/@bufbuild/buf-win32-x64": { - "version": "1.23.1", - "resolved": "https://registry.npmjs.org/@bufbuild/buf-win32-x64/-/buf-win32-x64-1.23.1.tgz", - "integrity": "sha512-TnnWxYecVTcyeTWkf5FJ3hQ2AlwWIA2TKUMxJyyh63g0ZBF0Z7D0x+tnmNxhOXtbU38muO6x362nsTFtds5bNg==", + "version": "1.26.1", + "resolved": "https://registry.npmjs.org/@bufbuild/buf-win32-x64/-/buf-win32-x64-1.26.1.tgz", + "integrity": "sha512-k9Dy3Z9P96wYR43lUhUo0jbjMSo001+MRBlsadEYiw85POqx6RWVaGyHLrxC2Ly7g+aGMisey050OjqfCWtKTA==", "cpu": [ "x64" ], @@ -178,99 +178,23 @@ "integrity": "sha512-/vwz7Jh05eS0qY8kczR/YyJd18d0C+PMtUkAealh4f6gwvhABLGCnktNJTcq/+UHxY0Cbv18r5uaJ4+7PPC+WQ==" }, "node_modules/@effect/io": { - "version": "0.36.1", - "resolved": "https://registry.npmjs.org/@effect/io/-/io-0.36.1.tgz", - "integrity": "sha512-qktbbvQX+tCQ2wxBcpr8g89ZAIAzRaq+/tWGogdVxfthWIhiLSagMeKYzLHqQ/MaXzN7sQSYLHbFgxcE+D8keQ==", + "version": "0.37.1", + "resolved": "https://registry.npmjs.org/@effect/io/-/io-0.37.1.tgz", + "integrity": "sha512-Ez3GfcG+gDDfAiBXtSjJpSrPU5Guiyw69LsYkMtIukFwyNwpHWLhYaVgfVbVjoQasil8KiFSQJSd5DbJL6nqPg==", "dependencies": { "@effect/data": "^0.17.0" } }, - "node_modules/@effect/match": { - "version": "0.29.1", - "resolved": "https://registry.npmjs.org/@effect/match/-/match-0.29.1.tgz", - "integrity": "sha512-ZstaS5ZLpX5RwLX8GuV9JeQuAKLCxFu+pU8iirVFYftc+8wUxcPUiGI1ryNupKW62m4IlcNrTz05JvtemNy82A==", - "dependencies": { - "@effect/data": "^0.16.3", - "@effect/schema": "^0.30.3" - } - }, - "node_modules/@effect/match/node_modules/@effect/data": { - "version": "0.16.3", - "resolved": "https://registry.npmjs.org/@effect/data/-/data-0.16.3.tgz", - "integrity": "sha512-RxRN3Vm8YPys0GjH/yjl2Z57kGCwJGjY6omgcDbeXhLVd+SUaVWbM6hJysKvb/gD9cuN1XwhXUnJ0shVErDelg==" - }, - "node_modules/@effect/match/node_modules/@effect/io": { - "version": "0.35.3", - "resolved": "https://registry.npmjs.org/@effect/io/-/io-0.35.3.tgz", - "integrity": "sha512-f0K2XQJvQY0DP6GQC76NUZgFh2QxKIpZZzIELyI8796uJtLt4CRx3s+YsJF68htgUttKm9hxkfcbtcJAg2Rmcw==", - "dependencies": { - "@effect/data": "^0.16.0" - } - }, - "node_modules/@effect/match/node_modules/@effect/schema": { - "version": "0.30.4", - "resolved": "https://registry.npmjs.org/@effect/schema/-/schema-0.30.4.tgz", - "integrity": "sha512-4MQkp4mStbJ2f5tRXc0cquCp1KgnmqgstGAjJAteed2dz84yoQ4kBD4gy7OCO8E1uChoqMbvB1ezxPrJ9EPEOw==", - "dependencies": { - "@effect/data": "^0.16.2", - "@effect/io": "^0.35.2", - "fast-check": "^3.11.0" - } - }, "node_modules/@effect/schema": { - "version": "0.31.0", - "resolved": "https://registry.npmjs.org/@effect/schema/-/schema-0.31.0.tgz", - "integrity": "sha512-bVAUl6Y0Fn0VL+gGHFsuqjtj4OvPl4bdEK/AFG3jsNor/cZK5vY1H8j1xDiCVJro2Wsu/GuISNu8wJQoXVRxEQ==", + "version": "0.32.0", + "resolved": "https://registry.npmjs.org/@effect/schema/-/schema-0.32.0.tgz", + "integrity": "sha512-4HJK/cFkVPdIjYICy0eRsL7JuuLJ6mE3aJC5rX9OuUIei/qfctFEEX2NaARjtGX7hACBrRcuJCNwiq+54TTjFw==", "dependencies": { - "@effect/data": "^0.17.0", - "@effect/io": "^0.36.0", + "@effect/data": "^0.17.1", + "@effect/io": "^0.37.0", "fast-check": "^3.12.0" } }, - "node_modules/@effect/stm": { - "version": "0.19.0", - "resolved": "https://registry.npmjs.org/@effect/stm/-/stm-0.19.0.tgz", - "integrity": "sha512-mwyz0gU3AmX4IpO+/xF19PC3ZZ2nRkZW07ii19SXqzrPk7TkbMmrWoJiDe4Uki5ZMe7nIHIHhCiHmC+cXXJEIA==", - "dependencies": { - "@effect/data": "^0.16.0", - "@effect/io": "^0.35.0" - } - }, - "node_modules/@effect/stm/node_modules/@effect/data": { - "version": "0.16.3", - "resolved": "https://registry.npmjs.org/@effect/data/-/data-0.16.3.tgz", - "integrity": "sha512-RxRN3Vm8YPys0GjH/yjl2Z57kGCwJGjY6omgcDbeXhLVd+SUaVWbM6hJysKvb/gD9cuN1XwhXUnJ0shVErDelg==" - }, - "node_modules/@effect/stm/node_modules/@effect/io": { - "version": "0.35.3", - "resolved": "https://registry.npmjs.org/@effect/io/-/io-0.35.3.tgz", - "integrity": "sha512-f0K2XQJvQY0DP6GQC76NUZgFh2QxKIpZZzIELyI8796uJtLt4CRx3s+YsJF68htgUttKm9hxkfcbtcJAg2Rmcw==", - "dependencies": { - "@effect/data": "^0.16.0" - } - }, - "node_modules/@effect/stream": { - "version": "0.31.1", - "resolved": "https://registry.npmjs.org/@effect/stream/-/stream-0.31.1.tgz", - "integrity": "sha512-5dUoKiMuUlPS+nBEgSfHSlBUqyTzF6MAY+fG5pa3lbTpeUVS3g8f8yMMYXVvAxETOli/0XTSHlkeyqOovmTKBQ==", - "dependencies": { - "@effect/data": "^0.16.0", - "@effect/io": "^0.35.1" - } - }, - "node_modules/@effect/stream/node_modules/@effect/data": { - "version": "0.16.3", - "resolved": "https://registry.npmjs.org/@effect/data/-/data-0.16.3.tgz", - "integrity": "sha512-RxRN3Vm8YPys0GjH/yjl2Z57kGCwJGjY6omgcDbeXhLVd+SUaVWbM6hJysKvb/gD9cuN1XwhXUnJ0shVErDelg==" - }, - "node_modules/@effect/stream/node_modules/@effect/io": { - "version": "0.35.3", - "resolved": "https://registry.npmjs.org/@effect/io/-/io-0.35.3.tgz", - "integrity": "sha512-f0K2XQJvQY0DP6GQC76NUZgFh2QxKIpZZzIELyI8796uJtLt4CRx3s+YsJF68htgUttKm9hxkfcbtcJAg2Rmcw==", - "dependencies": { - "@effect/data": "^0.16.0" - } - }, "node_modules/@substreams/core": { "version": "0.1.19", "resolved": "https://registry.npmjs.org/@substreams/core/-/core-0.1.19.tgz", @@ -281,22 +205,21 @@ } }, "node_modules/@substreams/manifest": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/@substreams/manifest/-/manifest-0.0.8.tgz", - "integrity": "sha512-mL3+BeSweyyaMMO9oHHth4EI7jRL31R9C3ayV2e2a4+26sNMKsAkSQEP/5FI1kItQUKOMJ2bzC/F+k9XGsA53w==", + "version": "0.0.9", + "resolved": "https://registry.npmjs.org/@substreams/manifest/-/manifest-0.0.9.tgz", + "integrity": "sha512-U/azNhOZle3S6lUCpnQ+P90V8GvtQWMMH8hEX3/TCPRz/hkOFm1PG0+6t6VYVgXX+yTCPrAVpYiyGUdfkaW9Bw==", "dependencies": { - "@bufbuild/buf": "1.23.1", + "@bufbuild/buf": "^1.25.0", "@bufbuild/protobuf": "^1.2.1", - "@effect/schema": "^0.31.0", - "@substreams/core": "^0.1.17", - "effect": "2.0.0-next.21", + "@effect/schema": "^0.32.0", + "@substreams/core": "^0.1.19", "yaml": "^2.3.1" } }, "node_modules/@substreams/node": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/@substreams/node/-/node-0.2.0.tgz", - "integrity": "sha512-wsoXkSQm2gvn0pFTpspkm+RMbomqhL8ayV1kmTRFvjSkx+B6xiNp2lQuOtTxJQWN7rPvMRXqeyYQ0z48JJxwoQ==", + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/@substreams/node/-/node-0.2.1.tgz", + "integrity": "sha512-wV7xB7+Bt/dnF2VFGea0DWwN1YURMjUxv9p+72bB/53G5Vgpho3HTspoBf9EnTMgeLlrXhtZ64JgaU7OTEBxSA==", "dependencies": { "@bufbuild/connect": "latest", "@bufbuild/connect-node": "latest", @@ -316,15 +239,8 @@ "node_modules/@types/node": { "version": "20.5.9", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.5.9.tgz", - "integrity": "sha512-PcGNd//40kHAS3sTlzKB9C9XL4K0sTup8nbG5lC14kzEteTNuAFh9u5nA0o5TWnSG2r/JNPRXFVcHJIIeRlmqQ==" - }, - "node_modules/@types/set-cookie-parser": { - "version": "2.4.3", - "resolved": "https://registry.npmjs.org/@types/set-cookie-parser/-/set-cookie-parser-2.4.3.tgz", - "integrity": "sha512-7QhnH7bi+6KAhBB+Auejz1uV9DHiopZqu7LfR/5gZZTkejJV5nYeZZpgfFoE0N8aDsXuiYpfKyfyMatCwQhyTQ==", - "dependencies": { - "@types/node": "*" - } + "integrity": "sha512-PcGNd//40kHAS3sTlzKB9C9XL4K0sTup8nbG5lC14kzEteTNuAFh9u5nA0o5TWnSG2r/JNPRXFVcHJIIeRlmqQ==", + "dev": true }, "node_modules/bintrees": { "version": "1.0.2", @@ -358,31 +274,6 @@ "url": "https://github.com/motdotla/dotenv?sponsor=1" } }, - "node_modules/effect": { - "version": "2.0.0-next.21", - "resolved": "https://registry.npmjs.org/effect/-/effect-2.0.0-next.21.tgz", - "integrity": "sha512-uBkHMWb+8M8lPt53jp8zjUOovvZDRiPAyfuBAAd+prfzitdI8AA1KGkPuUrPUWG50bjWK8OVemDW11jxZV9b1g==", - "dependencies": { - "@effect/data": "^0.16.3", - "@effect/io": "^0.35.3", - "@effect/match": "^0.29.0", - "@effect/stm": "^0.19.0", - "@effect/stream": "^0.31.0" - } - }, - "node_modules/effect/node_modules/@effect/data": { - "version": "0.16.3", - "resolved": "https://registry.npmjs.org/@effect/data/-/data-0.16.3.tgz", - "integrity": "sha512-RxRN3Vm8YPys0GjH/yjl2Z57kGCwJGjY6omgcDbeXhLVd+SUaVWbM6hJysKvb/gD9cuN1XwhXUnJ0shVErDelg==" - }, - "node_modules/effect/node_modules/@effect/io": { - "version": "0.35.3", - "resolved": "https://registry.npmjs.org/@effect/io/-/io-0.35.3.tgz", - "integrity": "sha512-f0K2XQJvQY0DP6GQC76NUZgFh2QxKIpZZzIELyI8796uJtLt4CRx3s+YsJF68htgUttKm9hxkfcbtcJAg2Rmcw==", - "dependencies": { - "@effect/data": "^0.16.0" - } - }, "node_modules/fast-check": { "version": "3.12.1", "resolved": "https://registry.npmjs.org/fast-check/-/fast-check-3.12.1.tgz", @@ -438,13 +329,9 @@ } }, "node_modules/headers-polyfill": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/headers-polyfill/-/headers-polyfill-3.2.1.tgz", - "integrity": "sha512-jpY9fNMWPWwkqRN9CpSRNqL9svpiuSmg4CsbPl4s43KltIDIVHlPv75UOXVgc/PTP6BzHSUvd9UMdxSW1I+ETQ==", - "dependencies": { - "@types/set-cookie-parser": "^2.4.3", - "set-cookie-parser": "^2.6.0" - } + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/headers-polyfill/-/headers-polyfill-3.2.2.tgz", + "integrity": "sha512-dMpZWcokEGf3wcPmv6X6vCxRpdUYIYTPKRYpfWZpqJCpKbX60kXNBE7yv3yj60PWZKQLlXJ0mkA9wmTVu/yfmg==" }, "node_modules/node-domexception": { "version": "1.0.0", @@ -507,11 +394,6 @@ } ] }, - "node_modules/set-cookie-parser": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.6.0.tgz", - "integrity": "sha512-RVnVQxTXuerk653XfuliOxBP81Sf0+qfQE73LIYKcyMYHG94AuH0kgrQpRDuTZnSmjpysHmzxJXKNfa6PjFhyQ==" - }, "node_modules/tdigest": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", diff --git a/package.json b/package.json index e1ca5d8..99d9fe9 100644 --- a/package.json +++ b/package.json @@ -41,13 +41,13 @@ "prepublishOnly": "npm run build" }, "dependencies": { - "@substreams/core": "^0.1.18", - "@substreams/manifest": "^0.0.8", - "@substreams/node": "^0.2.0", + "@substreams/core": "^0.1.19", + "@substreams/manifest": "^0.0.9", + "@substreams/node": "^0.2.1", "commander": "latest", "dotenv": "latest", - "node-fetch": "^3.3.2", - "prom-client": "^14.2.0", + "node-fetch": "latest", + "prom-client": "latest", "tslog": "latest" }, "devDependencies": { diff --git a/src/prometheus.ts b/src/prometheus.ts index 2334f0d..7358550 100644 --- a/src/prometheus.ts +++ b/src/prometheus.ts @@ -1,4 +1,4 @@ -import type { BlockScopedData, Clock } from "@substreams/core/proto"; +import type { BlockScopedData, SessionInit, Clock } from "@substreams/core/proto"; import type { BlockEmitter } from "@substreams/node"; import client, { Counter, Gauge, Summary, Histogram, type CounterConfiguration, type GaugeConfiguration, type SummaryConfiguration, type HistogramConfiguration } from "prom-client"; @@ -56,6 +56,7 @@ const substreams_sink_unknown_message = registerCounter("substreams_sink_unknown const substreams_sink_progress_message = registerCounter("substreams_sink_progress_message", "The number of progress message received", ["module"]); // Gauges +const trace_id = registerGauge("trace_id", "Substreams session trace id", ["trace_id"]); const head_block_number = registerGauge("head_block_number", "Last processed block number"); const head_block_time_drift = registerGauge("head_block_time_drift", "Head block time drift in seconds"); const head_block_timestamp = registerGauge("head_block_timestamp", "Head block timestamp"); @@ -79,8 +80,14 @@ function updateBlockDataMetrics(block: BlockScopedData) { substreams_sink_backprocessing_completion?.set(1); } +export function updateSessionMetrics(session: SessionInit) { + logger.info("trace_id", session.traceId); + trace_id?.labels({ trace_id: session.traceId }).set(1); +} + export function onPrometheusMetrics(emitter: BlockEmitter) { emitter.on("undo", () => substreams_sink_undo_message?.inc(1)); + emitter.on("session", session => updateSessionMetrics(session)); emitter.on("block", block => { updateBlockDataMetrics(block); if (block.clock) updateClockMetrics(block.clock); @@ -101,4 +108,4 @@ export function handleManifest(emitter: BlockEmitter, moduleHash: string, option stopBlockNum: String(emitter.request.stopBlockNum), productionMode: String(emitter.request.productionMode) }).set(1); -} \ No newline at end of file +}