From ba649b7e387fce1fe89df07235ebaaeae3867580 Mon Sep 17 00:00:00 2001 From: khavinshankar Date: Fri, 24 Nov 2023 19:48:27 +0530 Subject: [PATCH] dump observation data into s3 every hour + monitor it in sentry --- package-lock.json | 209 ++++++++---------------- package.json | 6 +- src/controller/ObservationController.ts | 37 ++++- src/controller/helper/makeDataDump.ts | 59 ++++--- 4 files changed, 144 insertions(+), 167 deletions(-) diff --git a/package-lock.json b/package-lock.json index f4d9907..1b22adb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,8 +10,8 @@ "license": "MIT", "dependencies": { "@prisma/client": "^4.15.0", - "@sentry/node": "^6.17.7", - "@sentry/tracing": "^6.17.7", + "@sentry/node": "^7.81.1", + "@sentry/tracing": "^7.81.1", "aws-sdk": "^2.1408.0", "axios": "^0.26.1", "connect-flash": "^0.1.1", @@ -167,98 +167,74 @@ "resolved": "https://registry.npmjs.org/@prisma/engines-version/-/engines-version-4.15.0-28.8fbc245156db7124f997f4cecdd8d1219e360944.tgz", "integrity": "sha512-sVOig4tjGxxlYaFcXgE71f/rtFhzyYrfyfNFUsxCIEJyVKU9rdOWIlIwQ2NQ7PntvGnn+x0XuFo4OC1jvPJKzg==" }, - "node_modules/@sentry/core": { - "version": "6.19.7", - "resolved": "https://registry.npmjs.org/@sentry/core/-/core-6.19.7.tgz", - "integrity": "sha512-tOfZ/umqB2AcHPGbIrsFLcvApdTm9ggpi/kQZFkej7kMphjT+SGBiQfYtjyg9jcRW+ilAR4JXC9BGKsdEQ+8Vw==", - "dependencies": { - "@sentry/hub": "6.19.7", - "@sentry/minimal": "6.19.7", - "@sentry/types": "6.19.7", - "@sentry/utils": "6.19.7", - "tslib": "^1.9.3" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/@sentry/hub": { - "version": "6.19.7", - "resolved": "https://registry.npmjs.org/@sentry/hub/-/hub-6.19.7.tgz", - "integrity": "sha512-y3OtbYFAqKHCWezF0EGGr5lcyI2KbaXW2Ik7Xp8Mu9TxbSTuwTe4rTntwg8ngPjUQU3SUHzgjqVB8qjiGqFXCA==", + "node_modules/@sentry-internal/tracing": { + "version": "7.81.1", + "resolved": "https://registry.npmjs.org/@sentry-internal/tracing/-/tracing-7.81.1.tgz", + "integrity": "sha512-E5xm27xrLXL10knH2EWDQsQYh5nb4SxxZzJ3sJwDGG9XGKzBdlp20UUhKqx00wixooVX9uCj3e4Jg8SvNB1hKg==", "dependencies": { - "@sentry/types": "6.19.7", - "@sentry/utils": "6.19.7", - "tslib": "^1.9.3" + "@sentry/core": "7.81.1", + "@sentry/types": "7.81.1", + "@sentry/utils": "7.81.1" }, "engines": { - "node": ">=6" + "node": ">=8" } }, - "node_modules/@sentry/minimal": { - "version": "6.19.7", - "resolved": "https://registry.npmjs.org/@sentry/minimal/-/minimal-6.19.7.tgz", - "integrity": "sha512-wcYmSJOdvk6VAPx8IcmZgN08XTXRwRtB1aOLZm+MVHjIZIhHoBGZJYTVQS/BWjldsamj2cX3YGbGXNunaCfYJQ==", + "node_modules/@sentry/core": { + "version": "7.81.1", + "resolved": "https://registry.npmjs.org/@sentry/core/-/core-7.81.1.tgz", + "integrity": "sha512-tU37yAmckOGCw/moWKSwekSCWWJP15O6luIq+u7wal22hE88F3Vc5Avo8SeF3upnPR+4ejaOFH+BJTr6bgrs6Q==", "dependencies": { - "@sentry/hub": "6.19.7", - "@sentry/types": "6.19.7", - "tslib": "^1.9.3" + "@sentry/types": "7.81.1", + "@sentry/utils": "7.81.1" }, "engines": { - "node": ">=6" + "node": ">=8" } }, "node_modules/@sentry/node": { - "version": "6.19.7", - "resolved": "https://registry.npmjs.org/@sentry/node/-/node-6.19.7.tgz", - "integrity": "sha512-gtmRC4dAXKODMpHXKfrkfvyBL3cI8y64vEi3fDD046uqYcrWdgoQsffuBbxMAizc6Ez1ia+f0Flue6p15Qaltg==", + "version": "7.81.1", + "resolved": "https://registry.npmjs.org/@sentry/node/-/node-7.81.1.tgz", + "integrity": "sha512-bKS3Mb95bar8AUEZSLKQ/RTSfFXo5sCSPNiBr5dDFuVljDFdkLq6NE3svG5bisrbENqfi0bqWsB4GZ7NHRTPbA==", "dependencies": { - "@sentry/core": "6.19.7", - "@sentry/hub": "6.19.7", - "@sentry/types": "6.19.7", - "@sentry/utils": "6.19.7", - "cookie": "^0.4.1", - "https-proxy-agent": "^5.0.0", - "lru_map": "^0.3.3", - "tslib": "^1.9.3" + "@sentry-internal/tracing": "7.81.1", + "@sentry/core": "7.81.1", + "@sentry/types": "7.81.1", + "@sentry/utils": "7.81.1", + "https-proxy-agent": "^5.0.0" }, "engines": { - "node": ">=6" + "node": ">=8" } }, "node_modules/@sentry/tracing": { - "version": "6.19.7", - "resolved": "https://registry.npmjs.org/@sentry/tracing/-/tracing-6.19.7.tgz", - "integrity": "sha512-ol4TupNnv9Zd+bZei7B6Ygnr9N3Gp1PUrNI761QSlHtPC25xXC5ssSD3GMhBgyQrcvpuRcCFHVNNM97tN5cZiA==", + "version": "7.81.1", + "resolved": "https://registry.npmjs.org/@sentry/tracing/-/tracing-7.81.1.tgz", + "integrity": "sha512-of9WMu0XgEBl9onTEk8SMaxj4BUadaUvHH96T1OpRMjdyuCM/3u2mjCkh3ekINr3Fu/uT2kJ/kO3goUxfcdXIQ==", "dependencies": { - "@sentry/hub": "6.19.7", - "@sentry/minimal": "6.19.7", - "@sentry/types": "6.19.7", - "@sentry/utils": "6.19.7", - "tslib": "^1.9.3" + "@sentry-internal/tracing": "7.81.1" }, "engines": { - "node": ">=6" + "node": ">=8" } }, "node_modules/@sentry/types": { - "version": "6.19.7", - "resolved": "https://registry.npmjs.org/@sentry/types/-/types-6.19.7.tgz", - "integrity": "sha512-jH84pDYE+hHIbVnab3Hr+ZXr1v8QABfhx39KknxqKWr2l0oEItzepV0URvbEhB446lk/S/59230dlUUIBGsXbg==", + "version": "7.81.1", + "resolved": "https://registry.npmjs.org/@sentry/types/-/types-7.81.1.tgz", + "integrity": "sha512-dvJvGyctiaPMIQqa46k56Re5IODWMDxiHJ1UjBs/WYDLrmWFPGrEbyJ8w8CYLhYA+7qqrCyIZmHbWSTRIxstHw==", "engines": { - "node": ">=6" + "node": ">=8" } }, "node_modules/@sentry/utils": { - "version": "6.19.7", - "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-6.19.7.tgz", - "integrity": "sha512-z95ECmE3i9pbWoXQrD/7PgkBAzJYR+iXtPuTkpBjDKs86O3mT+PXOT3BAn79w2wkn7/i3vOGD2xVr1uiMl26dA==", + "version": "7.81.1", + "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-7.81.1.tgz", + "integrity": "sha512-gq+MDXIirHKxNZ+c9/lVvCXd6y2zaZANujwlFggRH2u9SRiPaIXVilLpvMm4uJqmqBMEcY81ArujExtHvkbCqg==", "dependencies": { - "@sentry/types": "6.19.7", - "tslib": "^1.9.3" + "@sentry/types": "7.81.1" }, "engines": { - "node": ">=6" + "node": ">=8" } }, "node_modules/@tsconfig/node10": { @@ -2893,11 +2869,6 @@ "resolved": "https://registry.npmjs.org/long/-/long-5.2.0.tgz", "integrity": "sha512-9RTUNjK60eJbx3uz+TEGF7fUr29ZDxR5QzXcyDpeSfeH28S9ycINflOgOlppit5U+4kNTe83KQnMEerw7GmE8w==" }, - "node_modules/lru_map": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/lru_map/-/lru_map-0.3.3.tgz", - "integrity": "sha512-Pn9cox5CsMYngeDbmChANltQl+5pi6XmTrraMSzhPmMBbmgcxmqWry0U3PGapCU1yB4/LqCcom7qhHZiF/jGfQ==" - }, "node_modules/make-error": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", @@ -4409,11 +4380,6 @@ } } }, - "node_modules/tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - }, "node_modules/type-is": { "version": "1.6.18", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", @@ -4814,77 +4780,56 @@ "resolved": "https://registry.npmjs.org/@prisma/engines-version/-/engines-version-4.15.0-28.8fbc245156db7124f997f4cecdd8d1219e360944.tgz", "integrity": "sha512-sVOig4tjGxxlYaFcXgE71f/rtFhzyYrfyfNFUsxCIEJyVKU9rdOWIlIwQ2NQ7PntvGnn+x0XuFo4OC1jvPJKzg==" }, - "@sentry/core": { - "version": "6.19.7", - "resolved": "https://registry.npmjs.org/@sentry/core/-/core-6.19.7.tgz", - "integrity": "sha512-tOfZ/umqB2AcHPGbIrsFLcvApdTm9ggpi/kQZFkej7kMphjT+SGBiQfYtjyg9jcRW+ilAR4JXC9BGKsdEQ+8Vw==", + "@sentry-internal/tracing": { + "version": "7.81.1", + "resolved": "https://registry.npmjs.org/@sentry-internal/tracing/-/tracing-7.81.1.tgz", + "integrity": "sha512-E5xm27xrLXL10knH2EWDQsQYh5nb4SxxZzJ3sJwDGG9XGKzBdlp20UUhKqx00wixooVX9uCj3e4Jg8SvNB1hKg==", "requires": { - "@sentry/hub": "6.19.7", - "@sentry/minimal": "6.19.7", - "@sentry/types": "6.19.7", - "@sentry/utils": "6.19.7", - "tslib": "^1.9.3" + "@sentry/core": "7.81.1", + "@sentry/types": "7.81.1", + "@sentry/utils": "7.81.1" } }, - "@sentry/hub": { - "version": "6.19.7", - "resolved": "https://registry.npmjs.org/@sentry/hub/-/hub-6.19.7.tgz", - "integrity": "sha512-y3OtbYFAqKHCWezF0EGGr5lcyI2KbaXW2Ik7Xp8Mu9TxbSTuwTe4rTntwg8ngPjUQU3SUHzgjqVB8qjiGqFXCA==", - "requires": { - "@sentry/types": "6.19.7", - "@sentry/utils": "6.19.7", - "tslib": "^1.9.3" - } - }, - "@sentry/minimal": { - "version": "6.19.7", - "resolved": "https://registry.npmjs.org/@sentry/minimal/-/minimal-6.19.7.tgz", - "integrity": "sha512-wcYmSJOdvk6VAPx8IcmZgN08XTXRwRtB1aOLZm+MVHjIZIhHoBGZJYTVQS/BWjldsamj2cX3YGbGXNunaCfYJQ==", + "@sentry/core": { + "version": "7.81.1", + "resolved": "https://registry.npmjs.org/@sentry/core/-/core-7.81.1.tgz", + "integrity": "sha512-tU37yAmckOGCw/moWKSwekSCWWJP15O6luIq+u7wal22hE88F3Vc5Avo8SeF3upnPR+4ejaOFH+BJTr6bgrs6Q==", "requires": { - "@sentry/hub": "6.19.7", - "@sentry/types": "6.19.7", - "tslib": "^1.9.3" + "@sentry/types": "7.81.1", + "@sentry/utils": "7.81.1" } }, "@sentry/node": { - "version": "6.19.7", - "resolved": "https://registry.npmjs.org/@sentry/node/-/node-6.19.7.tgz", - "integrity": "sha512-gtmRC4dAXKODMpHXKfrkfvyBL3cI8y64vEi3fDD046uqYcrWdgoQsffuBbxMAizc6Ez1ia+f0Flue6p15Qaltg==", + "version": "7.81.1", + "resolved": "https://registry.npmjs.org/@sentry/node/-/node-7.81.1.tgz", + "integrity": "sha512-bKS3Mb95bar8AUEZSLKQ/RTSfFXo5sCSPNiBr5dDFuVljDFdkLq6NE3svG5bisrbENqfi0bqWsB4GZ7NHRTPbA==", "requires": { - "@sentry/core": "6.19.7", - "@sentry/hub": "6.19.7", - "@sentry/types": "6.19.7", - "@sentry/utils": "6.19.7", - "cookie": "^0.4.1", - "https-proxy-agent": "^5.0.0", - "lru_map": "^0.3.3", - "tslib": "^1.9.3" + "@sentry-internal/tracing": "7.81.1", + "@sentry/core": "7.81.1", + "@sentry/types": "7.81.1", + "@sentry/utils": "7.81.1", + "https-proxy-agent": "^5.0.0" } }, "@sentry/tracing": { - "version": "6.19.7", - "resolved": "https://registry.npmjs.org/@sentry/tracing/-/tracing-6.19.7.tgz", - "integrity": "sha512-ol4TupNnv9Zd+bZei7B6Ygnr9N3Gp1PUrNI761QSlHtPC25xXC5ssSD3GMhBgyQrcvpuRcCFHVNNM97tN5cZiA==", + "version": "7.81.1", + "resolved": "https://registry.npmjs.org/@sentry/tracing/-/tracing-7.81.1.tgz", + "integrity": "sha512-of9WMu0XgEBl9onTEk8SMaxj4BUadaUvHH96T1OpRMjdyuCM/3u2mjCkh3ekINr3Fu/uT2kJ/kO3goUxfcdXIQ==", "requires": { - "@sentry/hub": "6.19.7", - "@sentry/minimal": "6.19.7", - "@sentry/types": "6.19.7", - "@sentry/utils": "6.19.7", - "tslib": "^1.9.3" + "@sentry-internal/tracing": "7.81.1" } }, "@sentry/types": { - "version": "6.19.7", - "resolved": "https://registry.npmjs.org/@sentry/types/-/types-6.19.7.tgz", - "integrity": "sha512-jH84pDYE+hHIbVnab3Hr+ZXr1v8QABfhx39KknxqKWr2l0oEItzepV0URvbEhB446lk/S/59230dlUUIBGsXbg==" + "version": "7.81.1", + "resolved": "https://registry.npmjs.org/@sentry/types/-/types-7.81.1.tgz", + "integrity": "sha512-dvJvGyctiaPMIQqa46k56Re5IODWMDxiHJ1UjBs/WYDLrmWFPGrEbyJ8w8CYLhYA+7qqrCyIZmHbWSTRIxstHw==" }, "@sentry/utils": { - "version": "6.19.7", - "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-6.19.7.tgz", - "integrity": "sha512-z95ECmE3i9pbWoXQrD/7PgkBAzJYR+iXtPuTkpBjDKs86O3mT+PXOT3BAn79w2wkn7/i3vOGD2xVr1uiMl26dA==", + "version": "7.81.1", + "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-7.81.1.tgz", + "integrity": "sha512-gq+MDXIirHKxNZ+c9/lVvCXd6y2zaZANujwlFggRH2u9SRiPaIXVilLpvMm4uJqmqBMEcY81ArujExtHvkbCqg==", "requires": { - "@sentry/types": "6.19.7", - "tslib": "^1.9.3" + "@sentry/types": "7.81.1" } }, "@tsconfig/node10": { @@ -6938,11 +6883,6 @@ "resolved": "https://registry.npmjs.org/long/-/long-5.2.0.tgz", "integrity": "sha512-9RTUNjK60eJbx3uz+TEGF7fUr29ZDxR5QzXcyDpeSfeH28S9ycINflOgOlppit5U+4kNTe83KQnMEerw7GmE8w==" }, - "lru_map": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/lru_map/-/lru_map-0.3.3.tgz", - "integrity": "sha512-Pn9cox5CsMYngeDbmChANltQl+5pi6XmTrraMSzhPmMBbmgcxmqWry0U3PGapCU1yB4/LqCcom7qhHZiF/jGfQ==" - }, "make-error": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", @@ -8101,11 +8041,6 @@ "yn": "3.1.1" } }, - "tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - }, "type-is": { "version": "1.6.18", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", diff --git a/package.json b/package.json index 83f7f1f..8365f0d 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,8 @@ { "dependencies": { "@prisma/client": "^4.15.0", - "@sentry/node": "^6.17.7", - "@sentry/tracing": "^6.17.7", + "@sentry/node": "^7.81.1", + "@sentry/tracing": "^7.81.1", "aws-sdk": "^2.1408.0", "axios": "^0.26.1", "connect-flash": "^0.1.1", @@ -70,6 +70,6 @@ "typescript": "^5.2.2" }, "_moduleAliases": { - "@": "dist/src" + "@": "src" } } diff --git a/src/controller/ObservationController.ts b/src/controller/ObservationController.ts index fe85c96..a212f5c 100644 --- a/src/controller/ObservationController.ts +++ b/src/controller/ObservationController.ts @@ -37,6 +37,7 @@ var logData: { }[] = []; var statusData: ObservationStatus[] = []; var lastObservationData: LastObservationData = {}; +let observationData: { time: Date; data: Observation[][] }[] = []; // start updating after 1 minutes of starting the middleware let lastUpdatedToCare = new Date().getTime() - 59 * 60 * 1000; @@ -45,10 +46,30 @@ let lastUpdatedToCare = new Date().getTime() - 59 * 60 * 1000; // Update Interval is set to 1 hour const UPDATE_INTERVAL = 60 * 60 * 1000; +const S3_DATA_DUMP_INTERVAL = 1000 * 60 * 60; // For testing purposes, set update interval to 5 minutes // const UPDATE_INTERVAL = 5 * 60 * 1000; const DEFAULT_LISTING_LIMIT = 10; +setInterval(() => { + makeDataDumpToJson( + observationData, + `observations/${new Date().getTime()}.json`, + { + slug: "s3_observations_dump", + options: { + schedule: { + type: "interval", + unit: "minutes", + value: S3_DATA_DUMP_INTERVAL / (1000 * 60), + }, + }, + } + ); + + observationData = []; +}, S3_DATA_DUMP_INTERVAL); + const getTime = (date: string) => new Date(date.replace(" ", "T").concat("+0530")); @@ -350,12 +371,15 @@ const updateObservationsToCare = async () => { monitorPreset ); await makeDataDumpToJson( - payload, - v2Payload, - asset.externalId, - patient_id, - consultation_id, - b64Image + { + payload, + v2Payload, + assetExternalId: asset.externalId, + patient_id, + consultation_id, + b64Image, + }, + `${asset.externalId}--${new Date().getTime()}.json` ); } catch (err) { console.log("updateObservationsToCare:Data dump failed", err); @@ -504,6 +528,7 @@ export class ObservationController { addLogData(req.body); addStatusData(req.body); const observations = req.body; + observationData.push({ time: new Date(), data: observations }); // If req.body.observations is an array, then we need to loop through it and create a new observation for each one // If req.body.observations is a single object, then we need to create a new observation for it // If req.body.observations is undefined, then we need to return an error diff --git a/src/controller/helper/makeDataDump.ts b/src/controller/helper/makeDataDump.ts index a3be4ce..40be621 100644 --- a/src/controller/helper/makeDataDump.ts +++ b/src/controller/helper/makeDataDump.ts @@ -7,19 +7,30 @@ import { s3Provider, s3SecretAccessKey, } from "@/utils/configs"; -import type { DailyRoundObservation } from "@/types/observation"; -import type { OCRObservationV1Sanitized } from "@/types/ocr"; +import { captureCheckIn } from "@sentry/node"; dotenv.config({ path: "./.env" }); export const makeDataDumpToJson = async ( - v1Payload: DailyRoundObservation, - v2Payload: OCRObservationV1Sanitized, - assetExternalId: string, - patient_id: string, - consultation_id: string, - image: string | null + data: Record | any[], + key: string, + monitorOptions?: { + slug: string; + options?: any; + } ) => { + let checkInId: string | undefined = undefined; + + if (monitorOptions) { + checkInId = captureCheckIn( + { + monitorSlug: monitorOptions.slug, + status: "in_progress", + }, + monitorOptions.options + ); + } + try { const s3 = new AWS.S3({ accessKeyId: s3AccessKeyId, @@ -28,23 +39,14 @@ export const makeDataDumpToJson = async ( s3ForcePathStyle: s3Provider !== "AWS", }); - const dataDump = { - assetExternalId, - patient_id, - consultation_id, - v1Payload, - v2Payload, - image, - }; - if (!s3BucketName) { throw new Error("S3 Bucket Name not found"); } const params = { Bucket: s3BucketName, - Key: `${assetExternalId}--${new Date().getTime()}.json`, - Body: JSON.stringify(dataDump), + Key: key, + Body: JSON.stringify(data), ContentType: "application/json", }; @@ -53,16 +55,31 @@ export const makeDataDumpToJson = async ( params, function (err: Error, data: AWS.S3.ManagedUpload.SendData) { if (err) { - console.log("Auto OCR Upload error"); + console.log("Failed to upload data to S3"); reject(err); } else { - console.log("Auto OCR Upload Success"); + console.log("Successfully uploaded data to S3"); resolve(data); } } ); }); + + if (monitorOptions && checkInId) { + captureCheckIn({ + checkInId, + monitorSlug: monitorOptions.slug, + status: "ok", + }); + } } catch (err) { + if (monitorOptions && checkInId) { + captureCheckIn({ + checkInId, + monitorSlug: monitorOptions.slug, + status: "error", + }); + } console.log(err); } };