fix: OOM issues with memory optimized flush (#1251)

* start copy * use temp table to update * improved flush * cleanup * cleanup * cleanup * pgltie * cleanup * chore: changeset * nits
ponder-sh · Nov 18, 2024 · 2806076 · 2806076
1 parent a142bbd
commit 2806076
Show file tree

Hide file tree

Showing 6 changed files with 259 additions and 114 deletions.
diff --git a/.changeset/clean-months-end.md b/.changeset/clean-months-end.md
@@ -0,0 +1,5 @@
+---
+"@ponder/core": patch
+---
+
+Improved memory usage during historical indexing.
diff --git a/packages/core/package.json b/packages/core/package.json
@@ -67,6 +67,7 @@
     "kysely-pglite": "^0.6.0",
     "pg": "^8.11.3",
     "pg-connection-string": "^2.6.2",
+    "pg-copy-streams": "^6.0.6",
     "picocolors": "^1.0.0",
     "pino": "^8.16.2",
     "prom-client": "^15.0.0",
@@ -81,6 +82,7 @@
     "@types/glob": "^8.1.0",
     "@types/node": "^20.10.0",
     "@types/pg": "^8.10.9",
+    "@types/pg-copy-streams": "^1.2.5",
     "@types/react": "^18.2.38",
     "@viem/anvil": "^0.0.6",
     "@wagmi/cli": "^1.5.2",

diff --git a/packages/core/src/common/options.ts b/packages/core/src/common/options.ts
@@ -103,14 +103,14 @@ export const buildOptions = ({ cliOptions }: { cliOptions: CliOptions }) => {
 
     factoryAddressCountThreshold: 1_000,
 
-    // v8.getHeapStatistics().heap_size_limit / 4, bucketed closest to 128, 256, 512, 1024, 2048 mB
+    // v8.getHeapStatistics().heap_size_limit / 8, bucketed closest to 128, 256, 512, 1024, 2048 mB
     indexingCacheMaxBytes:
       2 **
         Math.min(
           Math.max(
             Math.round(
               Math.log2(
-                v8.getHeapStatistics().heap_size_limit / 1_024 / 1_024 / 4,
+                v8.getHeapStatistics().heap_size_limit / 1_024 / 1_024 / 8,
               ),
             ),
             7,

diff --git a/packages/core/src/database/index.ts b/packages/core/src/database/index.ts
@@ -45,6 +45,8 @@ import prometheus from "prom-client";
 import { HeadlessKysely } from "./kysely.js";
 
 export type Database = {
+  dialect: "pglite" | "postgres";
+  driver: PGliteDriver | PostgresDriver;
   qb: QueryBuilder;
   drizzle: Drizzle<Schema>;
   migrateSync(): Promise<void>;
@@ -452,6 +454,8 @@ export const createDatabase = (args: {
   };
 
   const database = {
+    dialect: dialect === "postgres" ? "postgres" : "pglite",
+    driver,
     qb,
     drizzle,
     async migrateSync() {