From c7c5ea07ed5d803ae669398eb041de8f440a5e97 Mon Sep 17 00:00:00 2001
From: Dan O'Neill <dev@jdoneill.com>
Date: Sat, 4 May 2024 17:26:22 -0700
Subject: [PATCH] optimize query use limit and offset to retrieve a portion of
 large datasets, value can be set as env variable for customization

---
 src/db/repo/data.js          | 4 +++-
 src/db/sql/createGeoJson.sql | 8 ++++----
 src/model/index.js           | 5 ++++-
 3 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/db/repo/data.js b/src/db/repo/data.js
index 2e3d61d..c0e3411 100644
--- a/src/db/repo/data.js
+++ b/src/db/repo/data.js
@@ -20,13 +20,15 @@ class DataRepository {
     }
   }
 
-  async createGeoJson(id, geom, srid, values) {
+  async createGeoJson(id, geom, srid, values, limit, offset) {
     try {
       const result = await this.db.oneOrNone(sql.createGeoJson, {
         id: id,
         geom: geom,
         srid: srid,
         table: values,
+        limit: limit,
+        offset: offset
       });
 
       return result;
diff --git a/src/db/sql/createGeoJson.sql b/src/db/sql/createGeoJson.sql
index ed70b1b..6ce21a6 100644
--- a/src/db/sql/createGeoJson.sql
+++ b/src/db/sql/createGeoJson.sql
@@ -4,8 +4,8 @@ SELECT jsonb_build_object(
 ) FROM (
   SELECT jsonb_build_object(
     'type',       'Feature',
-    'gid',         $[id],
-    'geometry',   ST_AsGeoJSON(ST_Transform($[geom:raw],$[srid:raw]))::jsonb,
-    'properties', to_jsonb(inputs) - $[geom]
+    'gid',         ${id},
+    'geometry',   ST_AsGeoJSON(ST_Transform(${geom:raw},${srid}))::jsonb,
+    'properties', to_jsonb(inputs) - ${geom}
   ) AS feature
-  FROM (SELECT * FROM $[table:raw]) inputs) features;
\ No newline at end of file
+  FROM (SELECT * FROM ${table:raw} WHERE ${id:raw} IN (SELECT ${id:raw} FROM ${table:raw} ORDER BY ${id:raw} LIMIT ${limit:raw} OFFSET ${offset:raw})) inputs) features;
diff --git a/src/model/index.js b/src/model/index.js
index 6f6e324..259fec3 100644
--- a/src/model/index.js
+++ b/src/model/index.js
@@ -10,6 +10,7 @@ class Model {
       const table = splitPath[1];
 
       const id = process.env.PG_OBJECTID || 'gid';
+      const pgLimit = process.env.PG_LIMIT || 10000000;
 
       if (!table)
         throw new Error('The "id" parameter must be in the form of "schema.table"');
@@ -22,8 +23,10 @@ class Model {
       
       const geom = result.f_geometry_column;
       const srid = result.srid;
+      const limit = parseInt(pgLimit);
+      const offset = 0;
 
-      const geojsonResult = await db.data.createGeoJson(id, geom, srid, schema + '.' + table);
+      const geojsonResult = await db.data.createGeoJson(id, geom, srid, schema + '.' + table, limit, offset);
       let geojson = geojsonResult.jsonb_build_object;
 
       geojson.description = 'PG Koop Feature Service';