Skip to content

Commit

Permalink
refactor(server): optimize query construction and db connection handling
Browse files Browse the repository at this point in the history
  • Loading branch information
@csulit
csulit committed Oct 18, 2024
1 parent d696255 commit f0a65dc
Show file tree
Hide file tree
Showing 3 changed files with 186 additions and 78 deletions.
24 changes: 13 additions & 11 deletions config/deno-kv.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ function cleanSpecialCharacters(input: string): string {
// Remove emojis and other special characters
const cleanedString = encodedString.replace(
/[\u{1F600}-\u{1F64F}\u{1F300}-\u{1F5FF}\u{1F680}-\u{1F6FF}\u{1F1E0}-\u{1F1FF}\u{2600}-\u{26FF}\u{2700}-\u{27BF}]/gu,
""
"",
);

// Remove extra whitespace
Expand Down Expand Up @@ -133,7 +133,7 @@ export async function listenQueue(kv: Deno.Kv) {
Object.keys(msg.data.dataLayer.attributes).length < 5
) {
throw new Error(
"Attributes are missing, undefined, or have fewer than 5 properties"
"Attributes are missing, undefined, or have fewer than 5 properties",
);
}

Expand All @@ -154,7 +154,7 @@ export async function listenQueue(kv: Deno.Kv) {
typeof msg.data.dataLayer.location !== "object"
) {
throw new Error(
"Location is missing, undefined, or not an object"
"Location is missing, undefined, or not an object",
);
}

Expand All @@ -164,7 +164,7 @@ export async function listenQueue(kv: Deno.Kv) {
const images = msg.data.images as { src: string }[];
const isCondominium =
msg.data.dataLayer.attributes.attribute_set_name ===
"Condominium";
"Condominium";
const isHouse =
msg.data.dataLayer.attributes.attribute_set_name === "House";
const isWarehouse =
Expand All @@ -185,8 +185,8 @@ export async function listenQueue(kv: Deno.Kv) {
};

const price = msg.data.dataLayer?.attributes?.price;
const priceFormatted =
msg.data.dataLayer?.attributes?.price_formatted;
const priceFormatted = msg.data.dataLayer?.attributes
?.price_formatted;

await transaction.queryArray({
args: [price, priceFormatted, listing.id],
Expand Down Expand Up @@ -216,8 +216,8 @@ export async function listenQueue(kv: Deno.Kv) {
JSON.stringify(
images.map((image) => image.src),
null,
2
)
2,
),
);

await transaction.commit();
Expand Down Expand Up @@ -273,8 +273,9 @@ export async function listenQueue(kv: Deno.Kv) {
const productOwnerName = msg.data.dataLayer.product_owner_name;
const location: Location = msg.data.dataLayer.location;
const dataLayerAttributes = msg.data.dataLayer.attributes;
const offerTypeId =
dataLayerAttributes.offer_type === "Rent" ? 2 : 1;
const offerTypeId = dataLayerAttributes.offer_type === "Rent"
? 2
: 1;
const sellerIsTrusted = dataLayerAttributes?.seller_is_trusted;
const locationData = await getLocation(transaction, {
...location,
Expand Down Expand Up @@ -385,7 +386,8 @@ export async function listenQueue(kv: Deno.Kv) {
offerTypeId,
newProperty,
],
text: `INSERT INTO Listing (title, url, project_name, description, is_scraped, address, price_formatted, price, offer_type_id, property_id)
text:
`INSERT INTO Listing (title, url, project_name, description, is_scraped, address, price_formatted, price, offer_type_id, property_id)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING id`,
});

Expand Down
11 changes: 8 additions & 3 deletions config/postgres.ts
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,17 @@
import { Pool } from "postgres";
import { Pool, type QueryArguments } from "postgres";

const POOL_CONNECTIONS = 20;

export const dbPool = new Pool(Deno.env.get("DATABASE_URL"), POOL_CONNECTIONS);

export async function runQuery(query: string) {
export async function runQueryObject(query: string, args?: QueryArguments) {
using client = await dbPool.connect();
return await client.queryObject(query);
return await client.queryObject(query, args);
}

export async function runQueryArray(query: string, args?: QueryArguments) {
using client = await dbPool.connect();
return await client.queryArray(query, args);
}

export default dbPool;
229 changes: 165 additions & 64 deletions server.ts
View file
Original file line number Diff line number Diff line change
@@ -1,76 +1,177 @@
import "jsr:@std/dotenv/load";
import { type Context, Hono } from "jsr:@hono/hono";

import { runQuery } from "./config/postgres.ts";
import { dbPool } from "./config/postgres.ts";
import { getKvInstance, listenQueue, sendMessage } from "./config/deno-kv.ts";

const app = new Hono();
const kv = await getKvInstance();

app.get("/", async (c: Context) => {
const postgres = await runQuery(`
SELECT
l.id AS listing_id,
l.title,
l.url,
l.project_name,
l.description,
l.is_scraped,
l.price,
l.price_formatted,
p.id AS property_id,
p.user_id,
p.floor_size,
p.lot_size,
p.building_size,
p.ceiling_height,
p.no_of_bedrooms,
p.no_of_bathrooms,
p.no_of_parking_spaces,
p.longitude,
p.latitude,
p.year_built,
p.primary_image_url,
p.images,
p.amenities,
p.property_features,
p.indoor_features,
p.outdoor_features,
p.ai_generated_description,
p.ai_generated_basic_features,
pt.type_name AS property_type_name,
wt.type_name AS warehouse_type_name,
l.address AS listing_address,
rg.region AS listing_region_name,
ct.city AS listing_city_name,
ar.area AS listing_area_name,
p.created_at AS property_created_at,
p.updated_at AS property_updated_at,
l.created_at AS listing_created_at,
l.updated_at AS listing_updated_at,
-- Price change log as an array ordered by latest changes
(
SELECT json_agg(
json_build_object(
'id', pcl.id,
'old_price', pcl.old_price,
'new_price', pcl.new_price,
'change_timestamp', pcl.change_timestamp
) ORDER BY pcl.change_timestamp DESC
)
FROM Price_Change_Log pcl
WHERE pcl.listing_id = l.id
) AS price_change_log
FROM
Listing l
JOIN Property p ON l.property_id = p.id
LEFT JOIN Property_Type pt ON p.property_type_id = pt.property_type_id
LEFT JOIN Warehouse_Type wt ON p.warehouse_type_id = wt.warehouse_type_id
LEFT JOIN Listing_Region rg ON p.listing_region_id = rg.id
LEFT JOIN Listing_City ct ON p.listing_city_id = ct.id
LEFT JOIN Listing_Area ar ON p.listing_area_id = ar.id
ORDER BY l.id DESC LIMIT 50;
`);
using client = await dbPool.connect();
const query = c.req.query() as unknown as {
page?: string;
page_size?: string;
property_type_id?: string;
listing_type_id?: string;
search_longitude?: string;
search_latitude?: string;
bounding_box?: string;
max_distance_km?: string;
};

if (!query.page || !query.page_size) {
query.page = "1";
query.page_size = "10";
}

if (!query.property_type_id) {
query.property_type_id = "1";
}

if (!query.listing_type_id) {
query.listing_type_id = "1";
}

const offset = (parseInt(query.page) - 1) * parseInt(query.page_size);

let boundingBoxCoords: number[] | null[] = [null, null, null, null];

if (query?.bounding_box) {
boundingBoxCoords = query.bounding_box.split("::").map((coord) =>
parseFloat(coord)
);
}

const searchLongitude = parseFloat(query?.search_longitude || "0");
const searchLatitude = parseFloat(query?.search_latitude || "0");

// Initialize the SQL WHERE clause with base conditions
let sqlWhereClause = `
pt.property_type_id = $1
AND lt.listing_type_id = $2
`;

// Initialize the SQL parameters array with base parameters
const sqlParams = [
parseInt(query.property_type_id),
parseInt(query.listing_type_id),
parseInt(query.page_size),
offset,
];

// Initialize the parameter counter for dynamic parameter numbering
let paramCounter = 5;

// Function to add a new condition to the WHERE clause
// deno-lint-ignore no-explicit-any
const addWhereCondition = (condition: string, ...params: any[]) => {
sqlWhereClause += ` AND ${condition}`;
sqlParams.push(...params);
paramCounter += params.length;
};

// Add bounding box condition if all coordinates are provided
if (boundingBoxCoords.every((coord) => coord !== null)) {
addWhereCondition(
`
p.latitude BETWEEN $${paramCounter} AND $${paramCounter + 1}
AND p.longitude BETWEEN $${paramCounter + 2} AND $${paramCounter + 3}
`,
...boundingBoxCoords as number[],
);
}

// Add max distance condition if required parameters are provided
if (query.max_distance_km && searchLongitude !== 0 && searchLatitude !== 0) {
addWhereCondition(
`
ST_DWithin(p.geog, ST_SetSRID(ST_MakePoint($${paramCounter}, $${
paramCounter + 1
}), 4326)::geography, ${parseFloat(query.max_distance_km)} * 1000)
`,
searchLongitude,
searchLatitude,
);
}

// Example of how to add a new condition in the future:
// if (someNewCondition) {
// addWhereCondition(`new_column = $${paramCounter}`, newValue);
// }

console.log({ sqlWhereClause, sqlParams, nextParamCounter: paramCounter });

const postgres = await client.queryObject({
args: sqlParams,
text: `
SELECT
l.id AS listing_id,
l.title,
l.url,
l.project_name,
l.description,
l.is_scraped,
l.price,
l.price_formatted,
p.id AS property_id,
p.user_id,
p.floor_size,
p.lot_size,
p.building_size,
p.ceiling_height,
p.no_of_bedrooms,
p.no_of_bathrooms,
p.no_of_parking_spaces,
p.longitude,
p.latitude,
p.year_built,
p.primary_image_url,
p.images,
p.amenities,
p.property_features,
p.indoor_features,
p.outdoor_features,
p.ai_generated_description,
p.ai_generated_basic_features,
pt.type_name AS property_type_name,
lt.type_name AS listing_type_name,
wt.type_name AS warehouse_type_name,
l.address AS listing_address,
rg.region AS listing_region_name,
ct.city AS listing_city_name,
ar.area AS listing_area_name,
p.created_at AS property_created_at,
p.updated_at AS property_updated_at,
l.created_at AS listing_created_at,
l.updated_at AS listing_updated_at,
-- Price change log as an array ordered by latest changes
(
SELECT json_agg(
json_build_object(
'id', pcl.id,
'old_price', pcl.old_price,
'new_price', pcl.new_price,
'change_timestamp', pcl.change_timestamp
) ORDER BY pcl.change_timestamp DESC
)
FROM Price_Change_Log pcl
WHERE pcl.listing_id = l.id
) AS price_change_log
FROM
Listing l
JOIN Property p ON l.property_id = p.id
LEFT JOIN Property_Type pt ON p.property_type_id = pt.property_type_id
LEFT JOIN Listing_Type lt ON l.offer_type_id = lt.listing_type_id
LEFT JOIN Warehouse_Type wt ON p.warehouse_type_id = wt.warehouse_type_id
LEFT JOIN Listing_Region rg ON p.listing_region_id = rg.id
LEFT JOIN Listing_City ct ON p.listing_city_id = ct.id
LEFT JOIN Listing_Area ar ON p.listing_area_id = ar.id
WHERE
${sqlWhereClause}
ORDER BY l.id DESC LIMIT $3 OFFSET $4;
`,
});
return c.json(postgres.rows);
});

Expand Down

0 comments on commit f0a65dc

Please sign in to comment.