From c90f5ebf7103b43ddf05f6703171008204b878c0 Mon Sep 17 00:00:00 2001 From: casulit Date: Tue, 29 Oct 2024 15:07:43 +0800 Subject: [PATCH] feat(server): Add endpoint for listing areas with data processing logic --- config/deno-kv.ts | 7 ++ server.ts | 299 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 305 insertions(+), 1 deletion(-) diff --git a/config/deno-kv.ts b/config/deno-kv.ts index 42e59d9..c98ac44 100644 --- a/config/deno-kv.ts +++ b/config/deno-kv.ts @@ -229,6 +229,13 @@ export async function listenQueue(kv: Deno.Kv) { RETURNING id`, }); } + + rawProperty.listing_region_id = (region.rows[0] as { id: number }) + .id.toString(); + rawProperty.listing_city_id = (city.rows[0] as { id: number }) + .id.toString(); + rawProperty.listing_area_id = (area.rows[0] as { id: number }) + .id.toString(); } if (rawProperties.rowCount && rawProperties.rowCount > 0) { diff --git a/server.ts b/server.ts index b920157..eb32113 100644 --- a/server.ts +++ b/server.ts @@ -3,7 +3,15 @@ import { type Context, Hono } from "npm:hono"; import { cors } from "npm:hono/cors"; import { dbPool } from "./config/postgres.ts"; -import { getKvInstance, listenQueue, sendMessage } from "./config/deno-kv.ts"; +import { + getKvInstance, + listenQueue, + type Listing, + type Property, + type RawLamudiData, + sendMessage, +} from "./config/deno-kv.ts"; +import type { PoolClient, Transaction } from "postgres"; const app = new Hono(); const kv = await getKvInstance(); @@ -487,6 +495,295 @@ app.get("/api/properties/cities", async (c: Context) => { }); }); +app.get("/api/properties/areas", async (c: Context) => { +let transaction: Transaction | null = null; + let client: PoolClient | null = null; + + try { + client = await dbPool.connect(); + using client2 = await dbPool.connect(); + transaction = client.createTransaction( + "create_listing_from_raw_lamudi_data", + ); + + await transaction.begin(); + + const rawProperties = await transaction.queryObject( + ` + SELECT + id, json_data, + json_data->'dataLayer'->>'title' AS raw_title, + CASE + WHEN json_data->'dataLayer'->'attributes'->>'attribute_set_name' = 'Condominium' THEN 1 + WHEN json_data->'dataLayer'->'attributes'->>'attribute_set_name' = 'House' THEN 2 + WHEN json_data->'dataLayer'->'attributes'->>'subcategory' = 'Warehouse' THEN 3 + WHEN json_data->'dataLayer'->'attributes'->>'attribute_set_name' = 'Land' THEN 4 + END AS property_type_id, + CASE + WHEN json_data->'dataLayer'->'attributes'->>'offer_type' = 'Buy' THEN 1 + WHEN json_data->'dataLayer'->'attributes'->>'offer_type' = 'Rent' THEN 2 + END AS offer_type_id, + json_data->'dataLayer'->>'agent_name' AS agent_name, + json_data->'dataLayer'->'attributes'->>'product_owner_name' AS product_owner_name, + json_data->'dataLayer'->'attributes'->>'listing_region_id' AS listing_region_id, + json_data->'dataLayer'->'location'->>'region' AS region, + json_data->'dataLayer'->'attributes'->>'listing_city_id' AS listing_city_id, + json_data->'dataLayer'->'location'->>'city' AS city, + json_data->'dataLayer'->'attributes'->>'listing_area' AS listing_area, + json_data->'dataLayer'->'attributes'->>'listing_area_id' AS listing_area_id, + COALESCE((json_data->'dataLayer'->'location'->>'rooms_total')::INTEGER, 0) AS rooms_total, + COALESCE((json_data->'dataLayer'->'attributes'->>'floor_size')::DOUBLE PRECISION, 0) AS floor_size, + COALESCE((json_data->'dataLayer'->'attributes'->>'lot_size')::DOUBLE PRECISION, 0) AS lot_size, + COALESCE((json_data->'dataLayer'->'attributes'->>'land_size')::DOUBLE PRECISION, 0) AS land_size, + COALESCE((json_data->'dataLayer'->'attributes'->>'building_size')::DOUBLE PRECISION, 0) AS building_size, + COALESCE((json_data->'dataLayer'->'attributes'->>'bedrooms')::INTEGER, 0) AS no_of_bedrooms, + COALESCE((json_data->'dataLayer'->'attributes'->>'bathrooms')::INTEGER, 0) AS no_of_bathrooms, + COALESCE((json_data->'dataLayer'->'attributes'->>'car_spaces')::INTEGER, 0) AS no_of_parking_spaces, + (json_data->'dataLayer'->'attributes'->>'location_longitude')::DOUBLE PRECISION AS longitude, + (json_data->'dataLayer'->'attributes'->>'location_latitude')::DOUBLE PRECISION AS latitude, + (json_data->'dataLayer'->'attributes'->>'year_built')::INTEGER AS year_built, + json_data->'dataLayer'->'attributes'->>'image_url' AS primary_image_url, + (json_data->'dataLayer'->'attributes'->>'indoor_features')::jsonb AS indoor_features, + (json_data->'dataLayer'->'attributes'->>'outdoor_features')::jsonb AS outdoor_features, + (json_data->'dataLayer'->'attributes'->>'other_features')::jsonb AS property_features, + json_data->'dataLayer'->'attributes'->>'listing_address' AS address, + json_data->'dataLayer'->'attributes'->>'project_name' AS project_name, + json_data->'dataLayer'->'attributes'->>'price' AS price, + json_data->'dataLayer'->'attributes'->>'price_formatted' AS price_formatted, + json_data->'dataLayer'->'description'->>'text' AS description, + CONCAT('https://lamudi.com.ph/', json_data->'dataLayer'->'attributes'->>'urlkey_details') AS full_url, + (json_data->>'images')::jsonb AS images + FROM lamudi_raw_data + WHERE is_process = FALSE + AND COALESCE((json_data->'dataLayer'->'attributes'->>'price')::INTEGER, 0) > 5000 + AND json_data->'dataLayer'->'location'->>'region' IS NOT NULL + AND json_data->'dataLayer'->'location'->>'city' IS NOT NULL + AND json_data->'dataLayer'->'attributes'->>'listing_area' IS NOT NULL + LIMIT 25 + `, + ); + + for (const rawProperty of rawProperties.rows) { + try { + let region = await transaction.queryObject(` + SELECT id, listing_region_id + FROM Listing_Region + WHERE listing_region_id = '${rawProperty.listing_region_id}' + `); + + let city = await transaction.queryObject(` + SELECT id, listing_city_id + FROM Listing_City + WHERE listing_city_id = '${rawProperty.listing_city_id}' + `); + + let area = await transaction.queryObject(` + SELECT id + FROM Listing_Area + WHERE listing_area_id = '${rawProperty.listing_area_id}' + `); + + if (region.rowCount === 0) { + region = await transaction.queryObject({ + args: [rawProperty.region, rawProperty.listing_region_id], + text: `INSERT INTO Listing_Region (region, listing_region_id) + VALUES ($1, $2) + RETURNING id, listing_region_id`, + }); + } + + if (city.rowCount === 0) { + const createdRegion = region.rows[0] as { + listing_region_id: number; + }; + + city = await transaction.queryObject({ + args: [ + rawProperty.city, + rawProperty.listing_city_id, + createdRegion.listing_region_id, + ], + text: + `INSERT INTO Listing_City (city, listing_city_id, listing_region_id) + VALUES ($1, $2, $3) + RETURNING id, listing_city_id`, + }); + } + + if (area.rowCount === 0 && rawProperty.listing_area_id) { + area = await transaction.queryObject({ + args: [rawProperty.listing_area, rawProperty.listing_area_id], + text: `INSERT INTO Listing_Area (area, listing_area_id) + VALUES ($1, $2) + RETURNING id`, + }); + } + + rawProperty.listing_region_id = (region.rows[0] as { id: number }).id.toString(); + rawProperty.listing_city_id = (city.rows[0] as { id: number }).id.toString(); + rawProperty.listing_area_id = (area.rows[0] as { id: number }).id.toString(); + } catch (error) { + throw error; + } + } + + if (rawProperties.rowCount && rawProperties.rowCount > 0) { + for (const rawProperty of rawProperties.rows) { + const images = rawProperty.images.map((image) => image.src); + + const listing = await transaction.queryObject({ + args: [rawProperty.full_url], + text: `SELECT url FROM Listing WHERE url = $1`, + }); + + if (listing.rowCount && listing.rowCount > 0) { + console.info("Listing already exists"); + + await transaction.queryObject({ + args: [rawProperty.id], + text: `UPDATE lamudi_raw_data SET is_process = TRUE WHERE id = $1`, + }); + + await transaction.queryObject({ + args: [ + rawProperty.price, + rawProperty.price_formatted, + rawProperty.full_url, + ], + text: `UPDATE Listing + SET price = $1, price_formatted = $2 + WHERE url = $3`, + }); + + await transaction.queryObject({ + args: [ + JSON.stringify(images), + rawProperty.agent_name, + rawProperty.product_owner_name, + rawProperty.project_name, + rawProperty.full_url, + ], + text: `UPDATE Property p + SET images = $1, + agent_name = $2, + product_owner_name = $3, + project_name = $4 + FROM Listing l + WHERE l.property_id = p.id AND l.url = $5`, + }); + + continue; + } + let property; + + console.log({ + listing_region_id: rawProperty.listing_region_id, + listing_city_id: rawProperty.listing_city_id, + listing_area_id: rawProperty.listing_area_id, + }) + + try { + property = await client2.queryObject({ + args: [ + rawProperty.floor_size, + rawProperty.lot_size, + rawProperty.building_size, + rawProperty.no_of_bedrooms, + rawProperty.no_of_bathrooms, + rawProperty.no_of_parking_spaces, + rawProperty.longitude, + rawProperty.latitude, + rawProperty.year_built ?? 0, + rawProperty.primary_image_url, + JSON.stringify(images), + JSON.stringify(rawProperty.property_features), + JSON.stringify(rawProperty.indoor_features), + JSON.stringify(rawProperty.outdoor_features), + rawProperty.property_type_id, + rawProperty.address ?? "-", + parseInt(rawProperty.listing_region_id), + parseInt(rawProperty.listing_city_id), + parseInt(rawProperty.listing_area_id), + rawProperty.project_name, + rawProperty.agent_name, + rawProperty.product_owner_name, + 0, // Add missing required field + JSON.stringify({}), // Add missing field + JSON.stringify({}), // Add missing field + JSON.stringify({}), // Add missing field + ], + text: `INSERT INTO Property + ( + floor_size, lot_size, building_size, no_of_bedrooms, + no_of_bathrooms, no_of_parking_spaces, longitude, + latitude, year_built, primary_image_url, images, + property_features, indoor_features, outdoor_features, + property_type_id, address, listing_region_id, listing_city_id, + listing_area_id, project_name, agent_name, product_owner_name, + ceiling_height, amenities, ai_generated_description, + ai_generated_basic_features + ) + VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, + $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, + $26 + ) + ON CONFLICT DO NOTHING + RETURNING id`, + }); + } catch (error) { + throw error; + } + console.log(property.rowCount); + + // if (property) { + // try { + // await transaction.queryObject({ + // args: [ + // rawProperty.raw_title, + // rawProperty.full_url, + // rawProperty.project_name, + // rawProperty.description, + // true, // is_scraped + // rawProperty.address, + // rawProperty.price_formatted, + // rawProperty.price, + // rawProperty.offer_type_id, + // property.rows[0].id, + // ], + // text: ` + // INSERT INTO Listing ( + // title, url, project_name, description, is_scraped, + // address, price_formatted, price, offer_type_id, property_id + // ) VALUES ( + // $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 + // ) + // `, + // }); + // } catch (error) { + // throw error; + // } + // } + } + } + + await transaction.commit(); + } catch (error) { + if (transaction) { + try { + await transaction.rollback(); + } catch (rollbackError) { + console.error("Error during rollback:", rollbackError); + } + } + console.error("Transaction error:", error); + } finally { + if (client) client.release(); + } + + return c.json({ data: "success" }); +}); + app.get("/api/properties/:id", async (c: Context) => { using client = await dbPool.connect(); const id = c.req.param("id");