Skip to content

Commit

Permalink
feat(server): Add endpoint for listing areas with data processing logic
Browse files Browse the repository at this point in the history
  • Loading branch information
@csulit
csulit committed Oct 29, 2024
1 parent 61d893a commit c90f5eb
Show file tree
Hide file tree
Showing 2 changed files with 305 additions and 1 deletion.
7 changes: 7 additions & 0 deletions config/deno-kv.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,13 @@ export async function listenQueue(kv: Deno.Kv) {
RETURNING id`,
});
}

rawProperty.listing_region_id = (region.rows[0] as { id: number })
.id.toString();
rawProperty.listing_city_id = (city.rows[0] as { id: number })
.id.toString();
rawProperty.listing_area_id = (area.rows[0] as { id: number })
.id.toString();
}

if (rawProperties.rowCount && rawProperties.rowCount > 0) {
Expand Down
299 changes: 298 additions & 1 deletion server.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,15 @@ import { type Context, Hono } from "npm:hono";
import { cors } from "npm:hono/cors";

import { dbPool } from "./config/postgres.ts";
import { getKvInstance, listenQueue, sendMessage } from "./config/deno-kv.ts";
import {
getKvInstance,
listenQueue,
type Listing,
type Property,
type RawLamudiData,
sendMessage,
} from "./config/deno-kv.ts";
import type { PoolClient, Transaction } from "postgres";

const app = new Hono();
const kv = await getKvInstance();
Expand Down Expand Up @@ -487,6 +495,295 @@ app.get("/api/properties/cities", async (c: Context) => {
});
});

app.get("/api/properties/areas", async (c: Context) => {
let transaction: Transaction | null = null;
let client: PoolClient | null = null;

try {
client = await dbPool.connect();
using client2 = await dbPool.connect();
transaction = client.createTransaction(
"create_listing_from_raw_lamudi_data",
);

await transaction.begin();

const rawProperties = await transaction.queryObject<RawLamudiData>(
`
SELECT
id, json_data,
json_data->'dataLayer'->>'title' AS raw_title,
CASE
WHEN json_data->'dataLayer'->'attributes'->>'attribute_set_name' = 'Condominium' THEN 1
WHEN json_data->'dataLayer'->'attributes'->>'attribute_set_name' = 'House' THEN 2
WHEN json_data->'dataLayer'->'attributes'->>'subcategory' = 'Warehouse' THEN 3
WHEN json_data->'dataLayer'->'attributes'->>'attribute_set_name' = 'Land' THEN 4
END AS property_type_id,
CASE
WHEN json_data->'dataLayer'->'attributes'->>'offer_type' = 'Buy' THEN 1
WHEN json_data->'dataLayer'->'attributes'->>'offer_type' = 'Rent' THEN 2
END AS offer_type_id,
json_data->'dataLayer'->>'agent_name' AS agent_name,
json_data->'dataLayer'->'attributes'->>'product_owner_name' AS product_owner_name,
json_data->'dataLayer'->'attributes'->>'listing_region_id' AS listing_region_id,
json_data->'dataLayer'->'location'->>'region' AS region,
json_data->'dataLayer'->'attributes'->>'listing_city_id' AS listing_city_id,
json_data->'dataLayer'->'location'->>'city' AS city,
json_data->'dataLayer'->'attributes'->>'listing_area' AS listing_area,
json_data->'dataLayer'->'attributes'->>'listing_area_id' AS listing_area_id,
COALESCE((json_data->'dataLayer'->'location'->>'rooms_total')::INTEGER, 0) AS rooms_total,
COALESCE((json_data->'dataLayer'->'attributes'->>'floor_size')::DOUBLE PRECISION, 0) AS floor_size,
COALESCE((json_data->'dataLayer'->'attributes'->>'lot_size')::DOUBLE PRECISION, 0) AS lot_size,
COALESCE((json_data->'dataLayer'->'attributes'->>'land_size')::DOUBLE PRECISION, 0) AS land_size,
COALESCE((json_data->'dataLayer'->'attributes'->>'building_size')::DOUBLE PRECISION, 0) AS building_size,
COALESCE((json_data->'dataLayer'->'attributes'->>'bedrooms')::INTEGER, 0) AS no_of_bedrooms,
COALESCE((json_data->'dataLayer'->'attributes'->>'bathrooms')::INTEGER, 0) AS no_of_bathrooms,
COALESCE((json_data->'dataLayer'->'attributes'->>'car_spaces')::INTEGER, 0) AS no_of_parking_spaces,
(json_data->'dataLayer'->'attributes'->>'location_longitude')::DOUBLE PRECISION AS longitude,
(json_data->'dataLayer'->'attributes'->>'location_latitude')::DOUBLE PRECISION AS latitude,
(json_data->'dataLayer'->'attributes'->>'year_built')::INTEGER AS year_built,
json_data->'dataLayer'->'attributes'->>'image_url' AS primary_image_url,
(json_data->'dataLayer'->'attributes'->>'indoor_features')::jsonb AS indoor_features,
(json_data->'dataLayer'->'attributes'->>'outdoor_features')::jsonb AS outdoor_features,
(json_data->'dataLayer'->'attributes'->>'other_features')::jsonb AS property_features,
json_data->'dataLayer'->'attributes'->>'listing_address' AS address,
json_data->'dataLayer'->'attributes'->>'project_name' AS project_name,
json_data->'dataLayer'->'attributes'->>'price' AS price,
json_data->'dataLayer'->'attributes'->>'price_formatted' AS price_formatted,
json_data->'dataLayer'->'description'->>'text' AS description,
CONCAT('https://lamudi.com.ph/', json_data->'dataLayer'->'attributes'->>'urlkey_details') AS full_url,
(json_data->>'images')::jsonb AS images
FROM lamudi_raw_data
WHERE is_process = FALSE
AND COALESCE((json_data->'dataLayer'->'attributes'->>'price')::INTEGER, 0) > 5000
AND json_data->'dataLayer'->'location'->>'region' IS NOT NULL
AND json_data->'dataLayer'->'location'->>'city' IS NOT NULL
AND json_data->'dataLayer'->'attributes'->>'listing_area' IS NOT NULL
LIMIT 25
`,
);

for (const rawProperty of rawProperties.rows) {
try {
let region = await transaction.queryObject(`
SELECT id, listing_region_id
FROM Listing_Region
WHERE listing_region_id = '${rawProperty.listing_region_id}'
`);

let city = await transaction.queryObject(`
SELECT id, listing_city_id
FROM Listing_City
WHERE listing_city_id = '${rawProperty.listing_city_id}'
`);

let area = await transaction.queryObject(`
SELECT id
FROM Listing_Area
WHERE listing_area_id = '${rawProperty.listing_area_id}'
`);

if (region.rowCount === 0) {
region = await transaction.queryObject({
args: [rawProperty.region, rawProperty.listing_region_id],
text: `INSERT INTO Listing_Region (region, listing_region_id)
VALUES ($1, $2)
RETURNING id, listing_region_id`,
});
}

if (city.rowCount === 0) {
const createdRegion = region.rows[0] as {
listing_region_id: number;
};

city = await transaction.queryObject({
args: [
rawProperty.city,
rawProperty.listing_city_id,
createdRegion.listing_region_id,
],
text:
`INSERT INTO Listing_City (city, listing_city_id, listing_region_id)
VALUES ($1, $2, $3)
RETURNING id, listing_city_id`,
});
}

if (area.rowCount === 0 && rawProperty.listing_area_id) {
area = await transaction.queryObject({
args: [rawProperty.listing_area, rawProperty.listing_area_id],
text: `INSERT INTO Listing_Area (area, listing_area_id)
VALUES ($1, $2)
RETURNING id`,
});
}

rawProperty.listing_region_id = (region.rows[0] as { id: number }).id.toString();
rawProperty.listing_city_id = (city.rows[0] as { id: number }).id.toString();
rawProperty.listing_area_id = (area.rows[0] as { id: number }).id.toString();
} catch (error) {
throw error;
}
}

if (rawProperties.rowCount && rawProperties.rowCount > 0) {
for (const rawProperty of rawProperties.rows) {
const images = rawProperty.images.map((image) => image.src);

const listing = await transaction.queryObject<Listing>({
args: [rawProperty.full_url],
text: `SELECT url FROM Listing WHERE url = $1`,
});

if (listing.rowCount && listing.rowCount > 0) {
console.info("Listing already exists");

await transaction.queryObject({
args: [rawProperty.id],
text: `UPDATE lamudi_raw_data SET is_process = TRUE WHERE id = $1`,
});

await transaction.queryObject({
args: [
rawProperty.price,
rawProperty.price_formatted,
rawProperty.full_url,
],
text: `UPDATE Listing
SET price = $1, price_formatted = $2
WHERE url = $3`,
});

await transaction.queryObject({
args: [
JSON.stringify(images),
rawProperty.agent_name,
rawProperty.product_owner_name,
rawProperty.project_name,
rawProperty.full_url,
],
text: `UPDATE Property p
SET images = $1,
agent_name = $2,
product_owner_name = $3,
project_name = $4
FROM Listing l
WHERE l.property_id = p.id AND l.url = $5`,
});

continue;
}
let property;

console.log({
listing_region_id: rawProperty.listing_region_id,
listing_city_id: rawProperty.listing_city_id,
listing_area_id: rawProperty.listing_area_id,
})

try {
property = await client2.queryObject<Property>({
args: [
rawProperty.floor_size,
rawProperty.lot_size,
rawProperty.building_size,
rawProperty.no_of_bedrooms,
rawProperty.no_of_bathrooms,
rawProperty.no_of_parking_spaces,
rawProperty.longitude,
rawProperty.latitude,
rawProperty.year_built ?? 0,
rawProperty.primary_image_url,
JSON.stringify(images),
JSON.stringify(rawProperty.property_features),
JSON.stringify(rawProperty.indoor_features),
JSON.stringify(rawProperty.outdoor_features),
rawProperty.property_type_id,
rawProperty.address ?? "-",
parseInt(rawProperty.listing_region_id),
parseInt(rawProperty.listing_city_id),
parseInt(rawProperty.listing_area_id),
rawProperty.project_name,
rawProperty.agent_name,
rawProperty.product_owner_name,
0, // Add missing required field
JSON.stringify({}), // Add missing field
JSON.stringify({}), // Add missing field
JSON.stringify({}), // Add missing field
],
text: `INSERT INTO Property
(
floor_size, lot_size, building_size, no_of_bedrooms,
no_of_bathrooms, no_of_parking_spaces, longitude,
latitude, year_built, primary_image_url, images,
property_features, indoor_features, outdoor_features,
property_type_id, address, listing_region_id, listing_city_id,
listing_area_id, project_name, agent_name, product_owner_name,
ceiling_height, amenities, ai_generated_description,
ai_generated_basic_features
)
VALUES (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13,
$14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25,
$26
)
ON CONFLICT DO NOTHING
RETURNING id`,
});
} catch (error) {
throw error;
}
console.log(property.rowCount);

// if (property) {
// try {
// await transaction.queryObject({
// args: [
// rawProperty.raw_title,
// rawProperty.full_url,
// rawProperty.project_name,
// rawProperty.description,
// true, // is_scraped
// rawProperty.address,
// rawProperty.price_formatted,
// rawProperty.price,
// rawProperty.offer_type_id,
// property.rows[0].id,
// ],
// text: `
// INSERT INTO Listing (
// title, url, project_name, description, is_scraped,
// address, price_formatted, price, offer_type_id, property_id
// ) VALUES (
// $1, $2, $3, $4, $5, $6, $7, $8, $9, $10
// )
// `,
// });
// } catch (error) {
// throw error;
// }
// }
}
}

await transaction.commit();
} catch (error) {
if (transaction) {
try {
await transaction.rollback();
} catch (rollbackError) {
console.error("Error during rollback:", rollbackError);
}
}
console.error("Transaction error:", error);
} finally {
if (client) client.release();
}

return c.json({ data: "success" });
});

app.get("/api/properties/:id", async (c: Context) => {
using client = await dbPool.connect();
const id = c.req.param("id");
Expand Down

0 comments on commit c90f5eb

Please sign in to comment.