diff --git a/job/jest.config.cjs b/job/jest.config.cjs index 71f629d..a570ae6 100644 --- a/job/jest.config.cjs +++ b/job/jest.config.cjs @@ -1,5 +1,5 @@ module.exports = { - displayName: 'Tests Typescript Application - Event', + displayName: 'Tests Typescript Application - Job', moduleDirectories: ['node_modules', 'src'], testMatch: ['**/tests/**/*.[jt]s?(x)', '**/?(*.)+(spec|test).[tj]s?(x)'], preset: 'ts-jest', diff --git a/job/src/services/bloomreach-discovery-catalog-ingestion.ts b/job/src/services/bloomreach-discovery-catalog-ingestion.ts index ed9fbea..f3591d7 100644 --- a/job/src/services/bloomreach-discovery-catalog-ingestion.ts +++ b/job/src/services/bloomreach-discovery-catalog-ingestion.ts @@ -1,5 +1,9 @@ import Bottleneck from 'bottleneck'; -import { Product } from '@commercetools/platform-sdk'; +import { + ClientResponse, + Product, + ProductPagedQueryResponse, +} from '@commercetools/platform-sdk'; import { logger } from '../utils/logger.utils'; import { createApiRoot } from '../client/create.client'; import { readConfiguration } from '../utils/config.utils'; @@ -39,11 +43,12 @@ export async function bloomreachDiscoveryCatalogIngestion() { bloomreachDiscoveryApiKey, bloomreachDiscoveryDomainKey, } = readConfiguration(); + + let _lastId: string | null = null; let _continue = true; - let offset = 0; const products: BloomreachProduct[] = []; - const limit = 500; + const limit = 3; const apiRoot = createApiRoot(); const limiter = new Bottleneck({ maxConcurrent: 1, @@ -51,87 +56,62 @@ export async function bloomreachDiscoveryCatalogIngestion() { }); const getProducts = limiter.wrap( - async (params: { limit: number; offset: number }) => { - logger.info(`Getting products... ${params.limit} ; ${params.offset}`); + async (params: { limit: number; lastId?: string }) => { return await apiRoot .products() - .get({ queryArgs: { limit: params.limit, offset: params.offset } }) + .get({ + queryArgs: { + limit: params.limit, + withTotal: false, + sort: 'id', + where: params.lastId ? `id > "${params.lastId}"` : undefined, + }, + }) .execute(); } ); - const getProductViews = (product: Product) => { - const views: BloomreachDiscoveryProductViews = {}; - const locales = product.masterData.current.name; - - for (const view in locales) { - views[view] = { - attributes: { - title: product.masterData.current.name[view] ?? '', - sku: product.masterData.current.masterVariant.sku ?? '', - description: product.masterData.current.description?.[view] ?? '', - slug: product.masterData.current.slug[view] ?? '', - price: - product.masterData.current.masterVariant.prices?.[0]?.value - .centAmount ?? 0, - image: - product.masterData.current.masterVariant.images?.[0]?.url ?? '', - }, - }; - } - - return views; - }; - - const getVariants = (product: Product) => { - const variants: BloomreachDiscoveryProductVariants = {}; - - product.masterData.current.variants.forEach((variant) => { - const attributesMap: Record = {}; - variant.attributes?.forEach((attribute) => { - if (attribute.value?.key) { - attributesMap[attribute.name] = attribute.value.label; - } - }); - - variants[variant.id] = { - attributes: { - ...attributesMap, - }, - }; - }); + // https://docs.commercetools.com/api/general-concepts#iterating-over-all-elements + while (_continue) { + let response: ClientResponse | null = null; - return variants; - }; + if (_lastId === null) { + response = await getProducts({ limit }); + } else { + response = await getProducts({ limit, lastId: _lastId }); + } - while (_continue) { - const response = await getProducts({ limit, offset }); - const data: BloomreachProduct[] = response.body.results.map((product) => { - return { - op: 'add', - path: `/products/${product.id}`, - value: { - attributes: { - title: product.masterData.current.name[locale] ?? '', - sku: product.masterData.current.masterVariant.sku ?? '', - description: product.masterData.current.description?.[locale] ?? '', - slug: product.masterData.current.slug[locale] ?? '', - price: - product.masterData.current.masterVariant.prices?.[0]?.value - .centAmount ?? 0, - image: - product.masterData.current.masterVariant.images?.[0]?.url ?? '', + const data: BloomreachProduct[] = + response?.body.results.map((product) => { + return { + op: 'add', + path: `/products/${product.id}`, + value: { + attributes: { + title: product.masterData.current.name[locale] ?? '', + sku: product.masterData.current.masterVariant.sku ?? '', + description: + product.masterData.current.description?.[locale] ?? '', + slug: product.masterData.current.slug[locale] ?? '', + price: + product.masterData.current.masterVariant.prices?.[0]?.value + .centAmount ?? 0, + image: + product.masterData.current.masterVariant.images?.[0]?.url ?? '', + }, + variants: getVariants(product), + views: getProductViews(product), }, - variants: getVariants(product), - views: getProductViews(product), - }, - }; - }); + }; + }) ?? []; + products.push(...data); - offset += limit; - if (products.length >= (response.body.total ?? 0)) { + if (products.length >= (response?.body.total ?? 0)) { _continue = false; } + _continue = response?.body.results.length == limit; + _lastId = + response?.body.results[response.body.results.length - 1]?.id ?? null; } const res = await fetch( @@ -149,3 +129,46 @@ export async function bloomreachDiscoveryCatalogIngestion() { const data = await res.json(); return data; } + +function getProductViews(product: Product) { + const views: BloomreachDiscoveryProductViews = {}; + const locales = product.masterData.current.name; + + for (const view in locales) { + views[view] = { + attributes: { + title: product.masterData.current.name[view] ?? '', + sku: product.masterData.current.masterVariant.sku ?? '', + description: product.masterData.current.description?.[view] ?? '', + slug: product.masterData.current.slug[view] ?? '', + price: + product.masterData.current.masterVariant.prices?.[0]?.value + .centAmount ?? 0, + image: product.masterData.current.masterVariant.images?.[0]?.url ?? '', + }, + }; + } + + return views; +} + +function getVariants(product: Product) { + const variants: BloomreachDiscoveryProductVariants = {}; + + product.masterData.current.variants.forEach((variant) => { + const attributesMap: Record = {}; + variant.attributes?.forEach((attribute) => { + if (attribute.value?.key) { + attributesMap[attribute.name] = attribute.value.label; + } + }); + + variants[variant.id] = { + attributes: { + ...attributesMap, + }, + }; + }); + + return variants; +}