refactor: minor updates

composable-com · Oct 12, 2023 · 53cb231 · 53cb231
1 parent b4a21f1
commit 53cb231
Show file tree

Hide file tree

Showing 2 changed files with 96 additions and 73 deletions.
diff --git a/job/jest.config.cjs b/job/jest.config.cjs
@@ -1,5 +1,5 @@
 module.exports = {
-  displayName: 'Tests Typescript Application - Event',
+  displayName: 'Tests Typescript Application - Job',
   moduleDirectories: ['node_modules', 'src'],
   testMatch: ['**/tests/**/*.[jt]s?(x)', '**/?(*.)+(spec|test).[tj]s?(x)'],
   preset: 'ts-jest',

diff --git a/job/src/services/bloomreach-discovery-catalog-ingestion.ts b/job/src/services/bloomreach-discovery-catalog-ingestion.ts
@@ -1,5 +1,9 @@
 import Bottleneck from 'bottleneck';
-import { Product } from '@commercetools/platform-sdk';
+import {
+  ClientResponse,
+  Product,
+  ProductPagedQueryResponse,
+} from '@commercetools/platform-sdk';
 import { logger } from '../utils/logger.utils';
 import { createApiRoot } from '../client/create.client';
 import { readConfiguration } from '../utils/config.utils';
@@ -39,99 +43,75 @@ export async function bloomreachDiscoveryCatalogIngestion() {
     bloomreachDiscoveryApiKey,
     bloomreachDiscoveryDomainKey,
   } = readConfiguration();
+
+  let _lastId: string | null = null;
   let _continue = true;
-  let offset = 0;
 
   const products: BloomreachProduct[] = [];
-  const limit = 500;
+  const limit = 3;
   const apiRoot = createApiRoot();
   const limiter = new Bottleneck({
     maxConcurrent: 1,
     minTime: 300,
   });
 
   const getProducts = limiter.wrap(
-    async (params: { limit: number; offset: number }) => {
-      logger.info(`Getting products... ${params.limit} ; ${params.offset}`);
+    async (params: { limit: number; lastId?: string }) => {
       return await apiRoot
         .products()
-        .get({ queryArgs: { limit: params.limit, offset: params.offset } })
+        .get({
+          queryArgs: {
+            limit: params.limit,
+            withTotal: false,
+            sort: 'id',
+            where: params.lastId ? `id > "${params.lastId}"` : undefined,
+          },
+        })
         .execute();
     }
   );
 
-  const getProductViews = (product: Product) => {
-    const views: BloomreachDiscoveryProductViews = {};
-    const locales = product.masterData.current.name;
-
-    for (const view in locales) {
-      views[view] = {
-        attributes: {
-          title: product.masterData.current.name[view] ?? '',
-          sku: product.masterData.current.masterVariant.sku ?? '',
-          description: product.masterData.current.description?.[view] ?? '',
-          slug: product.masterData.current.slug[view] ?? '',
-          price:
-            product.masterData.current.masterVariant.prices?.[0]?.value
-              .centAmount ?? 0,
-          image:
-            product.masterData.current.masterVariant.images?.[0]?.url ?? '',
-        },
-      };
-    }
-
-    return views;
-  };
-
-  const getVariants = (product: Product) => {
-    const variants: BloomreachDiscoveryProductVariants = {};
-
-    product.masterData.current.variants.forEach((variant) => {
-      const attributesMap: Record<string, string> = {};
-      variant.attributes?.forEach((attribute) => {
-        if (attribute.value?.key) {
-          attributesMap[attribute.name] = attribute.value.label;
-        }
-      });
-
-      variants[variant.id] = {
-        attributes: {
-          ...attributesMap,
-        },
-      };
-    });
+  // https://docs.commercetools.com/api/general-concepts#iterating-over-all-elements
+  while (_continue) {
+    let response: ClientResponse<ProductPagedQueryResponse> | null = null;
 
-    return variants;
-  };
+    if (_lastId === null) {
+      response = await getProducts({ limit });
+    } else {
+      response = await getProducts({ limit, lastId: _lastId });
+    }
 
-  while (_continue) {
-    const response = await getProducts({ limit, offset });
-    const data: BloomreachProduct[] = response.body.results.map((product) => {
-      return {
-        op: 'add',
-        path: `/products/${product.id}`,
-        value: {
-          attributes: {
-            title: product.masterData.current.name[locale] ?? '',
-            sku: product.masterData.current.masterVariant.sku ?? '',
-            description: product.masterData.current.description?.[locale] ?? '',
-            slug: product.masterData.current.slug[locale] ?? '',
-            price:
-              product.masterData.current.masterVariant.prices?.[0]?.value
-                .centAmount ?? 0,
-            image:
-              product.masterData.current.masterVariant.images?.[0]?.url ?? '',
+    const data: BloomreachProduct[] =
+      response?.body.results.map((product) => {
+        return {
+          op: 'add',
+          path: `/products/${product.id}`,
+          value: {
+            attributes: {
+              title: product.masterData.current.name[locale] ?? '',
+              sku: product.masterData.current.masterVariant.sku ?? '',
+              description:
+                product.masterData.current.description?.[locale] ?? '',
+              slug: product.masterData.current.slug[locale] ?? '',
+              price:
+                product.masterData.current.masterVariant.prices?.[0]?.value
+                  .centAmount ?? 0,
+              image:
+                product.masterData.current.masterVariant.images?.[0]?.url ?? '',
+            },
+            variants: getVariants(product),
+            views: getProductViews(product),
           },
-          variants: getVariants(product),
-          views: getProductViews(product),
-        },
-      };
-    });
+        };
+      }) ?? [];
+
     products.push(...data);
-    offset += limit;
-    if (products.length >= (response.body.total ?? 0)) {
+    if (products.length >= (response?.body.total ?? 0)) {
       _continue = false;
     }
+    _continue = response?.body.results.length == limit;
+    _lastId =
+      response?.body.results[response.body.results.length - 1]?.id ?? null;
   }
 
   const res = await fetch(
@@ -149,3 +129,46 @@ export async function bloomreachDiscoveryCatalogIngestion() {
   const data = await res.json();
   return data;
 }
+
+function getProductViews(product: Product) {
+  const views: BloomreachDiscoveryProductViews = {};
+  const locales = product.masterData.current.name;
+
+  for (const view in locales) {
+    views[view] = {
+      attributes: {
+        title: product.masterData.current.name[view] ?? '',
+        sku: product.masterData.current.masterVariant.sku ?? '',
+        description: product.masterData.current.description?.[view] ?? '',
+        slug: product.masterData.current.slug[view] ?? '',
+        price:
+          product.masterData.current.masterVariant.prices?.[0]?.value
+            .centAmount ?? 0,
+        image: product.masterData.current.masterVariant.images?.[0]?.url ?? '',
+      },
+    };
+  }
+
+  return views;
+}
+
+function getVariants(product: Product) {
+  const variants: BloomreachDiscoveryProductVariants = {};
+
+  product.masterData.current.variants.forEach((variant) => {
+    const attributesMap: Record<string, string> = {};
+    variant.attributes?.forEach((attribute) => {
+      if (attribute.value?.key) {
+        attributesMap[attribute.name] = attribute.value.label;
+      }
+    });
+
+    variants[variant.id] = {
+      attributes: {
+        ...attributesMap,
+      },
+    };
+  });
+
+  return variants;
+}