Fix issues with large search index responses. (#311)

* Fix issues with large search index responses. * Fix references to .hlx. * Try to improve home page. * Fix lint and other issues.
hlxsites · May 2, 2024 · bd2ecf7 · bd2ecf7
1 parent fd17bba
commit bd2ecf7
Show file tree

Hide file tree

Showing 5 changed files with 136 additions and 104 deletions.
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -3,5 +3,5 @@ Please always provide the [GitHub issue(s)](../issues) your PR is for, as well a
 Fix #<gh-issue-id>
 
 Test URLs:
-- Before: https://main--famous-smoke-cigaradvisor--hlxsites.hlx.page/cigaradvisor
-- After: https://<branch>--famous-smoke-cigaradvisor--hlxsites.hlx.page/cigaradvisor
+- Before: https://main--famous-smoke-cigaradvisor--hlxsites.aem.page/cigaradvisor
+- After: https://<branch>--famous-smoke-cigaradvisor--hlxsites.aem.page/cigaradvisor
diff --git a/README.md b/README.md
@@ -3,8 +3,8 @@
 Edge Delivery Services implementing https://www.famous-smoke.com/cigaradvisor
 
 ## Environments
-- Preview: https://main--famous-smoke-cigaradvisor--hlxsites.hlx.page/cigaradvisor/
-- Live: https://main--famous-smoke-cigaradvisor--hlxsites.hlx.live/cigaradvisor/
+- Preview: https://main--famous-smoke-cigaradvisor--hlxsites.aem.page/cigaradvisor/
+- Live: https://main--famous-smoke-cigaradvisor--hlxsites.aem.live/cigaradvisor/
 
 ## Installation
 
@@ -16,7 +16,7 @@ npm i
 
 ```sh
 npm run lint
-```
+``` 
 
 ## Local development
 

diff --git a/cigaradvisor/blocks/search-results/search-worker.js b/cigaradvisor/blocks/search-results/search-worker.js
@@ -1,76 +1,18 @@
-/* eslint-disable max-len */
-/* eslint-disable no-restricted-globals */
 const ARTICLE_INDEX_PATH = '/cigaradvisor/index/article-index.json';
 const SEARCH_INDEX_PATH = '/cigaradvisor/index/search-index.json';
-const articleIndexData = [];
-
-const searchIndexData = [];
-/**
- * Retrieves search index data from the server.
- * @returns {Promise<Object>} The search index data.
- */
-async function getSearchIndexData(path = SEARCH_INDEX_PATH, flag = false) {
-  if (searchIndexData.length === 0 || flag) {
-    const resp = await fetch(path);
-    let jsonData = '';
-    if (resp.ok) {
-      jsonData = await resp.json();
-    }
-    jsonData.data.forEach((a) => {
-      searchIndexData.push({ ...a });
-    });
-    // If there are more items to load, load them
-    if ((jsonData.total - jsonData.offset) > jsonData.limit) {
-      const offset = jsonData.offset + jsonData.limit;
-      const indexPath = `${SEARCH_INDEX_PATH}?offset=${offset}&limit=${jsonData.total - offset}`;
-      await getSearchIndexData(indexPath, true);
-    }
-  }
-  // Protected against callers modifying the objects
-  const ret = [];
-  if (searchIndexData) {
-    searchIndexData.forEach((a) => {
-      ret.push({ ...a });
-    });
-  }
-  return ret;
-}
 
-/**
- * Loads posts from the specified path asynchronously.
- * @param {string} path - The path to fetch the posts from.
- * @param {boolean} recurse - Indicates whether to recursively load more articles.
- * @returns {Promise<Array<Object>>} - A promise that resolves to an array of post objects.
- */
-async function loadPosts(path = ARTICLE_INDEX_PATH, recurse = false) {
-  if (articleIndexData.length === 0 || recurse) {
-    const resp = await fetch(path);
-    let jsonData = '';
-    if (resp.ok) {
-      jsonData = await resp.json();
-    }
-    jsonData.data.forEach((a) => {
-      articleIndexData.push({ ...a });
-    });
-    // If there are more articles to load, load them
-    if ((jsonData.total - jsonData.offset) > jsonData.limit) {
-      const offset = jsonData.offset + jsonData.limit;
-      const indexPath = `${ARTICLE_INDEX_PATH}?offset=${offset}&limit=${jsonData.total - offset}`;
-      await loadPosts(indexPath, true);
-    }
-  }
-  // Protected against callers modifying the objects
-  const ret = [];
-  if (articleIndexData) {
-    articleIndexData.forEach((a) => {
-      ret.push({ ...a });
-    });
+const BUCKET_SIZE = 500;
+const IGNORED_TERMS = [];
+
+async function getTotal(index) {
+  const resp = await fetch(`${index}?limit=1`);
+  if (resp.ok) {
+    const json = await resp.json();
+    return json.total;
   }
-  return ret;
+  return 0;
 }
 
-const IGNORED_TERMS = [];
-
 const doMatch = (property, term) => {
   const regex = new RegExp(term, 'gi');
   if (property) {
@@ -130,6 +72,10 @@ function filterData(fullTerm, data) {
     }
   });
 
+  return results;
+}
+
+function sortData(results) {
   return results.sort((l, r) => {
     if (l.rank < r.rank) {
       return -1;
@@ -147,17 +93,82 @@ function filterData(fullTerm, data) {
   }).map((r) => r.article);
 }
 
-self.onmessage = async function handleSearch(event) {
-  const data = await getSearchIndexData();
-  const allArticles = await loadPosts();
+async function doSearch(value) {
+  const total = await getTotal(SEARCH_INDEX_PATH);
+  const promises = [];
+  const buckets = Math.ceil(total / BUCKET_SIZE);
+
+  for (let i = 0; i < buckets; i += 1) {
+    promises.push(new Promise((resolve) => {
+      const offset = i * BUCKET_SIZE;
+      fetch(`${SEARCH_INDEX_PATH}?offset=${offset}&limit=${BUCKET_SIZE}`)
+        .then((resp) => {
+          if (resp.ok) {
+            return resp.json();
+          }
+          return {};
+        })
+        .then((json) => {
+          const { data } = json;
+          if (data) {
+            resolve(filterData(value, data));
+          }
+          resolve([]);
+        });
+    }));
+  }
+
+  return Promise.all(promises).then((results) => {
+    const matches = [];
+    results.forEach((r) => {
+      matches.push(...r);
+    });
+    return sortData(matches).map((m) => m.path);
+  });
+}
+
+async function getArticles(paths) {
+  const total = await getTotal(ARTICLE_INDEX_PATH);
+  const promises = [];
+  const buckets = Math.ceil(total / BUCKET_SIZE);
+
+  for (let i = 0; i < buckets; i += 1) {
+    promises.push(new Promise((resolve) => {
+      const offset = i * BUCKET_SIZE;
+      fetch(`${ARTICLE_INDEX_PATH}?offset=${offset}&limit=${BUCKET_SIZE}`)
+        .then((resp) => {
+          if (resp.ok) {
+            return resp.json();
+          }
+          return {};
+        })
+        .then((json) => {
+          const { data } = json;
+          if (data) {
+            resolve(data.filter((a) => paths.includes(a.path)));
+          }
+          resolve([]);
+        });
+    }));
+  }
+  return Promise.all(promises).then((results) => {
+    const matches = [];
+    results.forEach((a) => {
+      matches.push(...a);
+    });
+    return matches;
+  });
+}
+
+onmessage = async function handleSearch(event) {
   const { searchValue } = event.data;
-  const searchResults = filterData(searchValue, data);
+  const paths = await doSearch(searchValue);
+  const found = await getArticles(paths);
 
-  const paths = searchResults.map((r) => r.path);
-  const filteredArticles = allArticles.filter((a) => paths.includes(a.path)).sort((a, b) => {
+  found.sort((a, b) => {
     const indexA = paths.indexOf(a.path);
     const indexB = paths.indexOf(b.path);
     return indexA - indexB;
   });
-  self.postMessage({ results: filteredArticles });
+  postMessage({ results: found });
 };
diff --git a/cigaradvisor/docs/create-category.md b/cigaradvisor/docs/create-category.md
@@ -57,5 +57,5 @@ Now you should be all set to create and publish new pages in your previously cre
 
 After publishing, they should appear in the corresponding index and sitemap (it may take a moment to generate these):
 
-- `https://main--famous-smoke-cigaradvisor--hlxsites.hlx.page/cigaradvisor/index/article-index-<new-category>.json`
-- `https://main--famous-smoke-cigaradvisor--hlxsites.hlx.page/cigaradvisor/article-sitemap-<new-category>.xml`
+- `https://main--famous-smoke-cigaradvisor--hlxsites.aem.page/cigaradvisor/index/article-index-<new-category>.json`
+- `https://main--famous-smoke-cigaradvisor--hlxsites.aem.page/cigaradvisor/article-sitemap-<new-category>.xml`
diff --git a/cigaradvisor/scripts/scripts.js b/cigaradvisor/scripts/scripts.js
@@ -313,37 +313,58 @@ export function getRelativePath(path) {
 }
 
 const articleIndexData = [];
+
 /**
  * Loads posts from the specified path asynchronously.
  * @param {string} path - The path to fetch the posts from.
- * @param {boolean} recurse - Indicates whether to recursively load more articles.
  * @returns {Promise<Array<Object>>} - A promise that resolves to an array of post objects.
  */
-export async function loadPosts(path = ARTICLE_INDEX_PATH, recurse = false) {
-  if (articleIndexData.length === 0 || recurse) {
-    const resp = await fetch(path);
-    let jsonData = '';
-    if (resp.ok) {
-      jsonData = await resp.json();
-    }
-    jsonData.data.forEach((article) => {
-      articleIndexData.push({ ...article });
-    });
-    // If there are more articles to load, load them
-    if ((jsonData.total - jsonData.offset) > jsonData.limit) {
-      const offset = jsonData.offset + jsonData.limit;
-      const indexPath = `${ARTICLE_INDEX_PATH}?offset=${offset}&limit=${jsonData.total - offset}`;
-      await loadPosts(indexPath, true);
+export async function loadPosts(path = ARTICLE_INDEX_PATH) {
+  if (!articleIndexData.length) {
+    const limit = 500;
+    const first = await fetch(`${path}?limit=${limit}`)
+      .then((resp) => {
+        if (resp.ok) {
+          return resp.json();
+        }
+        return {};
+      });
+
+    const { total } = first;
+    if (total) {
+      articleIndexData.push(...first.data);
+      const promises = [];
+      const buckets = Math.ceil(total / limit);
+      for (let i = 1; i < buckets; i += 1) {
+        promises.push(new Promise((resolve) => {
+          const offset = i * limit;
+          fetch(`${path}?offset=${offset}&limit=${limit}`)
+            .then((resp) => {
+              if (resp.ok) {
+                return resp.json();
+              }
+              return {};
+            })
+            .then((json) => {
+              const { data } = json;
+              if (data) {
+                resolve(data);
+              }
+              resolve([]);
+            });
+        }));
+      }
+
+      await Promise.all(promises).then((values) => {
+        values.forEach((list) => {
+          articleIndexData.push(...list);
+        });
+      });
     }
   }
+
   // Protected against callers modifying the objects
-  const ret = [];
-  if (articleIndexData) {
-    articleIndexData.forEach((article) => {
-      ret.push({ ...article });
-    });
-  }
-  return ret;
+  return structuredClone(articleIndexData);
 }
 
 /**