diff --git a/.eslintignore b/.eslintignore index ddaac17dc9..6c7e174a0f 100644 --- a/.eslintignore +++ b/.eslintignore @@ -3,6 +3,7 @@ scripts/preact.js scripts/htm.js scripts/acdl tools/picker +tools/pdp-metadata scripts/dropins scripts/__dropins__ scripts/commerce-events-collector.js diff --git a/.hlxignore b/.hlxignore index f70015375c..a630daae5a 100644 --- a/.hlxignore +++ b/.hlxignore @@ -7,3 +7,4 @@ package-lock.json test/* postinstall.js tools/picker/src/* +tools/pdp-metadata/* diff --git a/tools/pdp-metadata/.gitignore b/tools/pdp-metadata/.gitignore new file mode 100644 index 0000000000..8d8f7fd14b --- /dev/null +++ b/tools/pdp-metadata/.gitignore @@ -0,0 +1 @@ +metadata.xlsx \ No newline at end of file diff --git a/tools/pdp-metadata/README.md b/tools/pdp-metadata/README.md new file mode 100644 index 0000000000..8c4faf6b46 --- /dev/null +++ b/tools/pdp-metadata/README.md @@ -0,0 +1,28 @@ +# PDP Metadata Generator + +## Overview +It is recommended to import product metadata into Edge Delivery so that it can be rendered server-side on product detail pages. +This is important so social media sites which do not parse JavaScript can pick it up. + +This project is designed to fetch product data from a catalog service, process it, and generate a metadata spreadsheet in XLSX format which can be used for the https://www.aem.live/docs/bulk-metadata feature in Edge Delivery. + +## Prerequisites +- Node.js installed on your machine. +- Access to the catalog service with the necessary API keys and configuration. + +## Installation +1. Clone the repository to your local machine. +2. Navigate to the project directory. +3. Run `npm install` to install the dependencies. + +## Configuration +Before running the application, you need to ensure the `configFile` variable in `pdp-metadata.js` points to the correct configuration JSON file URL. This file contains the required parameters to access Catalog Service and should have been setup as part of your project onboarding. + +## Running the Application +To start the application, run the following command in the terminal: + +```bash +npm start +``` + +This will fetch the product data, process it, and generate a file named `metadata.xlsx` in the project directory. diff --git a/tools/pdp-metadata/package-lock.json b/tools/pdp-metadata/package-lock.json new file mode 100644 index 0000000000..73d2796939 --- /dev/null +++ b/tools/pdp-metadata/package-lock.json @@ -0,0 +1,187 @@ +{ + "name": "pdp-metadata", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "pdp-metadata", + "version": "1.0.0", + "license": "Apache-2.0", + "dependencies": { + "he": "^1.2.0", + "xlsx": "^0.18.5" + } + }, + "node_modules/adler-32": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz", + "integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/cfb": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.2.tgz", + "integrity": "sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==", + "dependencies": { + "adler-32": "~1.3.0", + "crc-32": "~1.2.0" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/codepage": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz", + "integrity": "sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA==", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", + "bin": { + "crc32": "bin/crc32.njs" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/frac": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/frac/-/frac-1.1.2.tgz", + "integrity": "sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "bin": { + "he": "bin/he" + } + }, + "node_modules/ssf": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/ssf/-/ssf-0.11.2.tgz", + "integrity": "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==", + "dependencies": { + "frac": "~1.1.2" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/wmf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wmf/-/wmf-1.0.2.tgz", + "integrity": "sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/word": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/word/-/word-0.3.0.tgz", + "integrity": "sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/xlsx": { + "version": "0.18.5", + "resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz", + "integrity": "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==", + "dependencies": { + "adler-32": "~1.3.0", + "cfb": "~1.2.1", + "codepage": "~1.15.0", + "crc-32": "~1.2.1", + "ssf": "~0.11.2", + "wmf": "~1.0.1", + "word": "~0.3.0" + }, + "bin": { + "xlsx": "bin/xlsx.njs" + }, + "engines": { + "node": ">=0.8" + } + } + }, + "dependencies": { + "adler-32": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz", + "integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==" + }, + "cfb": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.2.tgz", + "integrity": "sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==", + "requires": { + "adler-32": "~1.3.0", + "crc-32": "~1.2.0" + } + }, + "codepage": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz", + "integrity": "sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA==" + }, + "crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==" + }, + "frac": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/frac/-/frac-1.1.2.tgz", + "integrity": "sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==" + }, + "he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==" + }, + "ssf": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/ssf/-/ssf-0.11.2.tgz", + "integrity": "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==", + "requires": { + "frac": "~1.1.2" + } + }, + "wmf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wmf/-/wmf-1.0.2.tgz", + "integrity": "sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==" + }, + "word": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/word/-/word-0.3.0.tgz", + "integrity": "sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==" + }, + "xlsx": { + "version": "0.18.5", + "resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz", + "integrity": "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==", + "requires": { + "adler-32": "~1.3.0", + "cfb": "~1.2.1", + "codepage": "~1.15.0", + "crc-32": "~1.2.1", + "ssf": "~0.11.2", + "wmf": "~1.0.1", + "word": "~0.3.0" + } + } + } +} diff --git a/tools/pdp-metadata/package.json b/tools/pdp-metadata/package.json new file mode 100644 index 0000000000..04c827a6ba --- /dev/null +++ b/tools/pdp-metadata/package.json @@ -0,0 +1,17 @@ +{ + "name": "pdp-metadata", + "private": true, + "type": "module", + "version": "1.0.0", + "description": "", + "main": "pdp-metadata.js", + "author": "", + "license": "Apache-2.0", + "scripts": { + "start": "node pdp-metadata.js" + }, + "dependencies": { + "he": "^1.2.0", + "xlsx": "^0.18.5" + } +} diff --git a/tools/pdp-metadata/pdp-metadata.js b/tools/pdp-metadata/pdp-metadata.js new file mode 100644 index 0000000000..fc63a6fa01 --- /dev/null +++ b/tools/pdp-metadata/pdp-metadata.js @@ -0,0 +1,164 @@ +import XLSX from 'xlsx'; +import fs from 'fs'; +import he from 'he'; +import productSearchQuery from './queries/products.graphql.js'; + +const basePath = 'https://main--aem-boilerplate-commerce--hlxsites.hlx.live'; +const configFile = `${basePath}/configs.json?sheet=prod`; + + +async function performCatalogServiceQuery(config, query, variables) { + const headers = { + 'Content-Type': 'application/json', + 'Magento-Environment-Id': config['commerce-environment-id'], + 'Magento-Website-Code': config['commerce-website-code'], + 'Magento-Store-View-Code': config['commerce-store-view-code'], + 'Magento-Store-Code': config['commerce-store-code'], + 'Magento-Customer-Group': config['commerce-customer-group'], + 'x-api-key': config['commerce-x-api-key'], + }; + + const apiCall = new URL(config['commerce-endpoint']); + apiCall.searchParams.append('query', query.replace(/(?:\r\n|\r|\n|\t|[\s]{4})/g, ' ') + .replace(/\s\s+/g, ' ')); + apiCall.searchParams.append('variables', variables ? JSON.stringify(variables) : null); + + const response = await fetch(apiCall, { + method: 'GET', + headers, + }); + + if (!response.ok) { + return null; + } + + const queryResponse = await response.json(); + + return queryResponse.data; +} + +/** + * Get products by page number + * @param {INT} pageNumber - pass the pagenumber to retrieved paginated results + */ +const getProducts = async (config, pageNumber) => { + const response = await performCatalogServiceQuery( + config, + productSearchQuery, + { currentPage: pageNumber }, + ); + + if (response && response.productSearch) { + const products = await Promise.all(response.productSearch.items.map(async (item) => { + const { + urlKey, + sku, + metaDescription, + name, + metaKeyword, + metaTitle, + description, + shortDescription, + } = item.productView; + const { url: imageUrl } = item.product.image ?? {}; + + let baseImageUrl = imageUrl; + if (baseImageUrl.startsWith('//')) { + baseImageUrl = `https:${baseImageUrl}`; + } + + let finalDescription = ''; + if (metaDescription) { + finalDescription = metaDescription; + } else if (shortDescription) { + finalDescription = shortDescription; + } else if (description) { + finalDescription = description; + } + finalDescription = he.decode(finalDescription.replace(/(<([^>]+)>)/ig, '')).trim(); + if (finalDescription.length > 200) { + finalDescription = `${finalDescription.substring(0, 197)}...`; + } + + return { + productView: { + ...item.productView, + image: baseImageUrl, + path: `/products/${urlKey}/${sku.toLowerCase()}`, + meta_keyword: (metaKeyword !== null) ? metaKeyword : '', + meta_title: he.decode((metaTitle !== '') ? metaTitle : name), + meta_description: finalDescription, + 'og:image': baseImageUrl, + 'og:image:secure_url': baseImageUrl, + 'twitter:image': baseImageUrl, + }, + }; + })); + const totalPages = response.productSearch.page_info.total_pages; + const currentPage = response.productSearch.page_info.current_page; + console.log(`Retrieved page ${currentPage} of ${totalPages} pages`); + if (currentPage !== totalPages) { + return [...products, ...(await getProducts(config, currentPage + 1))]; + } + return products; + } + return []; +}; + +(async () => { + const config = {}; + try { + const resp = await fetch(configFile).then((res) => res.json()); + resp.data.forEach((item) => { + config[item.key] = item.value; + }); + } catch (err) { + console.error(err); + return; + } + + const products = await getProducts(config, 1); + + const data = [ + [ + 'URL', + 'title', + 'description', + 'keywords', + 'og:type', + 'og:title', + 'og:description', + 'og:url', + 'og:image', + 'og:image:secure_url', + 'twitter:card', + 'twitter:title', + 'twitter:image' + ], + ]; + products.forEach(({ productView: metaData }) => { + data.push( + [ + metaData.path, // URL + metaData.meta_title, // title + metaData.meta_description, // description + metaData.meta_keyword, // keywords + 'og:product', // og:type + metaData.meta_title, // og:title + metaData.meta_description, // og:description + `${basePath}${metaData.path}`, // og:url + metaData['og:image'], // og:image + metaData['og:image:secure_url'], // og:image:secure_url + metaData.meta_description, // twitter:card + metaData.meta_title, // twitter:title + metaData['twitter:image'], // twitter:image + ], + ); + }); + + // Write XLSX file + const worksheet = XLSX.utils.aoa_to_sheet(data); + const workbook = { Sheets: { Sheet1: worksheet }, SheetNames: ['Sheet1'] }; + const xlsx = XLSX.write(workbook, { bookType: 'xlsx', type: 'buffer' }); + await fs.promises.writeFile('metadata.xlsx', xlsx); +})(); diff --git a/tools/pdp-metadata/queries/products.graphql.js b/tools/pdp-metadata/queries/products.graphql.js new file mode 100644 index 0000000000..d631215250 --- /dev/null +++ b/tools/pdp-metadata/queries/products.graphql.js @@ -0,0 +1,28 @@ +export default `query productSearch($currentPage: Int = 1) { + productSearch(current_page: $currentPage, page_size: 20, phrase: "") { + items { + productView { + __typename + sku + name + urlKey + shortDescription + description + metaDescription + metaKeyword + metaTitle + } + product { + image { + url + } + } + } + page_info { + current_page + page_size + total_pages + } + total_count + } +}`;