Skip to content

Commit

Permalink
updates
Browse files Browse the repository at this point in the history
* add function to get specs for sku
* update gitignore to ignore program output
* add some metrics to README
  • Loading branch information
dnoliver committed May 23, 2023
1 parent df0fd3b commit 31555c8
Show file tree
Hide file tree
Showing 3 changed files with 70 additions and 0 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
node_modules
db.*
9 changes: 9 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# Intel Ark Scrapper

```
time node index.js 2>&1
real 262m20.010s
user 0m0.295s
sys 0m0.983s
```
60 changes: 60 additions & 0 deletions index.js
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,46 @@ async function getListOfSKUsForSubProduct(driver) {
return result;
}

async function getSpecsForSKU(driver) {
let xpath = '//section[@class="blade specs-blade specifications"]';
let locator = By.xpath(xpath);
let specs = await driver.findElements(locator);

let result = {};

for (let i = 0; i < specs.length; i++) {
let xpath1 = `//section[@class="blade specs-blade specifications"][${
i + 1
}]//h2`;
let locator1 = By.xpath(xpath1);
let specTitle = (
await driver.findElement(locator1).getAttribute("innerText")
).trim();

result[specTitle] = {};

let xpath2 = `//section[@class="blade specs-blade specifications"][${
i + 1
}]//ul[@class="specs-list"]//li//span[@class="label"]`;
let locator2 = By.xpath(xpath2);
let specLabels = await driver.findElements(locator2);

let xpath3 = `//section[@class="blade specs-blade specifications"][${
i + 1
}]//ul[@class="specs-list"]//li//span[@class="value"]`;
let locator3 = By.xpath(xpath3);
let specValues = await driver.findElements(locator3);

for (let j = 0; j < specLabels.length; j++) {
let specLabel = (await specLabels[j].getAttribute("innerText")).trim();
let specValue = (await specValues[j].getAttribute("innerText")).trim();
result[specTitle][specLabel] = specValue;
}
}

return result;
}

async function main() {
await driver.get("https://ark.intel.com/");

Expand Down Expand Up @@ -146,6 +186,26 @@ async function main() {
}
}

for (let i in categoryList) {
let category = categoryList[i];
for (let j in category.products) {
let product = category.products[j];
for (let k in product.subproducts) {
let subproduct = product.subproducts[k];
for (let l in subproduct.skus) {
let sku = subproduct.skus[l];
console.log(sku.Url);
await driver.get(sku.Url);
try {
sku.specs = await getSpecsForSKU(driver);
} catch (e) {
console.error(e);
}
}
}
}
}

await fs.promises.writeFile("db.json", JSON.stringify(categoryList), "utf-8");
}

Expand Down

0 comments on commit 31555c8

Please sign in to comment.