add schema generation and erd diagram

.gitignore

@@ -1,2 +1,4 @@
 node_modules
-db.*
+db.json
+db.*.json
+schema.*.json

README.md

@@ -1,9 +1,47 @@
 # Intel Ark Scrapper
 
-```
+## Scrapper Execution
+
+```bash
 time node index.js 2>&1
 
 real    262m20.010s
 user    0m0.295s
 sys     0m0.983s
 ```
+
+## Schema Induction
+
+```bash
+node schema.js
+```
+
+## Product Types
+
+```bash
+jq '.[].name' db.json
+```
+
+## Product Brands
+
+```bash
+jq '.[0].products[].name' db.json
+```
+
+## Product Series
+
+```bash
+jq '.[0].products[0].subproducts[].name' db.json
+```
+
+## Products
+
+```bash
+jq '.[0].products[0].subproducts[0].skus[]."Product Name"' db.json
+```
+
+## Specification Categories
+
+```bash
+jq '.[0].products[0].subproducts[0].skus[0].specs | keys' db.json
+```

package.json

@@ -9,6 +9,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
+    "genson-js": "^0.0.8",
     "prettier": "^2.8.8",
     "selenium-webdriver": "^4.9.2"
   },

schema.js

@@ -0,0 +1,37 @@
+const { createSchema } = require('genson-js');
+const fs = require("fs");
+
+fs.readFile("db.json", { encoding: "utf-8" }, async function (err, data) {
+    const db = JSON.parse(data);
+    // 1st item is Processors
+    const ProcessorSchema = createSchema(db[0]);
+    // 2nd item is Server Products
+    const ServerProductSchema = createSchema(db[1]);
+    // 3rd item is Intel NUCs
+    const IntelNucSchema = createSchema(db[2]);
+    // 4th item is Wireless
+    const WirelessSchema = createSchema(db[3]);
+    // 5th item is Ethernet Products
+    const EthernetProductsSchema = createSchema(db[4]);
+    // 6th item is Intel® FPGAs
+    const IntelFPGAsSchema = createSchema(db[5]);
+    // 7th item is Memory and Storage
+    const MemoryandStorageSchema = createSchema(db[6]);
+    // 8th item is Chipsets
+    const ChipsetsSchema = createSchema(db[7]);
+    // 9th item is Graphics
+    const GraphicsSchema = createSchema(db[8]);
+
+    // Save Schemas 
+    await Promise.all([
+        fs.promises.writeFile("schema.processor.json", JSON.stringify(ProcessorSchema, null, '\t'), "utf-8"),
+        fs.promises.writeFile("schema.serverproducts.json", JSON.stringify(ServerProductSchema, null, '\t'), "utf-8"),
+        fs.promises.writeFile("schema.intelnucs.json", JSON.stringify(IntelNucSchema, null, '\t'), "utf-8"),
+        fs.promises.writeFile("schema.wireless.json", JSON.stringify(WirelessSchema, null, '\t'), "utf-8"),
+        fs.promises.writeFile("schema.ethernetproducts.json", JSON.stringify(EthernetProductsSchema, null, '\t'), "utf-8"),
+        fs.promises.writeFile("schema.intelfpgas.json", JSON.stringify(IntelFPGAsSchema, null, '\t'), "utf-8"),
+        fs.promises.writeFile("schema.memoryandstorage.json", JSON.stringify(MemoryandStorageSchema, null, '\t'), "utf-8"),
+        fs.promises.writeFile("schema.chipsets.json", JSON.stringify(ChipsetsSchema, null, '\t'), "utf-8"),
+        fs.promises.writeFile("schema.graphics.json", JSON.stringify(GraphicsSchema, null, '\t'), "utf-8")
+    ]);
+});

schema.sql

@@ -0,0 +1,84 @@
+CREATE TABLE ProductTypes (
+	Id 		INT NOT NULL,
+	Name 	TEXT NOT NULL,
+
+	CONSTRAINT PK_ProductTypes PRIMARY KEY (Id),
+	CONSTRAINT UQ_ProductTypes_Name UNIQUE (Name)
+);
+
+CREATE TABLE ProductBrands (
+	ProductType 	INT NOT NULL,
+	Id 				INT NOT NULL,
+	Name 			TEXT NOT NULL,
+
+	CONSTRAINT PK_ProductBrands PRIMARY KEY (ProductType, Id),
+	CONSTRAINT FK_ProductBrands_ProductTypes FOREIGN KEY (ProductType) REFERENCES ProductTypes,
+	CONSTRAINT UQ_ProductBrands_Name UNIQUE (Name)
+);
+
+CREATE TABLE ProductSeries (
+	ProductType 	INT NOT NULL,
+	ProductBrand 	INT NOT NULL,
+	Id 				INT NOT NULL,
+	Name 			TEXT NOT NULL,
+	Url				TEXT NOT NULL,
+
+	CONSTRAINT PK_ProductSeries PRIMARY KEY (ProductType, ProductBrand, Id),
+	CONSTRAINT FK_ProductSeries_ProductBrands FOREIGN KEY (ProductType, ProductBrand) REFERENCES ProductBrands (ProductType, Id),
+	CONSTRAINT UQ_ProductSeries_Name UNIQUE (Name),
+	CONSTRAINT UQ_ProductSeries_Url UNIQUE (Url)
+);
+
+CREATE TABLE Products (
+	ProductType 	INT NOT NULL,
+	ProductBrand 	INT NOT NULL,
+	ProductSerie 	INT NOT NULL,
+	Id 				INT NOT NULL,
+	Name 			TEXT NOT NULL,
+	Url				TEXT NOT NULL,
+
+	CONSTRAINT PK_Product PRIMARY KEY (ProductType, ProductBrand, ProductSerie, Id),
+	CONSTRAINT FK_Product_ProductSeries FOREIGN KEY (ProductType, ProductBrand, ProductSerie) REFERENCES ProductSeries (ProductType, ProductBrand, Id),
+	CONSTRAINT UQ_Product_Name UNIQUE (Name),
+	CONSTRAINT UQ_Product_Url UNIQUE (Url)
+);
+
+CREATE TABLE SpecificationCategories (
+	Id 		INT NOT NULL,
+	Name 	TEXT NOT NULL,
+
+	CONSTRAINT PK_SpecificationCategories PRIMARY KEY (Id),
+	CONSTRAINT UQ_SpecificationCategories_Name UNIQUE (Name)
+);
+
+CREATE TABLE SpecificationProperties (
+	SpecificationCategory	INT NOT NULL,
+	Id 						INT NOT NULL,
+	Name 					TEXT NOT NULL,
+
+	CONSTRAINT PK_SpecificationProperties PRIMARY KEY (SpecificationCategory, Id),
+	CONSTRAINT FK_SpecificationProperties_SpecificationCategories FOREIGN KEY (SpecificationCategory) REFERENCES SpecificationCategories (Id),
+	CONSTRAINT UQ_SpecificationProperties_Name UNIQUE (Name)
+);
+
+CREATE TABLE ProductsSpecifications (
+	ProductType 			INT NOT NULL,
+	ProductBrand 			INT NOT NULL,
+	ProductSerie 			INT NOT NULL,
+	ProductId				INT NOT NULL,
+	SpecificationCategory	INT NOT NULL,
+	SpecificationProperty	INT NOT NULL,
+	Value 					TEXT NOT NULL,
+
+	CONSTRAINT PK_ProductsSpecifications PRIMARY KEY (ProductType, ProductBrand, ProductSerie, ProductId, SpecificationCategory, SpecificationProperty),
+	CONSTRAINT FK_ProductsSpecifications_Products FOREIGN KEY (ProductType, ProductBrand, ProductSerie, ProductId) REFERENCES Products (ProductType, ProductBrand, ProductSerie, Id),
+	CONSTRAINT FK_ProductsSpecifications_SpecificationProperties FOREIGN KEY (SpecificationCategory, SpecificationProperty) REFERENCES SpecificationProperties (SpecificationCategory, Id)
+);
+
+DROP TABLE ProductTypes;
+DROP TABLE ProductBrands;
+DROP TABLE ProductSeries;
+DROP TABLE Products;
+DROP TABLE SpecificationCategories;
+DROP TABLE SpecificationProperties;
+DROP TABLE ProductsSpecifications;