added progress event

sainsburys-tech · Feb 22, 2021 · 806887e · 806887e
1 parent 5d13099
commit 806887e
Show file tree

Hide file tree

Showing 16 changed files with 1,055 additions and 2,402 deletions.
diff --git a/.gitignore b/.gitignore
@@ -60,5 +60,4 @@ examples
 
 # Project-specific ignores
 build
-src/test.ts
-tests
+integration-test
diff --git a/.mocharc.json b/.mocharc.json
@@ -0,0 +1,11 @@
+{
+  "extension": ["ts"],
+  "spec": ["tests/**/*.spec.ts"],
+  "require": [
+    "ts-node/register",
+    "tsconfig-paths/register",
+    "./tests/mocha-init.ts"
+  ],
+  "ignore": ["node_modules/*"],
+  "diff": true
+}
diff --git a/README.md b/README.md
@@ -1,18 +1,29 @@
-# snowflake-multisql [![node](https://img.shields.io/node/v/snowflake-multisql.svg)](https://www.npmjs.com/package/snowflake-multisql)
+# snowflake-multisql [![npm](https://img.shields.io/npm/v/snowflake-multisql.svg)](https://www.npmjs.com/package/snowflake-multisql) [![node](https://img.shields.io/node/v/snowflake-multisql.svg)](https://www.npmjs.com/package/snowflake-multisql)
 
-A Multi SQL, Promise-based and Typescript version to your [Snowflake](https://www.snowflake.net/) data warehouse.
+A Multi SQL Statement, Promise-based and Typescript version to your [Snowflake](https://www.snowflake.net/) data warehouse, since the original library doesn't support this multi statement calls.
 
-This library works on top of the excellent [Snowflake Promise](https://www.npmjs.com/package/snowflake-promise) for Node.js, which is a wrapper for [Snowflake SDK](https://www.npmjs.com/package/snowflake-sdk)
+Also adds nice features like:
 
-Unfortunately [Snowflake SDK](https://www.npmjs.com/package/snowflake-sdk) doesn't support multi statement calls.
+- replaces tags defined in the scripts by name using original data types (examples below).
+- Emits "progress" events so you can monitor long run calls
+- Preview parsed statement before sending to Snowflake.
+- Shows duration (in miliseconds) as well the
+- Shows data returned by Snowflake on each statement with the option to ommit them as well.
+- Utils: Loads an entire folder of "\*.sql" files or an array of specific files.
 
-## Installation
+PS: This library works on top of the excellent [Snowflake Promise](https://www.npmjs.com/package/snowflake-promise) for Node.js, which is a wrapper for [Snowflake SDK](https://www.npmjs.com/package/snowflake-sdk)
 
-- `yarn add snowflake-multisql` or `npm i snowflake-multisql`
+---
+
+### Installation
+
+- `yarn add snowflake-multisql`  
+  or
+- `npm i snowflake-multisql`
 
 ---
 
-## Connecting
+## How to use it
 
 The constructor extends SnowflakePromise class addin the new method **_executeAll_**.
 The unique method's param is deconstructed into the variables below:
@@ -21,7 +32,7 @@ The unique method's param is deconstructed into the variables below:
 {
   "sqlText": "your SQL Text string",
   "binds": "from the original library, replace tags by sequence",
-  "replaceTags": "new, replace tags by name, whatever order them appers",
+  "tags": "new, replace tags by name, whatever order them appers",
   "includeResults": true,
   "preview": true
 }
@@ -65,6 +76,15 @@ async function main() {
   `;
   const binds = ["AUTOMOBILE", 1234];
 
+  // NEW FEATURE: Feel free to monitor your running progress
+  snowflake.on("progress", (data) =>
+    console.log(`
+    progress: ${data.chunkOrder}/${data.chunksTotal}
+    duration: ${data.duration} ms,
+    totalDuration: ${data.totalDuration} ms
+  `)
+  );
+
   const rows = await snowflake.executeAll({
     sqlText,
     binds,
@@ -113,70 +133,27 @@ main();
   //  USE SCHEMA demo_schema;
   //  SELECT COUNT(*) FROM customer WHERE segment_id={%segment_id%};
 
-  const replaceTags = [
+  const tags = [
     { tag: "purchase_date", value: new Date(1610976670682) },
     { tag: "product_name", value: "AUTOMOBILE" },
     { tag: "segment_id", value: 1234 },
   ];
 
-  const rows = await snowflake.executeAll({
-    sqlText,
-    replaceTags,
-    includeResults: true,
-    preview: false // set it to true if you want to check the scripts first.
-  });
-
-  rows.map((row) => console.dir(row));
-}
-
-main();
-```
-
----
-
-## Advanced options with reaplceTags
-
-```typescript
-  import { Snowflake, loadFiles } from "snowflake-multisql"
-
-  const snowflake = new Snowflake({
-    account: "<account name>",
-    username: "<username>",
-    password: "<password>",
-    database: "DEMO_DATABASE",
-    schema: "DEMO_SCHEMA",
-    warehouse: "DEMO_WH",
-  });
-
-  // SWISS ARMY KNIFE as explained in the earlier example
-  const sqlText = await loadFiles({
-    filesPath: path.join(process.cwd(), "./sqlFolder"),
-  });
-
-  // file-1.sql content:
-  //  CREATE OR REPLACE TABLE temp_table_customer as
-  //  SELECT COUNT(*) FROM customer WHERE C_MKTSEGMENT={%tag_auto%};
-
-  // file-2.sql content:
-  //  SELECT * from temp_table_customer
-  //  WHERE product_name = {%tag_auto%}
-
-  // file-3.sql content:
-  //  USE SCHEMA demo_schema;
-  //  SELECT COUNT(*) FROM customer WHERE c_mktsegment={%tag_bike%};
-
-  const replaceTags = [
-    { tag: "tag_auto", value: "AUTOMOBILE" },
-    { tag: "tag_bike", value: "BIKE" },
-  ];
+  // NEW FEATURE: Feel free to monitor your running progress
+  snowflake.on("progress", (data) => console.log(`
+    progress: ${data.chunkOrder}/${data.chunksTotal}
+    duration: ${data.duration} ms,
+    totalDuration: ${data.totalDuration} ms
+  `));
 
   const rows = await snowflake.executeAll({
     sqlText,
-    replaceTags,
+    tags,
     includeResults: true,
+    preview: false // set it true for checking statements before sending to Snowflake.
   });
 
-  rows.map((row) => console.dir(row));
+  rows.map((rows) => console.dir(rows));
 }
 
 main();

diff --git a/jest.config.js b/jest.config.js
diff --git a/jest.config.js.old b/jest.config.js.old
diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "snowflake-multisql",
-  "version": "1.0.0",
-  "description": "Multi-SQL, Promise-based, TypeScript wrapper for Snowflake SDK",
+  "version": "1.2.0",
+  "description": "Multi SQL Statement, Promise-based, TypeScript wrapper for Snowflake SDK",
   "repository": "github:brugos/snowflake-multisql",
   "bugs": "https://github.com/brugos/snowflake-multisql/issues",
   "license": "MIT",
@@ -17,26 +17,38 @@
   "types": "build/src/index.d.ts",
   "scripts": {
     "prepare": "del-cli build && tsc",
-    "test": "jest --runInBand --silent",
-    "test:coverage": "jest --runInBand --coverage",
-    "test:coverage:map": "jest --runInBand --coverage && open ./coverage/lcov-report/snowflake-multisql.ts.html"
+    "test": "mocha --timeout 60000",
+    "test:watch": "yarn test:mocha -w",
+    "test:coverage": "yarn test && open && open ./coverage/lcov-report/index.ts.html"
   },
   "engines": {
     "node": ">=10"
   },
   "devDependencies": {
-    "@types/jest": "^26.0.20",
+    "@types/chai": "^4.2.15",
+    "@types/chai-as-promised": "^7.1.3",
+    "@types/chai-subset": "^1.3.3",
+    "@types/expect": "^24.3.0",
+    "@types/mocha": "^8.2.1",
     "@types/node": "^14.14.20",
+    "@types/sinon": "^9.0.10",
+    "chai": "^4.3.0",
+    "chai-as-promised": "^7.1.1",
+    "chai-subset": "^1.6.0",
     "del-cli": "^3.0.1",
     "dotenv": "^8.2.0",
     "eslint": "^7.17.0",
-    "jest": "^26.6.3",
-    "ts-jest": "^26.4.4",
+    "mocha": "^8.3.0",
+    "nyc": "^15.1.0",
+    "sinon": "^9.2.4",
+    "ts-mocha": "^8.0.0",
     "ts-node": "^9.1.1",
+    "tsconfig-paths": "^3.9.0",
     "tslog": "^3.0.5",
     "typescript": "^4.1.3"
   },
   "dependencies": {
-    "snowflake-promise": "^4.2.0"
+    "snowflake-promise": "^4.2.0",
+    "ts-mixer": "^5.4.0"
   }
 }
diff --git a/src/index.ts b/src/index.ts
@@ -1,4 +1,10 @@
-export { SnowflakeMultiSql as Snowflake } from "./snowflake-multisql";
+export {
+  SnowflakeMultiSql as Snowflake,
+  ITag,
+  IMultiSqlResult,
+  IPreview,
+  IExecuteAll,
+} from "./snowflake-multisql";
 export { loadFiles } from "./utils";
 export {
   ConnectionOptions,

diff --git a/src/snowflake-multisql.ts b/src/snowflake-multisql.ts
@@ -1,4 +1,6 @@
 import { Snowflake, ConnectionOptions } from "snowflake-promise";
+import { Mixin } from "ts-mixer";
+import { EventEmitter } from "events";
 
 export interface ITag {
   tag: string;
@@ -22,18 +24,16 @@ export interface IExecuteAll {
   preview?: boolean;
   includeResults?: boolean;
 }
-export class SnowflakeMultiSql extends Snowflake {
+export class SnowflakeMultiSql extends Mixin(Snowflake, EventEmitter) {
   constructor(conn: ConnectionOptions) {
     super(conn);
   }
   public async executeAll({
     sqlText,
     tags,
-    binds,
     preview,
     includeResults = false,
   }: IExecuteAll): Promise<IMultiSqlResult[]> {
-    // const replaced = this.replaceTags(sqlText, tags);
     const chunks = this.getChunks(sqlText);
     const chunksTotal = chunks.length;
     const results: IMultiSqlResult[] = [];
@@ -51,21 +51,23 @@ export class SnowflakeMultiSql extends Snowflake {
         };
         if (preview) {
           results.push(previewObj);
+          this.emit("progress", previewObj);
         } else {
           const startTime: number = new Date().valueOf();
           const data = await this.execute(sqlText, binds);
           const duration = new Date().valueOf() - startTime;
           totalDuration += duration;
 
-          const topush: IMultiSqlResult = {
+          const toPush: IMultiSqlResult = {
             ...previewObj,
             duration,
             totalDuration,
           };
           if (includeResults) {
-            topush.data = data;
+            toPush.data = data;
           }
-          results.push(topush);
+          results.push(toPush);
+          this.emit("progress", toPush);
         }
       }
     }

diff --git a/src/utils.ts b/src/utils.ts
@@ -18,8 +18,6 @@ export async function loadFiles({
   } else {
     files = [...fileNames];
   }
-  console.dir(files);
-
   const promises = [];
   files.map((fileName) => {
     if (isSql(fileName)) {