From 8f078eb4592f3be9a2373ba3e7024ba6f172fc99 Mon Sep 17 00:00:00 2001 From: Julio Brugos Date: Sun, 28 Feb 2021 16:49:29 +0000 Subject: [PATCH] inline tags --- README.md | 112 ++++++++++++++++--------------- package.json | 2 +- src/snowflake-multisql.ts | 88 ++++++++++++------------ tests/snowflake-multisql.spec.ts | 21 +++++- 4 files changed, 118 insertions(+), 105 deletions(-) diff --git a/README.md b/README.md index 479db6f..86897da 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ A Multi SQL Statement, Promise-based and Typescript version to your [Snowflake]( Also adds nice features like: - replaces **tags** defined in the scripts by name using original data types (examples below). +- Also replaces **inline tags** like `select * from {%table_name%}` so you can play around different scenarios between code and the sql script. Use cases: replace database name depending on the environment used (ex: db_name_dev, db_name_prod) - Emits **progress** events so you can monitor long run calls - **Preview** parsed statement before sending to Snowflake. - Shows **duration** (in miliseconds) for each chunk as well the **totalDuration**. @@ -48,58 +49,7 @@ The unique method's param is deconstructed into the variables below: --- -## Basic usage - -```typescript -const Snowflake = require("snowflake-multisql").Snowflake; -// or, for TypeScript: -import { Snowflake } from "snowflake-multisql"; - -async function main() { - const snowflake = new Snowflake({ - account: "", - username: "", - password: "", - database: "DEMO_DATABASE", - schema: "DEMO_SCHEMA", - warehouse: "DEMO_WH", - }); - - await snowflake.connect(); - - const sqlText = ` - CREATE OR REPLACE TABLE temp_table_customer as - SELECT COUNT(*) FROM customer WHERE C_MKTSEGMENT=:1; - - USE SCHEMA demo_schema; - SELECT COUNT(*) FROM customer WHERE C_MKTSEGMENT_ID=:2; - `; - const binds = ["AUTOMOBILE", 1234]; - - // NEW FEATURE: Feel free to monitor your running progress - snowflake.progress.on("news", (data) => - console.log(` - progress: ${data.chunkOrder}/${data.chunksTotal} - duration: ${data.duration} ms, - totalDuration: ${data.totalDuration} ms - `) - ); - - const rows = await snowflake.executeAll({ - sqlText, - binds, - includeResults: true, - }); - - console.log(rows); -} - -main(); -``` - ---- - -## Advanced options +## Usage ```typescript import { Snowflake, loadFiles } from "snowflake-multisql" @@ -123,7 +73,7 @@ main(); // file-1.sql content: // CREATE OR REPLACE TABLE temp_table_customer as - // SELECT COUNT(*) FROM customer WHERE PURCHASE_DATE={%purchase_date%}; + // SELECT COUNT(*) FROM {%table_name%} WHERE PURCHASE_DATE={%purchase_date%}; // file-2.sql content: // SELECT * from temp_table_customer @@ -131,12 +81,13 @@ main(); // file-3.sql content: // USE SCHEMA demo_schema; - // SELECT COUNT(*) FROM customer WHERE segment_id={%segment_id%}; + // SELECT COUNT(*) FROM {%table_name%} WHERE segment_id={%segment_id%}; const tags = [ { tag: "purchase_date", value: new Date(1610976670682) }, { tag: "product_name", value: "AUTOMOBILE" }, { tag: "segment_id", value: 1234 }, + { tag: "table_name", value: "customers", inline: true }, ]; // NEW FEATURE: Feel free to monitor your running progress @@ -161,7 +112,7 @@ main(); --- -## Using GENERIC TYPES +## Using GENERIC types ```typescript import { Snowflake, loadFiles } from "snowflake-multisql" @@ -197,3 +148,54 @@ main(); main(); ``` + +--- + +## If you prefer to manage the original binds by yourself + +```typescript +const Snowflake = require("snowflake-multisql").Snowflake; +// or, for TypeScript: +import { Snowflake } from "snowflake-multisql"; + +async function main() { + const snowflake = new Snowflake({ + account: "", + username: "", + password: "", + database: "DEMO_DATABASE", + schema: "DEMO_SCHEMA", + warehouse: "DEMO_WH", + }); + + await snowflake.connect(); + + const sqlText = ` + CREATE OR REPLACE TABLE temp_table_customer as + SELECT COUNT(*) FROM customer WHERE C_MKTSEGMENT=:1; + + USE SCHEMA demo_schema; + SELECT COUNT(*) FROM customer WHERE C_MKTSEGMENT_ID=:2; + `; + const binds = ["AUTOMOBILE", 1234]; + + // NEW FEATURE: Feel free to monitor your running progress + snowflake.progress.on("news", (data) => + console.log(` + progress: ${data.chunkOrder}/${data.chunksTotal} + duration: ${data.duration} ms, + totalDuration: ${data.totalDuration} ms + `) + ); + + const rows = await snowflake.executeAll({ + sqlText, + binds, + includeResults: true, + }); + + console.log(rows); +} + +main(); +``` diff --git a/package.json b/package.json index 042fed6..a33aa75 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "snowflake-multisql", - "version": "1.5.0", + "version": "1.6.0", "description": "Multi SQL Statement, Promise-based, TypeScript wrapper for Snowflake SDK", "repository": "github:brugos/snowflake-multisql", "bugs": "https://github.com/brugos/snowflake-multisql/issues", diff --git a/src/snowflake-multisql.ts b/src/snowflake-multisql.ts index ffe0fb1..2c6441d 100644 --- a/src/snowflake-multisql.ts +++ b/src/snowflake-multisql.ts @@ -17,6 +17,7 @@ export type IExecuteAllResult = IExecuteAllPreview & { export type ITag = { tag: string; value: any; + inline?: boolean; }; export type IExecuteAll = { sqlText: string; @@ -32,9 +33,23 @@ export class SnowflakeMultiSql extends Snowflake { super(conn); } + //public replaceInlineTags(fileText: string, binds: ITag[]): string { + private replaceInlineTags(params: IExecuteAll): IExecuteAll { + let text: string = params.sqlText; + params.tags = params.tags || []; + const inlineTags = params.tags.filter((tag) => tag.inline); + inlineTags?.map((bind) => { + const regex = new RegExp("{%" + bind.tag + "%}", "gi"); + params.sqlText = params.sqlText.replace(regex, bind.value); + }); + params.tags = params.tags.filter((tag) => !tag.inline); + return params; + } + public async executeAll>( params: IExecuteAll ): Promise[]> { + this.replaceInlineTags(params); const chunks = this.getChunks(params.sqlText); const chunksTotal = chunks.length; const results: IExecuteAllResult[] = []; @@ -75,21 +90,6 @@ export class SnowflakeMultiSql extends Snowflake { return results; } - /** - * Tag Replacement alternative - * replaces by name instead of sequence as per snowflake-sdk execute function - * @param fileText - * @param binds - */ - public replaceTags(fileText: string, binds: ITag[]): string { - let text: string = fileText; - binds?.map((bind) => { - const regex = new RegExp("{%" + bind.tag + "%}", "gi"); - text = text.replace(regex, bind.value); - }); - return text; - } - /** * Transforms nominal tags replacement into sequence as expected by the original library * @param sqlText @@ -97,15 +97,11 @@ export class SnowflakeMultiSql extends Snowflake { */ public tagsToBinds( sqlText: string, - tagValues: ITag[] = [] + tags: ITag[] = [] ): { sqlText: string; binds: any[] } { try { - const allRegex: RegExp = /{%\w+%}/gim; - const ret = { - sqlText, - binds: [], - }; - const rawTags = sqlText.match(allRegex) || []; + const ret = { sqlText, binds: [] }; + const rawTags = sqlText.match(/{%\w+%}/gim) || []; const sqlTextTags: { raw?: string; clean?: string; @@ -114,36 +110,13 @@ export class SnowflakeMultiSql extends Snowflake { clean: raw.replace("%}", "").replace("{%", ""), })); - // sqlTextTags.clean = sqlTextTags.raw.map((t) => cleanTextTag(t)); - // const cleanTextTag = (textTag) => - // textTag.replace("%}", "").replace("{%", ""); - - const checkTags = (textTags: string[], tagValues: ITag[]) => { - const uniqueTextTags = textTags.filter( - (txtTag, i) => textTags.indexOf(txtTag) === i - ); - // console.log(uniqueTextTags); - uniqueTextTags.map((uniqueTag) => { - if (!tagValues.find((tv) => tv.tag === uniqueTag)) - throw new Error( - "###" - // `it seems you've forgotten to list the tag ${uniqueTag}` - ); - }); - }; - - checkTags( - sqlTextTags.map((tt) => tt.clean), - tagValues - ); - let _i = 0; sqlTextTags.forEach((textTag, _i) => { ret.sqlText = ret.sqlText.replace( textTag.raw, ":".concat(String(_i + 1)) ); - const tagValue = tagValues?.find((obj) => obj.tag === textTag.clean); + const tagValue = tags?.find((obj) => obj.tag === textTag.clean); if (tagValue) ret.binds.push(tagValue?.value); }); @@ -153,6 +126,29 @@ export class SnowflakeMultiSql extends Snowflake { } } + private checkTagsExistence( + sqlTextTags: { raw?: string; clean?: string }[], + tagValues: ITag[] + ) { + const checkTags = (textTags: string[], tagValues: ITag[]) => { + const uniqueTextTags = textTags.filter( + (txtTag, i) => textTags.indexOf(txtTag) === i + ); + uniqueTextTags.map((uniqueTag) => { + if (!tagValues.find((tv) => tv.tag === uniqueTag)) + throw new Error( + "###" + // `it seems you've forgotten to list the tag ${uniqueTag}` + ); + }); + }; + + checkTags( + sqlTextTags.map((tt) => tt.clean), + tagValues + ); + } + /** * splits in chunks due to inability of Snowflake driver * to process multiple requests in a single call diff --git a/tests/snowflake-multisql.spec.ts b/tests/snowflake-multisql.spec.ts index 23d7ebd..4739727 100644 --- a/tests/snowflake-multisql.spec.ts +++ b/tests/snowflake-multisql.spec.ts @@ -14,11 +14,12 @@ describe("checks tagsToBinds function", () => { let snowflake: Snowflake; const sqlText: string = - "select * from {%tag0%} {%tag2%} {%tag0%} {%tag4%} where {%tag2%} {%tag4%}"; + "select * from {%inline1%} {%tag0%} {%tag2%} {%tag0%} {%tag4%} where {%tag2%} {%tag4%}"; const tags: ITag[] = [ { tag: "tag0", value: "hi" }, { tag: "tag2", value: 123 }, { tag: "tag4", value: new Date(1610976670682) }, + { tag: "inline1", value: "tblName", inline: true }, ]; type Conversion = { ID: string; @@ -72,9 +73,9 @@ describe("checks tagsToBinds function", () => { expect(spyProgress.called).to.be.true; }); - it("matches expected response", async () => { + it("check binds functionality", async () => { const expected = { - chunkText: "select * from :1 :2 :3 :4 where :5 :6", + chunkText: "select * from tblName :1 :2 :3 :4 where :5 :6", chunkOrder: 1, chunksTotal: 1, binds: [ @@ -94,6 +95,20 @@ describe("checks tagsToBinds function", () => { expect(ret[0]).to.containSubset(expected); }); + it("check tags absence", async () => { + const sqlText = "select * from tableName"; + const expected = { + chunkText: sqlText, + chunkOrder: 1, + chunksTotal: 1, + binds: [], + }; + const ret = await snowflake.executeAll({ + sqlText, + }); + expect(ret[0]).to.containSubset(expected); + }); + it("no tags sqltext, no tags params", () => { const sqlText = "select * from table"; const ret = snowflake.tagsToBinds(sqlText);