From d2cdf74d5e7015ec72906d08dc86695c1633f62c Mon Sep 17 00:00:00 2001 From: Jim Murphy Date: Fri, 3 Sep 2021 13:00:08 -0400 Subject: [PATCH] feat(add support for primary keys): add support for primary keys. Add generatePrimaryKeys option --- README.md | 2 +- src/etl-processes.ts | 71 ++++++++++++++++++---- src/sql.ts | 9 +-- src/tests/etl-processes.test.ts | 42 +++++++++++-- src/tests/sql.test.ts | 10 +++- src/tests/test-data.ts | 103 +++++++++++++++++++++++++------- 6 files changed, 190 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 468cb97..0222f56 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Supports two options, both of which are optional: * *createDatabase* - _true | false_ (Defaults to false) * *createTables* - _true | false_ (Defaults to false) - +* *generatePrimaryKeys* - _true | false_ (Defaults to false. Supports multiple primary keys. Append '_pk' to the column name in the workbook that will be the primary key) # Testing This package's tests are written using [Jest](https://jestjs.io/). To execute, run: diff --git a/src/etl-processes.ts b/src/etl-processes.ts index af2218c..b5991ab 100644 --- a/src/etl-processes.ts +++ b/src/etl-processes.ts @@ -1,18 +1,26 @@ -enum SQLType { - VARCHAR = 'VARCHAR', - BOOLEAN = 'BOOLEAN', - FLOAT = 'FLOAT', - INT = 'INT' +export enum SQLType { + VARCHAR = 'VARCHAR', + BOOLEAN = 'BOOLEAN', + FLOAT = 'FLOAT', + INT = 'INT' } +export enum SQLKeyword { + PRIMARY_KEY = 'PRIMARY KEY' +} export interface Column { - name: string; - type: string; + name: string; + type: string; } export interface Fields { - names: string[]; - values: T[]; + names: string[]; + values: T[]; +} + +interface FormatColumnsResult { + formattedColumns: string[]; + primaryKeyIndex: number[]; } export function getFields(data: T): Fields { @@ -53,13 +61,52 @@ export function getColumns(fields: Fields): Column[] { return tableColumns; } -export function formatColumns(columns: Column[]): string[] { +export function formatColumns(columns: Column[]): FormatColumnsResult { const formattedColumns: string[] = []; - columns.forEach((col: Column) => { + const primaryKeyIndex: number [] = []; + + columns.forEach((col: Column, index: number) => { const formatted = `${col.name.replace(/\s/g, '')} ${col.type}`; + + if (checkPrimaryKey(col.name)) { + primaryKeyIndex.push(index); + } + formattedColumns.push(formatted); }); - return formattedColumns; + return { formattedColumns, primaryKeyIndex: primaryKeyIndex }; +} + +export function checkPrimaryKey(col: string): boolean { + const primaryKeyIndicator = '_pk'; + + if (col.substring(col.length, col.length - 3).toUpperCase() === primaryKeyIndicator.toUpperCase()) { + return true; + } + + return false; +} + +export function formatPrimaryKey(formatColumnsResult: FormatColumnsResult): string[] { + if (formatColumnsResult.primaryKeyIndex.length === 1) { + const primaryColumn = formatColumnsResult.formattedColumns[formatColumnsResult.primaryKeyIndex[0]].concat(` ${SQLKeyword.PRIMARY_KEY}`); + + formatColumnsResult.formattedColumns[formatColumnsResult.primaryKeyIndex[0]] = primaryColumn; + + return formatColumnsResult.formattedColumns; + } + + const primaryKeys: string[] = []; + + formatColumnsResult.primaryKeyIndex.forEach(index => { + primaryKeys.push(formatColumnsResult.formattedColumns[index].substring(0, formatColumnsResult.formattedColumns[index].indexOf(' '))); + }); + + const primaryColumns = `${SQLKeyword.PRIMARY_KEY} (${primaryKeys})`; + + formatColumnsResult.formattedColumns.push(primaryColumns); + + return formatColumnsResult.formattedColumns; } \ No newline at end of file diff --git a/src/sql.ts b/src/sql.ts index 513b769..c76a82b 100644 --- a/src/sql.ts +++ b/src/sql.ts @@ -1,5 +1,5 @@ import { Pool } from 'pg'; -import { getFields, Fields, Column, getColumns, formatColumns } from './etl-processes'; +import { getFields, Fields, Column, getColumns, formatColumns, formatPrimaryKey } from './etl-processes'; import { readExcel } from './excel'; export interface Connection { @@ -13,16 +13,17 @@ export interface Connection { export interface Options { createDatabase?: boolean; createTables?: boolean; + generatePrimaryKeys?: boolean; } export function createDatabase(dbName: string): string { return `CREATE DATABASE ${dbName};`; } -export function createTable(tableName: string, data: T): string { +export function createTable(tableName: string, data: T, generatePrimaryKeys?: boolean): string { const fields: Fields = getFields(data); const columns: Column[] = getColumns(fields); - const formattedColumns: string[] = formatColumns(columns); + const formattedColumns: string[] = generatePrimaryKeys ? formatPrimaryKey(formatColumns(columns)) : formatColumns(columns).formattedColumns; return `CREATE TABLE ${tableName.replace(/\s/g, '')} ( ${formattedColumns} @@ -77,7 +78,7 @@ export async function excelToPostgresDb(connectionInfo: Connection, filePath: st let tableQuery = ''; sheets.forEach(async (sheet) => { - tableQuery = tableQuery.concat(createTable(sheet.title, sheet.data[0])); + tableQuery = tableQuery.concat(createTable(sheet.title, sheet.data[0], options?.generatePrimaryKeys)); insertQuery = insertQuery.concat(insert(sheet.title, sheet.data)); }); diff --git a/src/tests/etl-processes.test.ts b/src/tests/etl-processes.test.ts index 599dc5b..f43e3fe 100644 --- a/src/tests/etl-processes.test.ts +++ b/src/tests/etl-processes.test.ts @@ -1,16 +1,46 @@ -import { getFields, getColumns, formatColumns } from '../etl-processes'; -import { columns, etlProcesses } from './test-data'; +import { getFields, getColumns, formatColumns, checkPrimaryKey, formatPrimaryKey } from '../etl-processes'; +import { columns_one_pk, columns_multiple_pk, etlProcesses } from './test-data'; -describe('ETL processes tests', () => { +describe('ETL processes tests one primary key', () => { test('getFields', () => { - expect(getFields(etlProcesses.sheet)).toEqual(etlProcesses.fields); + expect(getFields(etlProcesses.sheet_one_pk)).toEqual(etlProcesses.fields_one_pk); }); test('getColumns', () => { - expect(getColumns(etlProcesses.fields)).toEqual(columns); + expect(getColumns(etlProcesses.fields_one_pk)).toEqual(columns_one_pk); }); test('formatColumns', () => { - expect(formatColumns(columns)).toEqual(etlProcesses.formattedColumns); + expect(formatColumns(columns_one_pk)).toEqual({formattedColumns: [...etlProcesses.formattedColumns], primaryKeyIndex: [0]}); + }); + + test('checkPrimaryKey', () => { + expect(checkPrimaryKey(columns_one_pk[0].name)).toEqual(true); + }); + + test('formatPrimaryKey', () => { + expect(formatPrimaryKey({ formattedColumns: etlProcesses.formattedColumns, primaryKeyIndex: [0] })).toEqual(etlProcesses.formattedColumnsOnePrimaryKey); + }); +}); + +describe('ETL processes tests multiple primary key', () => { + test('getFields', () => { + expect(getFields(etlProcesses.sheet_multiple_pk)).toEqual(etlProcesses.fields_multiple_pks); + }); + + test('getColumns', () => { + expect(getColumns(etlProcesses.fields_multiple_pks)).toEqual(columns_multiple_pk); + }); + + test('formatColumns', () => { + expect(formatColumns(columns_multiple_pk)).toEqual({ formattedColumns: [...etlProcesses.formattedColumns_multiple_pk], primaryKeyIndex: [0,1] }); + }); + + test('checkPrimaryKey', () => { + expect(checkPrimaryKey(columns_multiple_pk[0].name)).toEqual(true); + }); + + test('formatPrimaryKey', () => { + expect(formatPrimaryKey({ formattedColumns: etlProcesses.formattedColumns_multiple_pk, primaryKeyIndex: [0, 1] })).toEqual(etlProcesses.formattedColumnsMultiplePrimaryKeys); }); }); \ No newline at end of file diff --git a/src/tests/sql.test.ts b/src/tests/sql.test.ts index 4d32dcc..47b88a0 100644 --- a/src/tests/sql.test.ts +++ b/src/tests/sql.test.ts @@ -6,11 +6,15 @@ describe('SQL tests', () => { expect(createDatabase(sqlInfo.database)).toEqual(sqlResults.createDatabase); }); - test('create table', () => { - expect(createTable(sqlInfo.tableName, etlProcesses.sheet)).toEqual(sqlResults.createTable); + test('create table one primary key', () => { + expect(createTable(sqlInfo.tableName, etlProcesses.sheet_one_pk, true)).toEqual(sqlResults.createTableOnePrimaryKey); + }); + + test('create table multiple primary key', () => { + expect(createTable(sqlInfo.tableName, etlProcesses.sheet_multiple_pk, true)).toEqual(sqlResults.createTableMultiplePrimaryKeys); }); test('insert', () => { - expect(insert(sqlInfo.tableName, [etlProcesses.sheet])).toEqual(sqlResults.insert); + expect(insert(sqlInfo.tableName, [etlProcesses.sheet_one_pk])).toEqual(sqlResults.insert); }); }); \ No newline at end of file diff --git a/src/tests/test-data.ts b/src/tests/test-data.ts index a69061b..9ef50e0 100644 --- a/src/tests/test-data.ts +++ b/src/tests/test-data.ts @@ -1,47 +1,105 @@ +import { SQLKeyword, SQLType } from '../etl-processes'; + +const col_names_one_pk = [ + 'name_pk', + 'age', + 'isDev' +]; + +const col_names_multiple_pk = [ + 'name_pk', + 'age_pk', + 'isDev' +]; + const mockJSON = { - name: 'Person 1', + name_pk: 'Person 1', age: 18, isDev: true }; -export const columns = [ +export const columns_one_pk = [ + { + name: col_names_one_pk[0], + type: SQLType.VARCHAR + }, + { + name: col_names_one_pk[1], + type: SQLType.FLOAT + }, + { + name: col_names_one_pk[2], + type: SQLType.BOOLEAN + } +]; + +export const columns_multiple_pk = [ { - name: 'name', - type: 'VARCHAR' + name: col_names_multiple_pk[0], + type: SQLType.VARCHAR }, { - name: 'age', - type: 'FLOAT' + name: col_names_multiple_pk[1], + type: SQLType.FLOAT }, { - name: 'isDev', - type: 'BOOLEAN' + name: col_names_multiple_pk[2], + type: SQLType.BOOLEAN } ]; export const etlProcesses = { - sheet: { - name: mockJSON.name, + sheet_one_pk: { + name_pk: mockJSON.name_pk, age: mockJSON.age, isDev: mockJSON.isDev }, - fields: { + sheet_multiple_pk: { + name_pk: mockJSON.name_pk, + age_pk: mockJSON.age, + isDev: mockJSON.isDev + }, + fields_one_pk: { + names: [ + ...col_names_one_pk + ], + values: [ + mockJSON.name_pk, + mockJSON.age, + mockJSON.isDev + ] + }, + fields_multiple_pks: { names: [ - 'name', - 'age', - 'isDev' + ...col_names_multiple_pk ], values: [ - mockJSON.name, + mockJSON.name_pk, mockJSON.age, mockJSON.isDev ] }, formattedColumns: [ - `${columns[0].name} ${columns[0].type}`, - `${columns[1].name} ${columns[1].type}`, - `${columns[2].name} ${columns[2].type}` + `${columns_one_pk[0].name} ${columns_one_pk[0].type}`, + `${columns_one_pk[1].name} ${columns_one_pk[1].type}`, + `${columns_one_pk[2].name} ${columns_one_pk[2].type}` ], + formattedColumns_multiple_pk: [ + `${columns_multiple_pk[0].name} ${columns_multiple_pk[0].type}`, + `${columns_multiple_pk[1].name} ${columns_multiple_pk[1].type}`, + `${columns_multiple_pk[2].name} ${columns_multiple_pk[2].type}` + ], + formattedColumnsOnePrimaryKey: [ + `${columns_one_pk[0].name} ${columns_one_pk[0].type} ${SQLKeyword.PRIMARY_KEY}`, + `${columns_one_pk[1].name} ${columns_one_pk[1].type}`, + `${columns_one_pk[2].name} ${columns_one_pk[2].type}` + ], + formattedColumnsMultiplePrimaryKeys: [ + `${columns_multiple_pk[0].name} ${columns_multiple_pk[0].type}`, + `${columns_multiple_pk[1].name} ${columns_multiple_pk[1].type}`, + `${columns_multiple_pk[2].name} ${columns_multiple_pk[2].type}`, + `${SQLKeyword.PRIMARY_KEY} (${columns_multiple_pk[0].name},${columns_multiple_pk[1].name})` + ] }; export const sqlInfo = { @@ -51,8 +109,11 @@ export const sqlInfo = { export const sqlResults = { createDatabase: `CREATE DATABASE ${sqlInfo.database};`, - createTable: `CREATE TABLE ${sqlInfo.tableName} ( - ${etlProcesses.formattedColumns} + createTableOnePrimaryKey: `CREATE TABLE ${sqlInfo.tableName} ( + ${etlProcesses.formattedColumnsOnePrimaryKey} + );`, + createTableMultiplePrimaryKeys: `CREATE TABLE ${sqlInfo.tableName} ( + ${etlProcesses.formattedColumnsMultiplePrimaryKeys} );`, - insert: `INSERT INTO ${sqlInfo.tableName.replace(/\s/g, '')}(name,age,isDev) VALUES ('${mockJSON.name}',${mockJSON.age},${mockJSON.isDev});` + insert: `INSERT INTO ${sqlInfo.tableName.replace(/\s/g, '')}(${col_names_one_pk[0]},${col_names_one_pk[1]},${col_names_one_pk[2]}) VALUES ('${mockJSON.name_pk}',${mockJSON.age},${mockJSON.isDev});` }; \ No newline at end of file