Refactor SQL schema translation for text and object fields (#482)

* Refactor mysql schema translation for text and object fields * Refactor postgres schema translator & schema utils * Refactor Postgres schema column translation * Fix translation of date type in schemaTranslator * Add support for time and date columns in BigQuery, MSSQL, MySQL, and Postgres capabilities * Refactor columnToDbColumnSql method in spanner schema Column Translator * Fix translation of DATE type in schemaTranslator * updated supportedFieldTypes
wix · Dec 6, 2023 · 16ea92b · 16ea92b
1 parent fc6ba95
commit 16ea92b
Show file tree

Hide file tree

Showing 17 changed files with 218 additions and 141 deletions.
diff --git a/libs/external-db-bigquery/src/bigquery_capabilities.ts b/libs/external-db-bigquery/src/bigquery_capabilities.ts
@@ -18,4 +18,6 @@ export const ColumnsCapabilities = {
     image: { sortable: false, columnQueryOperators: [] },
     object: { sortable: false, columnQueryOperators: [] },
     datetime: { sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte] },
+    time: { sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte] },
+    date: { sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte] },
 }
diff --git a/libs/external-db-bigquery/src/sql_schema_translator.spec.ts b/libs/external-db-bigquery/src/sql_schema_translator.spec.ts
@@ -70,9 +70,17 @@ describe('Sql Schema Column Translator', () => {
         })
 
         describe('string fields', () => {
-            test('string', () => {
-                expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'text', subtype: 'string' }) ).toEqual({ mode: '', name: escapeId(ctx.fieldName), type: 'STRING' })
-            })
+            test.each([
+                'string',
+                'richcontent',
+                'image',
+                'video',
+                'audio',
+                'document',
+                'language',
+            ])('%s', (subtype) => {
+                expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'text', subtype }) ).toEqual({ mode: '', name: escapeId(ctx.fieldName), type: 'STRING' })
+              })
 
             test('string with length', () => {
                 expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'text', subtype: 'string', precision: '50' }) ).toEqual({ mode: '', name: escapeId(ctx.fieldName), type: 'STRING(50)' })
@@ -108,20 +116,15 @@ describe('Sql Schema Column Translator', () => {
 
         describe('JSON fields', () => {
             test.each([
-                ['object'],
-                ['image'],
-                ['document'],
-                ['video'],
-                ['audio'],
-                ['any'],
-                ['mediaGallery'],
-                ['address'],
-                ['pageLink'],
-                ['reference'],
-                ['multiReference'],
-                ['arrayString'],
-                ['arrayDocument'],
-                ['richContent'],
+                'object',
+                'any',
+                'mediaGallery',
+                'address',
+                'pageLink',
+                'reference',
+                'multiReference',
+                'arrayDocument',
+                'arrayString',
               ])('%s', (subtype) => {
                 expect(env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'object', subtype })).toEqual({ mode: '', name: escapeId(ctx.fieldName), type: 'JSON' })
               })
@@ -154,10 +157,17 @@ describe('Sql Schema Column Translator', () => {
         })
 
         describe('date time fields', () => {
+
+            test.each(['DATETIME', 'TIMESTAMP'])('%s', (t) => {
+                expect( env.schemaTranslator.translateType(t) ).toEqual('datetime')
+            })
+
+            test('time', () => {
+                expect( env.schemaTranslator.translateType('TIME') ).toEqual('time')
+            })
+
             test('date', () => {
-                ['DATE', 'DATETIME', 'TIME', 'TIMESTAMP'].forEach(t => {
-                    expect( env.schemaTranslator.translateType(t) ).toEqual('datetime')
-                })
+                 expect( env.schemaTranslator.translateType('DATE') ).toEqual('date')
             })
         })
 

diff --git a/libs/external-db-bigquery/src/sql_schema_translator.ts b/libs/external-db-bigquery/src/sql_schema_translator.ts
@@ -20,10 +20,13 @@ export default class SchemaColumnTranslator {
             case 'bigdecimal':
                 return 'number'
 
-            case 'timestamp':
-            case 'datetime':
             case 'time':
+                return 'time'
             case 'date':
+                return 'date'
+
+            case 'timestamp':
+            case 'datetime':
                 return 'datetime'
 
             case 'string':
@@ -90,6 +93,11 @@ export default class SchemaColumnTranslator {
             case 'text_medium':
             case 'text_large':
             case 'text_language':
+            case 'text_richcontent':
+            case 'text_image':
+            case 'text_video':
+            case 'text_audio':
+            case 'text_document':
                 return 'STRING'
 
             case 'boolean_':
@@ -98,19 +106,15 @@ export default class SchemaColumnTranslator {
 
             case 'object_':
             case 'object_object':
-            case 'object_image':
-            case 'object_document':
-            case 'object_video':
             case 'object_any':
-            case 'object_audio':
             case 'object_mediagallery':
             case 'object_address':
             case 'object_pagelink':
             case 'object_reference':
             case 'object_multireference':
             case 'object_arraystring':
             case 'object_arraydocument':
-            case 'object_richcontent':
+            case 'object_array':
                 return 'JSON'
 
             default:

diff --git a/libs/external-db-mssql/src/sql_schema_translator.spec.ts b/libs/external-db-mssql/src/sql_schema_translator.spec.ts
@@ -67,9 +67,18 @@ describe('Sql Schema Column Translator', () => {
         })
 
         describe('string fields', () => {
-            test('string', () => {
-                expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'text', subtype: 'string' }) ).toEqual(`${escapeId(ctx.fieldName)} VARCHAR(2048)`)
-            })
+
+            test.each([
+                'string',
+                'richcontent',
+                'image',
+                'video',
+                'audio',
+                'document',
+                'language',
+            ])('text %s', (subtype) => {
+                expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'text', subtype }) ).toEqual(`${escapeId(ctx.fieldName)} VARCHAR(2048)`)
+              })
 
             test('string with length', () => {
                 expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'text', subtype: 'string', precision: '2055' }) ).toEqual(`${escapeId(ctx.fieldName)} VARCHAR(2055)`)
@@ -86,12 +95,6 @@ describe('Sql Schema Column Translator', () => {
             test('text large', () => {
                 expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'text', subtype: 'large' }) ).toEqual(`${escapeId(ctx.fieldName)} TEXT`)
             })
-
-            test('text language', () => {
-                expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'text', subtype: 'language' }) ).toEqual(`${escapeId(ctx.fieldName)} TEXT`)
-            })
-
-
         })
 
         describe('other fields', () => {
@@ -127,10 +130,16 @@ describe('Sql Schema Column Translator', () => {
         })
 
         describe('date time fields', () => {
+            test('time', () => {
+                expect( env.schemaTranslator.translateType('TIME') ).toEqual('time')
+            })
+
             test('date', () => {
-                ['DATE', 'DATETIME', 'DATETIME2', 'TIME', 'DATETIMEOFFSET', 'SMALLDATETIME'].forEach(t => {
-                    expect( env.schemaTranslator.translateType(t) ).toEqual('datetime')
-                })
+                expect( env.schemaTranslator.translateType('DATE') ).toEqual('date')
+           })
+
+           test.each([ 'DATETIME', 'DATETIME2', 'DATETIMEOFFSET', 'SMALLDATETIME'])('%s', (t) => {
+            expect( env.schemaTranslator.translateType(t) ).toEqual('datetime')
             })
         })
 

diff --git a/libs/external-db-mssql/src/sql_schema_translator.ts b/libs/external-db-mssql/src/sql_schema_translator.ts
@@ -18,11 +18,14 @@ export default class SchemaColumnTranslator {
             case 'decimal':
             case 'numeric':
                 return 'number'
-
+
+            case 'time':
+                return 'time'
             case 'date':
+                return 'date'
+
             case 'datetime':
             case 'datetime2':
-            case 'time':
             case 'datetimeoffset':
             case 'smalldatetime':
                 return 'datetime'
@@ -83,12 +86,17 @@ export default class SchemaColumnTranslator {
                 return 'SMALLDATETIME'
 
             case 'text_string':
+            case 'text_richcontent':
+            case 'text_image':
+            case 'text_video':
+            case 'text_audio':
+            case 'text_document':
+            case 'text_language':
                 return `VARCHAR${this.parseLength(precision)}`
 
             case 'text_small':
             case 'text_medium':
             case 'text_large':
-            case 'text_language':
                 return 'TEXT'
 
 

diff --git a/libs/external-db-mysql/src/mysql_capabilities.ts b/libs/external-db-mysql/src/mysql_capabilities.ts
@@ -18,4 +18,6 @@ export const ColumnsCapabilities = {
     image: { sortable: false, columnQueryOperators: [] },
     object: { sortable: false, columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] },
     datetime: { sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte] },
+    time: { sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte] },
+    date: { sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte] },
 }
diff --git a/libs/external-db-mysql/src/sql_schema_translator.spec.ts b/libs/external-db-mysql/src/sql_schema_translator.spec.ts
@@ -67,9 +67,17 @@ describe('Sql Schema Column Translator', () => {
         })
 
         describe('string fields', () => {
-            test('string', () => {
-                expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'text', subtype: 'string' }) ).toEqual(`${escapeId(ctx.fieldName)} TEXT`)
-            })
+            test.each([
+                'string',
+                'richcontent',
+                'image',
+                'video',
+                'audio',
+                'document',
+                'language',
+            ])('%s', (subtype) => {
+                expect(env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'text', subtype })).toEqual(`${escapeId(ctx.fieldName)} TEXT`)
+              })
 
             test('string with length', () => {
                 expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'text', subtype: 'string', precision: '2055' }) ).toEqual(`${escapeId(ctx.fieldName)} VARCHAR(2055)`)
@@ -86,29 +94,19 @@ describe('Sql Schema Column Translator', () => {
             test('text large', () => {
                 expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'text', subtype: 'large' }) ).toEqual(`${escapeId(ctx.fieldName)} LONGTEXT`)
             })
-
-            test('text language', () => {
-
-                expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'text', subtype: 'language' }) ).toEqual(`${escapeId(ctx.fieldName)} TEXT`)
-            })
         })
 
         describe('JSON fields', () => {
             test.each([
-                ['object'],
-                ['image'],
-                ['document'],
-                ['video'],
-                ['audio'],
-                ['any'],
-                ['mediaGallery'],
-                ['address'],
-                ['pageLink'],
-                ['reference'],
-                ['multiReference'],
-                ['arrayString'],
-                ['arrayDocument'],
-                ['richContent'],
+                'object',
+                'any',
+                'mediaGallery',
+                'address',
+                'pageLink',
+                'reference',
+                'multiReference',
+                'arrayDocument',
+                'arrayString',
               ])('%s', (subtype) => {
                 expect(env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'object', subtype })).toEqual(`${escapeId(ctx.fieldName)} JSON`)
               })
@@ -141,11 +139,18 @@ describe('Sql Schema Column Translator', () => {
         })
 
         describe('date time fields', () => {
-            test('date', () => {
-                ['DATE', 'DATETIME', 'TIMESTAMP', 'TIME', 'YEAR'].forEach(t => {
-                    expect( env.schemaTranslator.translateType(t) ).toEqual('datetime')
-                })
+            test.each([ 'DATETIME', 'TIMESTAMP' ])('%s', (t) => {
+                expect( env.schemaTranslator.translateType(t) ).toEqual('datetime')
             })
+
+
+            test('time', () => {
+                expect( env.schemaTranslator.translateType('TIME') ).toEqual('time')
+            })
+
+            test('date', () => {
+                expect( env.schemaTranslator.translateType('DATE') ).toEqual('date')
+           })
         })
 
         describe('json fields', () => {

diff --git a/libs/external-db-mysql/src/sql_schema_translator.ts b/libs/external-db-mysql/src/sql_schema_translator.ts
@@ -18,14 +18,18 @@ export default class SchemaColumnTranslato {
             case 'float':
             case 'double':
             case 'decimal':
+            case 'year':
                 return 'number'
 
             case 'date':
+                return 'date'
+
             case 'datetime':
             case 'timestamp':
-            case 'time':
-            case 'year':
                 return 'datetime'
+
+            case 'time':
+                return 'time'
 
             case 'varchar':
             case 'text':
@@ -90,6 +94,11 @@ export default class SchemaColumnTranslato {
                 return 'TIMESTAMP DEFAULT CURRENT_TIMESTAMP'
 
             case 'text_string':
+            case 'text_richcontent':
+            case 'text_image':
+            case 'text_video':
+            case 'text_audio':
+            case 'text_document':
                 return precision ? `VARCHAR${this.parseLength(precision)}` : 'TEXT'
 
             case 'text_small':
@@ -108,19 +117,15 @@ export default class SchemaColumnTranslato {
 
             case 'object_':
             case 'object_object':
-            case 'object_image':
-            case 'object_document':
-            case 'object_video':
             case 'object_any':
-            case 'object_audio':
             case 'object_mediagallery':
             case 'object_address':
             case 'object_pagelink':
             case 'object_reference':
             case 'object_multireference':
             case 'object_arraystring':
             case 'object_arraydocument':
-            case 'object_richcontent':
+            case 'object_array':
                 return 'JSON'
 
             default:

diff --git a/libs/external-db-postgres/src/postgres_capabilities.ts b/libs/external-db-postgres/src/postgres_capabilities.ts
@@ -16,4 +16,6 @@ export const ColumnsCapabilities = {
     image: { sortable: false, columnQueryOperators: [] },
     object: { sortable: false, columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] },
     datetime: { sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte] },
+    time: { sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte] },
+    date: { sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte] },
 }