Hck 8696 fine tune the other sql queries that are executed by the synapse plugin during the re process

reverse_engineering/databaseService/databaseService.js

@@ -7,7 +7,16 @@ const { logAuthTokenInfo } = require('../helpers/logInfo');
 const { getConnection } = require('./helpers/connection');
 const {
 	queryForRetrievingTheTablesSelectedByTheUser,
-} = require('../queries/queryForRetrievingTheTablesSelectedByTheUser');
+} = require('../queries/selectedTablesSubQuery/QueryForRetrievingTheTablesSelectedByTheUser');
+const {
+	PartitionsSubQueryForRetrievingTheTablesSelectedByTheUser,
+} = require('../queries/selectedTablesSubQuery/PartitionsSubQueryForRetrievingTheTablesSelectedByTheUser');
+const {
+	DatabaseIndexesSubQueryForRetrievingTheTablesSelectedByTheUser,
+} = require('../queries/selectedTablesSubQuery/DatabaseIndexesSubQueryForRetrievingTheTablesSelectedByTheUser');
+const {
+	MemoryOptimizedTablesSubQueryForRetrievingTheTablesSelectedByTheUser,
+} = require('../queries/selectedTablesSubQuery/MemoryOptimizedTablesSubQueryForRetrievingTheTablesSelectedByTheUser');
 
 const QUERY_REQUEST_TIMEOUT = 60000;
 
@@ -194,38 +203,37 @@ const getViewDistributedColumns = async (connectionClient, dbName, tableName, ta
 	);
 };
 
-const getDatabaseIndexes = async (connectionClient, dbName, logger) => {
-	const currentDbConnectionClient = await getNewConnectionClientByDb(connectionClient, dbName);
-
+const getDatabaseIndexes = async ({ connectionClient, tablesInfo, dbName, logger }) => {
 	logger.log('info', { message: `Get '${dbName}' database indexes.` }, 'Reverse Engineering');
+	const currentDbConnectionClient = await getNewConnectionClientByDb(connectionClient, dbName);
+	const tablesSelectedByTheUser = new DatabaseIndexesSubQueryForRetrievingTheTablesSelectedByTheUser({
+		schemaToTablesMap: tablesInfo,
+	}).getQuery();
+	const queryRetrievingTheIndexes = `
+	WITH user_selected_tables AS (${tablesSelectedByTheUser.sql()})
+	SELECT
+		TableName = t.${tablesSelectedByTheUser.projection.tableName},
+		IndexName = ind.name,
+		ic.is_descending_key,
+		ic.is_included_column,
+		ic.column_store_order_ordinal,
+		COL_NAME(t.${tablesSelectedByTheUser.projection.tableId}, ic.column_id) as columnName,
+		S.name as schemaName,
+		ind.*
+	FROM sys.indexes ind
+	LEFT JOIN user_selected_tables t
+		ON ind.object_id = t.${tablesSelectedByTheUser.projection.tableId}
+	INNER JOIN sys.index_columns ic
+		ON ind.object_id = ic.object_id AND ind.index_id = ic.index_id
+	INNER JOIN sys.objects O ON O.object_id = t.${tablesSelectedByTheUser.projection.tableId}
+	INNER JOIN sys.schemas S ON S.schema_id = O.schema_id
+	WHERE
+		ind.is_primary_key = 0
+		AND ind.is_unique_constraint = 0
+		AND t.${tablesSelectedByTheUser.projection.isMsShipped} = 0
+	`;
 
-	return mapResponse(
-		await currentDbConnectionClient.query`
-		SELECT
-			TableName = t.name,
-			IndexName = ind.name,
-			ic.is_descending_key,
-			ic.is_included_column,
-			ic.column_store_order_ordinal,
-			COL_NAME(t.object_id, ic.column_id) as columnName,
-			S.name as schemaName,
-			p.data_compression_desc as dataCompression,
-			ind.*
-		FROM sys.indexes ind
-		LEFT JOIN sys.tables t
-			ON ind.object_id = t.object_id
-		INNER JOIN sys.index_columns ic
-			ON ind.object_id = ic.object_id AND ind.index_id = ic.index_id
-		INNER JOIN sys.partitions p
-			ON p.object_id = t.object_id AND ind.index_id = p.index_id
-		INNER JOIN sys.objects O ON O.object_id = t.object_id
-		INNER JOIN sys.schemas S ON S.schema_id = O.schema_id
-		WHERE
-			ind.is_primary_key = 0
-			AND ind.is_unique_constraint = 0
-			AND t.is_ms_shipped = 0
-		`,
-	);
+	return mapResponse(await currentDbConnectionClient.query(queryRetrievingTheIndexes));
 };
 
 const getViewsIndexes = async (connectionClient, dbName) => {
@@ -259,10 +267,12 @@ const getViewsIndexes = async (connectionClient, dbName) => {
 const getPartitions = async ({ connectionClient, tablesInfo, dbName, logger }) => {
 	logger.log('info', { message: `Get '${dbName}' database partitions.` }, 'Reverse Engineering');
 	const currentDbConnectionClient = await getNewConnectionClientByDb(connectionClient, dbName);
-	const tablesSelectedByTheUser = queryForRetrievingTheTablesSelectedByTheUser({ schemaToTablesMap: tablesInfo });
+	const tablesSelectedByTheUser = new PartitionsSubQueryForRetrievingTheTablesSelectedByTheUser({
+		schemaToTablesMap: tablesInfo,
+	}).getQuery();
 	const queryForRetrievingThePartitions = `
-    WITH user_selected_tables AS (${tablesSelectedByTheUser.sql()})
-    SELECT 
+		WITH user_selected_tables AS (${tablesSelectedByTheUser.sql()})
+		SELECT 
 			tbl.${tablesSelectedByTheUser.projection.schemaName} AS schemaName,
 			tbl.${tablesSelectedByTheUser.projection.tableName} AS tableName,
 			prt.partition_number,
@@ -300,23 +310,26 @@ const getTableColumnsDescription = async (connectionClient, dbName, tableName, s
 	`);
 };
 
-const getDatabaseMemoryOptimizedTables = async (connectionClient, dbName, logger) => {
+const getDatabaseMemoryOptimizedTables = async ({ connectionClient, tablesInfo, dbName, logger }) => {
 	try {
-		const currentDbConnectionClient = await getNewConnectionClientByDb(connectionClient, dbName);
-
 		logger.log('info', { message: `Get '${dbName}' database memory optimized indexes.` }, 'Reverse Engineering');
-
-		return mapResponse(currentDbConnectionClient.query`
+		const currentDbConnectionClient = await getNewConnectionClientByDb(connectionClient, dbName);
+		const tablesSelectedByTheUser = new MemoryOptimizedTablesSubQueryForRetrievingTheTablesSelectedByTheUser({
+			schemaToTablesMap: tablesInfo,
+		}).getQuery();
+		const queryForRetrievingMemoryOptimizedTables = `
+			WITH user_selected_tables AS (${tablesSelectedByTheUser.sql()})
 			SELECT
-				T.name,
-				T.durability,
-				T.durability_desc,
-				OBJECT_NAME(T.history_table_id) AS history_table,
-				SCHEMA_NAME(O.schema_id) AS history_schema,
-				T.temporal_type_desc
-			FROM sys.tables T LEFT JOIN sys.objects O ON T.history_table_id = O.object_id
-			WHERE T.is_memory_optimized=1
-		`);
+				T.${tablesSelectedByTheUser.projection.tableName},
+				T.${tablesSelectedByTheUser.projection.durability},
+				T.${tablesSelectedByTheUser.projection.durabilityDescription},
+				OBJECT_NAME(T.${tablesSelectedByTheUser.projection.historyTableId}) AS historyTable,
+				SCHEMA_NAME(O.schema_id) AS historySchema,
+				T.${tablesSelectedByTheUser.projection.temporalTypeDescription}
+			FROM user_selected_tables T LEFT JOIN sys.objects O ON T.${tablesSelectedByTheUser.projection.historyTableId} = O.object_id
+			WHERE T.${tablesSelectedByTheUser.projection.isMemoryOptimized}=1
+		`;
+		return mapResponse(currentDbConnectionClient.query(queryForRetrievingMemoryOptimizedTables));
 	} catch (error) {
 		logger.log('error', { message: error.message, stack: error.stack, error }, 'Retrieve memory optimized tables');
 
@@ -449,7 +462,7 @@ const getTableDefaultConstraintNames = async (connectionClient, dbName, tableNam
 	`);
 };
 
-const getDatabaseUserDefinedTypes = async (connectionClient, dbName, logger) => {
+const getDatabaseUserDefinedTypes = async ({ connectionClient, dbName, logger }) => {
 	const currentDbConnectionClient = await getNewConnectionClientByDb(connectionClient, dbName);
 
 	logger.log('info', { message: `Get '${dbName}' database UDTs.` }, 'Reverse Engineering');

reverse_engineering/queries/selectedTablesSubQuery/DatabaseIndexesSubQueryForRetrievingTheTablesSelectedByTheUser.js

@@ -0,0 +1,31 @@
+const { QueryForRetrievingTheTablesSelectedByTheUser } = require('./QueryForRetrievingTheTablesSelectedByTheUser');
+const { getProjectedPropertiesNames } = require('./getProjectedPropertiesNames');
+
+class DatabaseIndexesSubQueryForRetrievingTheTablesSelectedByTheUser extends QueryForRetrievingTheTablesSelectedByTheUser {
+	constructor({ schemaToTablesMap }) {
+		super();
+		this.schemaToTablesMap = schemaToTablesMap;
+	}
+
+	getQuery() {
+		const projection = {
+			'tbl.object_id': 'tableId',
+			'tbl.name': 'tableName',
+			'tbl.is_ms_shipped': 'isMsShipped',
+		};
+
+		const query = this.queryForRetrievingTheTablesSelectedByTheUser({
+			schemaToTablesMap: this.schemaToTablesMap,
+			projection,
+		});
+
+		return {
+			projection: getProjectedPropertiesNames({ projection }),
+			sql: () => query,
+		};
+	}
+}
+
+module.exports = {
+	DatabaseIndexesSubQueryForRetrievingTheTablesSelectedByTheUser,
+};

reverse_engineering/queries/selectedTablesSubQuery/MemoryOptimizedTablesSubQueryForRetrievingTheTablesSelectedByTheUser.js

@@ -0,0 +1,34 @@
+const { QueryForRetrievingTheTablesSelectedByTheUser } = require('./QueryForRetrievingTheTablesSelectedByTheUser');
+const { getProjectedPropertiesNames } = require('./getProjectedPropertiesNames');
+
+class MemoryOptimizedTablesSubQueryForRetrievingTheTablesSelectedByTheUser extends QueryForRetrievingTheTablesSelectedByTheUser {
+	constructor({ schemaToTablesMap }) {
+		super();
+		this.schemaToTablesMap = schemaToTablesMap;
+	}
+
+	getQuery() {
+		const projection = {
+			'tbl.name': 'tableName',
+			'tbl.durability': 'durability',
+			'tbl.durability_desc': 'durabilityDescription',
+			'tbl.history_table_id': 'historyTableId',
+			'tbl.temporal_type_desc': 'temporalTypeDescription',
+			'tbl.is_memory_optimized': 'isMemoryOptimized',
+		};
+
+		const query = this.queryForRetrievingTheTablesSelectedByTheUser({
+			schemaToTablesMap: this.schemaToTablesMap,
+			projection,
+		});
+
+		return {
+			projection: getProjectedPropertiesNames({ projection }),
+			sql: () => query,
+		};
+	}
+}
+
+module.exports = {
+	MemoryOptimizedTablesSubQueryForRetrievingTheTablesSelectedByTheUser,
+};

reverse_engineering/queries/selectedTablesSubQuery/PartitionsSubQueryForRetrievingTheTablesSelectedByTheUser.js

@@ -0,0 +1,31 @@
+const { QueryForRetrievingTheTablesSelectedByTheUser } = require('./QueryForRetrievingTheTablesSelectedByTheUser');
+const { getProjectedPropertiesNames } = require('./getProjectedPropertiesNames');
+
+class PartitionsSubQueryForRetrievingTheTablesSelectedByTheUser extends QueryForRetrievingTheTablesSelectedByTheUser {
+	constructor({ schemaToTablesMap }) {
+		super();
+		this.schemaToTablesMap = schemaToTablesMap;
+	}
+
+	getQuery() {
+		const projection = {
+			'tbl.object_id': 'tableId',
+			'tbl.name': 'tableName',
+			'sch.name': 'schemaName',
+		};
+
+		const query = this.queryForRetrievingTheTablesSelectedByTheUser({
+			schemaToTablesMap: this.schemaToTablesMap,
+			projection,
+		});
+
+		return {
+			projection: getProjectedPropertiesNames({ projection }),
+			sql: () => query,
+		};
+	}
+}
+
+module.exports = {
+	PartitionsSubQueryForRetrievingTheTablesSelectedByTheUser,
+};

reverse_engineering/queries/selectedTablesSubQuery/QueryForRetrievingTheTablesSelectedByTheUser.js

@@ -0,0 +1,47 @@
+class QueryForRetrievingTheTablesSelectedByTheUser {
+	#buildPredicateForTable({ schema, table }) {
+		return `(sch.name = '${schema}' AND tbl.name = '${table}')`;
+	}
+
+	#buildPredicateForTablesInSchema({ schema, tables }) {
+		return tables.map(table => this.#buildPredicateForTable({ schema, table })).join('OR');
+	}
+
+	#buildProjectionForProperty({ propertyName, projectionName }) {
+		return `${propertyName} AS ${projectionName}`;
+	}
+
+	#buildProjectionsForPropertiesSelectedByQuery({ projection }) {
+		const initialPropertiesNames = Object.keys(projection);
+
+		return initialPropertiesNames
+			.map(initialPropertyName =>
+				this.#buildProjectionForProperty({
+					propertyName: initialPropertyName,
+					projectionName: projection[initialPropertyName],
+				}),
+			)
+			.join(',');
+	}
+
+	queryForRetrievingTheTablesSelectedByTheUser({ schemaToTablesMap, projection }) {
+		const propertiesToSelectProjections = this.#buildProjectionsForPropertiesSelectedByQuery({
+			projection,
+		});
+		const predicate = Object.entries(schemaToTablesMap)
+			.map(([schema, tables]) => this.#buildPredicateForTablesInSchema({ schema, tables }))
+			.join('OR');
+		const whereClause = Object.entries(schemaToTablesMap).length > 0 ? `WHERE ${predicate}` : '';
+		return `
+			SELECT
+				${propertiesToSelectProjections}
+			FROM sys.tables tbl
+			JOIN sys.schemas sch ON sch.schema_id = tbl.schema_id
+			${whereClause}
+		  `;
+	}
+}
+
+module.exports = {
+	QueryForRetrievingTheTablesSelectedByTheUser,
+};

reverse_engineering/queries/selectedTablesSubQuery/getProjectedPropertiesNames.js

@@ -0,0 +1,6 @@
+const getProjectedPropertiesNames = ({ projection }) =>
+	Object.fromEntries(Object.values(projection).map(projectedName => [projectedName, projectedName]));
+
+module.exports = {
+	getProjectedPropertiesNames,
+};

reverse_engineering/reverseEngineeringService/reverseEngineeringService.js

@@ -336,11 +336,11 @@ const getMemoryOptimizedOptions = options => {
 
 	return {
 		memory_optimized: true,
-		durability: ['SCHEMA_ONLY', 'SCHEMA_AND_DATA'].includes(String(options.durability_desc).toUpperCase())
-			? String(options.durability_desc).toUpperCase()
+		durability: ['SCHEMA_ONLY', 'SCHEMA_AND_DATA'].includes(String(options.durabilityDescription).toUpperCase())
+			? String(options.durabilityDescription).toUpperCase()
 			: '',
-		systemVersioning: options.temporal_type_desc === 'SYSTEM_VERSIONED_TEMPORAL_TABLE',
-		historyTable: options.history_table ? `${options.history_schema}.${options.history_table}` : '',
+		systemVersioning: options.temporalTypeDescription === 'SYSTEM_VERSIONED_TEMPORAL_TABLE',
+		historyTable: options.historyTable ? `${options.historySchema}.${options.historyTable}` : '',
 	};
 };
 
@@ -377,7 +377,7 @@ const getIndexing = (indexingInfo, order) => {
 };
 
 const getOrder = indexingInfo => {
-	return indexingInfo.filter(column => column.column_store_order_ordinal).map(column => column.COLUMN_NAME);
+	return indexingInfo.filter(column => column.column_store_order_ordinal).map(column => column.columnName);
 };
 
 const getTableRole = (distribution, indexing) => {
@@ -404,11 +404,13 @@ const reverseCollectionsToJSON = logger => async (dbConnectionClient, tablesInfo
 	const dbName = dbConnectionClient.config.database;
 	progress(logger, `RE data from database "${dbName}"`, dbName);
 	const [databaseIndexes, databaseMemoryOptimizedTables, databaseUDT, dataBasePartitions] = await Promise.all([
-		getDatabaseIndexes(dbConnectionClient, dbName, logger).catch(logError(logger, 'Getting indexes')),
-		getDatabaseMemoryOptimizedTables(dbConnectionClient, dbName, logger).catch(
+		getDatabaseIndexes({ connectionClient: dbConnectionClient, tablesInfo, dbName, logger }).catch(
+			logError(logger, 'Getting indexes'),
+		),
+		getDatabaseMemoryOptimizedTables({ connectionClient: dbConnectionClient, tablesInfo, dbName, logger }).catch(
 			logError(logger, 'Getting memory optimized tables'),
 		),
-		getDatabaseUserDefinedTypes(dbConnectionClient, dbName, logger).catch(
+		getDatabaseUserDefinedTypes({ connectionClient: dbConnectionClient, dbName, logger }).catch(
 			logError(logger, 'Getting user defined types'),
 		),
 		getPartitions({ connectionClient: dbConnectionClient, tablesInfo, dbName, logger }).catch(