Merge pull request #91 from mountaindude/67-app-filter

#67 app filters
ptarmiganlabs · Mar 13, 2023 · 7f9b3fb · 7f9b3fb
2 parents c214881 + d7fe89a
commit 7f9b3fb
Show file tree

Hide file tree

Showing 9 changed files with 614 additions and 266 deletions.
diff --git a/README.md b/README.md
@@ -25,6 +25,7 @@ The tool will
     - [Load scripts](#load-scripts-1)
     - [Data connection definitions](#data-connection-definitions)
   - [Config file](#config-file)
+    - [App filters](#app-filters)
   - [Logging](#logging)
   - [Parallel extraction of lineage data](#parallel-extraction-of-lineage-data)
   - [Running Butler Spyglass](#running-butler-spyglass)
@@ -115,6 +116,9 @@ All parameters must be defined in the config file - run time errors will occur o
 | scriptExtract.exportDir | Directory where script files will be stored. |
 | dataConnectionExtract.enable | Controls whether data connections are extracted to JSON file not. true/false |
 | dataConnectionExtract.exportDir | Directory where data connections JSON file will be stored. |
+| appFilter.appNameExact | List of apps for which lineage and/or load scripts should be extracted. An exact match on app name is done. |
+| appFilter.appId | App ids for which lineage and/or load scripts should be extracted. |
+| appFilter.appTag | Lineage and/or load scripts will be extracted for apps with these tags set. |
 | configEngine.engineVersion | Version of the Qlik Sense engine running on the target server. Version 12.612.0 should work with any Qlik Sense server from 2020 February and later. |
 | configEngine.host | Host name, fully qualified domain name (=FQDN) or IP address of Qlik Sense Enterprise server where Qlik Engine Service (QES) is running. |
 | configEngine.port | Should be 4747, unless configured otherwise in the QMC. |
@@ -130,6 +134,10 @@ All parameters must be defined in the config file - run time errors will occur o
 | cert.clientCert | Client certificate, as exported from the QMC |
 | cert.clientCertKey | Client certificate key, as exported from the QMC |
 
+### App filters
+
+All apps will be processed (=lineage and/or load scripts extracted ) if no app filters at all are set in the config file.
+
 ## Logging
 
 Console logs are always enabled, with configurable logging level (in the YAML config file).  
@@ -228,6 +236,20 @@ ButlerSpyglass:
     enable: true                    # Should data connections definitions be saved to files? One JSON file with all data connections will be created.
     exportDir: ./out/dataconnection # Directory where data connection JSON definitions file will be stored.
 
+  # Filter out a selection of apps for which lineage and/or load scripts should be extracted.
+  # Filters are additive.
+  # If no filters are specified lineage/script will be extracted for all apps in the Sense server.
+  appFilter:
+    appNameExact:                   # Apps for which lineage/script should be extract. Exact matches are done on app name. 
+      - User retention
+      - Butler 8.4 demo app
+    appId:                          # App IDs for which lineage/script should be extracted.
+      - d1ace221-b80e-4754-98ea-3d0a9ebc9632
+      - bf4cbb34-cd3c-4fc4-b69d-6fa61d5a270e
+    appTag:                         # Lineage/script will be extracted for apps having these tags set.
+      - Test data
+      - apiCreated
+      
   configEngine:
     engineVersion: 12.612.0         # Qlik Associative Engine version to use with Enigma.js. ver 12.612.0 works with Feb 2020 and later
     host: 192.168.100.109
@@ -347,6 +369,20 @@ ButlerSpyglass:
     enable: true                      # Should data connections definitions be saved to files? One JSON file with all data connections will be created.
     exportDir: ./out/dataconnection   # Directory where data connection JSON definitions file will be stored.
 
+  # Filter out a selection of apps for which lineage and/or load scripts should be extracted.
+  # Filters are additive.
+  # If no filters are specified lineage/script will be extracted for all apps in the Sense server.
+  appFilter:
+    appNameExact:                   # Apps for which lineage/script should be extract. Exact matches are done on app name. 
+      - User retention
+      - Butler 8.4 demo app
+    appId:                          # App IDs for which lineage/script should be extracted.
+      - d1ace221-b80e-4754-98ea-3d0a9ebc9632
+      - bf4cbb34-cd3c-4fc4-b69d-6fa61d5a270e
+    appTag:                         # Lineage/script will be extracted for apps having these tags set.
+      - Test data
+      - apiCreated      
+
   configEngine:
     engineVersion: 12.612.0         # Qlik Associative Engine version to use with Enigma.js. ver 12.612.0 works with Feb 2020 and later 
     host: sense.ptarmiganlabs.com
@@ -428,24 +464,25 @@ This richness can be a problem though. If an inline table contains a thousand ro
 That's where the ```maxLengthDiscriminator``` config option (in the config YAML file) comes in handy. It makes it possible to set a limit to how many characters should be included for each row of lineage data.
 The setting is global for all apps, and applies to all rows of lineage data extracted from Sense.
 
-    AppId,Discriminator,Statement
-    10793a99-ef94-46ad-ae33-6a9efd260ab3,DSN=AUTOGENERATE;,
-    b7ef5bff-5a13-4d61-bae4-45b5fab722f9,RESIDENT RestConnectorMasterTable;,
-    916234f2-ffdb-4506-9c5b-193063da05ab,DSN=AUTOGENERATE;,
-    916234f2-ffdb-4506-9c5b-193063da05ab,DSN=https://bolin.su.se/data/stockholm/files/stockholm-historical-weather-observations-ver-1.0.2016/temperature/daily/raw/stockholma_daily_temp_obs_2013_2016_t1t2t3txtntm.txt;,
-    65c7ea3e-242d-423f-aa01-8acd24e4a7ed,DSN=AUTOGENERATE;,
-    8bd92672-8cf4-446e-8f5e-8c77205eefc3,DSN=AUTOGENERATE;,
-    0b94c6f9-82ce-40f1-a38c-8d277b01498e,RESIDENT RestConnectorMasterTable;,
-    0b94c6f9-82ce-40f1-a38c-8d277b01498e,{STORE - [Lib://App metadata/app_dump/20190125_apps.qvd] (qvd)};,
-    0b94c6f9-82ce-40f1-a38c-8d277b01498e,{STORE - [Lib://App metadata/app_dump/current_apps.qvd] (qvd)};,
-    0b94c6f9-82ce-40f1-a38c-8d277b01498e,monitor_apps_REST_app;,"RestConnectorMasterTable:
-    SQL SELECT
-        ""id"" ,
-        ""name"",
-        ""__KEY_root""
-    FROM JSON (wrap on) ""root"" PK ""__KEY_root"""
-    ,,
-    62dc4e60-8ba6-48af-9eac-9a076fd35819,DSN=AUTOGENERATE;,
+Here is an example lineage file. Note that both QVDs, SQL statements and inline tables are included in the lineage data. 
+
+    AppId,AppName,Discriminator,Statement
+    c840670c-7178-4a5e-8409-ba2da69127e2,Meetup.com,,
+    c840670c-7178-4a5e-8409-ba2da69127e2,Meetup.com,Healthcheck;,"RestConnectorMasterTable:
+        SQL SELECT 
+            ""col_1""
+        FROM CSV (header off, delimiter "","", quote """""""") ""CSV_source""
+        WITH CONNECTION(Url ""http://healthcheck.ptarmiganlabs.net:8000/ping/10a887bf-4580-4891-9c6f-2affbd380f16"")"
+    c840670c-7178-4a5e-8409-ba2da69127e2,Meetup.com,INLINE;,
+    c840670c-7178-4a5e-8409-ba2da69127e2,Meetup.com,RESIDENT __cityAliasesBase;,
+    c840670c-7178-4a5e-8409-ba2da69127e2,Meetup.com,RESIDENT __cityGeoBase;,
+    c840670c-7178-4a5e-8409-ba2da69127e2,Meetup.com,RESIDENT __countryAliasesBase;,
+    c840670c-7178-4a5e-8409-ba2da69127e2,Meetup.com,RESIDENT __countryGeoBase;,
+    c840670c-7178-4a5e-8409-ba2da69127e2,Meetup.com,\\fileshare1\testdata\meetupcom\categories.csv;,
+    c840670c-7178-4a5e-8409-ba2da69127e2,Meetup.com,\\pro\sensedata\staticcontent\appcontent\c840670c-7178-4a5e-8409-ba2da69127e2\cityaliases.qvd;,
+    c840670c-7178-4a5e-8409-ba2da69127e2,Meetup.com,\\pro\sensedata\staticcontent\appcontent\c840670c-7178-4a5e-8409-ba2da69127e2\citygeo.qvd;,
+    c840670c-7178-4a5e-8409-ba2da69127e2,Meetup.com,\\pro\sensedata\staticcontent\appcontent\c840670c-7178-4a5e-8409-ba2da69127e2\countryaliases.qvd;,
+    c840670c-7178-4a5e-8409-ba2da69127e2,Meetup.com,\\pro\sensedata\staticcontent\appcontent\c840670c-7178-4a5e-8409-ba2da69127e2\countrygeo.qvd;,
 
 ### Load script output files
 

diff --git a/butler-spyglass.js b/butler-spyglass.js
@@ -1,14 +1,13 @@
 const config = require('config');
 const fs = require('fs-extra');
 const Queue = require('better-queue');
-const enigma = require('enigma.js');
-const WebSocket = require('ws');
 const upath = require('upath');
 
 // Load our own code
-const extractApp = require('./src/extract_app');
+const { appExtractMetadata, resetExtractedAppCount } = require('./src/extract_app');
 const { getDataConnections } = require('./src/get_dataconnection');
 const { logger, getLoggingLevel } = require('./src/logger');
+const { getAppsToProcess } = require('./src/qrs');
 
 // Get app version from package.json file
 const appVersion = require('./package.json').version;
@@ -18,26 +17,6 @@ const appName = require('./package.json').name;
 const isPkg = typeof process.pkg !== 'undefined';
 const execPath = isPkg ? upath.dirname(process.execPath) : __dirname;
 
-// Helper function to read the contents of the certificate files:
-const readCert = (filename) => fs.readFileSync(filename);
-
-//  Engine config
-const configEngine = {
-    engineVersion: config.get('ButlerSpyglass.configEngine.engineVersion'),
-    host: config.get('ButlerSpyglass.configEngine.host'),
-    port: config.get('ButlerSpyglass.configEngine.port'),
-    isSecure: config.get('ButlerSpyglass.configEngine.useSSL'),
-    headers: config.get('ButlerSpyglass.configEngine.headers'),
-    ca: readCert(config.get('ButlerSpyglass.cert.clientCertCA')),
-    cert: readCert(config.get('ButlerSpyglass.cert.clientCert')),
-    key: readCert(config.get('ButlerSpyglass.cert.clientCertKey')),
-    rejectUnauthorized: config.get('ButlerSpyglass.configEngine.rejectUnauthorized'),
-};
-
-// Set up enigma.js configuration
-// eslint-disable-next-line import/no-dynamic-require
-const qixSchema = require(`enigma.js/schemas/${configEngine.engineVersion}`);
-
 logger.info(`--------------------------------------`);
 logger.info(`| ${appName}`);
 logger.info(`|  `);
@@ -75,7 +54,7 @@ const q = new Queue(
 
         // eslint-disable-next-line no-underscore-dangle
         const _self = this;
-        const newLocal = extractApp.appExtractMetadata(_self, q, taskItem, cb);
+        const newLocal = appExtractMetadata(_self, q, taskItem, cb);
 
         // cb();
     },
@@ -95,83 +74,51 @@ const q = new Queue(
 // })
 
 // Define function to be scheduled
-const scheduledExtract = function scheduledExtract() {
+const scheduledExtract = async function scheduledExtract() {
     // Write separator to separate this run from the previous one
     logger.info(`--------------------------------------`);
     logger.info(`Extraction run started`);
 
-    // Get data connections
-    if (config.get('ButlerSpyglass.dataConnectionExtract.enable') === true) {
-        getDataConnections();
-    }
-
     // Empty output folders
     fs.emptyDirSync(upath.resolve(upath.normalize(`${config.get('ButlerSpyglass.lineageExtract.exportDir')}/`)));
     fs.emptyDirSync(upath.resolve(upath.normalize(`${config.get('ButlerSpyglass.scriptExtract.exportDir')}/`)));
     fs.emptyDirSync(upath.resolve(upath.normalize(`${config.get('ButlerSpyglass.dataConnectionExtract.exportDir')}/`)));
 
-    // create a new session
-    const configEnigma = {
-        schema: qixSchema,
-        url: `wss://${configEngine.host}:${configEngine.port}`,
-        createSocket: (url) =>
-            new WebSocket(url, {
-                ca: [configEngine.ca],
-                key: configEngine.key,
-                cert: configEngine.cert,
-                headers: {
-                    'X-Qlik-User': 'UserDirectory=Internal;UserId=sa_repository',
-                },
-                rejectUnauthorized: false,
-            }),
-    };
-
-    const sessionAppList = enigma.create(configEnigma);
-
-    sessionAppList
-        .open()
-        .then((global) => {
-            // We can now interact with the global object, for example get the document list.
-            global
-                .getDocList()
-                .then((list) => {
-                    logger.silly(`Apps on this Engine that the configured user can open: ${JSON.stringify(list, null, 2)}`);
-                    logger.info(`Number of apps on server: ${list.length}`);
-
-                    // Reset # processed apps to zero
-                    extractApp.resetExtractedAppCount();
-
-                    // Send tasks to queue
-                    list.forEach((element) => {
-                        q.push(element).on('failed', (err) => {
-                            // Task failed!
-                            logger.error(`Task FAILED: ${err}`);
-                        });
-                    });
-
-                    q.on('progress', (progress) => {
-                        // logger.verbose(`========== Task progress: ${taskId}, ${completed}/${total} done`);
-                        logger.verbose(`========== Task progress: ${progress}`);
-                        // progress.eta - human readable string estimating time remaining
-                        // progress.pct - % complete (out of 100)
-                        // progress.complete - # completed so far
-                        // progress.total - # for completion
-                        // progress.message - status message
-                    });
-                })
-
-                .then(() => {
-                    try {
-                        sessionAppList.close();
-                    } catch (ex) {
-                        logger.error(`Error when closing sessionAppList: ${ex}`);
-                    }
-                });
-        })
-        .catch((error) => {
-            logger.error('Failed to open session and/or retrieve the app list:', error);
-            process.exit(1);
+    // Get data connections
+    if (config.get('ButlerSpyglass.dataConnectionExtract.enable') === true) {
+        await getDataConnections(execPath);
+    }
+
+    // Create list of apps for which lineage and/or load scripts should be extracted.
+    // If at least one filter is specified in the config file that filter will be used.
+    // If no filters are specified all apps that the user has access to will be processed.
+
+    const appsToProcess = await getAppsToProcess(logger, config, execPath);
+
+    logger.info(`Number of apps that will be processed: ${appsToProcess.length}`);
+    logger.silly(`Apps on this server that will be processed: ${JSON.stringify(appsToProcess, null, 2)}`);
+
+    // Reset # processed apps to zero
+    resetExtractedAppCount();
+
+    // eslint-disable-next-line no-restricted-syntax
+    // Send tasks to queue
+    appsToProcess.forEach((element) => {
+        q.push(element).on('failed', (err) => {
+            // Task failed!
+            logger.error(`Task FAILED: ${err}`);
         });
+    });
+
+    q.on('progress', (progress) => {
+        // logger.verbose(`========== Task progress: ${taskId}, ${completed}/${total} done`);
+        logger.verbose(`========== Task progress: ${progress}`);
+        // progress.eta - human readable string estimating time remaining
+        // progress.pct - % complete (out of 100)
+        // progress.complete - # completed so far
+        // progress.total - # for completion
+        // progress.message - status message
+    });
 };
 
 q.on('task_finish', (taskId, result) => {

diff --git a/changelog.md b/changelog.md
@@ -1,5 +1,12 @@
 # Changelog
 
+## [2.0.4](https://github.com/ptarmiganlabs/butler-spyglass/compare/butler-spyglass-v2.0.3...butler-spyglass-v2.0.4) (2023-03-12)
+
+
+### Build pipeline
+
+* Failing Docker image build ([ff8d030](https://github.com/ptarmiganlabs/butler-spyglass/commit/ff8d030799f9d9e2faa7a3eb2bfb8319dc0c83d3))
+
 ## [2.0.3](https://github.com/ptarmiganlabs/butler-spyglass/compare/butler-spyglass-v2.0.2...butler-spyglass-v2.0.3) (2023-03-12)
 
 

diff --git a/config/production-template.yaml b/config/production-template.yaml
@@ -27,6 +27,20 @@ ButlerSpyglass:
     enable: true                    # Should data connections definitions be saved to files? One JSON file with all data connections will be created.
     exportDir: ./out/dataconnection # Directory where data connection JSON definitions file will be stored.
 
+  # Filter out a selection of apps for which lineage and/or load scripts should be extracted.
+  # Filters are additive.
+  # If no filters are specified lineage/script will be extracted for all apps in the Sense server.
+  appFilter:
+    appNameExact:                   # Apps for which lineage/script should be extract. Exact matches are done on app name. 
+      - User retention
+      - Butler 8.4 demo app
+    appId:                          # App IDs for which lineage/script should be extracted.
+      - d1ace221-b80e-4754-98ea-3d0a9ebc9632
+      - bf4cbb34-cd3c-4fc4-b69d-6fa61d5a270e
+    appTag:                         # Lineage/script will be extracted for apps having these tags set.
+      - Test data
+      - apiCreated
+
   configEngine:
     engineVersion: 12.612.0         # Qlik Associative Engine version to use with Enigma.js. ver 12.612.0 works with Feb 2020 and later 
     host: sense.ptarmiganlabs.com