Trigger metabase db sync on data sync to populate filters (#242)

* Trigger metabase db sync on data sync to populate filters * Inject metabase credentials from .env file Co-authored-by: Yandry Perez Clemente <[email protected]>
faros-ai · Oct 28, 2022 · 535a101 · 535a101
1 parent 3ccfe5f
commit 535a101
Show file tree

Hide file tree

Showing 9 changed files with 161 additions and 13 deletions.
diff --git a/cli/src/airbyte/airbyte-client.ts b/cli/src/airbyte/airbyte-client.ts
@@ -1,6 +1,6 @@
 import retry from 'async-retry';
 import axios, {AxiosInstance} from 'axios';
-import { ceil } from 'lodash';
+import {ceil} from 'lodash';
 import ProgressBar from 'progress';
 import {VError} from 'verror';
 

diff --git a/cli/src/bitbucket/run.ts b/cli/src/bitbucket/run.ts
@@ -5,6 +5,7 @@ import VError from 'verror';
 
 import {Airbyte} from '../airbyte/airbyte-client';
 import {wrapApiError} from '../cli';
+import {Metabase} from '../metabase/metabase-client';
 import {
   display,
   Emoji,
@@ -27,6 +28,7 @@ const DEFAULT_API_URL = 'https://api.bitbucket.org/2.0';
 
 interface BitbucketConfig {
   readonly airbyte: Airbyte;
+  readonly metabase: Metabase;
   readonly serverUrl?: string;
   readonly username?: string;
   readonly password?: string;
@@ -62,8 +64,12 @@ export function makeBitbucketCommand(): Command {
 
   cmd.action(async (options) => {
     const airbyte = new Airbyte(options.airbyteUrl);
-
-    await runBitbucket({...options, airbyte});
+    const metabase = await Metabase.fromConfig({
+      url: options.metabaseUrl,
+      username: options.metabaseUsername,
+      password: options.metabasePassword,
+    });
+    await runBitbucket({...options, airbyte, metabase});
   });
 
   return cmd;
@@ -224,6 +230,14 @@ export async function runBitbucket(cfg: BitbucketConfig): Promise<void> {
     cfg.cutoffDays || DEFAULT_CUTOFF_DAYS,
     repos?.length || 0
   );
+
+  try {
+    await cfg.metabase.forceSync();
+  } catch (error) {
+    // main intent is to have filters immediately populated with values
+    // we do nothing on failure, basic functionalities are not impacted
+    // daily/hourly metabase db scans will eventually get us there
+  }
 }
 
 interface Workspace {

diff --git a/cli/src/cli.ts b/cli/src/cli.ts
@@ -6,11 +6,14 @@ import {makeBitbucketCommand, runBitbucket} from './bitbucket/run';
 import {makeGithubCommand, runGithub} from './github/run';
 import {makeGitlabCommand, runGitlab} from './gitlab/run';
 import {makeJiraCommand, runJira} from './jira/run';
+import {Metabase} from './metabase/metabase-client';
 import {display, terminalLink} from './utils';
 import {runSelect} from './utils/prompts';
 
 const DEFAULT_AIRBYTE_URL = 'http://localhost:8000';
 const DEFAULT_METABASE_URL = 'http://localhost:3000';
+const DEFAULT_METABASE_USER = '[email protected]';
+const DEFAULT_METABASE_PASSWORD = 'admin';
 
 export function wrapApiError(cause: unknown, msg: string): Error {
   // Omit verbose axios error
@@ -33,6 +36,12 @@ export async function main(): Promise<void> {
     .command('pick-source', {isDefault: true, hidden: true})
     .action(async (options) => {
       const airbyte = new Airbyte(options.airbyteUrl);
+      const metabase = await Metabase.fromConfig({
+        url: options.metabaseUrl,
+        username: options.metabaseUsername,
+        password: options.metabasePassword,
+      });
+
       let done = false;
       while (!done) {
         const source = await runSelect({
@@ -48,16 +57,16 @@ export async function main(): Promise<void> {
         });
         switch (source) {
           case 'GitHub (Cloud)':
-            await runGithub({airbyte});
+            await runGithub({airbyte, metabase});
             break;
           case 'GitLab (Cloud / Server)':
-            await runGitlab({airbyte});
+            await runGitlab({airbyte, metabase});
             break;
           case 'Bitbucket (Cloud / Server)':
-            await runBitbucket({airbyte});
+            await runBitbucket({airbyte, metabase});
             break;
           case 'Jira (Cloud)':
-            await runJira({airbyte});
+            await runJira({airbyte, metabase});
             break;
           case 'I\'m done!':
             done = true;
@@ -69,6 +78,16 @@ export async function main(): Promise<void> {
     cmd.option('--airbyte-url <string>', 'Airbyte URL', DEFAULT_AIRBYTE_URL);
     cmd
       .option('--metabase-url <string>', 'Metabase URL', DEFAULT_METABASE_URL)
+      .option(
+        '--metabase-username <string>',
+        'Metabase username',
+        DEFAULT_METABASE_USER
+      )
+      .option(
+        '--metabase-password <string>',
+        'Metabase password',
+        DEFAULT_METABASE_PASSWORD
+      )
       .hook('postAction', async (thisCommand) => {
         display(
           `Check out your metrics in ${await terminalLink(

diff --git a/cli/src/github/run.ts b/cli/src/github/run.ts
@@ -4,6 +4,7 @@ import VError from 'verror';
 
 import {Airbyte} from '../airbyte/airbyte-client';
 import {wrapApiError} from '../cli';
+import {Metabase} from '../metabase/metabase-client';
 import {
   display,
   Emoji,
@@ -26,6 +27,7 @@ const DEFAULT_CUTOFF_DAYS = 30;
 
 interface GithubConfig {
   readonly airbyte: Airbyte;
+  readonly metabase: Metabase;
   readonly token?: string;
   readonly repoList?: ReadonlyArray<string>;
   readonly cutoffDays?: number;
@@ -51,8 +53,13 @@ export function makeGithubCommand(): Command {
 
   cmd.action(async (options) => {
     const airbyte = new Airbyte(options.airbyteUrl);
+    const metabase = await Metabase.fromConfig({
+      url: options.metabaseUrl,
+      username: options.metabaseUsername,
+      password: options.metabasePassword,
+    });
 
-    await runGithub({...options, airbyte});
+    await runGithub({...options, airbyte, metabase});
   });
 
   return cmd;
@@ -143,6 +150,14 @@ export async function runGithub(cfg: GithubConfig): Promise<void> {
     cfg.cutoffDays || DEFAULT_CUTOFF_DAYS,
     repos?.length || 0
   );
+
+  try {
+    await cfg.metabase.forceSync();
+  } catch (error) {
+    // main intent is to have filters immediately populated with values
+    // we do nothing on failure, basic functionalities are not impacted
+    // daily/hourly metabase db scans will eventually get us there
+  }
 }
 
 async function promptForRepos(token: string): Promise<ReadonlyArray<string>> {

diff --git a/cli/src/gitlab/run.ts b/cli/src/gitlab/run.ts
@@ -4,6 +4,7 @@ import VError from 'verror';
 
 import {Airbyte} from '../airbyte/airbyte-client';
 import {wrapApiError} from '../cli';
+import {Metabase} from '../metabase/metabase-client';
 import {
   display,
   Emoji,
@@ -28,6 +29,7 @@ const DEFAULT_API_URL = 'gitlab.com';
 
 interface GitLabConfig {
   readonly airbyte: Airbyte;
+  readonly metabase: Metabase;
   readonly apiUrl?: string;
   readonly token?: string;
   readonly projectList?: ReadonlyArray<string>;
@@ -55,8 +57,13 @@ export function makeGitlabCommand(): Command {
 
   cmd.action(async (options) => {
     const airbyte = new Airbyte(options.airbyteUrl);
+    const metabase = await Metabase.fromConfig({
+      url: options.metabaseUrl,
+      username: options.metabaseUsername,
+      password: options.metabasePassword,
+    });
 
-    await runGitlab({...options, airbyte});
+    await runGitlab({...options, airbyte, metabase});
   });
 
   return cmd;
@@ -151,6 +158,14 @@ export async function runGitlab(cfg: GitLabConfig): Promise<void> {
     cfg.cutoffDays || DEFAULT_CUTOFF_DAYS,
     projects?.length || 0
   );
+
+  try {
+    await cfg.metabase.forceSync();
+  } catch (error) {
+    // main intent is to have filters immediately populated with values
+    // we do nothing on failure, basic functionalities are not impacted
+    // daily/hourly metabase db scans will eventually get us there
+  }
 }
 
 async function promptForProjects(

diff --git a/cli/src/jira/run.ts b/cli/src/jira/run.ts
@@ -4,6 +4,7 @@ import VError from 'verror';
 
 import {Airbyte} from '../airbyte/airbyte-client';
 import {wrapApiError} from '../cli';
+import {Metabase} from '../metabase/metabase-client';
 import {
   display,
   Emoji,
@@ -27,6 +28,7 @@ const DEFAULT_CUTOFF_DAYS = 30;
 
 interface JiraConfig {
   readonly airbyte: Airbyte;
+  readonly metabase: Metabase;
   readonly email?: string;
   readonly token?: string;
   readonly domain?: string;
@@ -55,8 +57,13 @@ export function makeJiraCommand(): Command {
 
   cmd.action(async (options) => {
     const airbyte = new Airbyte(options.airbyteUrl);
+    const metabase = await Metabase.fromConfig({
+      url: options.metabaseUrl,
+      username: options.metabaseUsername,
+      password: options.metabasePassword,
+    });
 
-    await runJira({...options, airbyte});
+    await runJira({...options, airbyte, metabase});
   });
 
   return cmd;
@@ -171,6 +178,14 @@ export async function runJira(cfg: JiraConfig): Promise<void> {
     cfg.cutoffDays || DEFAULT_CUTOFF_DAYS,
     projects?.length || 0
   );
+
+  try {
+    await cfg.metabase.forceSync();
+  } catch (error) {
+    // main intent is to have filters immediately populated with values
+    // we do nothing on failure, basic functionalities are not impacted
+    // daily/hourly metabase db scans will eventually get us there
+  }
 }
 
 async function promptForProjects(

diff --git a/cli/src/metabase/metabase-client.ts b/cli/src/metabase/metabase-client.ts
@@ -0,0 +1,56 @@
+import axios, {AxiosInstance} from 'axios';
+import {VError} from 'verror';
+
+export function wrapApiError(cause: unknown, msg: string): Error {
+  // Omit verbose axios error
+  const truncated = new VError((cause as Error).message);
+  return new VError(truncated, msg);
+}
+
+export interface MetabaseConfig {
+  readonly url: string;
+  readonly username: string;
+  readonly password: string;
+}
+
+export class Metabase {
+  constructor(private readonly api: AxiosInstance) {}
+
+  static async fromConfig(cfg: MetabaseConfig): Promise<Metabase> {
+    const token = await Metabase.sessionToken(cfg);
+    const api = axios.create({
+      baseURL: `${cfg.url}/api`,
+      headers: {
+        'X-Metabase-Session': token,
+      },
+    });
+    return new Metabase(api);
+  }
+
+  private static async sessionToken(cfg: MetabaseConfig): Promise<string> {
+    const {url, username, password} = cfg;
+    try {
+      const {data} = await axios.post(`${url}/api/session`, {
+        username,
+        password,
+      });
+      return data.id;
+    } catch (err) {
+      throw wrapApiError(err, 'failed to get session token');
+    }
+  }
+
+  async forceSync(): Promise<any> {
+    try {
+      // Objective is to get filter values populated
+      // Faros is always has DB id 2
+      // note that API call rescan_value was not sufficient
+      // hence the call to sync_schema instead
+      // https://www.metabase.com/docs/latest/api/database
+      const {data} = await this.api.post('database/2/sync_schema');
+      return data;
+    } catch (err) {
+      throw wrapApiError(err, 'unable to trigger rescan');
+    }
+  }
+}
diff --git a/init/resources/metabase/dashboards/git.json b/init/resources/metabase/dashboards/git.json
@@ -1600,8 +1600,7 @@
       "name": "Repository",
       "slug": "repository",
       "id": "81276e50",
-      "type": "string/=",
-      "sectionId": "string"
+      "type": "category"
     }
   ],
   "layout": [
@@ -2202,6 +2201,16 @@
       "visualization_settings": {}
     }
   ],
+  "fields": [
+    {
+      "field": {{ field "vcs_Repository.name" }},
+      "type": "type/Category"
+    },
+    {
+      "field": {{ field "tms_TaskBoard.name" }},
+      "type": "type/Category"
+    }
+  ],
   "path": "/Faros CE/Git",
   "priority": 14,
   "bookmark": true

diff --git a/start.sh b/start.sh
@@ -76,7 +76,12 @@ main() {
     docker pull farosai/faros-ce-cli:latest
     AIRBYTE_URL=$(grep "^WEBAPP_URL" .env| sed 's/^WEBAPP_URL=//')
     METABASE_PORT=$(grep "^METABASE_PORT" .env| sed 's/^METABASE_PORT=//')
-    docker run --network host -it farosai/faros-ce-cli pick-source --airbyte-url "$AIRBYTE_URL" --metabase-url "http://localhost:$METABASE_PORT"
+    METABASE_USER=$(grep "^METABASE_USER" .env| sed 's/^METABASE_USER=//')
+    METABASE_PASSWORD=$(grep "^METABASE_PASSWORD" .env| sed 's/^METABASE_PASSWORD=//')
+    docker run --network host -it farosai/faros-ce-cli pick-source --airbyte-url "$AIRBYTE_URL" \
+    --metabase-url "http://localhost:$METABASE_PORT" \
+    --metabase-username "$METABASE_USER" \
+    --metabase-password "$METABASE_PASSWORD"
   fi
 }