Merge pull request #888 from InfuseAI/feature/sc-31643/support-piperi…

…der-compare-commit

[Feature] Support piperider compare between git refs

piperider_cli/cli.py

@@ -430,6 +430,7 @@ def cloud_compare_reports(**kwargs):
 
 
 @cli.command(name='compare', short_help='Compare the change for the current branch.', cls=TrackCommand)
+@click.argument('ref', required=False, type=click.STRING)
 @click.option('--recipe', default=None, type=click.STRING, help='Select a different recipe.')
 @click.option('--upload', default=False, is_flag=True, help='Upload the report to PipeRider Cloud.')
 @click.option('--share', default=False, is_flag=True, help='Enable public share of the report to PipeRider Cloud.')
@@ -450,13 +451,33 @@ def cloud_compare_reports(**kwargs):
 ])
 @add_options(dbt_related_options)
 @add_options(debug_option)
-def compare_with_recipe(**kwargs):
+def compare_with_recipe(ref, **kwargs):
     """
     Generate the comparison report for your branch.
+
+    \b
+    # compare with main/master branch
+    piperider compare
+
+    \b
+    # compare with specific branch
+    piperider compare --base-branch <branch>
+
+    \b
+    # compare with any reference
+    piperider compare <git-ref>
+
+    \b
+    # compare with two references
+    piperider compare <git-ref>...<git-ref>
+
+    \b
+    Note: <git-ref> can be reference that git understands. e.g., branch, tag, commit, etc.
+
     """
 
     from piperider_cli.cli_utils.compare_with_recipe import compare_with_recipe as cmd
-    return cmd(**kwargs)
+    return cmd(ref, **kwargs)
 
 
 @cloud.command(short_help='Signup to PipeRider Cloud.', cls=TrackCommand)

piperider_cli/cli_utils/compare_with_recipe.py

@@ -1,4 +1,30 @@
-def compare_with_recipe(**kwargs):
+from rich.console import Console
+
+
+def parse_compare_ref(ref: str):
+    console = Console()
+
+    if ref is None:
+        return None, None
+
+    if '...' in ref:
+        base_ref = ref.split('...')[0]
+        target_ref = ref.split('...')[1]
+        if base_ref == '' or target_ref == '':
+            console.print('[bold red]Error:[/bold red] '
+                          'Please either provide a single git reference or a 3-dot diff comparison form.')
+            return None, None
+    elif '..' in ref:
+        console.print('[bold red]Error:[/bold red] Two-dot diff comparisons are not supported')
+        return None, None
+    else:
+        base_ref = ref
+        target_ref = None
+
+    return base_ref, target_ref
+
+
+def compare_with_recipe(ref, **kwargs):
     """
     Generate the comparison report for your branch.
     """
@@ -10,6 +36,8 @@ def compare_with_recipe(**kwargs):
     from piperider_cli.initializer import Initializer
     from piperider_cli.recipes import RecipeConfiguration, configure_recipe_execution_flags, is_recipe_dry_run
 
+    console = Console()
+
     recipe = kwargs.get('recipe')
     summary_file = kwargs.get('summary_file')
     force_upload = kwargs.get('upload')
@@ -19,10 +47,19 @@ def compare_with_recipe(**kwargs):
     debug = kwargs.get('debug', False)
     select = kwargs.get('select')
     modified = kwargs.get('modified')
-
-    base_branch = kwargs.get('base_branch')
     skip_datasource_connection = kwargs.get('skip_datasource')
 
+    base_ref, target_ref = parse_compare_ref(ref)
+    if ref is not None and base_ref is None:
+        return -1
+
+    if base_ref is not None and kwargs.get('base_branch') is not None:
+        console.print("[bold red]Error:[/bold red] "
+                      "'--base-branch' option and '[REF]' argument cannot be used together")
+        return -1
+    elif base_ref is None:
+        base_ref = kwargs.get('base_branch')
+
     # reconfigure recipe global flags
     configure_recipe_execution_flags(dry_run=kwargs.get('dry_run'), interactive=kwargs.get('interactive'))
 
@@ -55,7 +92,8 @@ def compare_with_recipe(**kwargs):
             recipe_name=recipe,
             select=select,
             modified=modified,
-            base_branch=base_branch,
+            base_ref=base_ref,
+            target_ref=target_ref,
             skip_datasource_connection=skip_datasource_connection,
             debug=debug)
         last = False

piperider_cli/dbt/list_task.py

@@ -26,11 +26,11 @@ def create_temp_dir():
     return tempfile.mkdtemp()
 
 
-def load_full_manifest(target_path: str):
+def load_full_manifest(target_path: str, project_dir: str = None):
     from dbt.adapters.factory import register_adapter
     from dbt.parser.manifest import ManifestLoader
 
-    runtime_config = PrepareRuntimeConfig(target_path)
+    runtime_config = PrepareRuntimeConfig(target_path, project_dir=project_dir)
     register_adapter(runtime_config)
 
     v = dbt_version
@@ -207,9 +207,9 @@ def convert_time_type(cls, agate_table: agate.Table, col_idx: int) -> str:
         pass
 
 
-def PrepareRuntimeConfig(target_path: str):
+def PrepareRuntimeConfig(target_path: str, project_dir: str = None):
     from piperider_cli.configuration import FileSystem
-    project_root = FileSystem.WORKING_DIRECTORY
+    project_root = project_dir if project_dir is not None else FileSystem.WORKING_DIRECTORY
     profiles_dir = FileSystem.DBT_PROFILES_DIR
 
     def _get_v13_runtime_config(flags):

piperider_cli/dbtutil.py

@@ -134,9 +134,11 @@ def _get_state_run_results(dbt_state_dir: str):
     return run_results
 
 
-def _get_state_manifest(dbt_state_dir: str):
+def _get_state_manifest(dbt_state_dir: str, project_dir: str = None):
     path = os.path.join(dbt_state_dir, 'manifest.json')
-    if os.path.isabs(path) is False:
+    if project_dir is not None:
+        path = os.path.join(project_dir, path)
+    elif os.path.isabs(path) is False:
         from piperider_cli.configuration import FileSystem
         path = os.path.join(FileSystem.WORKING_DIRECTORY, path)
     with open(path) as f:
@@ -656,8 +658,8 @@ def check_dbt_manifest(dbt_state_dir: str) -> bool:
     return os.path.exists(path)
 
 
-def get_dbt_manifest(dbt_state_dir: str):
-    return _get_state_manifest(dbt_state_dir)
+def get_dbt_manifest(dbt_state_dir: str, project_dir: str = None):
+    return _get_state_manifest(dbt_state_dir, project_dir=project_dir)
 
 
 def load_dbt_resources(target_path: str, select: tuple = None, state=None):

piperider_cli/recipe_executor.py

@@ -12,12 +12,12 @@
 
 class RecipeExecutor:
     @staticmethod
-    def exec(recipe_name: str, auto_generate_default_recipe: bool = True, select: tuple = None, modified: bool = False,
-             base_branch: str = None, skip_datasource_connection: bool = False, debug=False) -> RecipeConfiguration:
+    def exec(recipe_name: str, select: tuple = None, modified: bool = False, base_ref: str = None,
+             target_ref: str = None, skip_datasource_connection: bool = False, debug=False) -> RecipeConfiguration:
         config = Configuration.instance()
         recipe_path = select_recipe_file(recipe_name)
 
-        if recipe_name and (select or modified or base_branch or skip_datasource_connection):
+        if recipe_name and (select or modified or base_ref or skip_datasource_connection):
             console.print(
                 "[[bold yellow]Warning[/bold yellow]] "
                 "The recipe will be ignored when '--select', '--modified', '--base-branch', "
@@ -26,27 +26,28 @@ def exec(recipe_name: str, auto_generate_default_recipe: bool = True, select: tu
         if not skip_datasource_connection and select:
             console.print(
                 f"[[bold green]Select[/bold green]] Manually select the dbt nodes to run by '{','.join(select)}'")
-        if recipe_path is None or select or modified or base_branch or skip_datasource_connection:
-            if auto_generate_default_recipe:
-                dbt_project_path = None
-                if config.dataSources and config.dataSources[0].args.get('dbt'):
-                    dbt_project_path = os.path.relpath(config.dataSources[0].args.get('dbt', {}).get('projectDir'))
-                # generate a default recipe
-                console.rule("Recipe executor: generate recipe")
-                options = dict(base_branch=base_branch, skip_datasource_connection=skip_datasource_connection)
-                if select:
-                    options['select'] = select
-                recipe = generate_default_recipe(overwrite_existing=False,
-                                                 dbt_project_path=dbt_project_path,
-                                                 options=options)
-                if recipe is None:
-                    raise RecipeConfigException(
-                        message='Default recipe generation failed.',
-                        hint='Please provide a recipe file or feedback an issue to us'
-                    )
-                show_recipe_content(recipe)
-            else:
-                raise FileNotFoundError(f"Cannot find the recipe '{recipe_name}'")
+        if recipe_path is None or select or modified or base_ref or skip_datasource_connection:
+            dbt_project_path = None
+            if config.dataSources and config.dataSources[0].args.get('dbt'):
+                dbt_project_path = os.path.relpath(config.dataSources[0].args.get('dbt', {}).get('projectDir'))
+            # generate a default recipe
+            console.rule("Recipe executor: generate recipe")
+            options = dict(
+                base_ref=base_ref,
+                target_ref=target_ref,
+                skip_datasource_connection=skip_datasource_connection
+            )
+            if select:
+                options['select'] = select
+            recipe = generate_default_recipe(overwrite_existing=False,
+                                             dbt_project_path=dbt_project_path,
+                                             options=options)
+            if recipe is None:
+                raise RecipeConfigException(
+                    message='Default recipe generation failed.',
+                    hint='Please provide a recipe file or feedback an issue to us'
+                )
+            show_recipe_content(recipe)
         else:
             recipe = RecipeConfiguration.load(recipe_path)