Merge branch 'dbt-1.8-support' of https://github.com/AltimateAI/datap…

…ilot into dbt-1.8-support

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.0.10
+current_version = 0.0.11
 commit = True
 tag = True
 

.github/workflows/github-actions.yml

@@ -36,7 +36,7 @@ jobs:
             toxpython: 'python3.8'
             python_arch: 'x64'
             tox_env: 'py38-cover'
-            os: 'macos-latest'
+            os: 'macos-13'
           - name: 'py38-nocov (ubuntu)'
             python: '3.8'
             toxpython: 'python3.8'
@@ -54,7 +54,7 @@ jobs:
             toxpython: 'python3.8'
             python_arch: 'x64'
             tox_env: 'py38-nocov'
-            os: 'macos-latest'
+            os: 'macos-13'
           - name: 'py39-cover (ubuntu)'
             python: '3.9'
             toxpython: 'python3.9'
@@ -72,7 +72,7 @@ jobs:
             toxpython: 'python3.9'
             python_arch: 'x64'
             tox_env: 'py39-cover'
-            os: 'macos-latest'
+            os: 'macos-13'
           - name: 'py39-nocov (ubuntu)'
             python: '3.9'
             toxpython: 'python3.9'
@@ -90,7 +90,7 @@ jobs:
             toxpython: 'python3.9'
             python_arch: 'x64'
             tox_env: 'py39-nocov'
-            os: 'macos-latest'
+            os: 'macos-13'
           - name: 'py310-cover (ubuntu)'
             python: '3.10'
             toxpython: 'python3.10'
@@ -108,7 +108,7 @@ jobs:
             toxpython: 'python3.10'
             python_arch: 'x64'
             tox_env: 'py310-cover'
-            os: 'macos-latest'
+            os: 'macos-13'
           - name: 'py310-nocov (ubuntu)'
             python: '3.10'
             toxpython: 'python3.10'
@@ -126,7 +126,7 @@ jobs:
             toxpython: 'python3.10'
             python_arch: 'x64'
             tox_env: 'py310-nocov'
-            os: 'macos-latest'
+            os: 'macos-13'
           - name: 'py311-cover (ubuntu)'
             python: '3.11'
             toxpython: 'python3.11'
@@ -144,7 +144,7 @@ jobs:
             toxpython: 'python3.11'
             python_arch: 'x64'
             tox_env: 'py311-cover'
-            os: 'macos-latest'
+            os: 'macos-13'
           - name: 'py311-nocov (ubuntu)'
             python: '3.11'
             toxpython: 'python3.11'
@@ -162,7 +162,7 @@ jobs:
             toxpython: 'python3.11'
             python_arch: 'x64'
             tox_env: 'py311-nocov'
-            os: 'macos-latest'
+            os: 'macos-13'
           - name: 'py312-cover (ubuntu)'
             python: '3.12'
             toxpython: 'python3.12'
@@ -180,7 +180,7 @@ jobs:
             toxpython: 'python3.12'
             python_arch: 'x64'
             tox_env: 'py312-cover'
-            os: 'macos-latest'
+            os: 'macos-13'
           - name: 'py312-nocov (ubuntu)'
             python: '3.12'
             toxpython: 'python3.12'
@@ -198,7 +198,7 @@ jobs:
             toxpython: 'python3.12'
             python_arch: 'x64'
             tox_env: 'py312-nocov'
-            os: 'macos-latest'
+            os: 'macos-13'
           - name: 'pypy38-cover (ubuntu)'
             python: 'pypy-3.8'
             toxpython: 'pypy3.8'
@@ -216,7 +216,7 @@ jobs:
             toxpython: 'pypy3.8'
             python_arch: 'x64'
             tox_env: 'pypy38-cover'
-            os: 'macos-latest'
+            os: 'macos-13'
           - name: 'pypy38-nocov (ubuntu)'
             python: 'pypy-3.8'
             toxpython: 'pypy3.8'
@@ -234,7 +234,7 @@ jobs:
             toxpython: 'pypy3.8'
             python_arch: 'x64'
             tox_env: 'pypy38-nocov'
-            os: 'macos-latest'
+            os: 'macos-13'
           - name: 'pypy39-cover (ubuntu)'
             python: 'pypy-3.9'
             toxpython: 'pypy3.9'
@@ -252,7 +252,7 @@ jobs:
             toxpython: 'pypy3.9'
             python_arch: 'x64'
             tox_env: 'pypy39-cover'
-            os: 'macos-latest'
+            os: 'macos-13'
           - name: 'pypy39-nocov (ubuntu)'
             python: 'pypy-3.9'
             toxpython: 'pypy3.9'
@@ -270,7 +270,7 @@ jobs:
             toxpython: 'pypy3.9'
             python_arch: 'x64'
             tox_env: 'pypy39-nocov'
-            os: 'macos-latest'
+            os: 'macos-13'
           - name: 'pypy310-cover (ubuntu)'
             python: 'pypy-3.10'
             toxpython: 'pypy3.10'
@@ -288,7 +288,7 @@ jobs:
             toxpython: 'pypy3.10'
             python_arch: 'x64'
             tox_env: 'pypy310-cover'
-            os: 'macos-latest'
+            os: 'macos-13'
           - name: 'pypy310-nocov (ubuntu)'
             python: 'pypy-3.10'
             toxpython: 'pypy3.10'
@@ -306,7 +306,7 @@ jobs:
             toxpython: 'pypy3.10'
             python_arch: 'x64'
             tox_env: 'pypy310-nocov'
-            os: 'macos-latest'
+            os: 'macos-13'
     steps:
     - uses: actions/checkout@v4
       with:

docs/conf.py

@@ -15,7 +15,7 @@
 year = "2024"
 author = "Altimate Inc."
 copyright = f"{year}, {author}"
-version = release = "0.0.10"
+version = release = "0.0.11"
 
 pygments_style = "trac"
 templates_path = ["."]

setup.py

@@ -13,7 +13,7 @@ def read(*names, **kwargs):
 
 setup(
     name="altimate-datapilot-cli",
-    version="0.0.10",
+    version="0.0.11",
     license="MIT",
     description="Assistant for Data Teams",
     long_description="{}\n{}".format(
@@ -63,7 +63,6 @@ def read(*names, **kwargs):
     python_requires=">=3.8",
     install_requires=[
         "click==8.1.7",
-        "sqlglot==18.3.0",
         "dbt-artifacts-parser==0.7.0",
         "ruamel.yaml==0.18.6",
         "tabulate==0.9.0",

src/datapilot/__init__.py

@@ -1 +1 @@
-__version__ = "0.0.10"
+__version__ = "0.0.11"

src/datapilot/clients/altimate/client.py

@@ -62,7 +62,7 @@ def post(self, endpoint, data=None, timeout=None):
         response = requests.post(url, headers=headers, json=data, timeout=timeout)
         self.logger.debug(f"Received POST response with status: {response.status_code }")
 
-        return response
+        return response.json()
 
     def put(self, endpoint, data, timeout=None):
         url = f"{self.base_url}{endpoint}"
@@ -83,3 +83,7 @@ def get_signed_url(self, params=None):
     def validate_credentials(self):
         endpoint = "/dbt/v3/validate-credentials"
         return self.get(endpoint)
+
+    def start_dbt_ingestion(self, params=None):
+        endpoint = "/dbt/v1/start_dbt_ingestion"
+        return self.post(endpoint, data=params)

src/datapilot/clients/altimate/utils.py

@@ -46,18 +46,22 @@ def validate_credentials(
     return api_client.validate_credentials()
 
 
-def onboard_manifest(api_token, tenant, dbt_core_integration_id, manifest_path, backend_url) -> Dict:
+def onboard_file(api_token, tenant, dbt_core_integration_id, dbt_core_integration_environment, file_type, file_path, backend_url) -> Dict:
     api_client = APIClient(api_token, base_url=backend_url, tenant=tenant)
 
-    params = {"dbt_core_integration_id": dbt_core_integration_id, "file_type": "manifest"}
+    params = {
+        "dbt_core_integration_id": dbt_core_integration_id,
+        "dbt_core_integration_environment_type": dbt_core_integration_environment,
+        "file_type": file_type,
+    }
     signed_url_data = api_client.get_signed_url(params)
     if signed_url_data:
         signed_url = signed_url_data.get("url")
         file_id = signed_url_data.get("dbt_core_integration_file_id")
         api_client.log(f"Received signed URL: {signed_url}")
         api_client.log(f"Received File ID: {file_id}")
 
-        upload_response = upload_content_to_signed_url(manifest_path, signed_url)
+        upload_response = upload_content_to_signed_url(file_path, signed_url)
 
         if upload_response:
             verify_params = {"dbt_core_integration_file_id": file_id}
@@ -73,3 +77,20 @@ def onboard_manifest(api_token, tenant, dbt_core_integration_id, manifest_path,
             "ok": False,
             "message": "Error in uploading the manifest.                                                                                                                              ",
         }
+
+
+def start_dbt_ingestion(api_token, tenant, dbt_core_integration_id, dbt_core_integration_environment, backend_url):
+    api_client = APIClient(api_token, base_url=backend_url, tenant=tenant)
+    params = {
+        "dbt_core_integration_id": dbt_core_integration_id,
+        "dbt_core_integration_environment_type": dbt_core_integration_environment,
+    }
+    data = api_client.start_dbt_ingestion(params)
+    if data and data.get("ok"):
+        return {"ok": True}
+    else:
+        api_client.log("Error starting dbt ingestion worker")
+        return {
+            "ok": False,
+            "message": "Error starting dbt ingestion worker.                                                                                                                              ",
+        }

src/datapilot/core/platforms/dbt/cli/cli.py

@@ -3,7 +3,8 @@
 import click
 
 from datapilot.clients.altimate.utils import check_token_and_instance
-from datapilot.clients.altimate.utils import onboard_manifest
+from datapilot.clients.altimate.utils import onboard_file
+from datapilot.clients.altimate.utils import start_dbt_ingestion
 from datapilot.clients.altimate.utils import validate_credentials
 from datapilot.config.config import load_config
 from datapilot.core.platforms.dbt.constants import MODEL
@@ -87,9 +88,21 @@ def project_health(manifest_path, catalog_path, config_path=None, select=None):
 @click.option("--token", prompt="API Token", help="Your API token for authentication.")
 @click.option("--instance-name", prompt="Instance Name", help="Your tenant ID.")
 @click.option("--dbt_core_integration_id", prompt="DBT Core Integration ID", help="DBT Core Integration ID")
+@click.option(
+    "--dbt_core_integration_environment", default="PROD", prompt="DBT Core Integration Environment", help="DBT Core Integration Environment"
+)
 @click.option("--manifest-path", required=True, prompt="Manifest Path", help="Path to the manifest file.")
+@click.option("--catalog-path", required=False, prompt=False, help="Path to the catalog file.")
 @click.option("--backend-url", required=False, help="Altimate's Backend URL", default="https://api.myaltimate.com")
-def onboard(token, instance_name, dbt_core_integration_id, manifest_path, backend_url="https://api.myaltimate.com", env=None):
+def onboard(
+    token,
+    instance_name,
+    dbt_core_integration_id,
+    dbt_core_integration_environment,
+    manifest_path,
+    catalog_path,
+    backend_url="https://api.myaltimate.com",
+):
     """Onboard a manifest file to DBT."""
     check_token_and_instance(token, instance_name)
 
@@ -104,9 +117,27 @@ def onboard(token, instance_name, dbt_core_integration_id, manifest_path, backen
         click.echo(f"Error: {e}")
         return
 
-    response = onboard_manifest(token, instance_name, dbt_core_integration_id, manifest_path, backend_url)
-
+    response = onboard_file(
+        token, instance_name, dbt_core_integration_id, dbt_core_integration_environment, "manifest", manifest_path, backend_url
+    )
     if response["ok"]:
         click.echo("Manifest onboarded successfully!")
     else:
         click.echo(f"{response['message']}")
+
+    if not catalog_path:
+        return
+
+    response = onboard_file(
+        token, instance_name, dbt_core_integration_id, dbt_core_integration_environment, "catalog", catalog_path, backend_url
+    )
+    if response["ok"]:
+        click.echo("Catalog onboarded successfully!")
+    else:
+        click.echo(f"{response['message']}")
+
+    response = start_dbt_ingestion(token, instance_name, dbt_core_integration_id, dbt_core_integration_environment, backend_url)
+    if response["ok"]:
+        click.echo("Onboarding completed successfully!")
+    else:
+        click.echo(f"{response['message']}")