From 4192d60917b95e623108f8ddf67bb22213085f61 Mon Sep 17 00:00:00 2001 From: surya <118222278+suryaiyer95@users.noreply.github.com> Date: Tue, 28 May 2024 17:14:54 -0700 Subject: [PATCH 1/4] fix: github action (#37) * fix: github action * fix * fix: macos use x64 arch --- .github/workflows/github-actions.yml | 32 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 1a043cba..c905cae8 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -36,7 +36,7 @@ jobs: toxpython: 'python3.8' python_arch: 'x64' tox_env: 'py38-cover' - os: 'macos-latest' + os: 'macos-13' - name: 'py38-nocov (ubuntu)' python: '3.8' toxpython: 'python3.8' @@ -54,7 +54,7 @@ jobs: toxpython: 'python3.8' python_arch: 'x64' tox_env: 'py38-nocov' - os: 'macos-latest' + os: 'macos-13' - name: 'py39-cover (ubuntu)' python: '3.9' toxpython: 'python3.9' @@ -72,7 +72,7 @@ jobs: toxpython: 'python3.9' python_arch: 'x64' tox_env: 'py39-cover' - os: 'macos-latest' + os: 'macos-13' - name: 'py39-nocov (ubuntu)' python: '3.9' toxpython: 'python3.9' @@ -90,7 +90,7 @@ jobs: toxpython: 'python3.9' python_arch: 'x64' tox_env: 'py39-nocov' - os: 'macos-latest' + os: 'macos-13' - name: 'py310-cover (ubuntu)' python: '3.10' toxpython: 'python3.10' @@ -108,7 +108,7 @@ jobs: toxpython: 'python3.10' python_arch: 'x64' tox_env: 'py310-cover' - os: 'macos-latest' + os: 'macos-13' - name: 'py310-nocov (ubuntu)' python: '3.10' toxpython: 'python3.10' @@ -126,7 +126,7 @@ jobs: toxpython: 'python3.10' python_arch: 'x64' tox_env: 'py310-nocov' - os: 'macos-latest' + os: 'macos-13' - name: 'py311-cover (ubuntu)' python: '3.11' toxpython: 'python3.11' @@ -144,7 +144,7 @@ jobs: toxpython: 'python3.11' python_arch: 'x64' tox_env: 'py311-cover' - os: 'macos-latest' + os: 'macos-13' - name: 'py311-nocov (ubuntu)' python: '3.11' toxpython: 'python3.11' @@ -162,7 +162,7 @@ jobs: toxpython: 'python3.11' python_arch: 'x64' tox_env: 'py311-nocov' - os: 'macos-latest' + os: 'macos-13' - name: 'py312-cover (ubuntu)' python: '3.12' toxpython: 'python3.12' @@ -180,7 +180,7 @@ jobs: toxpython: 'python3.12' python_arch: 'x64' tox_env: 'py312-cover' - os: 'macos-latest' + os: 'macos-13' - name: 'py312-nocov (ubuntu)' python: '3.12' toxpython: 'python3.12' @@ -198,7 +198,7 @@ jobs: toxpython: 'python3.12' python_arch: 'x64' tox_env: 'py312-nocov' - os: 'macos-latest' + os: 'macos-13' - name: 'pypy38-cover (ubuntu)' python: 'pypy-3.8' toxpython: 'pypy3.8' @@ -216,7 +216,7 @@ jobs: toxpython: 'pypy3.8' python_arch: 'x64' tox_env: 'pypy38-cover' - os: 'macos-latest' + os: 'macos-13' - name: 'pypy38-nocov (ubuntu)' python: 'pypy-3.8' toxpython: 'pypy3.8' @@ -234,7 +234,7 @@ jobs: toxpython: 'pypy3.8' python_arch: 'x64' tox_env: 'pypy38-nocov' - os: 'macos-latest' + os: 'macos-13' - name: 'pypy39-cover (ubuntu)' python: 'pypy-3.9' toxpython: 'pypy3.9' @@ -252,7 +252,7 @@ jobs: toxpython: 'pypy3.9' python_arch: 'x64' tox_env: 'pypy39-cover' - os: 'macos-latest' + os: 'macos-13' - name: 'pypy39-nocov (ubuntu)' python: 'pypy-3.9' toxpython: 'pypy3.9' @@ -270,7 +270,7 @@ jobs: toxpython: 'pypy3.9' python_arch: 'x64' tox_env: 'pypy39-nocov' - os: 'macos-latest' + os: 'macos-13' - name: 'pypy310-cover (ubuntu)' python: 'pypy-3.10' toxpython: 'pypy3.10' @@ -288,7 +288,7 @@ jobs: toxpython: 'pypy3.10' python_arch: 'x64' tox_env: 'pypy310-cover' - os: 'macos-latest' + os: 'macos-13' - name: 'pypy310-nocov (ubuntu)' python: 'pypy-3.10' toxpython: 'pypy3.10' @@ -306,7 +306,7 @@ jobs: toxpython: 'pypy3.10' python_arch: 'x64' tox_env: 'pypy310-nocov' - os: 'macos-latest' + os: 'macos-13' steps: - uses: actions/checkout@v4 with: From ea36b135e0dd50696cc1673b536f81051d77e997 Mon Sep 17 00:00:00 2001 From: surya <118222278+suryaiyer95@users.noreply.github.com> Date: Tue, 28 May 2024 17:35:55 -0700 Subject: [PATCH 2/4] fix: remove sqlglot dependency (#33) --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 94719f59..d1721ef4 100755 --- a/setup.py +++ b/setup.py @@ -63,7 +63,6 @@ def read(*names, **kwargs): python_requires=">=3.8", install_requires=[ "click==8.1.7", - "sqlglot==18.3.0", "dbt-artifacts-parser==0.5.1", "ruamel.yaml==0.18.6", "tabulate==0.9.0", From a5ceedac826832d495d0daa06174305578dea23e Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Tue, 28 May 2024 18:01:19 -0700 Subject: [PATCH 3/4] =?UTF-8?q?Bump=20version:=200.0.10=20=E2=86=92=200.0.?= =?UTF-8?q?11?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- docs/conf.py | 2 +- setup.py | 2 +- src/datapilot/__init__.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index eda1c910..bbe58c70 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.0.10 +current_version = 0.0.11 commit = True tag = True diff --git a/docs/conf.py b/docs/conf.py index 12b81879..8b884ce9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,7 +15,7 @@ year = "2024" author = "Altimate Inc." copyright = f"{year}, {author}" -version = release = "0.0.10" +version = release = "0.0.11" pygments_style = "trac" templates_path = ["."] diff --git a/setup.py b/setup.py index d1721ef4..316ba9d1 100755 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ def read(*names, **kwargs): setup( name="altimate-datapilot-cli", - version="0.0.10", + version="0.0.11", license="MIT", description="Assistant for Data Teams", long_description="{}\n{}".format( diff --git a/src/datapilot/__init__.py b/src/datapilot/__init__.py index 9b36b86c..b2f01558 100644 --- a/src/datapilot/__init__.py +++ b/src/datapilot/__init__.py @@ -1 +1 @@ -__version__ = "0.0.10" +__version__ = "0.0.11" From e81e71e943618ac463245ec836df255f427ad76d Mon Sep 17 00:00:00 2001 From: Aditya Gajbhiye Date: Thu, 30 May 2024 01:54:04 +0100 Subject: [PATCH 4/4] feat: added catalog and env support (#36) * feat: added catalog support * feat: added env * WIP * WIP * WIP * feat: remove file ids * WIP * WIP * WIP * fix: imports --------- Co-authored-by: surya <118222278+suryaiyer95@users.noreply.github.com> --- src/datapilot/clients/altimate/client.py | 6 +++- src/datapilot/clients/altimate/utils.py | 27 ++++++++++++-- src/datapilot/core/platforms/dbt/cli/cli.py | 39 ++++++++++++++++++--- 3 files changed, 64 insertions(+), 8 deletions(-) diff --git a/src/datapilot/clients/altimate/client.py b/src/datapilot/clients/altimate/client.py index 4ad9e5b1..fd915174 100644 --- a/src/datapilot/clients/altimate/client.py +++ b/src/datapilot/clients/altimate/client.py @@ -62,7 +62,7 @@ def post(self, endpoint, data=None, timeout=None): response = requests.post(url, headers=headers, json=data, timeout=timeout) self.logger.debug(f"Received POST response with status: {response.status_code }") - return response + return response.json() def put(self, endpoint, data, timeout=None): url = f"{self.base_url}{endpoint}" @@ -83,3 +83,7 @@ def get_signed_url(self, params=None): def validate_credentials(self): endpoint = "/dbt/v3/validate-credentials" return self.get(endpoint) + + def start_dbt_ingestion(self, params=None): + endpoint = "/dbt/v1/start_dbt_ingestion" + return self.post(endpoint, data=params) diff --git a/src/datapilot/clients/altimate/utils.py b/src/datapilot/clients/altimate/utils.py index 9ed5548e..9a18a85f 100644 --- a/src/datapilot/clients/altimate/utils.py +++ b/src/datapilot/clients/altimate/utils.py @@ -46,10 +46,14 @@ def validate_credentials( return api_client.validate_credentials() -def onboard_manifest(api_token, tenant, dbt_core_integration_id, manifest_path, backend_url) -> Dict: +def onboard_file(api_token, tenant, dbt_core_integration_id, dbt_core_integration_environment, file_type, file_path, backend_url) -> Dict: api_client = APIClient(api_token, base_url=backend_url, tenant=tenant) - params = {"dbt_core_integration_id": dbt_core_integration_id, "file_type": "manifest"} + params = { + "dbt_core_integration_id": dbt_core_integration_id, + "dbt_core_integration_environment_type": dbt_core_integration_environment, + "file_type": file_type, + } signed_url_data = api_client.get_signed_url(params) if signed_url_data: signed_url = signed_url_data.get("url") @@ -57,7 +61,7 @@ def onboard_manifest(api_token, tenant, dbt_core_integration_id, manifest_path, api_client.log(f"Received signed URL: {signed_url}") api_client.log(f"Received File ID: {file_id}") - upload_response = upload_content_to_signed_url(manifest_path, signed_url) + upload_response = upload_content_to_signed_url(file_path, signed_url) if upload_response: verify_params = {"dbt_core_integration_file_id": file_id} @@ -73,3 +77,20 @@ def onboard_manifest(api_token, tenant, dbt_core_integration_id, manifest_path, "ok": False, "message": "Error in uploading the manifest. ", } + + +def start_dbt_ingestion(api_token, tenant, dbt_core_integration_id, dbt_core_integration_environment, backend_url): + api_client = APIClient(api_token, base_url=backend_url, tenant=tenant) + params = { + "dbt_core_integration_id": dbt_core_integration_id, + "dbt_core_integration_environment_type": dbt_core_integration_environment, + } + data = api_client.start_dbt_ingestion(params) + if data and data.get("ok"): + return {"ok": True} + else: + api_client.log("Error starting dbt ingestion worker") + return { + "ok": False, + "message": "Error starting dbt ingestion worker. ", + } diff --git a/src/datapilot/core/platforms/dbt/cli/cli.py b/src/datapilot/core/platforms/dbt/cli/cli.py index d62fffa1..acb8ebab 100644 --- a/src/datapilot/core/platforms/dbt/cli/cli.py +++ b/src/datapilot/core/platforms/dbt/cli/cli.py @@ -3,7 +3,8 @@ import click from datapilot.clients.altimate.utils import check_token_and_instance -from datapilot.clients.altimate.utils import onboard_manifest +from datapilot.clients.altimate.utils import onboard_file +from datapilot.clients.altimate.utils import start_dbt_ingestion from datapilot.clients.altimate.utils import validate_credentials from datapilot.config.config import load_config from datapilot.core.platforms.dbt.constants import MODEL @@ -87,9 +88,21 @@ def project_health(manifest_path, catalog_path, config_path=None, select=None): @click.option("--token", prompt="API Token", help="Your API token for authentication.") @click.option("--instance-name", prompt="Instance Name", help="Your tenant ID.") @click.option("--dbt_core_integration_id", prompt="DBT Core Integration ID", help="DBT Core Integration ID") +@click.option( + "--dbt_core_integration_environment", default="PROD", prompt="DBT Core Integration Environment", help="DBT Core Integration Environment" +) @click.option("--manifest-path", required=True, prompt="Manifest Path", help="Path to the manifest file.") +@click.option("--catalog-path", required=False, prompt=False, help="Path to the catalog file.") @click.option("--backend-url", required=False, help="Altimate's Backend URL", default="https://api.myaltimate.com") -def onboard(token, instance_name, dbt_core_integration_id, manifest_path, backend_url="https://api.myaltimate.com", env=None): +def onboard( + token, + instance_name, + dbt_core_integration_id, + dbt_core_integration_environment, + manifest_path, + catalog_path, + backend_url="https://api.myaltimate.com", +): """Onboard a manifest file to DBT.""" check_token_and_instance(token, instance_name) @@ -104,9 +117,27 @@ def onboard(token, instance_name, dbt_core_integration_id, manifest_path, backen click.echo(f"Error: {e}") return - response = onboard_manifest(token, instance_name, dbt_core_integration_id, manifest_path, backend_url) - + response = onboard_file( + token, instance_name, dbt_core_integration_id, dbt_core_integration_environment, "manifest", manifest_path, backend_url + ) if response["ok"]: click.echo("Manifest onboarded successfully!") else: click.echo(f"{response['message']}") + + if not catalog_path: + return + + response = onboard_file( + token, instance_name, dbt_core_integration_id, dbt_core_integration_environment, "catalog", catalog_path, backend_url + ) + if response["ok"]: + click.echo("Catalog onboarded successfully!") + else: + click.echo(f"{response['message']}") + + response = start_dbt_ingestion(token, instance_name, dbt_core_integration_id, dbt_core_integration_environment, backend_url) + if response["ok"]: + click.echo("Onboarding completed successfully!") + else: + click.echo(f"{response['message']}")