From ca2c1a6d845b3421beb1e049e20005d546023d58 Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Tue, 30 Jan 2024 22:21:31 +0530 Subject: [PATCH 01/11] Add manifest onboard cli --- src/datapilot/clients/__init__.py | 0 src/datapilot/clients/altimate/__init__.py | 0 src/datapilot/clients/altimate/client.py | 51 +++++++++++++++++++++ src/datapilot/core/platforms/dbt/cli/cli.py | 29 ++++++++++++ src/datapilot/utils/utils.py | 48 +++++++++++++++++++ 5 files changed, 128 insertions(+) create mode 100644 src/datapilot/clients/__init__.py create mode 100644 src/datapilot/clients/altimate/__init__.py create mode 100644 src/datapilot/clients/altimate/client.py diff --git a/src/datapilot/clients/__init__.py b/src/datapilot/clients/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/datapilot/clients/altimate/__init__.py b/src/datapilot/clients/altimate/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/datapilot/clients/altimate/client.py b/src/datapilot/clients/altimate/client.py new file mode 100644 index 00000000..0e42ed72 --- /dev/null +++ b/src/datapilot/clients/altimate/client.py @@ -0,0 +1,51 @@ +import logging + +import requests + + +class APIClient: + def __init__(self, api_token="", base_url="", tenant=""): + self.api_token = api_token + self.base_url = base_url + self.tenant = tenant + self.logger = logging.getLogger(self.__class__.__name__) + + def _get_headers(self): + headers = { + "Content-Type": "application/json", + } + + if self.api_token: + headers["Authorization"] = f"Bearer {self.api_token}" + + if self.tenant: + headers["x-tenant"] = self.tenant + + return headers + + def get(self, endpoint, params=None, timeout=None): + url = f"{self.base_url}{endpoint}" + headers = self._get_headers() + + self.logger.debug(f"Sending GET request for tenant {self.tenant} at url: {url}") + response = requests.get(url, headers=headers, params=params, timeout=timeout) + self.logger.debug(f"Received GET response with status: {response.status_code }") + return response.json() if response.status_code == 200 else None + + def post(self, endpoint, data=None, timeout=None): + url = f"{self.base_url}{endpoint}" + headers = self._get_headers() + + self.logger.debug(f"Sending POST request for tenant {self.tenant} at url: {url}") + response = requests.post(url, headers=headers, json=data, timeout=timeout) + self.logger.debug(f"Received POST response with status: {response.status_code }") + + return response + + def put(self, endpoint, data, timeout=None): + url = f"{self.base_url}{endpoint}" + + self.logger.debug(f"Sending PUT request for tenant {self.tenant} at url: {url}") + response = requests.put(url, data=data, timeout=timeout) + self.logger.debug(f"Received PUT response with status: {response.status_code}") + return response diff --git a/src/datapilot/core/platforms/dbt/cli/cli.py b/src/datapilot/core/platforms/dbt/cli/cli.py index d9b90084..d915e0be 100644 --- a/src/datapilot/core/platforms/dbt/cli/cli.py +++ b/src/datapilot/core/platforms/dbt/cli/cli.py @@ -1,4 +1,5 @@ import logging +import os import click @@ -9,6 +10,7 @@ from datapilot.core.platforms.dbt.formatting import generate_model_insights_table from datapilot.core.platforms.dbt.formatting import generate_project_insights_table from datapilot.utils.formatting.utils import tabulate_data +from datapilot.utils.utils import onboard_manifest logging.basicConfig(level=logging.INFO) @@ -64,3 +66,30 @@ def project_health(manifest_path, catalog_path, config_path=None): click.echo("Project Insights") click.echo("--" * 50) click.echo(tabulate_data(project_report, headers="keys")) + + +@dbt.command("onboard") +@click.option("--token", prompt="API Token", help="Your API token for authentication.") +@click.option("--tenant", prompt="Tenant", help="Your tenant ID.") +@click.option("--dbt_core_integration_id", prompt="DBT Core Integration ID", help="DBT Core Integration ID") +@click.option("--manifest-path", required=True, prompt="Manifest Path", help="Path to the manifest file.") +def onboard(token, tenant, dbt_core_integration_id, manifest_path, env=None): + """Onboard a manifest file to DBT.""" + if not token and env: + token = os.environ.get("ALTIMATE_API_KEY") + + if not tenant and env: + tenant = os.environ.get("ALTIMATE_INSTANCE_NAME") + + if not token or not tenant: + click.echo("Error: API Token is required.") + return + + # if not validate_credentials(token, tenant): + # print("Error: Invalid credentials.") + # return + + # This will throw error if manifest file is incorrect + # load_manifest(manifest_path) + + onboard_manifest(token, tenant, dbt_core_integration_id, manifest_path) diff --git a/src/datapilot/utils/utils.py b/src/datapilot/utils/utils.py index 77e7a463..3cfac689 100644 --- a/src/datapilot/utils/utils.py +++ b/src/datapilot/utils/utils.py @@ -3,6 +3,8 @@ from pathlib import Path from typing import Dict +from datapilot.clients.altimate.client import APIClient + def load_json(file_path: str) -> Dict: try: @@ -40,3 +42,49 @@ def get_dir_path(path: str) -> str: :return: """ return Path(path).parent + + +# Will need to change this +base_url = "http://localhost:5001" + + +def upload_content_to_signed_url(file_path, signed_url): + api_client = APIClient() + + with Path(file_path).open("rb") as file: + file_content = file.read() + + return api_client.put(signed_url, data=file_content) + + +def onboard_manifest(api_token, tenant, dbt_core_integration_id, manifest_path): + api_client = APIClient(api_token, base_url, tenant) + + endpoint = "/dbt/v1/signed_url" + params = {"dbt_core_integration_id": dbt_core_integration_id, "file_type": "manifest"} + signed_url_data = api_client.get(endpoint, params=params) + + if signed_url_data: + signed_url = signed_url_data.get("url") + file_id = signed_url_data.get("dbt_core_integration_file_id") + print(f"Received signed URL: {signed_url}") + print(f"Received File ID: {file_id}") + + upload_response = upload_content_to_signed_url(manifest_path, signed_url) + + if upload_response: + endpoint = "/dbt/v1/verify_upload" + verify_params = {"dbt_core_integration_file_id": file_id} + verify_response = api_client.post(endpoint, data=verify_params) + + if verify_response: + print("File successfully uploaded and verified.") + return + else: + print(f"Error verifying upload: {verify_response.status_code}, {verify_response.text}") + + else: + print(f"Error uploading file: {upload_response.status_code}, {upload_response.text}") + + else: + print("Error getting signed URL.") From 5241cf189ffae8501117e33c13cad9a10e20eeef Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Mon, 5 Feb 2024 15:26:36 +0530 Subject: [PATCH 02/11] api client changes --- src/datapilot/clients/altimate/client.py | 11 +++++++++++ src/datapilot/core/platforms/dbt/cli/cli.py | 3 ++- src/datapilot/utils/utils.py | 17 +++++------------ 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/datapilot/clients/altimate/client.py b/src/datapilot/clients/altimate/client.py index 0e42ed72..279a32b6 100644 --- a/src/datapilot/clients/altimate/client.py +++ b/src/datapilot/clients/altimate/client.py @@ -23,6 +23,9 @@ def _get_headers(self): return headers + def log(self, message): + self.logger.debug(message) + def get(self, endpoint, params=None, timeout=None): url = f"{self.base_url}{endpoint}" headers = self._get_headers() @@ -49,3 +52,11 @@ def put(self, endpoint, data, timeout=None): response = requests.put(url, data=data, timeout=timeout) self.logger.debug(f"Received PUT response with status: {response.status_code}") return response + + def verify_upload(self, params=None): + endpoint = "/dbt/v1/verify_upload" + verify_response = self.post(endpoint, data=params) + if verify_response: + self.logger.debug("File successfully uploaded and verified.") + else: + self.logger.debug(f"Error verifying upload: {verify_response.status_code}, {verify_response.text}") diff --git a/src/datapilot/core/platforms/dbt/cli/cli.py b/src/datapilot/core/platforms/dbt/cli/cli.py index 1e6eacbf..a6c066c5 100644 --- a/src/datapilot/core/platforms/dbt/cli/cli.py +++ b/src/datapilot/core/platforms/dbt/cli/cli.py @@ -9,6 +9,7 @@ from datapilot.core.platforms.dbt.executor import DBTInsightGenerator from datapilot.core.platforms.dbt.formatting import generate_model_insights_table from datapilot.core.platforms.dbt.formatting import generate_project_insights_table +from datapilot.core.platforms.dbt.utils import load_manifest from datapilot.utils.formatting.utils import tabulate_data from datapilot.utils.utils import onboard_manifest @@ -91,6 +92,6 @@ def onboard(token, tenant, dbt_core_integration_id, manifest_path, env=None): # return # This will throw error if manifest file is incorrect - # load_manifest(manifest_path) + load_manifest(manifest_path) onboard_manifest(token, tenant, dbt_core_integration_id, manifest_path) diff --git a/src/datapilot/utils/utils.py b/src/datapilot/utils/utils.py index 3cfac689..f2885376 100644 --- a/src/datapilot/utils/utils.py +++ b/src/datapilot/utils/utils.py @@ -67,24 +67,17 @@ def onboard_manifest(api_token, tenant, dbt_core_integration_id, manifest_path): if signed_url_data: signed_url = signed_url_data.get("url") file_id = signed_url_data.get("dbt_core_integration_file_id") - print(f"Received signed URL: {signed_url}") - print(f"Received File ID: {file_id}") + api_client.log(f"Received signed URL: {signed_url}") + api_client.log(f"Received File ID: {file_id}") upload_response = upload_content_to_signed_url(manifest_path, signed_url) if upload_response: - endpoint = "/dbt/v1/verify_upload" verify_params = {"dbt_core_integration_file_id": file_id} - verify_response = api_client.post(endpoint, data=verify_params) - - if verify_response: - print("File successfully uploaded and verified.") - return - else: - print(f"Error verifying upload: {verify_response.status_code}, {verify_response.text}") + api_client.verify_upload(verify_params) else: - print(f"Error uploading file: {upload_response.status_code}, {upload_response.text}") + api_client.log(f"Error uploading file: {upload_response.status_code}, {upload_response.text}") else: - print("Error getting signed URL.") + api_client.log("Error getting signed URL.") From 78b5ee8bbfe50a506da7d17d04da6f9b7bde54ca Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Tue, 6 Feb 2024 10:55:09 +0530 Subject: [PATCH 03/11] changes --- src/datapilot/clients/altimate/client.py | 8 ++++++++ src/datapilot/core/platforms/dbt/cli/cli.py | 10 ++++++++-- src/datapilot/utils/utils.py | 19 +++++++++---------- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/src/datapilot/clients/altimate/client.py b/src/datapilot/clients/altimate/client.py index 279a32b6..b0847991 100644 --- a/src/datapilot/clients/altimate/client.py +++ b/src/datapilot/clients/altimate/client.py @@ -60,3 +60,11 @@ def verify_upload(self, params=None): self.logger.debug("File successfully uploaded and verified.") else: self.logger.debug(f"Error verifying upload: {verify_response.status_code}, {verify_response.text}") + + def get_signed_url(self, params=None): + endpoint = "/dbt/v1/signed_url" + response: requests.Response = self.get(endpoint, params=params) + if response.status_code == 200: + self.logger.debug("Fetched signed url") + else: + self.logger.debug(f"Error fetching signed url: {response.status_code}, {response.text}") diff --git a/src/datapilot/core/platforms/dbt/cli/cli.py b/src/datapilot/core/platforms/dbt/cli/cli.py index a6c066c5..eedaa42e 100644 --- a/src/datapilot/core/platforms/dbt/cli/cli.py +++ b/src/datapilot/core/platforms/dbt/cli/cli.py @@ -75,7 +75,8 @@ def project_health(manifest_path, catalog_path, config_path=None): @click.option("--tenant", prompt="Tenant", help="Your tenant ID.") @click.option("--dbt_core_integration_id", prompt="DBT Core Integration ID", help="DBT Core Integration ID") @click.option("--manifest-path", required=True, prompt="Manifest Path", help="Path to the manifest file.") -def onboard(token, tenant, dbt_core_integration_id, manifest_path, env=None): +@click.option("--backend-url", required=False, prompt="Altimate's Backend URL", help="Altimate's Backend URL") +def onboard(token, tenant, dbt_core_integration_id, manifest_path, backend_url="https://api.myaltimate.com", env=None): """Onboard a manifest file to DBT.""" if not token and env: token = os.environ.get("ALTIMATE_API_KEY") @@ -94,4 +95,9 @@ def onboard(token, tenant, dbt_core_integration_id, manifest_path, env=None): # This will throw error if manifest file is incorrect load_manifest(manifest_path) - onboard_manifest(token, tenant, dbt_core_integration_id, manifest_path) + response = onboard_manifest(token, tenant, dbt_core_integration_id, manifest_path, backend_url) + + if response["ok"]: + click.echo("Manifest onboarded successfully!") + else: + click.echo(f"{response.message}") diff --git a/src/datapilot/utils/utils.py b/src/datapilot/utils/utils.py index f2885376..4a163ddc 100644 --- a/src/datapilot/utils/utils.py +++ b/src/datapilot/utils/utils.py @@ -3,6 +3,8 @@ from pathlib import Path from typing import Dict +from requests import Response + from datapilot.clients.altimate.client import APIClient @@ -44,11 +46,7 @@ def get_dir_path(path: str) -> str: return Path(path).parent -# Will need to change this -base_url = "http://localhost:5001" - - -def upload_content_to_signed_url(file_path, signed_url): +def upload_content_to_signed_url(file_path, signed_url) -> Response: api_client = APIClient() with Path(file_path).open("rb") as file: @@ -57,12 +55,11 @@ def upload_content_to_signed_url(file_path, signed_url): return api_client.put(signed_url, data=file_content) -def onboard_manifest(api_token, tenant, dbt_core_integration_id, manifest_path): - api_client = APIClient(api_token, base_url, tenant) +def onboard_manifest(api_token, tenant, dbt_core_integration_id, manifest_path, backend_url) -> dict: + api_client = APIClient(api_token, base_url=backend_url, tenant=tenant) - endpoint = "/dbt/v1/signed_url" params = {"dbt_core_integration_id": dbt_core_integration_id, "file_type": "manifest"} - signed_url_data = api_client.get(endpoint, params=params) + signed_url_data = api_client.get_signed_url(params) if signed_url_data: signed_url = signed_url_data.get("url") @@ -75,9 +72,11 @@ def onboard_manifest(api_token, tenant, dbt_core_integration_id, manifest_path): if upload_response: verify_params = {"dbt_core_integration_file_id": file_id} api_client.verify_upload(verify_params) - + return {"ok": True} else: api_client.log(f"Error uploading file: {upload_response.status_code}, {upload_response.text}") + return {"ok": False, "message": f"Error uploading file: {upload_response.status_code}, {upload_response.text}"} else: api_client.log("Error getting signed URL.") + return {"ok": False, "message": "Error getting signed URL."} From c7e5b553d19bf9ebcbafae16504cc355c342d3b0 Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Tue, 6 Feb 2024 11:01:38 +0530 Subject: [PATCH 04/11] more changes --- src/datapilot/clients/altimate/client.py | 5 +++-- src/datapilot/core/platforms/dbt/cli/cli.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/datapilot/clients/altimate/client.py b/src/datapilot/clients/altimate/client.py index b0847991..5cf50679 100644 --- a/src/datapilot/clients/altimate/client.py +++ b/src/datapilot/clients/altimate/client.py @@ -56,7 +56,7 @@ def put(self, endpoint, data, timeout=None): def verify_upload(self, params=None): endpoint = "/dbt/v1/verify_upload" verify_response = self.post(endpoint, data=params) - if verify_response: + if verify_response.status_code == 200: self.logger.debug("File successfully uploaded and verified.") else: self.logger.debug(f"Error verifying upload: {verify_response.status_code}, {verify_response.text}") @@ -64,7 +64,8 @@ def verify_upload(self, params=None): def get_signed_url(self, params=None): endpoint = "/dbt/v1/signed_url" response: requests.Response = self.get(endpoint, params=params) - if response.status_code == 200: + if response: self.logger.debug("Fetched signed url") + return response else: self.logger.debug(f"Error fetching signed url: {response.status_code}, {response.text}") diff --git a/src/datapilot/core/platforms/dbt/cli/cli.py b/src/datapilot/core/platforms/dbt/cli/cli.py index eedaa42e..34b68f74 100644 --- a/src/datapilot/core/platforms/dbt/cli/cli.py +++ b/src/datapilot/core/platforms/dbt/cli/cli.py @@ -100,4 +100,4 @@ def onboard(token, tenant, dbt_core_integration_id, manifest_path, backend_url=" if response["ok"]: click.echo("Manifest onboarded successfully!") else: - click.echo(f"{response.message}") + click.echo(f"{response['message']}") From 408bc03a949e10e272a3f4d1c1022f6dca015587 Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Tue, 6 Feb 2024 22:19:11 +0530 Subject: [PATCH 05/11] add creds validation --- src/datapilot/clients/altimate/client.py | 14 ++++++++++++-- src/datapilot/core/platforms/dbt/cli/cli.py | 7 ++++--- src/datapilot/utils/utils.py | 9 +++++++++ 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/datapilot/clients/altimate/client.py b/src/datapilot/clients/altimate/client.py index 5cf50679..263e9409 100644 --- a/src/datapilot/clients/altimate/client.py +++ b/src/datapilot/clients/altimate/client.py @@ -63,9 +63,19 @@ def verify_upload(self, params=None): def get_signed_url(self, params=None): endpoint = "/dbt/v1/signed_url" - response: requests.Response = self.get(endpoint, params=params) + response = self.get(endpoint, params=params) if response: self.logger.debug("Fetched signed url") return response else: - self.logger.debug(f"Error fetching signed url: {response.status_code}, {response.text}") + self.logger.debug("Error fetching signed url") + + def validate_credentials(self): + endpoint = "/dbt/v3/validate-credentials" + response = self.get(endpoint) + if response: + self.logger.debug("Credentials are valid") + return True + else: + self.logger.debug("Error validating credentials") + return False diff --git a/src/datapilot/core/platforms/dbt/cli/cli.py b/src/datapilot/core/platforms/dbt/cli/cli.py index 34b68f74..7e71fd46 100644 --- a/src/datapilot/core/platforms/dbt/cli/cli.py +++ b/src/datapilot/core/platforms/dbt/cli/cli.py @@ -12,6 +12,7 @@ from datapilot.core.platforms.dbt.utils import load_manifest from datapilot.utils.formatting.utils import tabulate_data from datapilot.utils.utils import onboard_manifest +from datapilot.utils.utils import validate_credentials logging.basicConfig(level=logging.INFO) @@ -88,9 +89,9 @@ def onboard(token, tenant, dbt_core_integration_id, manifest_path, backend_url=" click.echo("Error: API Token is required.") return - # if not validate_credentials(token, tenant): - # print("Error: Invalid credentials.") - # return + if not validate_credentials(token, backend_url, tenant): + click.echo("Error: Invalid credentials.") + return # This will throw error if manifest file is incorrect load_manifest(manifest_path) diff --git a/src/datapilot/utils/utils.py b/src/datapilot/utils/utils.py index 4a163ddc..24fb5a46 100644 --- a/src/datapilot/utils/utils.py +++ b/src/datapilot/utils/utils.py @@ -55,6 +55,15 @@ def upload_content_to_signed_url(file_path, signed_url) -> Response: return api_client.put(signed_url, data=file_content) +def validate_credentials( + token, + backend_url, + tenant, +) -> Response: + api_client = APIClient(api_token=token, base_url=backend_url, tenant=tenant) + return api_client.validate_credentials() + + def onboard_manifest(api_token, tenant, dbt_core_integration_id, manifest_path, backend_url) -> dict: api_client = APIClient(api_token, base_url=backend_url, tenant=tenant) From 4c8768537c43d5b7b2f1bc98979ef5a850469f14 Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Wed, 7 Feb 2024 16:50:40 +0530 Subject: [PATCH 06/11] remove error handling --- src/datapilot/clients/altimate/client.py | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/src/datapilot/clients/altimate/client.py b/src/datapilot/clients/altimate/client.py index 263e9409..f4d22f86 100644 --- a/src/datapilot/clients/altimate/client.py +++ b/src/datapilot/clients/altimate/client.py @@ -55,27 +55,12 @@ def put(self, endpoint, data, timeout=None): def verify_upload(self, params=None): endpoint = "/dbt/v1/verify_upload" - verify_response = self.post(endpoint, data=params) - if verify_response.status_code == 200: - self.logger.debug("File successfully uploaded and verified.") - else: - self.logger.debug(f"Error verifying upload: {verify_response.status_code}, {verify_response.text}") + self.post(endpoint, data=params) def get_signed_url(self, params=None): endpoint = "/dbt/v1/signed_url" - response = self.get(endpoint, params=params) - if response: - self.logger.debug("Fetched signed url") - return response - else: - self.logger.debug("Error fetching signed url") + return self.get(endpoint, params=params) def validate_credentials(self): endpoint = "/dbt/v3/validate-credentials" - response = self.get(endpoint) - if response: - self.logger.debug("Credentials are valid") - return True - else: - self.logger.debug("Error validating credentials") - return False + return self.get(endpoint) From e24ab118b36e883c1ca8f9e92d7ee27cbb89afe4 Mon Sep 17 00:00:00 2001 From: Michiel De Smet Date: Wed, 7 Feb 2024 21:05:33 +0100 Subject: [PATCH 07/11] Add requests library dependency --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 7a489430..e13b349f 100755 --- a/setup.py +++ b/setup.py @@ -67,6 +67,7 @@ def read(*names, **kwargs): "dbt-artifacts-parser==0.5.1", "configtree==0.6", "tabulate==0.9.0", + "requests==2.31.0", ], extras_require={ # eg: From 9d5bc240ab076f7ce869f2e69b26439274f8b49f Mon Sep 17 00:00:00 2001 From: surya <118222278+suryaiyer95@users.noreply.github.com> Date: Mon, 12 Feb 2024 12:30:44 -0800 Subject: [PATCH 08/11] Update src/datapilot/utils/utils.py --- src/datapilot/utils/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datapilot/utils/utils.py b/src/datapilot/utils/utils.py index 24fb5a46..4c7296b2 100644 --- a/src/datapilot/utils/utils.py +++ b/src/datapilot/utils/utils.py @@ -88,4 +88,4 @@ def onboard_manifest(api_token, tenant, dbt_core_integration_id, manifest_path, else: api_client.log("Error getting signed URL.") - return {"ok": False, "message": "Error getting signed URL."} + return {"ok": False, "message": "Error in uploading the manifest."} From e7cb5c1766d54b8214082fdca3a6f432bdf1eef8 Mon Sep 17 00:00:00 2001 From: suryaiyer95 Date: Mon, 12 Feb 2024 15:14:43 -0800 Subject: [PATCH 09/11] fix:ruff --- src/datapilot/clients/altimate/client.py | 27 ++++++++++++++++++++---- src/datapilot/utils/utils.py | 6 ++++-- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/datapilot/clients/altimate/client.py b/src/datapilot/clients/altimate/client.py index f4d22f86..4ad9e5b1 100644 --- a/src/datapilot/clients/altimate/client.py +++ b/src/datapilot/clients/altimate/client.py @@ -1,6 +1,10 @@ import logging import requests +from requests.exceptions import ConnectionError +from requests.exceptions import HTTPError +from requests.exceptions import RequestException +from requests.exceptions import Timeout class APIClient: @@ -30,10 +34,25 @@ def get(self, endpoint, params=None, timeout=None): url = f"{self.base_url}{endpoint}" headers = self._get_headers() - self.logger.debug(f"Sending GET request for tenant {self.tenant} at url: {url}") - response = requests.get(url, headers=headers, params=params, timeout=timeout) - self.logger.debug(f"Received GET response with status: {response.status_code }") - return response.json() if response.status_code == 200 else None + try: + self.logger.debug(f"Sending GET request for tenant {self.tenant} at url: {url}") + response = requests.get(url, headers=headers, params=params, timeout=timeout) + + # Check if the response was successful + response.raise_for_status() + self.logger.debug(f"Received GET response with status: {response.status_code}") + return response.json() + + except HTTPError as http_err: + self.logger.error(f"{http_err.response.json()} - Status code: {response.status_code}") + except ConnectionError as conn_err: + self.logger.error(f"Connection error occurred: {conn_err}") + except Timeout as timeout_err: + self.logger.error(f"Timeout error occurred: {timeout_err}") + except RequestException as req_err: + self.logger.error(f"Unexpected error occurred: {req_err}") + except Exception as err: + self.logger.error(f"An error occurred: {err}") def post(self, endpoint, data=None, timeout=None): url = f"{self.base_url}{endpoint}" diff --git a/src/datapilot/utils/utils.py b/src/datapilot/utils/utils.py index 4c7296b2..7f650e84 100644 --- a/src/datapilot/utils/utils.py +++ b/src/datapilot/utils/utils.py @@ -69,7 +69,6 @@ def onboard_manifest(api_token, tenant, dbt_core_integration_id, manifest_path, params = {"dbt_core_integration_id": dbt_core_integration_id, "file_type": "manifest"} signed_url_data = api_client.get_signed_url(params) - if signed_url_data: signed_url = signed_url_data.get("url") file_id = signed_url_data.get("dbt_core_integration_file_id") @@ -88,4 +87,7 @@ def onboard_manifest(api_token, tenant, dbt_core_integration_id, manifest_path, else: api_client.log("Error getting signed URL.") - return {"ok": False, "message": "Error in uploading the manifest."} + return { + "ok": False, + "message": "Error in uploading the manifest. ", + } From 06c61b4d1d9837180bbd287f05084ad9c1a29a77 Mon Sep 17 00:00:00 2001 From: suryaiyer95 Date: Mon, 12 Feb 2024 15:26:52 -0800 Subject: [PATCH 10/11] fix: ruff --- src/datapilot/clients/altimate/utils.py | 23 +++++++++++++++++++++ src/datapilot/core/platforms/dbt/cli/cli.py | 20 ++++++------------ 2 files changed, 29 insertions(+), 14 deletions(-) create mode 100644 src/datapilot/clients/altimate/utils.py diff --git a/src/datapilot/clients/altimate/utils.py b/src/datapilot/clients/altimate/utils.py new file mode 100644 index 00000000..548dd156 --- /dev/null +++ b/src/datapilot/clients/altimate/utils.py @@ -0,0 +1,23 @@ +import os +from typing import Optional + +import click + + +def check_token_and_instance( + token: Optional[str], + instance_name: Optional[str], +): + if not token: + token = os.environ.get("ALTIMATE_API_KEY") + + if not instance_name: + instance_name = os.environ.get("ALTIMATE_INSTANCE_NAME") + + if not token or not instance_name: + click.echo( + "Error: API TOKEN and instance name is required. Please provide a valid API token." + " You can pass it as command line arguments or set it using environment variables like " + "ALTIMATE_API_KEY and ALTIMATE_INSTANCE_NAME." + ) + return diff --git a/src/datapilot/core/platforms/dbt/cli/cli.py b/src/datapilot/core/platforms/dbt/cli/cli.py index 7e71fd46..7bbf9714 100644 --- a/src/datapilot/core/platforms/dbt/cli/cli.py +++ b/src/datapilot/core/platforms/dbt/cli/cli.py @@ -1,8 +1,8 @@ import logging -import os import click +from datapilot.clients.altimate.utils import check_token_and_instance from datapilot.config.config import load_config from datapilot.core.platforms.dbt.constants import MODEL from datapilot.core.platforms.dbt.constants import PROJECT @@ -73,30 +73,22 @@ def project_health(manifest_path, catalog_path, config_path=None): @dbt.command("onboard") @click.option("--token", prompt="API Token", help="Your API token for authentication.") -@click.option("--tenant", prompt="Tenant", help="Your tenant ID.") +@click.option("--instance-name", prompt="Instance Name", help="Your tenant ID.") @click.option("--dbt_core_integration_id", prompt="DBT Core Integration ID", help="DBT Core Integration ID") @click.option("--manifest-path", required=True, prompt="Manifest Path", help="Path to the manifest file.") @click.option("--backend-url", required=False, prompt="Altimate's Backend URL", help="Altimate's Backend URL") -def onboard(token, tenant, dbt_core_integration_id, manifest_path, backend_url="https://api.myaltimate.com", env=None): +def onboard(token, instance_name, dbt_core_integration_id, manifest_path, backend_url="https://api.myaltimate.com", env=None): """Onboard a manifest file to DBT.""" - if not token and env: - token = os.environ.get("ALTIMATE_API_KEY") + check_token_and_instance(token, instance_name) - if not tenant and env: - tenant = os.environ.get("ALTIMATE_INSTANCE_NAME") - - if not token or not tenant: - click.echo("Error: API Token is required.") - return - - if not validate_credentials(token, backend_url, tenant): + if not validate_credentials(token, backend_url, instance_name): click.echo("Error: Invalid credentials.") return # This will throw error if manifest file is incorrect load_manifest(manifest_path) - response = onboard_manifest(token, tenant, dbt_core_integration_id, manifest_path, backend_url) + response = onboard_manifest(token, instance_name, dbt_core_integration_id, manifest_path, backend_url) if response["ok"]: click.echo("Manifest onboarded successfully!") From d3de3c909e4922c3d645d61a167d45c753574395 Mon Sep 17 00:00:00 2001 From: suryaiyer95 Date: Mon, 12 Feb 2024 15:29:44 -0800 Subject: [PATCH 11/11] fix: ruff --- src/datapilot/clients/altimate/utils.py | 51 +++++++++++++++++++++ src/datapilot/core/platforms/dbt/cli/cli.py | 4 +- src/datapilot/utils/utils.py | 51 --------------------- 3 files changed, 53 insertions(+), 53 deletions(-) diff --git a/src/datapilot/clients/altimate/utils.py b/src/datapilot/clients/altimate/utils.py index 548dd156..9b0e8550 100644 --- a/src/datapilot/clients/altimate/utils.py +++ b/src/datapilot/clients/altimate/utils.py @@ -1,7 +1,11 @@ import os +from pathlib import Path from typing import Optional import click +from requests import Response + +from datapilot.clients.altimate.client import APIClient def check_token_and_instance( @@ -21,3 +25,50 @@ def check_token_and_instance( "ALTIMATE_API_KEY and ALTIMATE_INSTANCE_NAME." ) return + + +def upload_content_to_signed_url(file_path, signed_url) -> Response: + api_client = APIClient() + + with Path(file_path).open("rb") as file: + file_content = file.read() + + return api_client.put(signed_url, data=file_content) + + +def validate_credentials( + token, + backend_url, + tenant, +) -> Response: + api_client = APIClient(api_token=token, base_url=backend_url, tenant=tenant) + return api_client.validate_credentials() + + +def onboard_manifest(api_token, tenant, dbt_core_integration_id, manifest_path, backend_url) -> dict: + api_client = APIClient(api_token, base_url=backend_url, tenant=tenant) + + params = {"dbt_core_integration_id": dbt_core_integration_id, "file_type": "manifest"} + signed_url_data = api_client.get_signed_url(params) + if signed_url_data: + signed_url = signed_url_data.get("url") + file_id = signed_url_data.get("dbt_core_integration_file_id") + api_client.log(f"Received signed URL: {signed_url}") + api_client.log(f"Received File ID: {file_id}") + + upload_response = upload_content_to_signed_url(manifest_path, signed_url) + + if upload_response: + verify_params = {"dbt_core_integration_file_id": file_id} + api_client.verify_upload(verify_params) + return {"ok": True} + else: + api_client.log(f"Error uploading file: {upload_response.status_code}, {upload_response.text}") + return {"ok": False, "message": f"Error uploading file: {upload_response.status_code}, {upload_response.text}"} + + else: + api_client.log("Error getting signed URL.") + return { + "ok": False, + "message": "Error in uploading the manifest. ", + } diff --git a/src/datapilot/core/platforms/dbt/cli/cli.py b/src/datapilot/core/platforms/dbt/cli/cli.py index 7bbf9714..e3cc0a22 100644 --- a/src/datapilot/core/platforms/dbt/cli/cli.py +++ b/src/datapilot/core/platforms/dbt/cli/cli.py @@ -3,6 +3,8 @@ import click from datapilot.clients.altimate.utils import check_token_and_instance +from datapilot.clients.altimate.utils import onboard_manifest +from datapilot.clients.altimate.utils import validate_credentials from datapilot.config.config import load_config from datapilot.core.platforms.dbt.constants import MODEL from datapilot.core.platforms.dbt.constants import PROJECT @@ -11,8 +13,6 @@ from datapilot.core.platforms.dbt.formatting import generate_project_insights_table from datapilot.core.platforms.dbt.utils import load_manifest from datapilot.utils.formatting.utils import tabulate_data -from datapilot.utils.utils import onboard_manifest -from datapilot.utils.utils import validate_credentials logging.basicConfig(level=logging.INFO) diff --git a/src/datapilot/utils/utils.py b/src/datapilot/utils/utils.py index 7f650e84..77e7a463 100644 --- a/src/datapilot/utils/utils.py +++ b/src/datapilot/utils/utils.py @@ -3,10 +3,6 @@ from pathlib import Path from typing import Dict -from requests import Response - -from datapilot.clients.altimate.client import APIClient - def load_json(file_path: str) -> Dict: try: @@ -44,50 +40,3 @@ def get_dir_path(path: str) -> str: :return: """ return Path(path).parent - - -def upload_content_to_signed_url(file_path, signed_url) -> Response: - api_client = APIClient() - - with Path(file_path).open("rb") as file: - file_content = file.read() - - return api_client.put(signed_url, data=file_content) - - -def validate_credentials( - token, - backend_url, - tenant, -) -> Response: - api_client = APIClient(api_token=token, base_url=backend_url, tenant=tenant) - return api_client.validate_credentials() - - -def onboard_manifest(api_token, tenant, dbt_core_integration_id, manifest_path, backend_url) -> dict: - api_client = APIClient(api_token, base_url=backend_url, tenant=tenant) - - params = {"dbt_core_integration_id": dbt_core_integration_id, "file_type": "manifest"} - signed_url_data = api_client.get_signed_url(params) - if signed_url_data: - signed_url = signed_url_data.get("url") - file_id = signed_url_data.get("dbt_core_integration_file_id") - api_client.log(f"Received signed URL: {signed_url}") - api_client.log(f"Received File ID: {file_id}") - - upload_response = upload_content_to_signed_url(manifest_path, signed_url) - - if upload_response: - verify_params = {"dbt_core_integration_file_id": file_id} - api_client.verify_upload(verify_params) - return {"ok": True} - else: - api_client.log(f"Error uploading file: {upload_response.status_code}, {upload_response.text}") - return {"ok": False, "message": f"Error uploading file: {upload_response.status_code}, {upload_response.text}"} - - else: - api_client.log("Error getting signed URL.") - return { - "ok": False, - "message": "Error in uploading the manifest. ", - }