Skip to content

Commit

Permalink
Lint
Browse files Browse the repository at this point in the history
  • Loading branch information
@maximearmstrong
maximearmstrong committed Nov 6, 2024
1 parent 840cb37 commit 162616e
Show file tree
Hide file tree
Showing 3 changed files with 41 additions and 56 deletions.
10 changes: 4 additions & 6 deletions python_modules/libraries/dagster-fivetran/dagster_fivetran/experimental/resources.py
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
import logging
from typing import Any, Mapping, Optional
from urllib.parse import urljoin
import time
import os
import time
from typing import Any, Mapping, Optional

import requests
from dagster import get_dagster_logger, __version__, Failure
from dagster import Failure, __version__, get_dagster_logger
from dagster._annotations import experimental
from dagster._config.pythonic_config import ConfigurableResource
from dagster._utils.cached_method import cached_method
Expand All @@ -15,7 +14,6 @@

from dagster_fivetran.experimental.translator import FivetranWorkspaceData


FIVETRAN_API_BASE = "https://api.fivetran.com"
FIVETRAN_API_VERSION = "v1"
FIVETRAN_CONNECTOR_ENDPOINT = "connectors"
Expand Down Expand Up @@ -141,7 +139,7 @@ def get_groups(self) -> Mapping[str, Any]:
Returns:
Dict[str, Any]: Parsed json data from the response to this request.
"""
return self._make_request("GET", f"groups")
return self._make_request("GET", "groups")


class FivetranWorkspace(ConfigurableResource):
Expand Down
79 changes: 35 additions & 44 deletions python_modules/libraries/dagster-fivetran/dagster_fivetran_tests/experimental/conftest.py
View file
Original file line number Diff line number Diff line change
@@ -1,25 +1,21 @@
from typing import Iterator, Callable
import contextlib
from typing import Callable, Iterator

import pytest

import responses

from dagster_fivetran.experimental.resources import FIVETRAN_API_BASE, FIVETRAN_API_VERSION, FIVETRAN_CONNECTOR_ENDPOINT
from dagster_fivetran.experimental.resources import (
FIVETRAN_API_BASE,
FIVETRAN_API_VERSION,
FIVETRAN_CONNECTOR_ENDPOINT,
)

SAMPLE_GROUPS = {
"code": "Success",
"message": "Operation performed.",
"data": {
"items": [
{
"id": "group_id",
"name": "Group_Name",
"created_at": "2024-01-01T00:00:00Z"
}
],
"nextCursor": "cursor_value"
}
"items": [{"id": "group_id", "name": "Group_Name", "created_at": "2024-01-01T00:00:00Z"}],
"nextCursor": "cursor_value",
},
}

SAMPLE_CONNECTORS_FOR_GROUP = {
Expand All @@ -37,27 +33,24 @@
{
"code": "resync_table_warning",
"message": "Resync Table Warning",
"details": "string"
"details": "string",
}
],
"warnings": [
{
"code": "resync_table_warning",
"message": "Resync Table Warning",
"details": "string"
"details": "string",
}
],
"schema_status": "ready",
"update_state": "delayed",
"setup_state": "connected",
"sync_state": "scheduled",
"is_historical_sync": False,
"rescheduled_for": "2024-12-01T15:43:29.013729Z"
},
"config": {
"property1": {},
"property2": {}
"rescheduled_for": "2024-12-01T15:43:29.013729Z",
},
"config": {"property1": {}, "property2": {}},
"daily_sync_time": "14:00",
"succeeded_at": "2024-12-01T15:43:29.013729Z",
"sync_frequency": 360,
Expand All @@ -68,7 +61,7 @@
"title": "Test Title",
"status": "PASSED",
"message": "Test Passed",
"details": "Test Details"
"details": "Test Details",
}
],
"source_sync_details": {},
Expand All @@ -80,7 +73,7 @@
"networking_method": "Directly",
"connect_card": {
"token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJkIjp7ImxvZ2luIjp0cnVlLCJ1c2VyIjoiX2FjY291bnR3b3J0aHkiLCJhY2NvdW50IjoiX21vb25iZWFtX2FjYyIsImdyb3VwIjoiX21vb25iZWFtIiwiY29ubmVjdG9yIjoiY29iYWx0X2VsZXZhdGlvbiIsIm1ldGhvZCI6IlBiZkNhcmQiLCJpZGVudGl0eSI6ZmFsc2V9LCJpYXQiOjE2Njc4MzA2MzZ9.YUMGUbzxW96xsKJLo4bTorqzx8Q19GTrUi3WFRFM8BU",
"uri": "https://fivetran.com/connect-card/setup?auth=eyJ0eXAiOiJKV1QiLCJh..."
"uri": "https://fivetran.com/connect-card/setup?auth=eyJ0eXAiOiJKV1QiLCJh...",
},
"pause_after_trial": False,
"data_delay_threshold": 0,
Expand All @@ -89,13 +82,13 @@
"local_processing_agent_id": "string",
"connect_card_config": {
"redirect_uri": "https://your.site/path",
"hide_setup_guide": True
"hide_setup_guide": True,
},
"hybrid_deployment_agent_id": "string"
"hybrid_deployment_agent_id": "string",
}
],
"nextCursor": "cursor_value"
}
"nextCursor": "cursor_value",
},
}

SAMPLE_DESTINATION_DETAILS = {
Expand All @@ -115,7 +108,7 @@
"title": "Test Title",
"status": "PASSED",
"message": "Test Passed",
"details": "Test Details"
"details": "Test Details",
}
],
"local_processing_agent_id": "local_processing_agent_id",
Expand All @@ -139,9 +132,9 @@
"databricks_connection_type": "Directly | PrivateLink | SshTunnel | ProxyAgent",
"secret_value": "service_principal_secret_value",
"oauth2_client_id": "string",
"personal_access_token": "string"
}
}
"personal_access_token": "string",
},
},
}

SAMPLE_CONNECTOR_DETAILS = {
Expand All @@ -157,22 +150,22 @@
{
"code": "resync_table_warning",
"message": "Resync Table Warning",
"details": "string"
"details": "string",
}
],
"warnings": [
{
"code": "resync_table_warning",
"message": "Resync Table Warning",
"details": "string"
"details": "string",
}
],
"schema_status": "ready",
"update_state": "delayed",
"setup_state": "connected",
"sync_state": "scheduled",
"is_historical_sync": False,
"rescheduled_for": "2024-12-01T15:43:29.013729Z"
"rescheduled_for": "2024-12-01T15:43:29.013729Z",
},
"daily_sync_time": "14:00",
"succeeded_at": "2024-03-17T12:31:40.870504Z",
Expand All @@ -184,7 +177,7 @@
"title": "Test Title",
"status": "PASSED",
"message": "Test Passed",
"details": "Test Details"
"details": "Test Details",
}
],
"source_sync_details": {},
Expand All @@ -196,32 +189,30 @@
"networking_method": "Directly",
"connect_card": {
"token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJkIjp7ImxvZ2luIjp0cnVlLCJ1c2VyIjoiX2FjY291bnR3b3J0aHkiLCJhY2NvdW50IjoiX21vb25iZWFtX2FjYyIsImdyb3VwIjoiX21vb25iZWFtIiwiY29ubmVjdG9yIjoiY29iYWx0X2VsZXZhdGlvbiIsIm1ldGhvZCI6IlBiZkNhcmQiLCJpZGVudGl0eSI6ZmFsc2V9LCJpYXQiOjE2Njc4MzA2MzZ9.YUMGUbzxW96xsKJLo4bTorqzx8Q19GTrUi3WFRFM8BU",
"uri": "https://fivetran.com/connect-card/setup?auth=eyJ0eXAiOiJKV1QiLCJh..."
"uri": "https://fivetran.com/connect-card/setup?auth=eyJ0eXAiOiJKV1QiLCJh...",
},
"pause_after_trial": False,
"data_delay_threshold": 0,
"data_delay_sensitivity": "NORMAL",
"schedule_type": "auto",
"local_processing_agent_id": "local_processing_agent_id",
"connect_card_config": {
"redirect_uri": "https://your.site/path",
"hide_setup_guide": True
},
"connect_card_config": {"redirect_uri": "https://your.site/path", "hide_setup_guide": True},
"hybrid_deployment_agent_id": "hybrid_deployment_agent_id",
"config": {
"api_key": "your_15five_api_key"
}
}
"config": {"api_key": "your_15five_api_key"},
},
}


@pytest.fixture(name="connector_id")
def connector_id_fixture() -> str:
return "connector_id"


@pytest.fixture(name="destination_id")
def destination_id_fixture() -> str:
return "destination_id"


@pytest.fixture(name="group_id")
def group_id_fixture() -> str:
return "group_id"
Expand Down Expand Up @@ -266,4 +257,4 @@ def _method() -> Iterator[responses.RequestsMock]:

yield response

return _method
return _method
8 changes: 2 additions & 6 deletions ..._modules/libraries/dagster-fivetran/dagster_fivetran_tests/experimental/test_resources.py
View file
Original file line number Diff line number Diff line change
@@ -1,24 +1,20 @@
import responses
import uuid
from typing import Callable

import responses
from dagster_fivetran.experimental import FivetranWorkspace


@responses.activate
def test_basic_resource_request(
connector_id: str,
destination_id: str,
group_id: str,
workspace_data_api_mocks_fn: Callable
connector_id: str, destination_id: str, group_id: str, workspace_data_api_mocks_fn: Callable
) -> None:
api_key = uuid.uuid4().hex
api_secret = uuid.uuid4().hex

resource = FivetranWorkspace(api_key=api_key, api_secret=api_secret)

with workspace_data_api_mocks_fn() as response:

client = resource.get_client()
client.get_connector_details(connector_id=connector_id)
client.get_connectors_for_group(group_id=group_id)
Expand Down

0 comments on commit 162616e

Please sign in to comment.