From ca23d1bc74c3a4b9901a35c081ff47b16a4ce7a2 Mon Sep 17 00:00:00 2001 From: Maxime Armstrong Date: Tue, 19 Nov 2024 20:21:15 -0500 Subject: [PATCH 1/7] [docs] Add docs for reworked Fivetran API --- docs/content/integrations/fivetran/fivetran.mdx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/content/integrations/fivetran/fivetran.mdx b/docs/content/integrations/fivetran/fivetran.mdx index e69de29bb2d1d..3d8d60b6dbd98 100644 --- a/docs/content/integrations/fivetran/fivetran.mdx +++ b/docs/content/integrations/fivetran/fivetran.mdx @@ -0,0 +1,6 @@ +--- +title: "Using Dagster with Fivetran" +description: Represent your Fivetran connectors in Dagster +--- + +# Using Dagster with Fivetran \ No newline at end of file From b52beada50a39d2fb7eba81bac06975cf1d94fc1 Mon Sep 17 00:00:00 2001 From: Maxime Armstrong Date: Wed, 27 Nov 2024 09:24:26 -0500 Subject: [PATCH 2/7] Add intro and prerequisites --- .../integrations/fivetran/fivetran.mdx | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/content/integrations/fivetran/fivetran.mdx b/docs/content/integrations/fivetran/fivetran.mdx index 3d8d60b6dbd98..af021f94fafc0 100644 --- a/docs/content/integrations/fivetran/fivetran.mdx +++ b/docs/content/integrations/fivetran/fivetran.mdx @@ -3,4 +3,30 @@ title: "Using Dagster with Fivetran" description: Represent your Fivetran connectors in Dagster --- -# Using Dagster with Fivetran \ No newline at end of file +# Using Dagster with Fivetran + +This guide provides instructions for using Dagster with Fivetran using the `dagster-fivetran` library. Your Fivetran connector tables, can be represented as assets in the Dagster asset graph, allowing you to track lineage and dependencies between Fivetran assets and data assets you are already modeling in Dagster. You can also use Dagster to orchestrate Fivetran connectors, allowing you to trigger sync these on a cadence or based on upstream data changes. + +--- + +## What you'll learn + +- How to represent Fivetran assets in the Dagster asset graph, including lineage to other Dagster assets. +- How to customize asset definition metadata for these Fivetran assets. +- How to materialize Fivetran connector tables from Dagster. +- How to customize how Fivetran connector tables are materialized. + +--- + +## Prerequisites + +To get started, you'll need the following: + +- The `dagster` and `dagster-fivetran` libraries installed in your environment +- Familiarity with asset definitions and the Dagster asset graph +- Familiarity with Dagster resources +- Familiarity with Fivetran concepts, like connectors and connector tables +- A Fivetran workspace +- A Fivetran API key and API secret. For more information, see [Getting Started](https://fivetran.com/docs/rest-api/getting-started) in the Fivetran REST API documentation. + +--- \ No newline at end of file From d48e1d65e69688c2cc0083f2fcf613afda23ffed Mon Sep 17 00:00:00 2001 From: Maxime Armstrong Date: Wed, 27 Nov 2024 09:39:23 -0500 Subject: [PATCH 3/7] Update fivetran docs --- docs/content/integrations/fivetran/fivetran.mdx | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/docs/content/integrations/fivetran/fivetran.mdx b/docs/content/integrations/fivetran/fivetran.mdx index af021f94fafc0..4a0047b101cde 100644 --- a/docs/content/integrations/fivetran/fivetran.mdx +++ b/docs/content/integrations/fivetran/fivetran.mdx @@ -5,9 +5,7 @@ description: Represent your Fivetran connectors in Dagster # Using Dagster with Fivetran -This guide provides instructions for using Dagster with Fivetran using the `dagster-fivetran` library. Your Fivetran connector tables, can be represented as assets in the Dagster asset graph, allowing you to track lineage and dependencies between Fivetran assets and data assets you are already modeling in Dagster. You can also use Dagster to orchestrate Fivetran connectors, allowing you to trigger sync these on a cadence or based on upstream data changes. - ---- +This guide provides instructions for using Dagster with Fivetran using the `dagster-fivetran` library. Your Fivetran connector tables can be represented as assets in the Dagster asset graph, allowing you to track lineage and dependencies between Fivetran assets and data assets you are already modeling in Dagster. You can also use Dagster to orchestrate Fivetran connectors, allowing you to trigger syncs for these on a cadence or based on upstream data changes. ## What you'll learn @@ -16,11 +14,9 @@ This guide provides instructions for using Dagster with Fivetran using the `dags - How to materialize Fivetran connector tables from Dagster. - How to customize how Fivetran connector tables are materialized. ---- - -## Prerequisites -To get started, you'll need the following: +
+ Prerequisites - The `dagster` and `dagster-fivetran` libraries installed in your environment - Familiarity with asset definitions and the Dagster asset graph @@ -29,4 +25,4 @@ To get started, you'll need the following: - A Fivetran workspace - A Fivetran API key and API secret. For more information, see [Getting Started](https://fivetran.com/docs/rest-api/getting-started) in the Fivetran REST API documentation. ---- \ No newline at end of file +
From 6a484f7534abe286d3b4d6fd97e04cc36c09c6f9 Mon Sep 17 00:00:00 2001 From: Maxime Armstrong Date: Wed, 27 Nov 2024 13:26:02 -0500 Subject: [PATCH 4/7] Add custom translator section in Fivetran docs --- .../integrations/fivetran/fivetran.mdx | 93 ++++++++++++++++++- .../apidocs/libraries/dagster-fivetran.rst | 29 +++--- .../fivetran/customize-fivetran-asset-defs.py | 60 ++++++++++++ .../fivetran/representing-fivetran-assets.py | 14 +++ 4 files changed, 182 insertions(+), 14 deletions(-) create mode 100644 examples/docs_snippets/docs_snippets/integrations/fivetran/customize-fivetran-asset-defs.py create mode 100644 examples/docs_snippets/docs_snippets/integrations/fivetran/representing-fivetran-assets.py diff --git a/docs/content/integrations/fivetran/fivetran.mdx b/docs/content/integrations/fivetran/fivetran.mdx index 4a0047b101cde..ab85f63c93027 100644 --- a/docs/content/integrations/fivetran/fivetran.mdx +++ b/docs/content/integrations/fivetran/fivetran.mdx @@ -14,7 +14,6 @@ This guide provides instructions for using Dagster with Fivetran using the `dags - How to materialize Fivetran connector tables from Dagster. - How to customize how Fivetran connector tables are materialized. -
Prerequisites @@ -26,3 +25,95 @@ This guide provides instructions for using Dagster with Fivetran using the `dags - A Fivetran API key and API secret. For more information, see [Getting Started](https://fivetran.com/docs/rest-api/getting-started) in the Fivetran REST API documentation.
+ +## Represent Fivetran assets in the asset graph + +To load Fivetran assets into the Dagster asset graph, you must first construct a resource, which allows Dagster to communicate with your Fivetran workspace. You'll need to supply your account ID, API key and API secret. See [Getting Started](https://fivetran.com/docs/rest-api/getting-started) in the Fivetran REST API documentation for more information on how to create your API key and API secret. + +Dagster can automatically load all connector tables from your Sigma workspace as asset specs. Call the function, which returns list of s representing your Fivetran assets. You can then include these asset specs in your object: + +```python file=/integrations/fivetran/representing-fivetran-assets.py +from dagster_fivetran import FivetranWorkspace, load_fivetran_asset_specs + +import dagster as dg + +fivetran_workspace = FivetranWorkspace( + account_id=dg.EnvVar("FIVETRAN_ACCOUNT_ID"), + api_key=dg.EnvVar("FIVETRAN_API_KEY"), + api_secret=dg.EnvVar("FIVETRAN_API_SECRET"), +) + +fivetran_specs = load_fivetran_asset_specs(fivetran_workspace) +defs = dg.Definitions( + assets=[*fivetran_specs], resources={"fivetran": fivetran_workspace} +) +``` + +### Customize asset definition metadata for Fivetran assets + +By default, Dagster will generate asset specs for each Fivetran asset and populate default metadata. You can further customize asset properties by passing a custom subclass to the function, the decorator or the factory. + +```python file=/integrations/fivetran/customize-fivetran-asset-defs.py +from dagster_fivetran import ( + DagsterFivetranTranslator, + FivetranConnectorTableProps, + FivetranWorkspace, + build_fivetran_assets_definitions, + fivetran_assets, + load_fivetran_asset_specs, +) + +import dagster as dg + +fivetran_workspace = FivetranWorkspace( + account_id=dg.EnvVar("FIVETRAN_ACCOUNT_ID"), + api_key=dg.EnvVar("FIVETRAN_API_KEY"), + api_secret=dg.EnvVar("FIVETRAN_API_SECRET"), +) + + +# A translator class lets us customize properties of the built +# Fivetran assets, such as the owners or asset key +class MyCustomFivetranTranslator(DagsterFivetranTranslator): + def get_asset_spec(self, props: FivetranConnectorTableProps) -> dg.AssetSpec: + # We create the default asset spec using super() + default_spec = super().get_asset_spec(props) + # We customize the metadata, asset key prefix and team owner tag for all assets + return default_spec.replace_attributes( + key=default_spec.key.with_prefix("prefix"), + owners=["team:my_team"], + ).merge_attributes(metadata={"custom": "metadata"}) + + +# When loading asset specs +fivetran_specs = load_fivetran_asset_specs( + fivetran_workspace, dagster_fivetran_translator=MyCustomFivetranTranslator() +) + + +# Alternatively, when creating assets definition for a given connector using the `@fivetran_assets` decorator +@fivetran_assets( + connector_id="fivetran_connector_id", + name="fivetran_connector_id", + group_name="fivetran_connector_id", + workspace=fivetran_workspace, + dagster_fivetran_translator=MyCustomFivetranTranslator(), +) +def fivetran_connector_assets( + context: dg.AssetExecutionContext, fivetran: FivetranWorkspace +): + yield from fivetran.sync_and_poll(context=context) + + +# Alternatively, when creating all assets definitions for the Fivetran workspace +# using the `build_fivetran_assets_definitions` factory +fivetran_assets = build_fivetran_assets_definitions( + fivetran_workspace, dagster_fivetran_translator=MyCustomFivetranTranslator() +) + +defs = dg.Definitions( + assets=[*fivetran_assets], resources={"tableau": fivetran_workspace} +) +``` + +Note that `super()` is called in each of the overridden methods to generate the default asset spec. It is best practice to generate the default asset spec before customizing it. diff --git a/docs/sphinx/sections/api/apidocs/libraries/dagster-fivetran.rst b/docs/sphinx/sections/api/apidocs/libraries/dagster-fivetran.rst index 7116c528ce7da..c62d85554acce 100644 --- a/docs/sphinx/sections/api/apidocs/libraries/dagster-fivetran.rst +++ b/docs/sphinx/sections/api/apidocs/libraries/dagster-fivetran.rst @@ -6,24 +6,18 @@ This library provides a Dagster integration with `Fivetran dg.AssetSpec: + # We create the default asset spec using super() + default_spec = super().get_asset_spec(props) + # We customize the metadata, asset key prefix and team owner tag for all assets + return default_spec.replace_attributes( + key=default_spec.key.with_prefix("prefix"), + owners=["team:my_team"], + ).merge_attributes(metadata={"custom": "metadata"}) + + +# When loading asset specs +fivetran_specs = load_fivetran_asset_specs( + fivetran_workspace, dagster_fivetran_translator=MyCustomFivetranTranslator() +) + + +# Alternatively, when creating assets definition for a given connector using the `@fivetran_assets` decorator +@fivetran_assets( + connector_id="fivetran_connector_id", + name="fivetran_connector_id", + group_name="fivetran_connector_id", + workspace=fivetran_workspace, + dagster_fivetran_translator=MyCustomFivetranTranslator(), +) +def fivetran_connector_assets( + context: dg.AssetExecutionContext, fivetran: FivetranWorkspace +): + yield from fivetran.sync_and_poll(context=context) + + +# Alternatively, when creating all assets definitions for the Fivetran workspace +# using the `build_fivetran_assets_definitions` factory +fivetran_assets = build_fivetran_assets_definitions( + fivetran_workspace, dagster_fivetran_translator=MyCustomFivetranTranslator() +) + +defs = dg.Definitions( + assets=[*fivetran_assets], resources={"tableau": fivetran_workspace} +) diff --git a/examples/docs_snippets/docs_snippets/integrations/fivetran/representing-fivetran-assets.py b/examples/docs_snippets/docs_snippets/integrations/fivetran/representing-fivetran-assets.py new file mode 100644 index 0000000000000..efca6dda3d9bf --- /dev/null +++ b/examples/docs_snippets/docs_snippets/integrations/fivetran/representing-fivetran-assets.py @@ -0,0 +1,14 @@ +from dagster_fivetran import FivetranWorkspace, load_fivetran_asset_specs + +import dagster as dg + +fivetran_workspace = FivetranWorkspace( + account_id=dg.EnvVar("FIVETRAN_ACCOUNT_ID"), + api_key=dg.EnvVar("FIVETRAN_API_KEY"), + api_secret=dg.EnvVar("FIVETRAN_API_SECRET"), +) + +fivetran_specs = load_fivetran_asset_specs(fivetran_workspace) +defs = dg.Definitions( + assets=[*fivetran_specs], resources={"fivetran": fivetran_workspace} +) From 0bca81ecee752c9f6e490c3571c1640b43a2cabc Mon Sep 17 00:00:00 2001 From: Maxime Armstrong Date: Wed, 27 Nov 2024 14:13:04 -0500 Subject: [PATCH 5/7] Add multiple fivetran workspaces section --- .../integrations/fivetran/fivetran.mdx | 35 ++++++++++++++++++- .../fivetran/customize-fivetran-asset-defs.py | 2 +- .../fivetran/multiple-fivetran-workspaces.py | 26 ++++++++++++++ 3 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 examples/docs_snippets/docs_snippets/integrations/fivetran/multiple-fivetran-workspaces.py diff --git a/docs/content/integrations/fivetran/fivetran.mdx b/docs/content/integrations/fivetran/fivetran.mdx index ab85f63c93027..ef9539e838b08 100644 --- a/docs/content/integrations/fivetran/fivetran.mdx +++ b/docs/content/integrations/fivetran/fivetran.mdx @@ -112,8 +112,41 @@ fivetran_assets = build_fivetran_assets_definitions( ) defs = dg.Definitions( - assets=[*fivetran_assets], resources={"tableau": fivetran_workspace} + assets=[*fivetran_assets], resources={"fivetran": fivetran_workspace} ) ``` Note that `super()` is called in each of the overridden methods to generate the default asset spec. It is best practice to generate the default asset spec before customizing it. + +### Load Fivetran assets from multiple workspaces + +Definitions from multiple Fivetran workspaces can be combined by instantiating multiple resources and merging their specs. This lets you view all your Fivetran assets in a single asset graph: + +```python file=/integrations/fivetran/multiple-fivetran-workspaces.py +from dagster_fivetran import FivetranWorkspace, load_fivetran_asset_specs + +import dagster as dg + +sales_fivetran_workspace = FivetranWorkspace( + account_id=dg.EnvVar("FIVETRAN_SALES_ACCOUNT_ID"), + api_key=dg.EnvVar("FIVETRAN_SALES_API_KEY"), + api_secret=dg.EnvVar("FIVETRAN_SALES_API_SECRET"), +) +marketing_fivetran_workspace = FivetranWorkspace( + account_id=dg.EnvVar("FIVETRAN_MARKETING_ACCOUNT_ID"), + api_key=dg.EnvVar("FIVETRAN_MARKETING_API_KEY"), + api_secret=dg.EnvVar("FIVETRAN_MARKETING_API_SECRET"), +) + +sales_fivetran_specs = load_fivetran_asset_specs(sales_fivetran_workspace) +marketing_fivetran_specs = load_fivetran_asset_specs(marketing_fivetran_workspace) + +# Merge the specs into a single set of definitions +defs = dg.Definitions( + assets=[*sales_fivetran_specs, *marketing_fivetran_specs], + resources={ + "marketing_fivetran": marketing_fivetran_workspace, + "sales_fivetran": sales_fivetran_workspace, + }, +) +``` diff --git a/examples/docs_snippets/docs_snippets/integrations/fivetran/customize-fivetran-asset-defs.py b/examples/docs_snippets/docs_snippets/integrations/fivetran/customize-fivetran-asset-defs.py index 716adccc270b5..b5bb5195b89b4 100644 --- a/examples/docs_snippets/docs_snippets/integrations/fivetran/customize-fivetran-asset-defs.py +++ b/examples/docs_snippets/docs_snippets/integrations/fivetran/customize-fivetran-asset-defs.py @@ -56,5 +56,5 @@ def fivetran_connector_assets( ) defs = dg.Definitions( - assets=[*fivetran_assets], resources={"tableau": fivetran_workspace} + assets=[*fivetran_assets], resources={"fivetran": fivetran_workspace} ) diff --git a/examples/docs_snippets/docs_snippets/integrations/fivetran/multiple-fivetran-workspaces.py b/examples/docs_snippets/docs_snippets/integrations/fivetran/multiple-fivetran-workspaces.py new file mode 100644 index 0000000000000..d28203bf3bcdf --- /dev/null +++ b/examples/docs_snippets/docs_snippets/integrations/fivetran/multiple-fivetran-workspaces.py @@ -0,0 +1,26 @@ +from dagster_fivetran import FivetranWorkspace, load_fivetran_asset_specs + +import dagster as dg + +sales_fivetran_workspace = FivetranWorkspace( + account_id=dg.EnvVar("FIVETRAN_SALES_ACCOUNT_ID"), + api_key=dg.EnvVar("FIVETRAN_SALES_API_KEY"), + api_secret=dg.EnvVar("FIVETRAN_SALES_API_SECRET"), +) +marketing_fivetran_workspace = FivetranWorkspace( + account_id=dg.EnvVar("FIVETRAN_MARKETING_ACCOUNT_ID"), + api_key=dg.EnvVar("FIVETRAN_MARKETING_API_KEY"), + api_secret=dg.EnvVar("FIVETRAN_MARKETING_API_SECRET"), +) + +sales_fivetran_specs = load_fivetran_asset_specs(sales_fivetran_workspace) +marketing_fivetran_specs = load_fivetran_asset_specs(marketing_fivetran_workspace) + +# Merge the specs into a single set of definitions +defs = dg.Definitions( + assets=[*sales_fivetran_specs, *marketing_fivetran_specs], + resources={ + "marketing_fivetran": marketing_fivetran_workspace, + "sales_fivetran": sales_fivetran_workspace, + }, +) From 7e0bdced800020e4fe1441c60ea8977a15d0dfbc Mon Sep 17 00:00:00 2001 From: Maxime Armstrong Date: Wed, 27 Nov 2024 14:23:37 -0500 Subject: [PATCH 6/7] Add materialize fivetran assets section --- .../integrations/fivetran/fivetran.mdx | 39 ++++++++++++++++++- .../sync-and-materialize-fivetran-assets.py | 27 +++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 examples/docs_snippets/docs_snippets/integrations/fivetran/sync-and-materialize-fivetran-assets.py diff --git a/docs/content/integrations/fivetran/fivetran.mdx b/docs/content/integrations/fivetran/fivetran.mdx index ef9539e838b08..abeb983e46cad 100644 --- a/docs/content/integrations/fivetran/fivetran.mdx +++ b/docs/content/integrations/fivetran/fivetran.mdx @@ -30,7 +30,7 @@ This guide provides instructions for using Dagster with Fivetran using the `dags To load Fivetran assets into the Dagster asset graph, you must first construct a resource, which allows Dagster to communicate with your Fivetran workspace. You'll need to supply your account ID, API key and API secret. See [Getting Started](https://fivetran.com/docs/rest-api/getting-started) in the Fivetran REST API documentation for more information on how to create your API key and API secret. -Dagster can automatically load all connector tables from your Sigma workspace as asset specs. Call the function, which returns list of s representing your Fivetran assets. You can then include these asset specs in your object: +Dagster can automatically load all connector tables from your Fivetran workspace as asset specs. Call the function, which returns list of s representing your Fivetran assets. You can then include these asset specs in your object: ```python file=/integrations/fivetran/representing-fivetran-assets.py from dagster_fivetran import FivetranWorkspace, load_fivetran_asset_specs @@ -49,6 +49,41 @@ defs = dg.Definitions( ) ``` + +### Sync and materialize Fivetran assets + +You can use Dagster to sync Fivetran connectors and materialize Fivetran connector tables. + +```python file=/integrations/fivetran/sync-and-materialize-fivetran-assets.py +from dagster_fivetran import FivetranWorkspace, fivetran_assets + +import dagster as dg + +fivetran_workspace = FivetranWorkspace( + account_id=dg.EnvVar("FIVETRAN_ACCOUNT_ID"), + api_key=dg.EnvVar("FIVETRAN_API_KEY"), + api_secret=dg.EnvVar("FIVETRAN_API_SECRET"), +) + + +@fivetran_assets( + connector_id="fivetran_connector_id", + name="fivetran_connector_id", + group_name="fivetran_connector_id", + workspace=fivetran_workspace, +) +def fivetran_connector_assets( + context: dg.AssetExecutionContext, fivetran: FivetranWorkspace +): + yield from fivetran.sync_and_poll(context=context) + + +defs = dg.Definitions( + assets=[fivetran_connector_assets], + resources={"fivetran": fivetran_workspace}, +) +``` + ### Customize asset definition metadata for Fivetran assets By default, Dagster will generate asset specs for each Fivetran asset and populate default metadata. You can further customize asset properties by passing a custom subclass to the function, the decorator or the factory. @@ -149,4 +184,4 @@ defs = dg.Definitions( "sales_fivetran": sales_fivetran_workspace, }, ) -``` +``` \ No newline at end of file diff --git a/examples/docs_snippets/docs_snippets/integrations/fivetran/sync-and-materialize-fivetran-assets.py b/examples/docs_snippets/docs_snippets/integrations/fivetran/sync-and-materialize-fivetran-assets.py new file mode 100644 index 0000000000000..8c54672ab7e94 --- /dev/null +++ b/examples/docs_snippets/docs_snippets/integrations/fivetran/sync-and-materialize-fivetran-assets.py @@ -0,0 +1,27 @@ +from dagster_fivetran import FivetranWorkspace, fivetran_assets + +import dagster as dg + +fivetran_workspace = FivetranWorkspace( + account_id=dg.EnvVar("FIVETRAN_ACCOUNT_ID"), + api_key=dg.EnvVar("FIVETRAN_API_KEY"), + api_secret=dg.EnvVar("FIVETRAN_API_SECRET"), +) + + +@fivetran_assets( + connector_id="fivetran_connector_id", + name="fivetran_connector_id", + group_name="fivetran_connector_id", + workspace=fivetran_workspace, +) +def fivetran_connector_assets( + context: dg.AssetExecutionContext, fivetran: FivetranWorkspace +): + yield from fivetran.sync_and_poll(context=context) + + +defs = dg.Definitions( + assets=[fivetran_connector_assets], + resources={"fivetran": fivetran_workspace}, +) From e67667fc0a7504a9b90c90a8bce8252baa3c7764 Mon Sep 17 00:00:00 2001 From: Maxime Armstrong Date: Wed, 27 Nov 2024 14:27:47 -0500 Subject: [PATCH 7/7] Update materialize fivetran assets section --- .../integrations/fivetran/fivetran.mdx | 22 +++++++++++++------ .../fivetran/customize-fivetran-asset-defs.py | 4 ++-- .../sync-and-materialize-fivetran-assets.py | 13 +++++++++-- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/docs/content/integrations/fivetran/fivetran.mdx b/docs/content/integrations/fivetran/fivetran.mdx index abeb983e46cad..079fff497e343 100644 --- a/docs/content/integrations/fivetran/fivetran.mdx +++ b/docs/content/integrations/fivetran/fivetran.mdx @@ -49,13 +49,16 @@ defs = dg.Definitions( ) ``` - ### Sync and materialize Fivetran assets -You can use Dagster to sync Fivetran connectors and materialize Fivetran connector tables. +You can use Dagster to sync Fivetran connectors and materialize Fivetran connector tables. You can use the decorator to create the assets definition for a given connector, or the factory to create all assets definitions for your Fivetran workspace. ```python file=/integrations/fivetran/sync-and-materialize-fivetran-assets.py -from dagster_fivetran import FivetranWorkspace, fivetran_assets +from dagster_fivetran import ( + FivetranWorkspace, + build_fivetran_assets_definitions, + fivetran_assets, +) import dagster as dg @@ -66,6 +69,7 @@ fivetran_workspace = FivetranWorkspace( ) +# Creating assets definition for a given connector using the `@fivetran_assets` decorator @fivetran_assets( connector_id="fivetran_connector_id", name="fivetran_connector_id", @@ -78,8 +82,12 @@ def fivetran_connector_assets( yield from fivetran.sync_and_poll(context=context) +# Alternatively, creating all assets definitions for the Fivetran workspace +# using the `build_fivetran_assets_definitions` factory +all_fivetran_assets = build_fivetran_assets_definitions(fivetran_workspace) + defs = dg.Definitions( - assets=[fivetran_connector_assets], + assets=[*all_fivetran_assets], resources={"fivetran": fivetran_workspace}, ) ``` @@ -142,12 +150,12 @@ def fivetran_connector_assets( # Alternatively, when creating all assets definitions for the Fivetran workspace # using the `build_fivetran_assets_definitions` factory -fivetran_assets = build_fivetran_assets_definitions( +all_fivetran_assets = build_fivetran_assets_definitions( fivetran_workspace, dagster_fivetran_translator=MyCustomFivetranTranslator() ) defs = dg.Definitions( - assets=[*fivetran_assets], resources={"fivetran": fivetran_workspace} + assets=[*all_fivetran_assets], resources={"fivetran": fivetran_workspace} ) ``` @@ -184,4 +192,4 @@ defs = dg.Definitions( "sales_fivetran": sales_fivetran_workspace, }, ) -``` \ No newline at end of file +``` diff --git a/examples/docs_snippets/docs_snippets/integrations/fivetran/customize-fivetran-asset-defs.py b/examples/docs_snippets/docs_snippets/integrations/fivetran/customize-fivetran-asset-defs.py index b5bb5195b89b4..19931cec57102 100644 --- a/examples/docs_snippets/docs_snippets/integrations/fivetran/customize-fivetran-asset-defs.py +++ b/examples/docs_snippets/docs_snippets/integrations/fivetran/customize-fivetran-asset-defs.py @@ -51,10 +51,10 @@ def fivetran_connector_assets( # Alternatively, when creating all assets definitions for the Fivetran workspace # using the `build_fivetran_assets_definitions` factory -fivetran_assets = build_fivetran_assets_definitions( +all_fivetran_assets = build_fivetran_assets_definitions( fivetran_workspace, dagster_fivetran_translator=MyCustomFivetranTranslator() ) defs = dg.Definitions( - assets=[*fivetran_assets], resources={"fivetran": fivetran_workspace} + assets=[*all_fivetran_assets], resources={"fivetran": fivetran_workspace} ) diff --git a/examples/docs_snippets/docs_snippets/integrations/fivetran/sync-and-materialize-fivetran-assets.py b/examples/docs_snippets/docs_snippets/integrations/fivetran/sync-and-materialize-fivetran-assets.py index 8c54672ab7e94..4300c1a56df6f 100644 --- a/examples/docs_snippets/docs_snippets/integrations/fivetran/sync-and-materialize-fivetran-assets.py +++ b/examples/docs_snippets/docs_snippets/integrations/fivetran/sync-and-materialize-fivetran-assets.py @@ -1,4 +1,8 @@ -from dagster_fivetran import FivetranWorkspace, fivetran_assets +from dagster_fivetran import ( + FivetranWorkspace, + build_fivetran_assets_definitions, + fivetran_assets, +) import dagster as dg @@ -9,6 +13,7 @@ ) +# Creating assets definition for a given connector using the `@fivetran_assets` decorator @fivetran_assets( connector_id="fivetran_connector_id", name="fivetran_connector_id", @@ -21,7 +26,11 @@ def fivetran_connector_assets( yield from fivetran.sync_and_poll(context=context) +# Alternatively, creating all assets definitions for the Fivetran workspace +# using the `build_fivetran_assets_definitions` factory +all_fivetran_assets = build_fivetran_assets_definitions(fivetran_workspace) + defs = dg.Definitions( - assets=[fivetran_connector_assets], + assets=[*all_fivetran_assets], resources={"fivetran": fivetran_workspace}, )