From d9361c3c55e7239cd67855dfb84bb7b0e3a5e226 Mon Sep 17 00:00:00 2001 From: Chris DeCarolis Date: Thu, 21 Nov 2024 15:38:17 -0800 Subject: [PATCH] [dagster-airlift] Operator Migration guides --- docs/content/_navigation.json | 10 ++++ docs/content/integrations/airlift.mdx | 6 +++ .../operator-migration/bash-operator-dbt.mdx | 53 +++++++++++++++++++ .../airlift/operator-migration/overview.mdx | 12 +++++ .../operator_migration/bash_operator_dbt.py | 4 +- 5 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 docs/content/integrations/airlift/operator-migration/bash-operator-dbt.mdx create mode 100644 docs/content/integrations/airlift/operator-migration/overview.mdx diff --git a/docs/content/_navigation.json b/docs/content/_navigation.json index 865424ed7718a..ea8cc5d6e7bae 100644 --- a/docs/content/_navigation.json +++ b/docs/content/_navigation.json @@ -971,6 +971,16 @@ { "title": "Migration with Airflow 1", "path": "/integrations/airlift/airflow-1-migration" + }, + { + "title": "Operator Migration Reference", + "path": "/integrations/airlift/operator-migration/overview", + "children": [ + { + "title": "Migrating a BashOperator for dbt", + "path": "/integrations/airlift/operator-migration/bash-operator-dbt" + } + ] } ] }, diff --git a/docs/content/integrations/airlift.mdx b/docs/content/integrations/airlift.mdx index 8cfc96209068e..4f702b052f2bb 100644 --- a/docs/content/integrations/airlift.mdx +++ b/docs/content/integrations/airlift.mdx @@ -39,6 +39,12 @@ In this tutorial, we'll use `dagster-airlift` to observe DAGs from multiple Airf [Click here to get started](/integrations/airlift/federation-tutorial/overview). +## Airlift Operator Migation Reference + +In this reference, we'll explain how to migrate common Airflow operators to Dagster. + +[Click here to get started](/integrations/airlift/operator-migration/overview). + ## References diff --git a/docs/content/integrations/airlift/operator-migration/bash-operator-dbt.mdx b/docs/content/integrations/airlift/operator-migration/bash-operator-dbt.mdx new file mode 100644 index 0000000000000..dade16cfca7e5 --- /dev/null +++ b/docs/content/integrations/airlift/operator-migration/bash-operator-dbt.mdx @@ -0,0 +1,53 @@ +# Operator migration guides: Migrating usage of `BashOperator` for `dbt` + +In this page, we'll explain migrating an Airflow `BashOperator` that runs a `dbt` command to Dagster. + +### Background + +In Airflow, you might have a `BashOperator` that runs a `dbt` command. For example, you might have a task that runs `dbt run` to build your dbt models. + +```python file=/integrations/airlift/operator_migration/bash_operator_dbt.py +from airflow.operators.bash import BashOperator + +run_dbt_model = BashOperator(task_id="build_dbt_models", bash_command="dbt run") +``` + +### Dagster equivalent + +The Dagster equivalent is to instead use the `dagster-dbt` library to run commands against your dbt project. Here would be the equivalent code in Dagster: + +```python file=/integrations/airlift/operator_migration/using_dbt_assets.py +from dagster_dbt import DbtCliResource, DbtProject, dbt_assets + +from dagster import AssetExecutionContext + +project = DbtProject(project_dir="path/to/dbt_project") + + +@dbt_assets(manifest=project.manifest_path, project=project) +def my_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource): + yield from dbt.cli(["run"], context=context).stream() +``` + +### Migrating the operator + +Migrating the operator breaks down into a few steps: + +1. Making the dbt project available to both your Airflow and Dagster deployments. +2. Writing a `@dbt_asset`-decorated function which runs your dbt commands. +3. Using `dagster-airlift` to proxy execution of the original task to Dagster. + +### Step 1: Making the dbt project available & building manifest + +First, you'll need to make the dbt project available to the Dagster runtime and build the manifest. + +- If you're building your Dagster deployment in a monorepo alongside your dbt and Airflow projects, you can follow this guide: [Monorepo setup](/integrations/dbt/reference#deploying-a-dagster-project-with-a-dbt-project). +- If you're deploying within a separate repository, you can follow this guide: [Separate repository setup](/integrations/dbt/reference#deploying-a-dbt-project-from-a-separate-git-repository). + +### Step 2: Writing a `@dbt_asset`-decorated function + +Once your dbt project is available, you can write a function that runs your dbt commands using the decorator and . Most dbt CLI commands and flags are supported - to see a full guide on using `dbt_assets`, check out the [dagster-dbt guide](/integrations/dbt/quickstart). + +### Step 3: Using `dagster-airlift` to proxy execution + +Finally, you can use `dagster-airlift` to proxy the execution of the original task to Dagster. The [dagster-airlift migration guide](/integrations/airlift/tutorial/overview) details this process. diff --git a/docs/content/integrations/airlift/operator-migration/overview.mdx b/docs/content/integrations/airlift/operator-migration/overview.mdx new file mode 100644 index 0000000000000..19ed7d5eb6a5d --- /dev/null +++ b/docs/content/integrations/airlift/operator-migration/overview.mdx @@ -0,0 +1,12 @@ +# Operator Migration Reference + +This page contains a collection of reference materials for migrating usage of common Airflow operator types to Dagster. + +## References + + + + diff --git a/examples/docs_snippets/docs_snippets/integrations/airlift/operator_migration/bash_operator_dbt.py b/examples/docs_snippets/docs_snippets/integrations/airlift/operator_migration/bash_operator_dbt.py index 71b011cc4f000..7f500a680aebd 100644 --- a/examples/docs_snippets/docs_snippets/integrations/airlift/operator_migration/bash_operator_dbt.py +++ b/examples/docs_snippets/docs_snippets/integrations/airlift/operator_migration/bash_operator_dbt.py @@ -1,5 +1,3 @@ from airflow.operators.bash import BashOperator -run_dbt_model = BashOperator( - task_id="build_dbt_models", bash_command="dbt run", dag=... -) +run_dbt_model = BashOperator(task_id="build_dbt_models", bash_command="dbt run")