From 56221090f0b045a46a5e84d2bf290391ee7b6ab5 Mon Sep 17 00:00:00 2001 From: Avinash Prasad Date: Fri, 6 Oct 2023 19:12:22 +0530 Subject: [PATCH] Add PAD-Bigquery quickstart --- airbyte_dbt_prefect/.gitignore | 163 +++++++++++++++ airbyte_dbt_prefect/README.md | 185 ++++++++++++++++++ airbyte_dbt_prefect/dbt_project/.gitignore | 10 + airbyte_dbt_prefect/dbt_project/README.md | 15 ++ .../dbt_project/analyses/.gitkeep | 0 .../dbt_project/dbt_project.yml | 37 ++++ .../dbt_project/macros/.gitkeep | 0 .../models/example/my_first_dbt_model.sql | 27 +++ .../models/example/my_second_dbt_model.sql | 6 + .../dbt_project/models/example/schema.yml | 21 ++ airbyte_dbt_prefect/dbt_project/profiles.yml | 14 ++ .../dbt_project/seeds/.gitkeep | 0 .../dbt_project/snapshots/.gitkeep | 0 .../dbt_project/tests/.gitkeep | 0 airbyte_dbt_prefect/infra/.gitignore | 34 ++++ .../infra/airbyte/.terraform.lock.hcl | 25 +++ airbyte_dbt_prefect/infra/airbyte/main.tf | 68 +++++++ airbyte_dbt_prefect/infra/airbyte/provider.tf | 20 ++ .../infra/airbyte/variables.tf | 9 + .../orchestration/orchestration/__init__.py | 0 .../orchestration/orchestration/assets.py | 30 +++ .../orchestration/orchestration/flow.py | 10 + .../orchestration/pyproject.toml | 7 + airbyte_dbt_prefect/setup.py | 12 ++ 24 files changed, 693 insertions(+) create mode 100644 airbyte_dbt_prefect/.gitignore create mode 100644 airbyte_dbt_prefect/README.md create mode 100644 airbyte_dbt_prefect/dbt_project/.gitignore create mode 100644 airbyte_dbt_prefect/dbt_project/README.md create mode 100644 airbyte_dbt_prefect/dbt_project/analyses/.gitkeep create mode 100644 airbyte_dbt_prefect/dbt_project/dbt_project.yml create mode 100644 airbyte_dbt_prefect/dbt_project/macros/.gitkeep create mode 100644 airbyte_dbt_prefect/dbt_project/models/example/my_first_dbt_model.sql create mode 100644 airbyte_dbt_prefect/dbt_project/models/example/my_second_dbt_model.sql create mode 100644 airbyte_dbt_prefect/dbt_project/models/example/schema.yml create mode 100644 airbyte_dbt_prefect/dbt_project/profiles.yml create mode 100644 airbyte_dbt_prefect/dbt_project/seeds/.gitkeep create mode 100644 airbyte_dbt_prefect/dbt_project/snapshots/.gitkeep create mode 100644 airbyte_dbt_prefect/dbt_project/tests/.gitkeep create mode 100644 airbyte_dbt_prefect/infra/.gitignore create mode 100644 airbyte_dbt_prefect/infra/airbyte/.terraform.lock.hcl create mode 100644 airbyte_dbt_prefect/infra/airbyte/main.tf create mode 100644 airbyte_dbt_prefect/infra/airbyte/provider.tf create mode 100644 airbyte_dbt_prefect/infra/airbyte/variables.tf create mode 100644 airbyte_dbt_prefect/orchestration/orchestration/__init__.py create mode 100644 airbyte_dbt_prefect/orchestration/orchestration/assets.py create mode 100644 airbyte_dbt_prefect/orchestration/orchestration/flow.py create mode 100644 airbyte_dbt_prefect/orchestration/pyproject.toml create mode 100644 airbyte_dbt_prefect/setup.py diff --git a/airbyte_dbt_prefect/.gitignore b/airbyte_dbt_prefect/.gitignore new file mode 100644 index 0000000..bc83b30 --- /dev/null +++ b/airbyte_dbt_prefect/.gitignore @@ -0,0 +1,163 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +#Desktop Services Store +.DS_Store \ No newline at end of file diff --git a/airbyte_dbt_prefect/README.md b/airbyte_dbt_prefect/README.md new file mode 100644 index 0000000..99f33db --- /dev/null +++ b/airbyte_dbt_prefect/README.md @@ -0,0 +1,185 @@ +# Airbyte-dbt-Prefect Integration (with BigQuery) + +Welcome to the "Airbyte-dbt-Prefect Integration" repository! This repository offers a quickstart template for building a full data stack using Airbyte, Prefect, dbt, and BigQuery. Effortlessly extract data from Postgres, transfer it to BigQuery, and execute essential transformations using dbt, all smoothly orchestrated through Prefect. While this template doesn't dive into the specifics of data or transformations, its objective is to highlight the seamless collaboration of these powerful tools. + +This quickstart is designed to minimize setup hassles and propel you forward. + +## Table of Contents + +- [Prerequisites](#prerequisites) +- [Setting an environment for your project](#1-setting-an-environment-for-your-project) +- [Setting Up Airbyte Connectors with Terraform](#2-setting-up-airbyte-connectors-with-terraform) +- [Setting Up the dbt Project](#3-setting-up-the-dbt-project) +- [Orchestrating with Prefect](#4-orchestrating-with-prefect) +- [Next Steps](#next-steps) + +## Prerequisites + +Before you embark on this integration, ensure you have the following set up and ready: + +1. **Python 3.10 or later**: If not installed, download and install it from [Python's official website](https://www.python.org/downloads/). + +2. **Docker and Docker Compose (Docker Desktop)**: Install [Docker](https://docs.docker.com/get-docker/) following the official documentation for your specific OS. + +3. **Airbyte OSS version**: Deploy the open-source version of Airbyte. Follow the installation instructions from the [Airbyte Documentation](https://docs.airbyte.com/quickstart/deploy-airbyte/). + +4. **Terraform**: Terraform will help you provision and manage the Airbyte resources. If you haven't installed it, follow the [official Terraform installation guide](https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli). + + +## 1. Setting an environment for your project + +Get the project up and running on your local machine by following these steps: + +1. **Clone the repository (Clone only this quickstart)**: + ```bash + git clone --filter=blob:none --sparse https://github.com/airbytehq/quickstarts.git + ``` + + ```bash + cd quickstarts + ``` + + ```bash + git sparse-checkout add airbyte_dbt_prefect + ``` + + +2. **Navigate to the directory**: + ```bash + cd airbyte_dbt_prefect + ``` + +3. **Set Up a Virtual Environment**: + - For Mac: + ```bash + python3 -m venv venv + source venv/bin/activate + ``` + - For Windows: + ```bash + python -m venv venv + .\venv\Scripts\activate + ``` + +4. **Install Dependencies**: + ```bash + pip install -e ".[dev]" + ``` + +## 2. Setting Up Airbyte Connectors with Terraform + +Airbyte allows you to create connectors for sources and destinations, facilitating data synchronization between various platforms. In this project, we're harnessing the power of Terraform to automate the creation of these connectors and the connections between them. Here's how you can set this up: + +1. **Navigate to the Airbyte Configuration Directory**: + + Change to the relevant directory containing the Terraform configuration for Airbyte: + ```bash + cd infra/airbyte + ``` + +2. **Modify Configuration Files**: + + Within the `infra/airbyte` directory, you'll find three crucial Terraform files: + - `provider.tf`: Defines the Airbyte provider. + - `main.tf`: Contains the main configuration for creating Airbyte resources. + - `variables.tf`: Holds various variables, including credentials. + + Adjust the configurations in these files to suit your project's needs. Specifically, provide credentials for your Postgres and BigQuery connections. You can utilize the `variables.tf` file to manage these credentials. + +3. **Initialize Terraform**: + + This step prepares Terraform to create the resources defined in your configuration files. + ```bash + terraform init + ``` + +4. **Review the Plan**: + + Before applying any changes, review the plan to understand what Terraform will do. + ```bash + terraform plan + ``` + +5. **Apply Configuration**: + + After reviewing and confirming the plan, apply the Terraform configurations to create the necessary Airbyte resources. + ```bash + terraform apply + ``` + +6. **Verify in Airbyte UI**: + + Once Terraform completes its tasks, navigate to the Airbyte UI. Here, you should see your source and destination connectors, as well as the connection between them, set up and ready to go. + +## 3. Setting Up the dbt Project + +[dbt (data build tool)](https://www.getdbt.com/) allows you to transform your data by writing, documenting, and executing SQL workflows. Setting up the dbt project requires specifying connection details for your data platform, in this case, BigQuery. Here’s a step-by-step guide to help you set this up: + +1. **Navigate to the dbt Project Directory**: + + Change to the directory containing the dbt configuration: + ```bash + cd dbt_project + ``` + +2. **Update Connection Details**: + + You'll find a `profiles.yml` file within the directory. This file contains configurations for dbt to connect with your data platform. Update this file with your BigQuery connection details. + +3. **Utilize Environment Variables (Optional but Recommended)**: + + To keep your credentials secure, you can leverage environment variables. An example is provided within the `profiles.yml` file. + +4. **Test the Connection**: + + Once you’ve updated the connection details, you can test the connection to your BigQuery instance using: + ```bash + dbt debug + ``` + + If everything is set up correctly, this command should report a successful connection to BigQuery. + +## 4. Orchestrating with Prefect + +Prefect is a modern workflow orchestrator designed to streamline the creation, testing, and monitoring of data workflows. In this section, we'll guide you through setting up Prefect to manage both the Airbyte and dbt workflows: + +1. Navigate to the Orchestration Directory: + +Change your working directory to the one containing the Prefect orchestration configurations: +```bash +cd orchestration +``` + +2. Set Environment Variables: + +Prefect requires specific environment variables to interact effectively with other tools like dbt and Airbyte. Configure the following variables: +```bash +export DBT_PARSE_PROJECT_ON_LOAD=1 +export AIRBYTE_PASSWORD=password +``` +Note: The AIRBYTE_PASSWORD is set to password as a default for local Airbyte instances. If you've changed this during your Airbyte setup, ensure you use the appropriate password here. + +3. Run the Prefect Flow: + +With the environment variables properly configured, you can run your Prefect workflow: +```bash +python flows.py +``` + +In this Prefect-based setup, you use Python scripts to define and execute your workflows, making it easier to manage and monitor data pipelines involving Airbyte and dbt. + +## Next Steps + +Once you've set up and launched this initial integration, the real power lies in its adaptability and extensibility. Here’s a roadmap to help you customize and harness this project tailored to your specific data needs: + +1. **Create dbt Sources for Airbyte Data**: + + Your raw data extracted via Airbyte can be represented as sources in dbt. Start by [creating new dbt sources](https://docs.getdbt.com/docs/build/sources) to represent this data, allowing for structured transformations down the line. + +2. **Add Your dbt Transformations**: + + With your dbt sources in place, you can now build upon them. Add your custom SQL transformations in dbt, ensuring that you treat the sources as an upstream dependency. This ensures that your transformations work on the most up-to-date raw data. + +3. **Extend the Project**: + + The real beauty of this integration is its extensibility. Whether you want to add more data sources, integrate additional tools, or enhance your transformation logic – the floor is yours. With the foundation set, sky's the limit for how you want to extend and refine your data processes. \ No newline at end of file diff --git a/airbyte_dbt_prefect/dbt_project/.gitignore b/airbyte_dbt_prefect/dbt_project/.gitignore new file mode 100644 index 0000000..a8c5a8b --- /dev/null +++ b/airbyte_dbt_prefect/dbt_project/.gitignore @@ -0,0 +1,10 @@ + +target/ +dbt_packages/ +logs/ + +#Desktop Services Store +.DS_Store + +#User cookie +.user.yml \ No newline at end of file diff --git a/airbyte_dbt_prefect/dbt_project/README.md b/airbyte_dbt_prefect/dbt_project/README.md new file mode 100644 index 0000000..7874ac8 --- /dev/null +++ b/airbyte_dbt_prefect/dbt_project/README.md @@ -0,0 +1,15 @@ +Welcome to your new dbt project! + +### Using the starter project + +Try running the following commands: +- dbt run +- dbt test + + +### Resources: +- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) +- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers +- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support +- Find [dbt events](https://events.getdbt.com) near you +- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices diff --git a/airbyte_dbt_prefect/dbt_project/analyses/.gitkeep b/airbyte_dbt_prefect/dbt_project/analyses/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airbyte_dbt_prefect/dbt_project/dbt_project.yml b/airbyte_dbt_prefect/dbt_project/dbt_project.yml new file mode 100644 index 0000000..179fef0 --- /dev/null +++ b/airbyte_dbt_prefect/dbt_project/dbt_project.yml @@ -0,0 +1,37 @@ + +# Name your project! Project names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'dbt_project' +version: '1.0.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. +profile: 'dbt_project' + +# These configurations specify where dbt should look for different types of files. +# The `model-paths` config, for example, states that models in this project can be +# found in the "models/" directory. You probably won't need to change these! +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_packages" + + +# Configuring models +# Full documentation: https://docs.getdbt.com/docs/configuring-models + +# In this example config, we tell dbt to build all models in the example/ +# directory as views. These settings can be overridden in the individual model +# files using the `{{ config(...) }}` macro. +models: + dbt_project: + # Config indicated by + and applies to all files under models/example/ + example: + +materialized: view diff --git a/airbyte_dbt_prefect/dbt_project/macros/.gitkeep b/airbyte_dbt_prefect/dbt_project/macros/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airbyte_dbt_prefect/dbt_project/models/example/my_first_dbt_model.sql b/airbyte_dbt_prefect/dbt_project/models/example/my_first_dbt_model.sql new file mode 100644 index 0000000..f31a12d --- /dev/null +++ b/airbyte_dbt_prefect/dbt_project/models/example/my_first_dbt_model.sql @@ -0,0 +1,27 @@ + +/* + Welcome to your first dbt model! + Did you know that you can also configure models directly within SQL files? + This will override configurations stated in dbt_project.yml + + Try changing "table" to "view" below +*/ + +{{ config(materialized='table') }} + +with source_data as ( + + select 1 as id + union all + select null as id + +) + +select * +from source_data + +/* + Uncomment the line below to remove records with null `id` values +*/ + +-- where id is not null diff --git a/airbyte_dbt_prefect/dbt_project/models/example/my_second_dbt_model.sql b/airbyte_dbt_prefect/dbt_project/models/example/my_second_dbt_model.sql new file mode 100644 index 0000000..c91f879 --- /dev/null +++ b/airbyte_dbt_prefect/dbt_project/models/example/my_second_dbt_model.sql @@ -0,0 +1,6 @@ + +-- Use the `ref` function to select from other models + +select * +from {{ ref('my_first_dbt_model') }} +where id = 1 diff --git a/airbyte_dbt_prefect/dbt_project/models/example/schema.yml b/airbyte_dbt_prefect/dbt_project/models/example/schema.yml new file mode 100644 index 0000000..2a53081 --- /dev/null +++ b/airbyte_dbt_prefect/dbt_project/models/example/schema.yml @@ -0,0 +1,21 @@ + +version: 2 + +models: + - name: my_first_dbt_model + description: "A starter dbt model" + columns: + - name: id + description: "The primary key for this table" + tests: + - unique + - not_null + + - name: my_second_dbt_model + description: "A starter dbt model" + columns: + - name: id + description: "The primary key for this table" + tests: + - unique + - not_null diff --git a/airbyte_dbt_prefect/dbt_project/profiles.yml b/airbyte_dbt_prefect/dbt_project/profiles.yml new file mode 100644 index 0000000..64c2cf6 --- /dev/null +++ b/airbyte_dbt_prefect/dbt_project/profiles.yml @@ -0,0 +1,14 @@ +dbt_project: + outputs: + dev: + dataset: my_dataset + job_execution_timeout_seconds: 300 + job_retries: 1 + keyfile: "{{ env_var('DBT_BIGQUERY_KEYFILE_PATH', '') }}" + location: my_dataset_location + method: service-account + priority: interactive + project: my_project_id + threads: 1 + type: bigquery + target: dev \ No newline at end of file diff --git a/airbyte_dbt_prefect/dbt_project/seeds/.gitkeep b/airbyte_dbt_prefect/dbt_project/seeds/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airbyte_dbt_prefect/dbt_project/snapshots/.gitkeep b/airbyte_dbt_prefect/dbt_project/snapshots/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airbyte_dbt_prefect/dbt_project/tests/.gitkeep b/airbyte_dbt_prefect/dbt_project/tests/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airbyte_dbt_prefect/infra/.gitignore b/airbyte_dbt_prefect/infra/.gitignore new file mode 100644 index 0000000..dc82526 --- /dev/null +++ b/airbyte_dbt_prefect/infra/.gitignore @@ -0,0 +1,34 @@ +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +# Exclude all .tfvars files, which are likely to contain sensitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +*.tfvars +*.tfvars.json + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc \ No newline at end of file diff --git a/airbyte_dbt_prefect/infra/airbyte/.terraform.lock.hcl b/airbyte_dbt_prefect/infra/airbyte/.terraform.lock.hcl new file mode 100644 index 0000000..e8d9f86 --- /dev/null +++ b/airbyte_dbt_prefect/infra/airbyte/.terraform.lock.hcl @@ -0,0 +1,25 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/airbytehq/airbyte" { + version = "0.3.3" + constraints = "0.3.3" + hashes = [ + "h1:a6g5uWP/pt1/popVNlKwnTssWNfdYY4KVFPMisN/yvU=", + "zh:0efa470b34d9b912b47efe4469c51713bfc3c2413e52c17e1e903f2a3cddb2f6", + "zh:1bddd69fa2c2d4f3e239d60555446df9bc4ce0c0cabbe7e092fe1d44989ab004", + "zh:2e20540403a0010007b53456663fb037b24e30f6c8943f65da1bcf7fa4dfc8a6", + "zh:2f415369ad884e8b7115a5c5ff229d052f7af1fca27abbfc8ebef379ed11aec4", + "zh:46fd9a906f4b6461112dcc5a5aa01a3fcd7a19a72d4ad0b2e37790da37701fe1", + "zh:83503ebb77bb6d6941c42ba323cf22380d08a1506554a2dcc8ac54e74c0886a1", + "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", + "zh:8fd770eff726826d3a63b9e3733c5455b5cde004027b04ee3f75888eb8538c90", + "zh:b0fc890ed4f9b077bf70ed121cc3550e7a07d16e7798ad517623274aa62ad7b0", + "zh:c2a01612362da9b73cd5958f281e1aa7ff09af42182e463097d11ed78e778e72", + "zh:c64b2bb1887a0367d64ba3393d4b3a16c418cf5b1792e2e7aae7c0b5413eb334", + "zh:ce14ebbf0ed91913ec62655a511763dec62b5779de9a209bd6f1c336640cddc0", + "zh:e0662ca837eee10f7733ea9a501d995281f56bd9b410ae13ad03eb106011db14", + "zh:e103d480fc6066004bc98e9e04a141a1f55b918cc2912716beebcc6fc4c872fb", + "zh:e2507049098f0f1b21cb56870f4a5ef624bcf6d3959e5612eada1f8117341648", + ] +} diff --git a/airbyte_dbt_prefect/infra/airbyte/main.tf b/airbyte_dbt_prefect/infra/airbyte/main.tf new file mode 100644 index 0000000..0925b9d --- /dev/null +++ b/airbyte_dbt_prefect/infra/airbyte/main.tf @@ -0,0 +1,68 @@ +// Airbyte Terraform provider documentation: https://registry.terraform.io/providers/airbytehq/airbyte/latest/docs + +// Sources +resource "airbyte_source_postgres" "postgres" { + configuration = { + database = "...my_database..." + host = "...my_host..." + username = "...my_username..." + password = "...my_password..." + port = 5432 + source_type = "postgres" + schemas = [ + "...my_schema..." + ] + ssl_mode = { + source_postgres_ssl_modes_allow = { + mode = "allow" + } + } + tunnel_method = { + source_postgres_ssh_tunnel_method_no_tunnel = { + tunnel_method = "NO_TUNNEL" + } + } + replication_method = { + source_postgres_replication_method_standard = { + method = "Standard" + } + } + } + name = "Postgres" + workspace_id = var.workspace_id +} + +// Destinations +resource "airbyte_destination_bigquery" "bigquery" { + configuration = { + dataset_id = "...my_dataset_id..." + dataset_location = "...my_dataset_location..." + destination_type = "bigquery" + project_id = "...my_project_id..." + credentials_json = "...my_credentials_json_file_path..." + loading_method = { + destination_bigquery_loading_method_standard_inserts = { + method = "Standard" + } + } + } + name = "BigQuery" + workspace_id = var.workspace_id +} + +// Connections +resource "airbyte_connection" "postgres_to_bigquery" { + name = "Postgres to BigQuery" + source_id = airbyte_source_postgres.postgres.source_id + destination_id = airbyte_destination_bigquery.bigquery.destination_id + configurations = { + streams = [ + { + name = "...my_table_name_1..." + }, + { + name = "...my_table_name_2..." + }, + ] + } +} \ No newline at end of file diff --git a/airbyte_dbt_prefect/infra/airbyte/provider.tf b/airbyte_dbt_prefect/infra/airbyte/provider.tf new file mode 100644 index 0000000..98224e9 --- /dev/null +++ b/airbyte_dbt_prefect/infra/airbyte/provider.tf @@ -0,0 +1,20 @@ +// Airbyte Terraform provider documentation: https://registry.terraform.io/providers/airbytehq/airbyte/latest/docs + +terraform { + required_providers { + airbyte = { + source = "airbytehq/airbyte" + version = "0.3.3" + } + } +} + +provider "airbyte" { + // If running locally (Airbyte OSS) with docker-compose using the airbyte-proxy, + // include the actual password/username you've set up (or use the defaults below) + username = "airbyte" + password = "password" + + // if running locally (Airbyte OSS), include the server url to the airbyte-api-server + server_url = "http://localhost:8006/v1" +} \ No newline at end of file diff --git a/airbyte_dbt_prefect/infra/airbyte/variables.tf b/airbyte_dbt_prefect/infra/airbyte/variables.tf new file mode 100644 index 0000000..88498e0 --- /dev/null +++ b/airbyte_dbt_prefect/infra/airbyte/variables.tf @@ -0,0 +1,9 @@ +variable "workspace_id" { + type = string +} + + + + + + diff --git a/airbyte_dbt_prefect/orchestration/orchestration/__init__.py b/airbyte_dbt_prefect/orchestration/orchestration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/airbyte_dbt_prefect/orchestration/orchestration/assets.py b/airbyte_dbt_prefect/orchestration/orchestration/assets.py new file mode 100644 index 0000000..9b9c730 --- /dev/null +++ b/airbyte_dbt_prefect/orchestration/orchestration/assets.py @@ -0,0 +1,30 @@ +import os +from prefect import Flow, task, Parameter +from prefect.run_configs.local import LocalRun +from prefect.storage.local import Local +from prefect_dbtdk.tasks import DbtCloudRunTask, AirbyteTask + +from .constants import dbt_manifest_path + +@task +def dbt_project_dbt_assets(): + return "Run your dbt tasks here using Prefect tasks." + +@task +def define_flow(): + dbt_run_task = DbtCloudRunTask( + project_id="YOUR_PROJECT_ID", # Replace with your dbt Cloud project ID + api_token="YOUR_API_TOKEN", # Replace with your dbt Cloud API token + job_name="YOUR_JOB_NAME" # Replace with your dbt Cloud job name + ) + + result = dbt_run_task.run() + return result + +airbyte_instance = AirbyteTask( + host="localhost", + port="8000", + # If using basic auth, include username and password: + username="airbyte", + password=os.getenv("AIRBYTE_PASSWORD") +) \ No newline at end of file diff --git a/airbyte_dbt_prefect/orchestration/orchestration/flow.py b/airbyte_dbt_prefect/orchestration/orchestration/flow.py new file mode 100644 index 0000000..02a8e0b --- /dev/null +++ b/airbyte_dbt_prefect/orchestration/orchestration/flow.py @@ -0,0 +1,10 @@ +from prefect import Flow + +from .assets import dbt_project_dbt_assets, airbyte_assets, define_flow + +with Flow("my-flow") as flow: + dbt_assets = dbt_project_dbt_assets() + airbyte_assets = airbyte_assets() + dbt_run_result = define_flow() + +dbt_assets.set_upstream(airbyte_assets) diff --git a/airbyte_dbt_prefect/orchestration/pyproject.toml b/airbyte_dbt_prefect/orchestration/pyproject.toml new file mode 100644 index 0000000..c7ce686 --- /dev/null +++ b/airbyte_dbt_prefect/orchestration/pyproject.toml @@ -0,0 +1,7 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[tool.prefect] +module_name = "orchestration.flows" +code_location_name = "orchestration" diff --git a/airbyte_dbt_prefect/setup.py b/airbyte_dbt_prefect/setup.py new file mode 100644 index 0000000..3478855 --- /dev/null +++ b/airbyte_dbt_prefect/setup.py @@ -0,0 +1,12 @@ +from setuptools import find_packages, setup + +setup( + name="orchestration", + version="0.0.1", + packages=find_packages(), + install_requires=[ + "prefect", + "dbt_prefect", # Add this line for dbt-Prefect integration + # Other dependencies as needed + ], +)