diff --git a/.flake8 b/.flake8 index 3aa19a9..8e25972 100644 --- a/.flake8 +++ b/.flake8 @@ -1,12 +1,8 @@ [flake8] max-line-length = 88 docstring-convention = google -# Will finish these later -extend-ignore = D101,D107 +ignore = D107, W503 per-file-ignores = - tests/*.py: D1,D2,D4 - examples/*.py: D1,D2,D4 - __version__.py: D100 __init__.py: F401 exclude = airflow @@ -16,6 +12,7 @@ exclude = setup.py build dist + tests .venv .tox .mypy_cache diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7534157..75a3dac 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,6 +10,8 @@ repos: hooks: - id: flake8 additional_dependencies: [flake8-docstrings] + args: ["--config", ".flake8"] + exclude: tests/ - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.902 diff --git a/airflow_dbt_python/__version__.py b/airflow_dbt_python/__version__.py index 8201c2f..c847a74 100644 --- a/airflow_dbt_python/__version__.py +++ b/airflow_dbt_python/__version__.py @@ -1,3 +1,4 @@ +"""The module's version information.""" __author__ = "Tomás Farías Santana" __copyright__ = "Copyright 2021 Tomás Farías Santana" __title__ = "airflow-dbt-python" diff --git a/airflow_dbt_python/hooks/dbt_s3.py b/airflow_dbt_python/hooks/dbt_s3.py index 75d694f..37e46f1 100644 --- a/airflow_dbt_python/hooks/dbt_s3.py +++ b/airflow_dbt_python/hooks/dbt_s3.py @@ -7,6 +7,12 @@ class DbtS3Hook(S3Hook): + """Subclass of S3Hook with methods to pull dbt-related files. + + A dbt hook should provide a method to pull a dbt profiles file (profiles.yml) and + all the files corresponding to a project. + """ + def get_dbt_profiles( self, s3_profiles_url: str, profiles_dir: Optional[str] = None ) -> Path: diff --git a/examples/__init__.py b/examples/__init__.py index e69de29..8f6e936 100644 --- a/examples/__init__.py +++ b/examples/__init__.py @@ -0,0 +1 @@ +"""Usage examples of airflow-dbt-python.""" diff --git a/examples/basic_dag.py b/examples/basic_dag.py index ce57e57..165e204 100644 --- a/examples/basic_dag.py +++ b/examples/basic_dag.py @@ -1,6 +1,4 @@ -""" -Sample basic DAG which dbt runs a project -""" +"""Sample basic DAG which dbt runs a project.""" import datetime as dt from airflow import DAG diff --git a/examples/complete_dbt_workflow_dag.py b/examples/complete_dbt_workflow_dag.py index 74ff87c..1f76d95 100644 --- a/examples/complete_dbt_workflow_dag.py +++ b/examples/complete_dbt_workflow_dag.py @@ -1,5 +1,6 @@ -""" -Sample DAG showcasing a complete dbt workflow +"""Sample DAG showcasing a complete dbt workflow. + +The complete workflow includes a sequence of source, seed, and several run commands. """ import datetime as dt diff --git a/examples/dbt_project_in_s3_dag.py b/examples/dbt_project_in_s3_dag.py index 039a41b..7ed7eb7 100644 --- a/examples/dbt_project_in_s3_dag.py +++ b/examples/dbt_project_in_s3_dag.py @@ -1,6 +1,4 @@ -""" -Sample basic DAG which showcases a dbt project being pulled from S3 -""" +"""Sample basic DAG which showcases a dbt project being pulled from S3.""" import datetime as dt from airflow import DAG diff --git a/examples/use_dbt_artifacts_dag.py b/examples/use_dbt_artifacts_dag.py index eeb3e87..8f2cc43 100644 --- a/examples/use_dbt_artifacts_dag.py +++ b/examples/use_dbt_artifacts_dag.py @@ -1,6 +1,4 @@ -""" -Sample DAG to showcase pulling dbt artifacts from XCOM -""" +"""Sample DAG to showcase pulling dbt artifacts from XCOM.""" import datetime as dt from airflow import DAG @@ -10,9 +8,7 @@ def process_dbt_artifacts(**context): - """ - Report which model or models took the longest to compile and execute - """ + """Report which model or models took the longest to compile and execute.""" run_results = context["ti"].xcom_pull( key="run_results.json", task_ids="dbt_run_daily" ) diff --git a/tests/conftest.py b/tests/conftest.py index fbfbb53..3d9e703 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,8 @@ +"""Conftest file including setting common fixtures. + +Common fixtures include a connection to a postgres database, a set of sample model and + seed files, dbt configuration files, and temporary directories for everything. +""" import boto3 import pytest from dbt.version import __version__ as DBT_VERSION diff --git a/tests/test_dbt_base.py b/tests/test_dbt_base.py index 456b84d..52410d4 100644 --- a/tests/test_dbt_base.py +++ b/tests/test_dbt_base.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtBaseOperator.""" from pathlib import Path from unittest.mock import patch diff --git a/tests/test_dbt_build.py b/tests/test_dbt_build.py index 90994a3..f84e792 100644 --- a/tests/test_dbt_build.py +++ b/tests/test_dbt_build.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtBuildOperator.""" import json from unittest.mock import patch diff --git a/tests/test_dbt_clean.py b/tests/test_dbt_clean.py index fdf5316..606fd06 100644 --- a/tests/test_dbt_clean.py +++ b/tests/test_dbt_clean.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtCleanOperator.""" from unittest.mock import patch from airflow_dbt_python.operators.dbt import DbtCleanOperator, DbtCompileOperator diff --git a/tests/test_dbt_compile.py b/tests/test_dbt_compile.py index 24b427d..9eda40b 100644 --- a/tests/test_dbt_compile.py +++ b/tests/test_dbt_compile.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtCompileOperator.""" from unittest.mock import patch from dbt.contracts.results import RunStatus diff --git a/tests/test_dbt_debug.py b/tests/test_dbt_debug.py index f23e60a..9ef2e01 100644 --- a/tests/test_dbt_debug.py +++ b/tests/test_dbt_debug.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtDebugOperator.""" from unittest.mock import patch from dbt.version import __version__ as DBT_VERSION diff --git a/tests/test_dbt_deps.py b/tests/test_dbt_deps.py index 88a0463..f9ffc48 100644 --- a/tests/test_dbt_deps.py +++ b/tests/test_dbt_deps.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtDepsOperator.""" from unittest.mock import patch import pytest diff --git a/tests/test_dbt_list.py b/tests/test_dbt_list.py index c34e1e7..06bef0a 100644 --- a/tests/test_dbt_list.py +++ b/tests/test_dbt_list.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtListOperator.""" from itertools import chain from unittest.mock import patch diff --git a/tests/test_dbt_parse.py b/tests/test_dbt_parse.py index a679d67..54b1db5 100644 --- a/tests/test_dbt_parse.py +++ b/tests/test_dbt_parse.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtParseOperator.""" from pathlib import Path from unittest.mock import patch diff --git a/tests/test_dbt_run.py b/tests/test_dbt_run.py index 805559d..1e13139 100644 --- a/tests/test_dbt_run.py +++ b/tests/test_dbt_run.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtRunOperator.""" import json from unittest.mock import patch diff --git a/tests/test_dbt_run_operation.py b/tests/test_dbt_run_operation.py index 3d8b7f4..908cc2a 100644 --- a/tests/test_dbt_run_operation.py +++ b/tests/test_dbt_run_operation.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtRunOperationOperator.""" from unittest.mock import patch import pytest diff --git a/tests/test_dbt_s3_hook.py b/tests/test_dbt_s3_hook.py index 936be43..7322eb2 100644 --- a/tests/test_dbt_s3_hook.py +++ b/tests/test_dbt_s3_hook.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtS3Hook.""" import pytest try: diff --git a/tests/test_dbt_seed.py b/tests/test_dbt_seed.py index c97ecf4..0f7a6ee 100644 --- a/tests/test_dbt_seed.py +++ b/tests/test_dbt_seed.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtSeedOperator.""" import json from unittest.mock import patch diff --git a/tests/test_dbt_snapshot.py b/tests/test_dbt_snapshot.py index 6bdd31f..0eeb9f1 100644 --- a/tests/test_dbt_snapshot.py +++ b/tests/test_dbt_snapshot.py @@ -1,9 +1,10 @@ +"""Unit test module for DbtSnapshotOperator.""" from unittest.mock import patch import pytest -from airflow import AirflowException from dbt.contracts.results import RunStatus +from airflow import AirflowException from airflow_dbt_python.operators.dbt import DbtSnapshotOperator diff --git a/tests/test_dbt_source.py b/tests/test_dbt_source.py index 3639b3d..d1a3de8 100644 --- a/tests/test_dbt_source.py +++ b/tests/test_dbt_source.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtSourceOperator.""" from pathlib import Path from unittest.mock import patch diff --git a/tests/test_dbt_test.py b/tests/test_dbt_test.py index dbad6bb..1c72a73 100644 --- a/tests/test_dbt_test.py +++ b/tests/test_dbt_test.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtTestOperator.""" from unittest.mock import patch import pytest