From ea2dcac223ba0041838a95d25e80d577e288701f Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Mon, 18 Oct 2021 19:48:59 +0200 Subject: [PATCH] docs: Update docstrings in all files except tests All flake8-docstring errors have been cleared except for those coming from test files since test files are not part of the documentation pipeline. Eventually will work on them though just so that we can eliminate the final ignore. --- .flake8 | 7 ++----- .pre-commit-config.yaml | 2 ++ airflow_dbt_python/__version__.py | 1 + airflow_dbt_python/hooks/dbt_s3.py | 6 ++++++ examples/__init__.py | 1 + examples/basic_dag.py | 4 +--- examples/complete_dbt_workflow_dag.py | 5 +++-- examples/dbt_project_in_s3_dag.py | 4 +--- examples/use_dbt_artifacts_dag.py | 8 ++------ tests/conftest.py | 5 +++++ tests/test_dbt_base.py | 1 + tests/test_dbt_build.py | 1 + tests/test_dbt_clean.py | 1 + tests/test_dbt_compile.py | 1 + tests/test_dbt_debug.py | 1 + tests/test_dbt_deps.py | 1 + tests/test_dbt_list.py | 1 + tests/test_dbt_parse.py | 1 + tests/test_dbt_run.py | 1 + tests/test_dbt_run_operation.py | 1 + tests/test_dbt_s3_hook.py | 1 + tests/test_dbt_seed.py | 1 + tests/test_dbt_snapshot.py | 3 ++- tests/test_dbt_source.py | 1 + tests/test_dbt_test.py | 1 + 25 files changed, 40 insertions(+), 20 deletions(-) diff --git a/.flake8 b/.flake8 index 3aa19a9..8e25972 100644 --- a/.flake8 +++ b/.flake8 @@ -1,12 +1,8 @@ [flake8] max-line-length = 88 docstring-convention = google -# Will finish these later -extend-ignore = D101,D107 +ignore = D107, W503 per-file-ignores = - tests/*.py: D1,D2,D4 - examples/*.py: D1,D2,D4 - __version__.py: D100 __init__.py: F401 exclude = airflow @@ -16,6 +12,7 @@ exclude = setup.py build dist + tests .venv .tox .mypy_cache diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7534157..75a3dac 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,6 +10,8 @@ repos: hooks: - id: flake8 additional_dependencies: [flake8-docstrings] + args: ["--config", ".flake8"] + exclude: tests/ - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.902 diff --git a/airflow_dbt_python/__version__.py b/airflow_dbt_python/__version__.py index 8201c2f..c847a74 100644 --- a/airflow_dbt_python/__version__.py +++ b/airflow_dbt_python/__version__.py @@ -1,3 +1,4 @@ +"""The module's version information.""" __author__ = "Tomás Farías Santana" __copyright__ = "Copyright 2021 Tomás Farías Santana" __title__ = "airflow-dbt-python" diff --git a/airflow_dbt_python/hooks/dbt_s3.py b/airflow_dbt_python/hooks/dbt_s3.py index 75d694f..37e46f1 100644 --- a/airflow_dbt_python/hooks/dbt_s3.py +++ b/airflow_dbt_python/hooks/dbt_s3.py @@ -7,6 +7,12 @@ class DbtS3Hook(S3Hook): + """Subclass of S3Hook with methods to pull dbt-related files. + + A dbt hook should provide a method to pull a dbt profiles file (profiles.yml) and + all the files corresponding to a project. + """ + def get_dbt_profiles( self, s3_profiles_url: str, profiles_dir: Optional[str] = None ) -> Path: diff --git a/examples/__init__.py b/examples/__init__.py index e69de29..8f6e936 100644 --- a/examples/__init__.py +++ b/examples/__init__.py @@ -0,0 +1 @@ +"""Usage examples of airflow-dbt-python.""" diff --git a/examples/basic_dag.py b/examples/basic_dag.py index ce57e57..165e204 100644 --- a/examples/basic_dag.py +++ b/examples/basic_dag.py @@ -1,6 +1,4 @@ -""" -Sample basic DAG which dbt runs a project -""" +"""Sample basic DAG which dbt runs a project.""" import datetime as dt from airflow import DAG diff --git a/examples/complete_dbt_workflow_dag.py b/examples/complete_dbt_workflow_dag.py index 74ff87c..1f76d95 100644 --- a/examples/complete_dbt_workflow_dag.py +++ b/examples/complete_dbt_workflow_dag.py @@ -1,5 +1,6 @@ -""" -Sample DAG showcasing a complete dbt workflow +"""Sample DAG showcasing a complete dbt workflow. + +The complete workflow includes a sequence of source, seed, and several run commands. """ import datetime as dt diff --git a/examples/dbt_project_in_s3_dag.py b/examples/dbt_project_in_s3_dag.py index 039a41b..7ed7eb7 100644 --- a/examples/dbt_project_in_s3_dag.py +++ b/examples/dbt_project_in_s3_dag.py @@ -1,6 +1,4 @@ -""" -Sample basic DAG which showcases a dbt project being pulled from S3 -""" +"""Sample basic DAG which showcases a dbt project being pulled from S3.""" import datetime as dt from airflow import DAG diff --git a/examples/use_dbt_artifacts_dag.py b/examples/use_dbt_artifacts_dag.py index eeb3e87..8f2cc43 100644 --- a/examples/use_dbt_artifacts_dag.py +++ b/examples/use_dbt_artifacts_dag.py @@ -1,6 +1,4 @@ -""" -Sample DAG to showcase pulling dbt artifacts from XCOM -""" +"""Sample DAG to showcase pulling dbt artifacts from XCOM.""" import datetime as dt from airflow import DAG @@ -10,9 +8,7 @@ def process_dbt_artifacts(**context): - """ - Report which model or models took the longest to compile and execute - """ + """Report which model or models took the longest to compile and execute.""" run_results = context["ti"].xcom_pull( key="run_results.json", task_ids="dbt_run_daily" ) diff --git a/tests/conftest.py b/tests/conftest.py index fbfbb53..3d9e703 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,8 @@ +"""Conftest file including setting common fixtures. + +Common fixtures include a connection to a postgres database, a set of sample model and + seed files, dbt configuration files, and temporary directories for everything. +""" import boto3 import pytest from dbt.version import __version__ as DBT_VERSION diff --git a/tests/test_dbt_base.py b/tests/test_dbt_base.py index 456b84d..52410d4 100644 --- a/tests/test_dbt_base.py +++ b/tests/test_dbt_base.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtBaseOperator.""" from pathlib import Path from unittest.mock import patch diff --git a/tests/test_dbt_build.py b/tests/test_dbt_build.py index 90994a3..f84e792 100644 --- a/tests/test_dbt_build.py +++ b/tests/test_dbt_build.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtBuildOperator.""" import json from unittest.mock import patch diff --git a/tests/test_dbt_clean.py b/tests/test_dbt_clean.py index fdf5316..606fd06 100644 --- a/tests/test_dbt_clean.py +++ b/tests/test_dbt_clean.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtCleanOperator.""" from unittest.mock import patch from airflow_dbt_python.operators.dbt import DbtCleanOperator, DbtCompileOperator diff --git a/tests/test_dbt_compile.py b/tests/test_dbt_compile.py index 24b427d..9eda40b 100644 --- a/tests/test_dbt_compile.py +++ b/tests/test_dbt_compile.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtCompileOperator.""" from unittest.mock import patch from dbt.contracts.results import RunStatus diff --git a/tests/test_dbt_debug.py b/tests/test_dbt_debug.py index f23e60a..9ef2e01 100644 --- a/tests/test_dbt_debug.py +++ b/tests/test_dbt_debug.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtDebugOperator.""" from unittest.mock import patch from dbt.version import __version__ as DBT_VERSION diff --git a/tests/test_dbt_deps.py b/tests/test_dbt_deps.py index 88a0463..f9ffc48 100644 --- a/tests/test_dbt_deps.py +++ b/tests/test_dbt_deps.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtDepsOperator.""" from unittest.mock import patch import pytest diff --git a/tests/test_dbt_list.py b/tests/test_dbt_list.py index c34e1e7..06bef0a 100644 --- a/tests/test_dbt_list.py +++ b/tests/test_dbt_list.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtListOperator.""" from itertools import chain from unittest.mock import patch diff --git a/tests/test_dbt_parse.py b/tests/test_dbt_parse.py index a679d67..54b1db5 100644 --- a/tests/test_dbt_parse.py +++ b/tests/test_dbt_parse.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtParseOperator.""" from pathlib import Path from unittest.mock import patch diff --git a/tests/test_dbt_run.py b/tests/test_dbt_run.py index 805559d..1e13139 100644 --- a/tests/test_dbt_run.py +++ b/tests/test_dbt_run.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtRunOperator.""" import json from unittest.mock import patch diff --git a/tests/test_dbt_run_operation.py b/tests/test_dbt_run_operation.py index 3d8b7f4..908cc2a 100644 --- a/tests/test_dbt_run_operation.py +++ b/tests/test_dbt_run_operation.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtRunOperationOperator.""" from unittest.mock import patch import pytest diff --git a/tests/test_dbt_s3_hook.py b/tests/test_dbt_s3_hook.py index 936be43..7322eb2 100644 --- a/tests/test_dbt_s3_hook.py +++ b/tests/test_dbt_s3_hook.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtS3Hook.""" import pytest try: diff --git a/tests/test_dbt_seed.py b/tests/test_dbt_seed.py index c97ecf4..0f7a6ee 100644 --- a/tests/test_dbt_seed.py +++ b/tests/test_dbt_seed.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtSeedOperator.""" import json from unittest.mock import patch diff --git a/tests/test_dbt_snapshot.py b/tests/test_dbt_snapshot.py index 6bdd31f..0eeb9f1 100644 --- a/tests/test_dbt_snapshot.py +++ b/tests/test_dbt_snapshot.py @@ -1,9 +1,10 @@ +"""Unit test module for DbtSnapshotOperator.""" from unittest.mock import patch import pytest -from airflow import AirflowException from dbt.contracts.results import RunStatus +from airflow import AirflowException from airflow_dbt_python.operators.dbt import DbtSnapshotOperator diff --git a/tests/test_dbt_source.py b/tests/test_dbt_source.py index 3639b3d..d1a3de8 100644 --- a/tests/test_dbt_source.py +++ b/tests/test_dbt_source.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtSourceOperator.""" from pathlib import Path from unittest.mock import patch diff --git a/tests/test_dbt_test.py b/tests/test_dbt_test.py index dbad6bb..1c72a73 100644 --- a/tests/test_dbt_test.py +++ b/tests/test_dbt_test.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtTestOperator.""" from unittest.mock import patch import pytest