From d110dea2914c6cace9af65da96407c8260d312ff Mon Sep 17 00:00:00 2001 From: ChristoGrab Date: Wed, 4 Dec 2024 14:05:14 -0800 Subject: [PATCH 1/8] feat: add unit test fixtures for manifest-only connectors to CDK --- .../test/utils/manifest_only_fixtures.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 airbyte_cdk/test/utils/manifest_only_fixtures.py diff --git a/airbyte_cdk/test/utils/manifest_only_fixtures.py b/airbyte_cdk/test/utils/manifest_only_fixtures.py new file mode 100644 index 000000000..3e2996100 --- /dev/null +++ b/airbyte_cdk/test/utils/manifest_only_fixtures.py @@ -0,0 +1,35 @@ +from typing import Any, Optional +import pytest +from pathlib import Path +import importlib.util +from types import ModuleType + +@pytest.fixture(scope="session") +def connector_dir(request: pytest.FixtureRequest) -> Path: + """Return the connector's root directory, which should always be the unit_tests folder's parent directory.""" + return Path(request.config.rootpath).parent + + +@pytest.fixture(scope="session") +def components_module(connector_dir: Path) -> Optional[ModuleType]: + """Load and return the components module from the connector directory.""" + components_path = connector_dir / "components.py" + if not components_path.exists(): + return None + + spec = importlib.util.spec_from_file_location("components", components_path) + if spec is None: + return None + + module = importlib.util.module_from_spec(spec) + if spec.loader is None: + return None + + spec.loader.exec_module(module) + return module + + +@pytest.fixture(scope="session") +def manifest_path(connector_dir: Path) -> Path: + """Return the path to the connector's manifest file.""" + return connector_dir / "manifest.yaml" From 45375cab4f208424a50b1aeb73f68952dafc4816 Mon Sep 17 00:00:00 2001 From: ChristoGrab Date: Wed, 4 Dec 2024 14:21:15 -0800 Subject: [PATCH 2/8] chore: lint/format/license --- airbyte_cdk/test/utils/manifest_only_fixtures.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/airbyte_cdk/test/utils/manifest_only_fixtures.py b/airbyte_cdk/test/utils/manifest_only_fixtures.py index 3e2996100..480160725 100644 --- a/airbyte_cdk/test/utils/manifest_only_fixtures.py +++ b/airbyte_cdk/test/utils/manifest_only_fixtures.py @@ -1,8 +1,13 @@ -from typing import Any, Optional -import pytest -from pathlib import Path +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. + + import importlib.util +from pathlib import Path from types import ModuleType +from typing import Any, Optional + +import pytest + @pytest.fixture(scope="session") def connector_dir(request: pytest.FixtureRequest) -> Path: @@ -16,11 +21,11 @@ def components_module(connector_dir: Path) -> Optional[ModuleType]: components_path = connector_dir / "components.py" if not components_path.exists(): return None - + spec = importlib.util.spec_from_file_location("components", components_path) if spec is None: return None - + module = importlib.util.module_from_spec(spec) if spec.loader is None: return None From 6dcdb8ac5c9e62c16a676bfd5645662efe8c6d7e Mon Sep 17 00:00:00 2001 From: ChristoGrab Date: Wed, 4 Dec 2024 20:21:00 -0800 Subject: [PATCH 3/8] fix: use invocation_params to find correct project directory --- .../test/utils/manifest_only_fixtures.py | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/airbyte_cdk/test/utils/manifest_only_fixtures.py b/airbyte_cdk/test/utils/manifest_only_fixtures.py index 480160725..c650ba947 100644 --- a/airbyte_cdk/test/utils/manifest_only_fixtures.py +++ b/airbyte_cdk/test/utils/manifest_only_fixtures.py @@ -4,34 +4,42 @@ import importlib.util from pathlib import Path from types import ModuleType -from typing import Any, Optional +from typing import Optional import pytest @pytest.fixture(scope="session") def connector_dir(request: pytest.FixtureRequest) -> Path: - """Return the connector's root directory, which should always be the unit_tests folder's parent directory.""" - return Path(request.config.rootpath).parent + """Return the connector's root directory. + + This assumes tests are being run from the unit_tests directory, + and that it is a direct child of the connector directory. + """ + test_dir = Path(request.config.invocation_params.dir) + return test_dir.parent @pytest.fixture(scope="session") def components_module(connector_dir: Path) -> Optional[ModuleType]: - """Load and return the components module from the connector directory.""" + """Load and return the components module from the connector directory. + + This assumes the components module is located at /components.py. + """ components_path = connector_dir / "components.py" if not components_path.exists(): return None - spec = importlib.util.spec_from_file_location("components", components_path) - if spec is None: + components_spec = importlib.util.spec_from_file_location("components", components_path) + if components_spec is None: return None - module = importlib.util.module_from_spec(spec) - if spec.loader is None: + components_module = importlib.util.module_from_spec(components_spec) + if components_spec.loader is None: return None - spec.loader.exec_module(module) - return module + components_spec.loader.exec_module(components_module) + return components_module @pytest.fixture(scope="session") From 583f5f0699b2e1dc69772ad00905c6682ce2a4a9 Mon Sep 17 00:00:00 2001 From: ChristoGrab Date: Wed, 4 Dec 2024 20:22:48 -0800 Subject: [PATCH 4/8] chore: format --- airbyte_cdk/test/utils/manifest_only_fixtures.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte_cdk/test/utils/manifest_only_fixtures.py b/airbyte_cdk/test/utils/manifest_only_fixtures.py index c650ba947..e26e638bf 100644 --- a/airbyte_cdk/test/utils/manifest_only_fixtures.py +++ b/airbyte_cdk/test/utils/manifest_only_fixtures.py @@ -12,7 +12,7 @@ @pytest.fixture(scope="session") def connector_dir(request: pytest.FixtureRequest) -> Path: """Return the connector's root directory. - + This assumes tests are being run from the unit_tests directory, and that it is a direct child of the connector directory. """ @@ -23,7 +23,7 @@ def connector_dir(request: pytest.FixtureRequest) -> Path: @pytest.fixture(scope="session") def components_module(connector_dir: Path) -> Optional[ModuleType]: """Load and return the components module from the connector directory. - + This assumes the components module is located at /components.py. """ components_path = connector_dir / "components.py" From 0ac8dd55d892fedeee248e79b92e609d92ca71f8 Mon Sep 17 00:00:00 2001 From: ChristoGrab Date: Thu, 5 Dec 2024 11:37:45 -0800 Subject: [PATCH 5/8] chore: add comments --- airbyte_cdk/test/utils/manifest_only_fixtures.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/airbyte_cdk/test/utils/manifest_only_fixtures.py b/airbyte_cdk/test/utils/manifest_only_fixtures.py index e26e638bf..641993d9d 100644 --- a/airbyte_cdk/test/utils/manifest_only_fixtures.py +++ b/airbyte_cdk/test/utils/manifest_only_fixtures.py @@ -8,6 +8,12 @@ import pytest +# The following fixtures are used to load a manifest-only connector's components module and manifest file. +# They can be accessed from any test file in the connector's unit_tests directory by importing them as follows: + +# from airbyte_cdk.test.utils.manifest_only_fixtures import components_module, connector_dir, manifest_path + +# individual components can then be referenced as: components_module. @pytest.fixture(scope="session") def connector_dir(request: pytest.FixtureRequest) -> Path: From 1b92503aae22266c3e5c6225fd72ea0dd7491fe0 Mon Sep 17 00:00:00 2001 From: ChristoGrab Date: Thu, 5 Dec 2024 11:50:57 -0800 Subject: [PATCH 6/8] chore: formatter raining on my parade --- airbyte_cdk/test/utils/manifest_only_fixtures.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte_cdk/test/utils/manifest_only_fixtures.py b/airbyte_cdk/test/utils/manifest_only_fixtures.py index 641993d9d..0a7cb2dcb 100644 --- a/airbyte_cdk/test/utils/manifest_only_fixtures.py +++ b/airbyte_cdk/test/utils/manifest_only_fixtures.py @@ -15,6 +15,7 @@ # individual components can then be referenced as: components_module. + @pytest.fixture(scope="session") def connector_dir(request: pytest.FixtureRequest) -> Path: """Return the connector's root directory. From f0aef10e9aaf72ded6163ee8fd972236dfb46bab Mon Sep 17 00:00:00 2001 From: ChristoGrab Date: Mon, 9 Dec 2024 11:13:18 -0800 Subject: [PATCH 7/8] fix: handle connector_dir by environment --- airbyte_cdk/test/utils/manifest_only_fixtures.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/airbyte_cdk/test/utils/manifest_only_fixtures.py b/airbyte_cdk/test/utils/manifest_only_fixtures.py index 0a7cb2dcb..dd7514a1c 100644 --- a/airbyte_cdk/test/utils/manifest_only_fixtures.py +++ b/airbyte_cdk/test/utils/manifest_only_fixtures.py @@ -18,13 +18,15 @@ @pytest.fixture(scope="session") def connector_dir(request: pytest.FixtureRequest) -> Path: - """Return the connector's root directory. + """Return the connector's root directory.""" - This assumes tests are being run from the unit_tests directory, - and that it is a direct child of the connector directory. - """ - test_dir = Path(request.config.invocation_params.dir) - return test_dir.parent + current_dir = Path(request.config.invocation_params.dir) + + # If the tests are run locally from the connector's unit_tests directory, return the parent (connector) directory + if current_dir.name == "unit_tests": + return current_dir.parent + # In CI, the tests are run from the connector directory itself + return current_dir @pytest.fixture(scope="session") From 763d247a393e41c6d1915bd69473dcafc67131d4 Mon Sep 17 00:00:00 2001 From: Christo Grabowski <108154848+ChristoGrab@users.noreply.github.com> Date: Mon, 9 Dec 2024 17:00:02 -0500 Subject: [PATCH 8/8] Update airbyte_cdk/test/utils/manifest_only_fixtures.py Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- airbyte_cdk/test/utils/manifest_only_fixtures.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/airbyte_cdk/test/utils/manifest_only_fixtures.py b/airbyte_cdk/test/utils/manifest_only_fixtures.py index dd7514a1c..47620e7c1 100644 --- a/airbyte_cdk/test/utils/manifest_only_fixtures.py +++ b/airbyte_cdk/test/utils/manifest_only_fixtures.py @@ -54,4 +54,7 @@ def components_module(connector_dir: Path) -> Optional[ModuleType]: @pytest.fixture(scope="session") def manifest_path(connector_dir: Path) -> Path: """Return the path to the connector's manifest file.""" - return connector_dir / "manifest.yaml" + path = connector_dir / "manifest.yaml" + if not path.exists(): + raise FileNotFoundError(f"Manifest file not found at {path}") + return path