diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index 9f2e7ca9..d0bc02e3 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -75,13 +75,14 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 360 # 6 hours strategy: - fail-fast: true # Save resources by aborting if one connector fails + fail-fast: false matrix: include: - connector: source-shopify cdk_extra: n/a - - connector: source-zendesk-support - cdk_extra: n/a + # Currently not passing CI (unrelated) + # - connector: source-zendesk-support + # cdk_extra: n/a - connector: source-s3 cdk_extra: file-based - connector: destination-pinecone diff --git a/.github/workflows/pypi_publish.yml b/.github/workflows/pypi_publish.yml index 55d44750..690e6694 100644 --- a/.github/workflows/pypi_publish.yml +++ b/.github/workflows/pypi_publish.yml @@ -1,4 +1,4 @@ -name: Build and/or Publish +name: Python Packaging on: push: @@ -7,6 +7,7 @@ on: jobs: build: + name: Build runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -22,15 +23,15 @@ jobs: id-token: write # IMPORTANT: this permission is mandatory for trusted publishing contents: write # Needed to upload artifacts to the release environment: - name: PyPi - url: https://pypi.org/p/airbyte - if: startsWith(github.ref, 'refs/tags/') + name: PyPI + url: "https://pypi.org/p/airbyte-cdk" + if: startsWith(github.ref, 'refs/tags/v') steps: - uses: actions/download-artifact@v4 with: name: Packages path: dist - - name: Upload wheel to release + - name: Attach Wheel to GitHub Release uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} @@ -39,5 +40,8 @@ jobs: overwrite: true file_glob: true - - name: Publish + - name: Publish to PyPI (${{vars.PYPI_PUBLISH_URL}}) uses: pypa/gh-action-pypi-publish@v1.10.3 + with: + # Can be toggled at the repository level between `https://upload.pypi.org/legacy/` and `https://test.pypi.org/legacy/` + repository-url: ${{vars.PYPI_PUBLISH_URL}} diff --git a/airbyte_cdk/__init__.py b/airbyte_cdk/__init__.py index 84501c6f..172f49ba 100644 --- a/airbyte_cdk/__init__.py +++ b/airbyte_cdk/__init__.py @@ -46,7 +46,7 @@ # Imports should also be placed in `if TYPE_CHECKING` blocks if they are only used as type # hints - again, to avoid circular dependencies. # Once those issues are resolved, the below can be sorted with isort. -from importlib import metadata +import dunamai as _dunamai from .destinations import Destination from .models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, Status, Type, FailureType, AirbyteStream, AdvancedAuth, DestinationSyncMode, ConnectorSpecification, OAuthConfigSpecification, OrchestratorType, ConfiguredAirbyteStream, SyncMode, AirbyteLogMessage, Level, AirbyteRecordMessage @@ -281,4 +281,13 @@ "Source", "StreamSlice", ] -__version__ = metadata.version("airbyte_cdk") + +__version__ = _dunamai.get_version( + "airbyte-cdk", + third_choice=_dunamai.Version.from_any_vcs, +).serialize() +"""Version generated by poetry dynamic versioning during publish. + +When running in development, dunamai will calculate a new prerelease version +from existing git release tag info. +""" diff --git a/airbyte_cdk/sources/declarative/manifest_declarative_source.py b/airbyte_cdk/sources/declarative/manifest_declarative_source.py index 05fbee7a..05a80321 100644 --- a/airbyte_cdk/sources/declarative/manifest_declarative_source.py +++ b/airbyte_cdk/sources/declarative/manifest_declarative_source.py @@ -256,7 +256,10 @@ def _validate_source(self) -> None: manifest_version, "manifest" ) - if cdk_major < manifest_major or ( + if cdk_version.startswith("0.0.0"): + # Skipping version compatibility check on unreleased dev branch + pass + elif cdk_major < manifest_major or ( cdk_major == manifest_major and cdk_minor < manifest_minor ): raise ValidationError( diff --git a/poetry.lock b/poetry.lock index 9d4d01bf..404cabb6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -999,6 +999,20 @@ files = [ {file = "dpath-2.2.0.tar.gz", hash = "sha256:34f7e630dc55ea3f219e555726f5da4b4b25f2200319c8e6902c394258dd6a3e"}, ] +[[package]] +name = "dunamai" +version = "1.22.0" +description = "Dynamic version generation" +optional = false +python-versions = ">=3.5" +files = [ + {file = "dunamai-1.22.0-py3-none-any.whl", hash = "sha256:eab3894b31e145bd028a74b13491c57db01986a7510482c9b5fff3b4e53d77b7"}, + {file = "dunamai-1.22.0.tar.gz", hash = "sha256:375a0b21309336f0d8b6bbaea3e038c36f462318c68795166e31f9873fdad676"}, +] + +[package.dependencies] +packaging = ">=20.9" + [[package]] name = "emoji" version = "2.14.0" @@ -5247,4 +5261,4 @@ vector-db-based = ["cohere", "langchain", "openai", "tiktoken"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "ea323704b6919c3f47c5f65ff15e14fe3870521a0b71164d29706a71c8debbdf" +content-hash = "f098ec85ef745d825378e98f5a2140c1748c85e2c15ee48c2b07c5ce28113af9" diff --git a/pyproject.toml b/pyproject.toml index c2a4c4d0..a0eb6325 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,16 +1,15 @@ [build-system] -requires = ["poetry-core>=1.0.0"] -build-backend = "poetry.core.masonry.api" +requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning>=1.0.0,<2.0.0"] +build-backend = "poetry_dynamic_versioning.backend" [tool.poetry] name = "airbyte-cdk" -version = "6.5.2" description = "A framework for writing Airbyte Connectors." authors = ["Airbyte "] license = "MIT" readme = "README.md" -homepage = "https://github.com/airbytehq/airbyte" -repository = "https://github.com/airbytehq/airbyte" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte-python-cdk" documentation = "https://docs.airbyte.io/" classifiers = [ "Development Status :: 3 - Alpha", @@ -22,6 +21,11 @@ classifiers = [ ] keywords = ["airbyte", "connector-development-kit", "cdk"] +# Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning +version = "0.0.0" # Version will be calculated dynamically. + +[tool.poetry-dynamic-versioning] +enable = true [tool.poetry.dependencies] python = "^3.10" @@ -30,6 +34,7 @@ backoff = "*" cachetools = "*" Deprecated = "~1.2" dpath = "^2.1.6" +dunamai = "^1.22.0" genson = "1.2.2" isodate = "~0.6.1" Jinja2 = "~3.1.2" diff --git a/pyproject.toml-bak b/pyproject.toml-bak deleted file mode 100644 index af6c4aab..00000000 --- a/pyproject.toml-bak +++ /dev/null @@ -1,127 +0,0 @@ -[build-system] -requires = ["poetry-core>=1.0.0"] -build-backend = "poetry.core.masonry.api" - -[tool.poetry] -name = "airbyte-cdk" -version = "6.5.2" -description = "A framework for writing Airbyte Connectors." -authors = ["Airbyte "] -license = "MIT" -readme = "README.md" -homepage = "https://github.com/airbytehq/airbyte" -repository = "https://github.com/airbytehq/airbyte" -documentation = "https://docs.airbyte.io/" -classifiers = [ - "Development Status :: 3 - Alpha", - "Intended Audience :: Developers", - "Topic :: Scientific/Engineering", - "Topic :: Software Development :: Libraries :: Python Modules", - "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 3.10", -] -keywords = ["airbyte", "connector-development-kit", "cdk"] - - -[tool.poetry.dependencies] -python = "^3.10" -airbyte-protocol-models-dataclasses = "^0.13" -backoff = "*" -cachetools = "*" -Deprecated = "~1.2" -dpath = "^2.1.6" -genson = "1.2.2" -isodate = "~0.6.1" -Jinja2 = "~3.1.2" -jsonref = "~0.2" -jsonschema = "~3.2.0" -pandas = "2.2.2" -pendulum = "<3.0.0" -psutil = "6.1.0" -pydantic = "^2.7" -pyrate-limiter = "~3.1.0" -python-dateutil = "*" -PyYAML = "^6.0.1" -requests = "*" -requests_cache = "*" -wcmatch = "8.4" -# Extras depedencies -avro = { version = "~1.11.2", optional = true } -cohere = { version = "4.21", optional = true } -fastavro = { version = "~1.8.0", optional = true } -langchain = { version = "0.1.16", optional = true } -langchain_core = { version = "0.1.42", optional = true } -markdown = { version = "*", optional = true } -openai = { version = "0.27.9", extras = ["embeddings"], optional = true } -pdf2image = { version = "1.16.3", optional = true } -"pdfminer.six" = { version = "20221105", optional = true } -pyarrow = { version = "~15.0.0", optional = true } -pytesseract = { version = "0.3.10", optional = true } -python-calamine = { version = "0.2.3", optional = true } -python-snappy = { version = "0.7.3", optional = true } -Sphinx = { version = "~4.2", optional = true } -sphinx-rtd-theme = { version = "~1.0", optional = true } -tiktoken = { version = "0.4.0", optional = true } -nltk = { version = "3.8.1", optional = true } -# This will ensure that even when you run poetry install or pip install, the compatible version of numpy will always be chosen. -# airbyte-ci will try to install latest version when --use-local-cdk is used, resulting in the conflict. -numpy = "<2" -unstructured = { version = "0.10.27", extras = ["docx", "pptx"], optional = true } -"unstructured.pytesseract" = { version = ">=0.3.12", optional = true } -pyjwt = "^2.8.0" -cryptography = "^42.0.5" -pytz = "2024.1" -orjson = "^3.10.7" -serpyco-rs = "^1.10.2" -sqlalchemy = {version = "^2.0,!=2.0.36", optional = true } -xmltodict = "^0.13.0" - -[tool.poetry.group.dev.dependencies] -freezegun = "*" -mypy = "*" -asyncio = "3.4.3" -poethepoet = "^0.24.2" -pyproject-flake8 = "^6.1.0" -pytest = "^7" -pytest-memray = "^1.6.0" -pytest-cov = "*" -pytest-httpserver = "*" -pytest-mock = "*" -requests-mock = "*" - -[tool.poetry.extras] -file-based = ["avro", "fastavro", "pyarrow", "unstructured", "pdf2image", "pdfminer.six", "unstructured.pytesseract", "pytesseract", "markdown", "python-calamine", "python-snappy"] -sphinx-docs = ["Sphinx", "sphinx-rtd-theme"] -vector-db-based = ["langchain", "openai", "cohere", "tiktoken"] -sql = ["sqlalchemy"] - -[tool.ruff] -# Setting python version to at least 3.10 avoids `from __future__ import annotations`. -target-version = "py310" -# This is consistent with airbytehq/airbyte root pyproject.toml Black rule defined. -line-length = 140 - -[tool.poe.tasks] -# Build tasks -assemble = {cmd = "bin/generate-component-manifest-dagger.sh", help = "Generate component manifest files."} -build-package = {cmd = "poetry build", help = "Build the python package: source and wheels archives."} -build = {sequence = ["assemble", "build-package"], help = "Run all tasks to build the package."} - -# Check tasks -lint = {cmd = "pflake8 --config ../../pyproject.toml ./", help = "Lint with flake8."} -type-check = {cmd = "bin/run-mypy-on-modified-files.sh", help = "Type check modified files with mypy."} -unit-test-with-cov = {cmd = "pytest -s unit_tests -c pytest.ini --cov=airbyte_cdk --cov-report=term --cov-config ../../pyproject.toml", help = "Run unit tests and create a coverage report."} -# TODO: find a version of the modified mypy check that works both locally and in CI. -check-lockfile = {cmd = "poetry check", help = "Check the poetry lock file."} -check-local = {sequence = ["lint", "type-check", "check-lockfile", "unit-test-with-cov"], help = "Lint all code, type-check modified files, and run unit tests."} -check-ci = {sequence = ["check-lockfile", "build", "lint", "unit-test-with-cov"], help = "Build the package, lint and run unit tests. Does not include type-checking."} - -# Build and check -pre-push = {sequence = ["build", "check-local"], help = "Run all build and check tasks."} - -[tool.airbyte_ci] -python_versions = ["3.10", "3.11"] -optional_poetry_groups = ["dev"] -poetry_extras = ["file-based", "sphinx-docs", "vector-db-based"] -poe_tasks = ["check-ci"] -mount_docker_socket = true