From 8825ccb8bf2c1343f91a5f3c6ea0c7a05376f1ee Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:16:35 -0400 Subject: [PATCH 1/2] Add query tag back to incremental models (#1212) * add query tag back to incremental models * fix query tag test --- .../macros/materializations/incremental.sql | 1 + tests/functional/query_tag/test_query_tags.py | 34 +++++++------------ 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/dbt/include/snowflake/macros/materializations/incremental.sql b/dbt/include/snowflake/macros/materializations/incremental.sql index 12645cd11..d73525d6d 100644 --- a/dbt/include/snowflake/macros/materializations/incremental.sql +++ b/dbt/include/snowflake/macros/materializations/incremental.sql @@ -58,6 +58,7 @@ {% materialization incremental, adapter='snowflake', supported_languages=['sql', 'python'] -%} + {% set original_query_tag = set_query_tag() %} {#-- Set vars --#} {%- set full_refresh_mode = (should_full_refresh()) -%} diff --git a/tests/functional/query_tag/test_query_tags.py b/tests/functional/query_tag/test_query_tags.py index 421aae1b6..4ddafabb2 100644 --- a/tests/functional/query_tag/test_query_tags.py +++ b/tests/functional/query_tag/test_query_tags.py @@ -1,6 +1,7 @@ import pytest from dbt.tests.util import run_dbt + snapshots__snapshot_query_tag_sql = """ {% snapshot snapshot_query_tag %} {{ @@ -14,16 +15,15 @@ }} select 1 as id, 'blue' as color {% endsnapshot %} - """ + models__table_model_query_tag_sql = """ {{ config(materialized = 'table') }} - select 1 as id - """ + models__models_config_yml = """ version: 2 @@ -33,23 +33,21 @@ - name: id data_tests: - unique - """ + models__view_model_query_tag_sql = """ {{ config(materialized = 'view') }} - select 1 as id - """ + models__incremental_model_query_tag_sql = """ {{ config(materialized = 'incremental', unique_key = 'id') }} - select 1 as id - """ + macros__check_tag_sql = """ {% macro check_query_tag() %} @@ -61,12 +59,12 @@ {% endif %} {% endmacro %} - """ + seeds__seed_query_tag_csv = """id 1 -""" +""".strip() class TestQueryTag: @@ -95,20 +93,14 @@ def seeds(self): def project_config_update(self, prefix): return { "config-version": 2, - "models": { - "tests": {"query_tag": prefix, "post-hook": "{{ check_tag() }}"}, - }, - "seeds": { - "tests": {"query_tag": prefix, "post-hook": "{{ check_tag() }}"}, - }, - "snapshots": { - "tests": {"query_tag": prefix, "post-hook": "{{ check_tag() }}"}, - }, + "models": {"query_tag": prefix, "post-hook": "{{ check_query_tag() }}"}, + "seeds": {"query_tag": prefix, "post-hook": "{{ check_query_tag() }}"}, + "snapshots": {"query_tag": prefix, "post-hook": "{{ check_query_tag() }}"}, "tests": {"test": {"query_tag": prefix, "post-hook": "{{ check_query_tag() }}"}}, } def build_all_with_query_tags(self, project, prefix): - run_dbt(["build", "--vars", '{{"check_tag": "{}"}}'.format(prefix)]) + run_dbt(["build", "--vars", '{{"query_tag": "{}"}}'.format(prefix)]) def test_snowflake_query_tag(self, project, prefix): self.build_all_with_query_tags(project, prefix) @@ -130,7 +122,7 @@ def profiles_config_update(self, prefix): return {"query_tag": prefix} def build_all_with_query_tags(self, project, prefix): - run_dbt(["build", "--vars", '{{"check_tag": "{}"}}'.format(prefix)]) + run_dbt(["build", "--vars", '{{"query_tag": "{}"}}'.format(prefix)]) def test_snowflake_query_tag(self, project, prefix): self.build_all_with_query_tags(project, prefix) From 78f86674bc726000686a484c4d2f0e2f9d350ed1 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:48:05 -0400 Subject: [PATCH 2/2] Drop support for Python 3.8 (#1211) * drop support for python 3.8 --- .../Breaking Changes-20241016-183143.yaml | 6 +++ .github/scripts/integration-test-matrix.js | 4 +- .github/workflows/main.yml | 8 +-- .pre-commit-config.yaml | 1 - CONTRIBUTING.md | 2 +- Makefile | 10 ++-- dev-requirements.txt | 3 +- docker/Dockerfile | 4 +- docker/dev.Dockerfile | 52 +++++++++---------- setup.py | 8 ++- tox.ini | 6 +-- 11 files changed, 53 insertions(+), 51 deletions(-) create mode 100644 .changes/unreleased/Breaking Changes-20241016-183143.yaml diff --git a/.changes/unreleased/Breaking Changes-20241016-183143.yaml b/.changes/unreleased/Breaking Changes-20241016-183143.yaml new file mode 100644 index 000000000..26cc4b6de --- /dev/null +++ b/.changes/unreleased/Breaking Changes-20241016-183143.yaml @@ -0,0 +1,6 @@ +kind: Breaking Changes +body: Drop support for Python 3.8 +time: 2024-10-16T18:31:43.4167-04:00 +custom: + Author: mikealfare + Issue: "1211" diff --git a/.github/scripts/integration-test-matrix.js b/.github/scripts/integration-test-matrix.js index 81386c54e..756c21d5e 100644 --- a/.github/scripts/integration-test-matrix.js +++ b/.github/scripts/integration-test-matrix.js @@ -1,6 +1,6 @@ module.exports = ({ context }) => { - const defaultPythonVersion = "3.8"; - const supportedPythonVersions = ["3.8", "3.9", "3.10", "3.11", "3.12"]; + const defaultPythonVersion = "3.9"; + const supportedPythonVersions = ["3.9", "3.10", "3.11", "3.12"]; const supportedAdapters = ["snowflake"]; // if PR, generate matrix based on files changed and PR labels diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 95ce18033..4913917f4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -50,7 +50,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Install python dependencies run: | @@ -71,7 +71,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12'] env: TOXENV: "unit" @@ -127,7 +127,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Install python dependencies run: | @@ -175,7 +175,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-12, windows-latest] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12'] dist-type: ['whl', 'gz'] steps: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e14455e28..807e32a9a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,6 @@ repos: - id: black args: - --line-length=99 - - --target-version=py38 - --target-version=py39 - --target-version=py310 - --target-version=py311 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 45e0054c5..5b68aa03a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -79,7 +79,7 @@ WARNING: The parameters in your `test.env` file must link to a valid Snowflake a There are a few methods for running tests locally. #### `tox` -`tox` automatically runs unit tests against several Python versions using its own virtualenvs. Run `tox -p` to run unit tests for Python 3.8, Python 3.9, Python 3.10, and `flake8` in parallel. Run `tox -e py38` to invoke tests on Python version 3.8 only (use py38, py39, or py310). Tox recipes are found in `tox.ini`. +`tox` automatically runs unit tests against several Python versions using its own virtualenvs. Run `tox -p` to run unit tests for Python 3.9 and Python 3.10, and `flake8` in parallel. Run `tox -e py39` to invoke tests on Python version 3.9 only (use py39 or py310). Tox recipes are found in `tox.ini`. #### `pytest` You may run a specific test or group of tests using `pytest` directly. Activate a Python virtualenv active with dev dependencies installed. Then, run tests like so: diff --git a/Makefile b/Makefile index b42de9147..9a2376f47 100644 --- a/Makefile +++ b/Makefile @@ -12,20 +12,20 @@ dev-uninstall: ## Uninstalls all packages while maintaining the virtual environm pip uninstall -y dbt-snowflake .PHONY: unit -unit: ## Runs unit tests with py38. +unit: ## Runs unit tests with py39. @\ - tox -e py38 + tox -e py39 .PHONY: test -test: ## Runs unit tests with py38 and code checks against staged changes. +test: ## Runs unit tests with py39 and code checks against staged changes. @\ - tox -p -e py38; \ + tox -p -e py39; \ pre-commit run --all-files .PHONY: integration integration: ## Runs snowflake integration tests with py38. @\ - tox -e py38-snowflake -- + tox -e py39-snowflake -- .PHONY: clean @echo "cleaning repo" diff --git a/dev-requirements.txt b/dev-requirements.txt index f3d120eec..906003768 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -6,8 +6,7 @@ git+https://github.com/dbt-labs/dbt-common.git # dev ipdb~=0.13.13 -pre-commit~=3.7.0;python_version>="3.9" -pre-commit~=3.5.0;python_version<"3.9" +pre-commit~=3.7.0 # test ddtrace==2.3.0 diff --git a/docker/Dockerfile b/docker/Dockerfile index d256dcac4..17315b12d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # this image gets published to GHCR for production use ARG py_version=3.11.2 -FROM python:$py_version-slim-bullseye as base +FROM python:$py_version-slim-bullseye AS base RUN apt-get update \ && apt-get dist-upgrade -y \ @@ -25,7 +25,7 @@ ENV LANG=C.UTF-8 RUN python -m pip install --upgrade "pip==24.0" "setuptools==69.2.0" "wheel==0.43.0" --no-cache-dir -FROM base as dbt-snowflake +FROM base AS dbt-snowflake ARG commit_ref=main diff --git a/docker/dev.Dockerfile b/docker/dev.Dockerfile index 0fc667048..44f86f005 100644 --- a/docker/dev.Dockerfile +++ b/docker/dev.Dockerfile @@ -1,43 +1,43 @@ # this image does not get published, it is intended for local development only, see `Makefile` for usage -FROM ubuntu:22.04 as base +FROM ubuntu:24.04 AS base # prevent python installation from asking for time zone region ARG DEBIAN_FRONTEND=noninteractive # add python repository RUN apt-get update \ - && apt-get install -y software-properties-common=0.99.22.9 \ - && add-apt-repository -y ppa:deadsnakes/ppa \ - && apt-get clean \ - && rm -rf \ - /var/lib/apt/lists/* \ - /tmp/* \ - /var/tmp/* + && apt-get install -y software-properties-common=0.99.48 \ + && add-apt-repository -y ppa:deadsnakes/ppa \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* # install python RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - build-essential=12.9ubuntu3 \ - git-all=1:2.34.1-1ubuntu1.10 \ - python3.8=3.8.19-1+jammy1 \ - python3.8-dev=3.8.19-1+jammy1 \ - python3.8-distutils=3.8.19-1+jammy1 \ - python3.8-venv=3.8.19-1+jammy1 \ - python3-pip=22.0.2+dfsg-1ubuntu0.4 \ - python3-wheel=0.37.1-2ubuntu0.22.04.1 \ - && apt-get clean \ - && rm -rf \ - /var/lib/apt/lists/* \ - /tmp/* \ - /var/tmp/* + && apt-get install -y --no-install-recommends \ + build-essential=12.10ubuntu1 \ + git-all=1:2.43.0-1ubuntu7.1 \ + python3.9=3.9.20-1+noble1 \ + python3.9-dev=3.9.20-1+noble1 \ + python3.9-distutils=3.9.20-1+noble1 \ + python3.9-venv=3.9.20-1+noble1 \ + python3-pip=24.0+dfsg-1ubuntu1 \ + python3-wheel=0.42.0-2 \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* \ # update the default system interpreter to the newly installed version -RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1 -FROM base as dbt-snowflake-dev +FROM base AS dbt-snowflake-dev -HEALTHCHECK CMD python3 --version || exit 1 +HEALTHCHECK CMD python --version || exit 1 # send stdout/stderr to terminal ENV PYTHONUNBUFFERED=1 @@ -47,4 +47,4 @@ WORKDIR /opt/code VOLUME /opt/code # create a virtual environment -RUN python3 -m venv /opt/venv +RUN python -m venv /opt/venv diff --git a/setup.py b/setup.py index f542b9fcb..c0716341d 100644 --- a/setup.py +++ b/setup.py @@ -5,9 +5,9 @@ import sys # require python 3.8 or newer -if sys.version_info < (3, 8): +if sys.version_info < (3, 9): print("Error: dbt does not support this version of Python.") - print("Please upgrade to Python 3.8 or higher.") + print("Please upgrade to Python 3.9 or higher.") sys.exit(1) @@ -72,12 +72,10 @@ def _plugin_version() -> str: "Operating System :: Microsoft :: Windows", "Operating System :: MacOS :: MacOS X", "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", ], - python_requires=">=3.8", + python_requires=">=3.9", ) diff --git a/tox.ini b/tox.ini index d6f040a61..f6952efaf 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,8 @@ [tox] skipsdist = True -envlist = py38,py39,py310,py311,py312 +envlist = py39,py310,py311,py312 -[testenv:{unit,py38,py39,py310,py311,py312,py}] +[testenv:{unit,py39,py310,py311,py312,py}] description = unit testing skip_install = true passenv = @@ -13,7 +13,7 @@ deps = -rdev-requirements.txt -e. -[testenv:{integration,py38,py39,py310,py311,py312,py}-{snowflake}] +[testenv:{integration,py39,py310,py311,py312,py}-{snowflake}] description = adapter plugin integration testing skip_install = true passenv =