Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add and configure pre-commit #62

Merged
merged 16 commits into from
Apr 19, 2024
Merged
Show file tree
Hide file tree
Changes from 15 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 10 additions & 20 deletions .github/workflows/code-quality.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,14 @@ on:
workflow_dispatch:
inputs:
dbt_adapters_branch:
description: "The branch of dbt-adapters to evaluate"
type: string
required: true
default: "main"
description: "The branch of dbt-adapters to evaluate"
type: string
default: "main"
workflow_call:
inputs:
dbt_adapters_branch:
description: "The branch of dbt-adapters to evaluate"
type: string
required: true
mikealfare marked this conversation as resolved.
Show resolved Hide resolved
default: "main"

permissions: read-all
Expand All @@ -27,34 +25,26 @@ permissions: read-all
concurrency:
group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}
cancel-in-progress: true
env:
# set DBT_ADAPTERS_BRANCH to the input value if the event is a workflow_dispatch (workflow_call uses the same event_name),
# otherwise use 'main'
DBT_ADAPTERS_BRANCH: ${{ github.event_name == 'workflow_dispatch' && inputs.dbt_adapters_branch || 'main' }}

jobs:
code-quality:
name: Code Quality
runs-on: ubuntu-latest


steps:
- name: Check out repository
uses: actions/checkout@v4
with:
persist-credentials: false

- name: Update Adapters and Core branches
if: ${{ contains(github.event_name, 'workflow_') }}
shell: bash
run: |
./.github/scripts/update_dev_packages.sh \
$DBT_ADAPTERS_BRANCH \
"main"
run: ./.github/scripts/update_dev_packages.sh ${{ inputs.dbt_adapters_branch }} "main"

- name: Setup `hatch`
uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main

- name: Run linters
run: hatch run lint:all
uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@config/add-precommit
mikealfare marked this conversation as resolved.
Show resolved Hide resolved

- name: Run typechecks
run: hatch run typecheck:all
- name: Run code quality
shell: bash
run: hatch run code-quality
mikealfare marked this conversation as resolved.
Show resolved Hide resolved
11 changes: 1 addition & 10 deletions .github/workflows/integration-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ on:
required: false
default: "main"


permissions: read-all

# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise
Expand Down Expand Up @@ -87,7 +86,7 @@ jobs:
PGDATABASE: postgres

- name: Setup `hatch`
uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main
uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@config/add-precommit
mikealfare marked this conversation as resolved.
Show resolved Hide resolved
with:
python-version: ${{ matrix.python-version }}

Expand All @@ -101,14 +100,6 @@ jobs:
POSTGRES_TEST_DATABASE: dbt
POSTGRES_TEST_THREADS: 4

- name: Publish results
mikealfare marked this conversation as resolved.
Show resolved Hide resolved
uses: dbt-labs/dbt-adapters/.github/actions/publish-results@main
if: always()
with:
source-file: "results.csv"
file-name: "integration_results"
python-version: ${{ matrix.python-version }}

psycopg2-check:
name: "Test psycopg2 build version"
runs-on: ${{ matrix.scenario.platform }}
Expand Down
10 changes: 1 addition & 9 deletions .github/workflows/unit-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,18 +32,10 @@ jobs:
persist-credentials: false

- name: Setup `hatch`
uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main
uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@config/add-precommit
mikealfare marked this conversation as resolved.
Show resolved Hide resolved
with:
python-version: ${{ matrix.python-version }}

- name: Run unit tests
run: hatch run unit-tests:all
shell: bash

- name: Publish results
mikealfare marked this conversation as resolved.
Show resolved Hide resolved
uses: dbt-labs/dbt-adapters/.github/actions/publish-results@main
if: always()
with:
source-file: "results.csv"
file-name: "unit_results"
python-version: ${{ matrix.python-version }}
55 changes: 55 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
default_language_version:
mikealfare marked this conversation as resolved.
Show resolved Hide resolved
python: python3

repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
hooks:
- id: check-yaml
args: [--unsafe]
- id: check-json
- id: end-of-file-fixer
- id: trailing-whitespace
- id: check-case-conflict

- repo: https://github.com/dbt-labs/pre-commit-hooks
rev: v0.1.0a1
hooks:
- id: dbt-core-in-adapters-check

- repo: https://github.com/psf/black
rev: 24.4.0
hooks:
- id: black
args:
- --line-length=99
- --target-version=py38
- --target-version=py39
- --target-version=py310
- --target-version=py311

- repo: https://github.com/pycqa/flake8
rev: 7.0.0
hooks:
- id: flake8
exclude: tests/
args:
- --max-line-length=99
- --select=E,F,W
- --ignore=E203,E501,E741,W503,W504
- --per-file-ignores=*/__init__.py:F401

- repo: https://github.com/pre-commit/mirrors-mypy
rev: v1.9.0
hooks:
- id: mypy
args:
- --explicit-package-bases
- --ignore-missing-imports
- --pretty
- --show-error-codes
files: ^dbt/adapters/postgres
additional_dependencies:
- types-PyYAML
- types-protobuf
- types-pytz
1 change: 0 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,3 @@ and is generated by [Changie](https://github.com/miniscruff/changie).
### Under the Hood

* Update dependabot configuration to cover GHA

2 changes: 1 addition & 1 deletion CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ Remember to commit and push the file that's created.

### Signing the CLA

> **_NOTE:_** All contributors to `dbt-postgres` must sign the
> **_NOTE:_** All contributors to `dbt-postgres` must sign the
> [Contributor License Agreement](https://docs.getdbt.com/docs/contributor-license-agreements)(CLA).

Maintainers will be unable to merge contributions until the contributor signs the CLA.
Expand Down
30 changes: 17 additions & 13 deletions dbt/adapters/postgres/relation.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,19 +20,23 @@

@dataclass(frozen=True, eq=False, repr=False)
class PostgresRelation(BaseRelation):
renameable_relations: FrozenSet[RelationType] = field(default_factory=lambda: frozenset(
{
RelationType.View,
RelationType.Table,
RelationType.MaterializedView,
}
))
replaceable_relations: FrozenSet[RelationType] = field(default_factory=lambda: frozenset(
{
RelationType.View,
RelationType.Table,
}
))
renameable_relations: FrozenSet[RelationType] = field(
default_factory=lambda: frozenset(
{
RelationType.View,
RelationType.Table,
RelationType.MaterializedView,
}
)
)
replaceable_relations: FrozenSet[RelationType] = field(
default_factory=lambda: frozenset(
{
RelationType.View,
RelationType.Table,
}
)
)

def __post_init__(self):
# Check for length of Postgres table/view names.
Expand Down
2 changes: 1 addition & 1 deletion dbt/adapters/postgres/relation_configs/index.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ class PostgresIndexMethod(StrEnum):

@classmethod
def default(cls) -> "PostgresIndexMethod":
return cls.btree
return cls("btree")


@dataclass(frozen=True, eq=True, unsafe_hash=True)
Expand Down
72 changes: 9 additions & 63 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -59,56 +59,30 @@ path = "dbt/adapters/postgres/__version__.py"
dependencies = [
"dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git",
"dbt-common @ git+https://github.com/dbt-labs/dbt-common.git",
'pre-commit==3.7.0;python_version>="3.9"',
'pre-commit==3.5.0;python_version=="3.8"',
]

[tool.hatch.envs.lint]
mikealfare marked this conversation as resolved.
Show resolved Hide resolved
detached = true
dependencies = [
"black>=24.3",
"flake8",
"Flake8-pyproject",
]
[tool.hatch.envs.lint.scripts]
all = [
"black",
"flake8",
]
black = "python -m black ."
flake8 = "python -m flake8 ."

[tool.hatch.envs.typecheck]
dependencies = [
"mypy",
"types-protobuf",
"types-pytz",
]
[tool.hatch.envs.typecheck.scripts]
all = "python -m mypy ."
[tool.hatch.envs.default.scripts]
dev = "pre-commit install"
code-quality = "pre-commit run --all-files"

[tool.hatch.envs.unit-tests]
dependencies = [
"dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git",
"dbt-common @ git+https://github.com/dbt-labs/dbt-common.git",
"dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core",
"freezegun",
"pytest",
"pytest-dotenv",
"pytest-mock",
"pytest-xdist",
]
extra-dependencies = [
# TODO: remove `dbt-core` dependencies from unit tests
"dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git",
"dbt-common @ git+https://github.com/dbt-labs/dbt-common.git",
"dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core",
]
[tool.hatch.envs.unit-tests.scripts]
all = "python -m pytest {args:tests/unit}"

[tool.hatch.envs.integration-tests]
template = "unit-tests"
extra-dependencies = [
# TODO: remove `dbt-core` dependencies from integration tests
"dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git",
"dbt-common @ git+https://github.com/dbt-labs/dbt-common.git",
"dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core",
"dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter",
]
[tool.hatch.envs.integration-tests.env-vars]
Expand All @@ -125,8 +99,6 @@ dependencies = [
"twine",
"check-wheel-contents",
]


[tool.hatch.envs.build.scripts]
check-all = [
"- check-wheel",
Expand All @@ -143,35 +115,9 @@ check-sdist = [
"pip freeze | grep dbt-postgres",
]

[tool.black]
line-length = 99
target-version = ['py38']

[tool.flake8]
select = ["E", "W", "F"]
ignore = ["E203", "E501", "E741", "W503", "W504"]
exclude = ["tests", "venv", ".hatch_venvs"]
per-file-ignores = ["*/__init__.py: F401"]

[tool.mypy]
namespace_packages = true
show_error_codes = true
explicit_package_bases = true
ignore_missing_imports = true
pretty = true
files = [
"dbt/adapters/postgres",
"tests/unit",
]
exclude = [
"tests/functional",
"venv",
".hatch_venvs",
]

[tool.pytest]
env_files = ["test.env"]
testpaths = [
"tests/functional",
"tests/unit",
]
]
11 changes: 0 additions & 11 deletions tests/functional/artifacts/test_artifacts.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
from datetime import datetime
import jsonschema
import os

Expand All @@ -12,16 +11,6 @@
)
import pytest

from tests.functional.artifacts.expected_manifest import (
expected_references_manifest,
expected_seeded_manifest,
expected_versions_manifest,
)
from tests.functional.artifacts.expected_run_results import (
expected_references_run_results,
expected_run_results,
expected_versions_run_results,
)
from tests.functional.utils import run_dbt, run_dbt_and_capture


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from dbt.artifacts.schemas.base import get_artifact_schema_version
from dbt.artifacts.schemas.run import RunResultsArtifact
from dbt.contracts.graph.manifest import WritableManifest
from dbt.exceptions import IncompatibleSchemaError
from dbt.artifacts.exceptions import IncompatibleSchemaError
from dbt.tests.util import get_manifest
import pytest

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import json
import os

from dbt.tests.util import write_file
from dbt_common.exceptions import CompilationError
Expand Down
5 changes: 1 addition & 4 deletions tests/functional/dbt_debug/test_dbt_debug.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
import pytest
import os
import re
import yaml

from tests.functional.utils import run_dbt, run_dbt_and_capture
from tests.functional.utils import run_dbt_and_capture

MODELS__MODEL_SQL = """
seled 1 as id
Expand Down
Loading
Loading