Merge pull request #189 from z3z1ma/refactor/clean-up-codebase

refactor!: clean up codebase, simplify where possible

chore: fix workbench req txt

fix: get release working

.github/workflows/constraints.txt

@@ -1,3 +1,2 @@
 pip==24.0
-poetry==1.3.2
-virtualenv==20.21.0
+uv==0.5.13

.github/workflows/lint.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Install pre-commit hooks
         run: |
           pip install -U pip==23.1.0
-          pip install -U pre-commit==3.4.0
+          pip install -U pre-commit==3.4.0 uv==0.5.13
           pre-commit install
       - name: Run pre-commit hooks
         run: |

.github/workflows/release.yml

@@ -25,10 +25,10 @@ jobs:
           pip install --constraint=.github/workflows/constraints.txt pip
           pip --version
 
-      - name: Install Poetry
+      - name: Install uv
         run: |
-          pip install --constraint=.github/workflows/constraints.txt poetry
-          poetry --version
+          pip install --constraint=.github/workflows/constraints.txt uv
+          uv --version
 
       - name: Check if there is a parent commit
         id: check-parent-commit
@@ -42,18 +42,16 @@ jobs:
         uses: salsify/[email protected]
         with:
           version-command: |
-            bash -o pipefail -c "poetry version | awk '{ print \$2 }'"
+            sed -nE 's/^version = "([^"]+)"/\1/p' pyproject.toml
 
       - name: Bump version for developmental release
         if: "! steps.check-version.outputs.tag"
         run: |
-          poetry version patch &&
-          version="$(poetry version | awk '{ print $2 }')" &&
-          poetry version "${version:?}.dev.$(date +%s)"
+          sed -i -E "s/^version\s*=\s*\"[^\"]+\"/version = \"1.0.0.dev.$(date +%Y%m%d)\"/" pyproject.toml
 
       - name: Build package
         run: |
-          poetry build --ansi
+          uv build
 
       - name: Publish package on PyPI
         if: steps.check-version.outputs.tag

.github/workflows/tests.yml

@@ -29,23 +29,23 @@ jobs:
           pip install --constraint=.github/workflows/constraints.txt pip
           pip --version
 
-      - name: Install Poetry
+      - name: Install uv
         run: |
-          pip install --constraint=.github/workflows/constraints.txt poetry
-          poetry --version
-
-      - name: Install dbt-core
-        run: |
-          poetry add dbt-core==${{ matrix.dbt-version }}
+          pip install --constraint=.github/workflows/constraints.txt uv
+          uv --version
 
       - name: Install required packages
         run: | # install duckdb extras to be able to parse manifest
-          poetry install -E duckdb
+          uv sync --extra duckdb --extra dev
+
+      - name: Install dbt-core
+        run: |
+          uv pip install dbt-core==${{ matrix.dbt-version }}
 
       - name: Parse manifest
         run: |
-          poetry run dbt parse --project-dir demo_duckdb --profiles-dir demo_duckdb -t test
+          uv run dbt parse --project-dir demo_duckdb --profiles-dir demo_duckdb -t test
 
       - name: Run pytest
         run: |
-          poetry run python -m pytest
+          uv run pytest

.gitignore

@@ -131,3 +131,6 @@ dmypy.json
 
 # Nix
 .devenv
+
+# Makefile touch target
+.uv-installed-*

.pre-commit-config.yaml

@@ -1,36 +1,31 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
-      - id: end-of-file-fixer
-      - id: trailing-whitespace
+      - id: check-ast
       - id: check-json
       - id: check-yaml
+      - id: check-toml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
       - id: detect-private-key
       - id: debug-statements
-  - repo: https://github.com/rhysd/actionlint
-    rev: v1.7.1
-    hooks:
-      - id: actionlint-docker
-  - repo: https://github.com/shellcheck-py/shellcheck-py
-    rev: v0.10.0.1
+  - repo: https://github.com/commitizen-tools/commitizen
+    rev: v4.1.0
     hooks:
-      - id: shellcheck
-  - repo: https://github.com/psf/black
-    rev: 24.4.2
+      - id: commitizen
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.8.0
     hooks:
-      - id: black
-        args: ["--config", "pyproject.toml"]
-  - repo: https://github.com/pycqa/isort
-    rev: 5.13.2
+      - id: ruff-format
+        args: ["--preview"]
+      - id: ruff
+        args: [--fix]
+  - repo: https://github.com/gitleaks/gitleaks
+    rev: v8.22.1
     hooks:
-      - id: isort
-        args: ["-sp", "pyproject.toml"]
-  - repo: https://github.com/hadolint/hadolint
-    rev: v2.13.0-beta
+      - id: gitleaks
+  - repo: https://github.com/rhysd/actionlint
+    rev: v1.7.5
     hooks:
-      - id: hadolint-docker
-#  - repo: https://github.com/pycqa/flake8
-#    rev: 4.0.1
-#    hooks:
-#      - id: flake8
+      - id: actionlint

Makefile

@@ -0,0 +1,59 @@
+# Author: Alex B
+# Description: Makefile for setting up the development environment. Includes convenient aliases for common tasks.
+
+PY_VERSION := $(shell cat .python-version)
+
+.PHONY: uv venv pre-commit dev all test lint format scan scan-new-baseline \
+        scan-without-baseline clean
+
+check: format lint test
+
+.uv-installed-$(PY_VERSION): .python-version
+	@if [ ! -f .python-version ]; then echo 'Please create a .python-version file with the desired Python version'; exit 1; fi
+	@if command -v uv > /dev/null; then echo 'Verified uv is installed'; else echo 'Please install uv by running `curl -LsSf https://astral.sh/uv/install.sh | sh` or visit https://docs.astral.sh/uv/ for more information'; exit 1; fi
+	@uv tool update-shell
+	@uv python install
+	@rm -f .uv-installed-*
+	@touch .uv-installed-$(PY_VERSION)
+
+
+uv: .uv-installed-$(PY_VERSION)
+
+.venv: .uv-installed-$(PY_VERSION)
+	@uv venv .venv
+
+venv: .venv
+
+.git/hooks/pre-commit: .uv-installed-$(PY_VERSION)
+	@uv tool install pre-commit
+	@uv tool run pre-commit install
+
+pre-commit: .git/hooks/pre-commit
+
+dev: .venv .git/hooks/pre-commit
+	@uv sync --extra=dev --extra=duckdb
+
+clean:
+	@rm -rf .venv target demo_duckdb/target demo_sqlite/target
+
+lint: .uv-installed-$(PY_VERSION)
+	@uvx ruff check
+
+format: .uv-installed-$(PY_VERSION)
+	@uvx ruff check --fix --select I
+	@uvx ruff format --preview
+
+test: .uv-installed-$(PY_VERSION)
+	@uv run pytest tests/
+
+scan: .uv-installed-$(PY_VERSION)
+	@uvx bandit -r src -b tests/bandit_baseline.json
+
+scan-new-baseline: .uv-installed-$(PY_VERSION)
+	@uvx bandit -r src -f json -o tests/bandit_baseline.json
+
+scan-without-baseline: .uv-installed-$(PY_VERSION)
+	@uvx bandit -r src
+
+requirements.txt: .uv-installed-$(PY_VERSION)
+	@uv export -o requirements.txt --no-hashes --frozen

demo_duckdb/dbt_project.yml

@@ -5,23 +5,29 @@ version: "0.1"
 
 profile: "jaffle_shop"
 
-model-paths: [ "models" ]
-seed-paths: [ "seeds" ]
-test-paths: [ "tests" ]
-analysis-paths: [ "analysis" ]
-macro-paths: [ "macros" ]
+model-paths: ["models"]
+seed-paths: ["seeds"]
+test-paths: ["tests"]
+analysis-paths: ["analysis"]
+macro-paths: ["macros"]
 
 target-path: "target"
 clean-targets:
   - "target"
   - "dbt_modules"
   - "logs"
 
-require-dbt-version: [ ">=1.0.0", "<2.0.0" ]
+require-dbt-version: [">=1.0.0", "<2.0.0"]
 
 models:
-  +dbt-osmosis: schema.yml
-  jaffle_shop:
+  jaffle_shop_duckdb:
+    +dbt-osmosis: "{node.database}/{node.schema}/{node.name}.yml"
     materialized: table
     staging:
       materialized: view
+
+seeds:
+  jaffle_shop_duckdb:
+    +dbt-osmosis: "{node.database}/{node.schema}/{node.name}.yml"
+
+flags: {}

demo_duckdb/models/jaffle_shop/main/customers.yml

@@ -0,0 +1,26 @@
+version: 2
+models:
+  - name: customers
+    description: This table has basic information about a customer, as well as some derived facts based on a customer's orders
+
+    columns:
+      - name: customer_id
+        description: This is a unique identifier for a customer
+        tests:
+          - unique
+          - not_null
+
+      - name: first_name
+        description: Customer's first name. PII.
+      - name: last_name
+        description: Customer's last name. PII.
+      - name: first_order
+        description: Date (UTC) of a customer's first order
+      - name: most_recent_order
+        description: Date (UTC) of a customer's most recent order
+      - name: number_of_orders
+        description: Count of the number of orders a customer has placed
+      - name: customer_lifetime_value
+        data_type: DOUBLE
+      - name: customer_average_value
+        data_type: DECIMAL(18,3)

demo_duckdb/models/schema.yml → demo_duckdb/models/jaffle_shop/main/orders.yml

@@ -1,38 +1,5 @@
 version: 2
-
 models:
-  - name: customers
-    description: This table has basic information about a customer, as well as some derived facts based on a customer's orders
-
-    columns:
-      - name: customer_id
-        description: This is a unique identifier for a customer
-        tests:
-          - unique
-          - not_null
-
-      - name: first_name
-        description: Customer's first name. PII.
-
-      - name: last_name
-        description: Customer's last name. PII.
-
-      - name: first_order
-        description: Date (UTC) of a customer's first order
-
-      - name: most_recent_order
-        description: Date (UTC) of a customer's most recent order
-
-      - name: number_of_orders
-        description: Count of the number of orders a customer has placed
-
-      - name: customer_lifetime_value
-        data_type: DOUBLE
-        description: ''
-
-      - name: customer_rank
-        data_type: VARCHAR
-        description: ''
   - name: orders
     description: This table has basic information about orders, as well as some derived facts based on payments
 
@@ -42,7 +9,6 @@ models:
           - unique
           - not_null
         description: This is a unique identifier for an order
-
       - name: customer_id
         description: Foreign key to the customers table
         tests:
@@ -53,12 +19,12 @@ models:
 
       - name: order_date
         description: Date (UTC) that the order was placed
-
       - name: status
         tests:
           - accepted_values:
               values: ['placed', 'shipped', 'completed', 'return_pending', 'returned']
 
+        description: '{{ doc("orders_status") }}'
       - name: credit_card_amount
         description: Amount of the order (AUD) paid for by credit card
         tests:

demo_duckdb/models/jaffle_shop/main/orders_prefix.yml

@@ -0,0 +1,23 @@
+version: 2
+models:
+  - name: orders_prefix
+    columns:
+      - name: o_order_id
+        data_type: INTEGER
+      - name: o_customer_id
+        data_type: INTEGER
+      - name: o_order_date
+        data_type: DATE
+      - name: o_status
+        description: '{{ doc("orders_status") }}'
+        data_type: VARCHAR
+      - name: o_credit_card_amount
+        data_type: DOUBLE
+      - name: o_coupon_amount
+        data_type: DOUBLE
+      - name: o_bank_transfer_amount
+        data_type: DOUBLE
+      - name: o_gift_card_amount
+        data_type: DOUBLE
+      - name: o_amount
+        data_type: DOUBLE