more or less complete rewrite of application

add reports for sonarqube
override any region from profile
get rid of backend selector, use enum
allow everything to be frozen
squelch some output from Makefile
correct remote_path_options model
pass remote_options as a dict
handle case when no global vars are set
handle model based options properly
handle empty global vars into hooks
download modules prior to looking for required providers
Added error handling for jinja2 syntax errors
change resolved path to use definition name, not path
add support for multiple version constraints
enable strict_locking flag
fix issue when multiple provider verions are specified

also adds tests for
- base backend validate empty function
- backends custom enum method
- exceptions raised during s3 tests
- add target for tests with artifacts
- adds tests to complete authenticators coverage.
- add tests for the app_state model.
- add tests for cli-options
- add a missing test condition to the FS copier
- add proper poetry install to makefile
- add tests for Definition model and methods
- add tests for definitions collection
- add tests for definitions plan
- Add test for defintion prepare
- Remove legacy tests

.gitignore

@@ -9,3 +9,6 @@ __pycache__
 htmlcov
 .pytest_cache
 registry.terraform.io/
+.scannerwork/
+coverage.xml
+reports/*

.isort.cfg

@@ -1,3 +1,4 @@
 [settings]
-known_third_party = atlassian,boto3,botocore,click,deepdiff,google,hcl2,jinja2,lark,mergedeep,moto,pydantic,pytest,yaml
+known_third_party = atlassian,boto3,botocore,click,google,hcl2,jinja2,lark,moto,packaging,pydantic,pydantic_core,pydantic_settings,pytest,yaml
 profile = black
+skip = ["*/__init__.py"]

CHANGELOG.md

@@ -0,0 +1,18 @@
+# Changes 0.12.0 -> 0.13.0 (2024-06...)
+
+## Notes
+This version reconsiders many of the core elements of the worker application.
+
+The application now relies on pydantic to define the models for what is consumed from the configuration file and the command line. Previously there was a heinous mix of classes which just had attributes and methods seemingly hung on
+them at random. This was a nightmare to maintain and understand.
+
+- the CopierFactory was reorganized to be more modular
+- providers became a package, with a model used to validation all provider configurations
+- all of the configuration file logic was moved into the `commands` module, the logic to handle managing the cofig is all contained there instead of being spread out many places
+- added tfworker.util.log to handle interaction with the user, via the CLI or via a logger in the future
+- made significant strides towards consolidating all exceptions / error handling
+- made significant strides in only having the primary cli.py and commands/ actually cause the program to terminate
+- validation of options and inputs is now handled centrally in the pydantic models instead of being spread out *everywhere*
+
+
+- ... @TODO find time to update this :)

Makefile

@@ -1,21 +1,28 @@
 init:
 	poetry install
 
+init-dev:
+	poetry install --with dev
+
 default: lint test
 
-lint: init
+lint: init-dev
 	poetry run flake8 --ignore E501,W503 tfworker tests
 
-format: init
+format: init-dev
 	poetry run black tfworker tests
-	poetry run seed-isort-config || echo "known_third_party setting changed. Please commit pyproject.toml"
+	@poetry run seed-isort-config || echo "known_third_party setting changed. Please commit pyproject.toml"
 	poetry run isort tfworker tests
 
-test: init
+test: init-dev
 	poetry run pytest -p no:warnings --disable-socket
 	poetry run coverage report --fail-under=60 -m --skip-empty
 
-dep-test: init
+ci-test: init-dev
+	poetry run pytest --disable-socket --junitxml=reports/junit.xml
+	poetry run coverage xml -o reports/coverage.xml
+
+dep-test: init-dev
 	poetry run pytest --disable-socket
 	poetry run coverage report --fail-under=60 -m --skip-empty
 

pyproject.toml

@@ -28,16 +28,18 @@ classifiers = [
 
 [tool.poetry.dependencies]
 python = "^3.11"
+atlassian-python-api = "^3.41"
 boto3 = "^1.34"
 click = "^8.1"
-jinja2 = "^3.1"
 google-cloud-storage = "^2.17"
+jinja2 = "^3.1"
+mergedeep = "^1.3"
+pydantic = "^2.7"
 python-hcl2 = "^4.3"
 pyyaml = "^6.0"
-mergedeep = "^1.3"
 setuptools = "^70.0"
-atlassian-python-api = "^3.41"
-pydantic = "^2.7"
+pydantic-settings = "^2.3.4"
+packaging = "^24.1"
 
 [tool.poetry.scripts]
 worker = 'tfworker.cli:cli'
@@ -46,24 +48,29 @@ worker = 'tfworker.cli:cli'
 optional = true
 
 [tool.poetry.group.dev.dependencies]
-pytest-timeout = "2.3.1"
-ipython = "^8.24"
-pytest = "^8.2"
 black = "^24.4"
-isort = "^5.13"
-seed-isort-config = "^2.2"
+coverage = "^7.5"
+deepdiff = "^7.0"
 flake8 = "^7.0"
-wheel = "^0.43"
+ipython = "^8.24"
+isort = "^5.13"
+pytest = "^8.2"
+pytest-cov = "^5.0"
 pytest-depends = "^1.0"
+pytest-mock = "^3.14"
 pytest-socket = "^0.7"
-coverage = "^7.5"
-pytest-cov = "^5.0"
-moto = {extras = ["sts","dynamodb", "s3"], version = "^5.0"}
-deepdiff = "^7.0"
+pytest-timeout = "2.3.1"
+seed-isort-config = "^2.2"
 Sphinx = "^7.3"
+wheel = "^0.43"
+moto = {version = "^5.0.10", extras = ["sts", "dynamodb", "s3"]}
+
 
 [tool.pytest.ini_options]
-addopts = "--capture=sys --cov=tfworker --cov-report="
+addopts = "--capture=sys --cov=tfworker --cov-report= -m 'not performance'"
+markers = [
+    "performance: mark a test as a performance test"
+]
 
 [build-system]
 requires = ["poetry>=0.12"]

tests/__init__.py

@@ -1,13 +1 @@
-# Copyright 2020 Richard Maynard ([email protected])
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.