From 12dd750a731ed9c96c3c53c269261e3f3429ba5a Mon Sep 17 00:00:00 2001 From: Richard Maynard Date: Mon, 27 May 2024 20:20:20 -0500 Subject: [PATCH] refactor the s3 backend This refactoring included breaking down several of the long methods in the s3 backend to smaller functions in order to enhance unit testing and ensure that this section of code remains solid. This also updates moto from v4.x to v5.x to take advantage of new functionality and simplified use. There are many test additions for the s3 backend, but also a lot of changes to try and reduce the reliance on complicated fixtures and provide smaller more consise tests. --- Makefile | 4 +- poetry.lock | 228 +++++++++++++++-------- pyproject.toml | 3 +- tests/authenticators/test_aws_auth.py | 8 +- tests/backends/test_s3.py | 193 +++++++++++++++++++ tests/conftest.py | 14 +- tests/test_plugins.py | 1 + tests/util/test_system.py | 5 +- tfworker/backends/base.py | 11 +- tfworker/backends/s3.py | 257 ++++++++++++++++---------- tfworker/commands/base.py | 3 +- tfworker/util/system.py | 1 + 12 files changed, 541 insertions(+), 187 deletions(-) diff --git a/Makefile b/Makefile index 5bc9c7f..01691d3 100644 --- a/Makefile +++ b/Makefile @@ -12,11 +12,11 @@ format: init poetry run isort tfworker tests test: init - poetry run pytest -p no:warnings + poetry run pytest -p no:warnings --disable-socket poetry run coverage report --fail-under=60 -m --skip-empty dep-test: init - poetry run pytest + poetry run pytest --disable-socket poetry run coverage report --fail-under=60 -m --skip-empty clean: diff --git a/poetry.lock b/poetry.lock index 993c5a5..9e9e607 100644 --- a/poetry.lock +++ b/poetry.lock @@ -389,63 +389,63 @@ files = [ [[package]] name = "coverage" -version = "7.5.1" +version = "7.5.2" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.8" files = [ - {file = "coverage-7.5.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0884920835a033b78d1c73b6d3bbcda8161a900f38a488829a83982925f6c2e"}, - {file = "coverage-7.5.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:39afcd3d4339329c5f58de48a52f6e4e50f6578dd6099961cf22228feb25f38f"}, - {file = "coverage-7.5.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a7b0ceee8147444347da6a66be737c9d78f3353b0681715b668b72e79203e4a"}, - {file = "coverage-7.5.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a9ca3f2fae0088c3c71d743d85404cec8df9be818a005ea065495bedc33da35"}, - {file = "coverage-7.5.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fd215c0c7d7aab005221608a3c2b46f58c0285a819565887ee0b718c052aa4e"}, - {file = "coverage-7.5.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4bf0655ab60d754491004a5efd7f9cccefcc1081a74c9ef2da4735d6ee4a6223"}, - {file = "coverage-7.5.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:61c4bf1ba021817de12b813338c9be9f0ad5b1e781b9b340a6d29fc13e7c1b5e"}, - {file = "coverage-7.5.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:db66fc317a046556a96b453a58eced5024af4582a8dbdc0c23ca4dbc0d5b3146"}, - {file = "coverage-7.5.1-cp310-cp310-win32.whl", hash = "sha256:b016ea6b959d3b9556cb401c55a37547135a587db0115635a443b2ce8f1c7228"}, - {file = "coverage-7.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:df4e745a81c110e7446b1cc8131bf986157770fa405fe90e15e850aaf7619bc8"}, - {file = "coverage-7.5.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:796a79f63eca8814ca3317a1ea443645c9ff0d18b188de470ed7ccd45ae79428"}, - {file = "coverage-7.5.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4fc84a37bfd98db31beae3c2748811a3fa72bf2007ff7902f68746d9757f3746"}, - {file = "coverage-7.5.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6175d1a0559986c6ee3f7fccfc4a90ecd12ba0a383dcc2da30c2b9918d67d8a3"}, - {file = "coverage-7.5.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fc81d5878cd6274ce971e0a3a18a8803c3fe25457165314271cf78e3aae3aa2"}, - {file = "coverage-7.5.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:556cf1a7cbc8028cb60e1ff0be806be2eded2daf8129b8811c63e2b9a6c43bca"}, - {file = "coverage-7.5.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9981706d300c18d8b220995ad22627647be11a4276721c10911e0e9fa44c83e8"}, - {file = "coverage-7.5.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d7fed867ee50edf1a0b4a11e8e5d0895150e572af1cd6d315d557758bfa9c057"}, - {file = "coverage-7.5.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ef48e2707fb320c8f139424a596f5b69955a85b178f15af261bab871873bb987"}, - {file = "coverage-7.5.1-cp311-cp311-win32.whl", hash = "sha256:9314d5678dcc665330df5b69c1e726a0e49b27df0461c08ca12674bcc19ef136"}, - {file = "coverage-7.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:5fa567e99765fe98f4e7d7394ce623e794d7cabb170f2ca2ac5a4174437e90dd"}, - {file = "coverage-7.5.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b6cf3764c030e5338e7f61f95bd21147963cf6aa16e09d2f74f1fa52013c1206"}, - {file = "coverage-7.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2ec92012fefebee89a6b9c79bc39051a6cb3891d562b9270ab10ecfdadbc0c34"}, - {file = "coverage-7.5.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16db7f26000a07efcf6aea00316f6ac57e7d9a96501e990a36f40c965ec7a95d"}, - {file = "coverage-7.5.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:beccf7b8a10b09c4ae543582c1319c6df47d78fd732f854ac68d518ee1fb97fa"}, - {file = "coverage-7.5.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8748731ad392d736cc9ccac03c9845b13bb07d020a33423fa5b3a36521ac6e4e"}, - {file = "coverage-7.5.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7352b9161b33fd0b643ccd1f21f3a3908daaddf414f1c6cb9d3a2fd618bf2572"}, - {file = "coverage-7.5.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:7a588d39e0925f6a2bff87154752481273cdb1736270642aeb3635cb9b4cad07"}, - {file = "coverage-7.5.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:68f962d9b72ce69ea8621f57551b2fa9c70509af757ee3b8105d4f51b92b41a7"}, - {file = "coverage-7.5.1-cp312-cp312-win32.whl", hash = "sha256:f152cbf5b88aaeb836127d920dd0f5e7edff5a66f10c079157306c4343d86c19"}, - {file = "coverage-7.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:5a5740d1fb60ddf268a3811bcd353de34eb56dc24e8f52a7f05ee513b2d4f596"}, - {file = "coverage-7.5.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e2213def81a50519d7cc56ed643c9e93e0247f5bbe0d1247d15fa520814a7cd7"}, - {file = "coverage-7.5.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5037f8fcc2a95b1f0e80585bd9d1ec31068a9bcb157d9750a172836e98bc7a90"}, - {file = "coverage-7.5.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c3721c2c9e4c4953a41a26c14f4cef64330392a6d2d675c8b1db3b645e31f0e"}, - {file = "coverage-7.5.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca498687ca46a62ae590253fba634a1fe9836bc56f626852fb2720f334c9e4e5"}, - {file = "coverage-7.5.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cdcbc320b14c3e5877ee79e649677cb7d89ef588852e9583e6b24c2e5072661"}, - {file = "coverage-7.5.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:57e0204b5b745594e5bc14b9b50006da722827f0b8c776949f1135677e88d0b8"}, - {file = "coverage-7.5.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8fe7502616b67b234482c3ce276ff26f39ffe88adca2acf0261df4b8454668b4"}, - {file = "coverage-7.5.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:9e78295f4144f9dacfed4f92935fbe1780021247c2fabf73a819b17f0ccfff8d"}, - {file = "coverage-7.5.1-cp38-cp38-win32.whl", hash = "sha256:1434e088b41594baa71188a17533083eabf5609e8e72f16ce8c186001e6b8c41"}, - {file = "coverage-7.5.1-cp38-cp38-win_amd64.whl", hash = "sha256:0646599e9b139988b63704d704af8e8df7fa4cbc4a1f33df69d97f36cb0a38de"}, - {file = "coverage-7.5.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4cc37def103a2725bc672f84bd939a6fe4522310503207aae4d56351644682f1"}, - {file = "coverage-7.5.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fc0b4d8bfeabd25ea75e94632f5b6e047eef8adaed0c2161ada1e922e7f7cece"}, - {file = "coverage-7.5.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d0a0f5e06881ecedfe6f3dd2f56dcb057b6dbeb3327fd32d4b12854df36bf26"}, - {file = "coverage-7.5.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9735317685ba6ec7e3754798c8871c2f49aa5e687cc794a0b1d284b2389d1bd5"}, - {file = "coverage-7.5.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d21918e9ef11edf36764b93101e2ae8cc82aa5efdc7c5a4e9c6c35a48496d601"}, - {file = "coverage-7.5.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c3e757949f268364b96ca894b4c342b41dc6f8f8b66c37878aacef5930db61be"}, - {file = "coverage-7.5.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:79afb6197e2f7f60c4824dd4b2d4c2ec5801ceb6ba9ce5d2c3080e5660d51a4f"}, - {file = "coverage-7.5.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d1d0d98d95dd18fe29dc66808e1accf59f037d5716f86a501fc0256455219668"}, - {file = "coverage-7.5.1-cp39-cp39-win32.whl", hash = "sha256:1cc0fe9b0b3a8364093c53b0b4c0c2dd4bb23acbec4c9240b5f284095ccf7981"}, - {file = "coverage-7.5.1-cp39-cp39-win_amd64.whl", hash = "sha256:dde0070c40ea8bb3641e811c1cfbf18e265d024deff6de52c5950677a8fb1e0f"}, - {file = "coverage-7.5.1-pp38.pp39.pp310-none-any.whl", hash = "sha256:6537e7c10cc47c595828b8a8be04c72144725c383c4702703ff4e42e44577312"}, - {file = "coverage-7.5.1.tar.gz", hash = "sha256:54de9ef3a9da981f7af93eafde4ede199e0846cd819eb27c88e2b712aae9708c"}, + {file = "coverage-7.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:554c7327bf0fd688050348e22db7c8e163fb7219f3ecdd4732d7ed606b417263"}, + {file = "coverage-7.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d0305e02e40c7cfea5d08d6368576537a74c0eea62b77633179748d3519d6705"}, + {file = "coverage-7.5.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:829fb55ad437d757c70d5b1c51cfda9377f31506a0a3f3ac282bc6a387d6a5f1"}, + {file = "coverage-7.5.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:894b1acded706f1407a662d08e026bfd0ff1e59e9bd32062fea9d862564cfb65"}, + {file = "coverage-7.5.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe76d6dee5e4febefa83998b17926df3a04e5089e3d2b1688c74a9157798d7a2"}, + {file = "coverage-7.5.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:c7ebf2a37e4f5fea3c1a11e1f47cea7d75d0f2d8ef69635ddbd5c927083211fc"}, + {file = "coverage-7.5.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:20e611fc36e1a0fc7bbf957ef9c635c8807d71fbe5643e51b2769b3cc0fb0b51"}, + {file = "coverage-7.5.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7c5c5b7ae2763533152880d5b5b451acbc1089ade2336b710a24b2b0f5239d20"}, + {file = "coverage-7.5.2-cp310-cp310-win32.whl", hash = "sha256:1e4225990a87df898e40ca31c9e830c15c2c53b1d33df592bc8ef314d71f0281"}, + {file = "coverage-7.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:976cd92d9420e6e2aa6ce6a9d61f2b490e07cb468968adf371546b33b829284b"}, + {file = "coverage-7.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5997d418c219dcd4dcba64e50671cca849aaf0dac3d7a2eeeb7d651a5bd735b8"}, + {file = "coverage-7.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ec27e93bbf5976f0465e8936f02eb5add99bbe4e4e7b233607e4d7622912d68d"}, + {file = "coverage-7.5.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f11f98753800eb1ec872562a398081f6695f91cd01ce39819e36621003ec52a"}, + {file = "coverage-7.5.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e34680049eecb30b6498784c9637c1c74277dcb1db75649a152f8004fbd6646"}, + {file = "coverage-7.5.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e12536446ad4527ac8ed91d8a607813085683bcce27af69e3b31cd72b3c5960"}, + {file = "coverage-7.5.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:3d3f7744b8a8079d69af69d512e5abed4fb473057625588ce126088e50d05493"}, + {file = "coverage-7.5.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:431a3917e32223fcdb90b79fe60185864a9109631ebc05f6c5aa03781a00b513"}, + {file = "coverage-7.5.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a7c6574225f34ce45466f04751d957b5c5e6b69fca9351db017c9249786172ce"}, + {file = "coverage-7.5.2-cp311-cp311-win32.whl", hash = "sha256:2b144d142ec9987276aeff1326edbc0df8ba4afbd7232f0ca10ad57a115e95b6"}, + {file = "coverage-7.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:900532713115ac58bc3491b9d2b52704a05ed408ba0918d57fd72c94bc47fba1"}, + {file = "coverage-7.5.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9a42970ce74c88bdf144df11c52c5cf4ad610d860de87c0883385a1c9d9fa4ab"}, + {file = "coverage-7.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:26716a1118c6ce2188283b4b60a898c3be29b480acbd0a91446ced4fe4e780d8"}, + {file = "coverage-7.5.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:60b66b0363c5a2a79fba3d1cd7430c25bbd92c923d031cae906bdcb6e054d9a2"}, + {file = "coverage-7.5.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5d22eba19273b2069e4efeff88c897a26bdc64633cbe0357a198f92dca94268"}, + {file = "coverage-7.5.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3bb5b92a0ab3d22dfdbfe845e2fef92717b067bdf41a5b68c7e3e857c0cff1a4"}, + {file = "coverage-7.5.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1aef719b6559b521ae913ddeb38f5048c6d1a3d366865e8b320270b7bc4693c2"}, + {file = "coverage-7.5.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8809c0ea0e8454f756e3bd5c36d04dddf222989216788a25bfd6724bfcee342c"}, + {file = "coverage-7.5.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1acc2e2ef098a1d4bf535758085f508097316d738101a97c3f996bccba963ea5"}, + {file = "coverage-7.5.2-cp312-cp312-win32.whl", hash = "sha256:97de509043d3f0f2b2cd171bdccf408f175c7f7a99d36d566b1ae4dd84107985"}, + {file = "coverage-7.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:8941e35a0e991a7a20a1fa3e3182f82abe357211f2c335a9e6007067c3392fcf"}, + {file = "coverage-7.5.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5662bf0f6fb6757f5c2d6279c541a5af55a39772c2362ed0920b27e3ce0e21f7"}, + {file = "coverage-7.5.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3d9c62cff2ffb4c2a95328488fd7aa96a7a4b34873150650fe76b19c08c9c792"}, + {file = "coverage-7.5.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74eeaa13e8200ad72fca9c5f37395fb310915cec6f1682b21375e84fd9770e84"}, + {file = "coverage-7.5.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f29bf497d51a5077994b265e976d78b09d9d0dff6ca5763dbb4804534a5d380"}, + {file = "coverage-7.5.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f96aa94739593ae0707eda9813ce363a0a0374a810ae0eced383340fc4a1f73"}, + {file = "coverage-7.5.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:51b6cee539168a912b4b3b040e4042b9e2c9a7ad9c8546c09e4eaeff3eacba6b"}, + {file = "coverage-7.5.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:59a75e6aa5c25b50b5a1499f9718f2edff54257f545718c4fb100f48d570ead4"}, + {file = "coverage-7.5.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:29da75ce20cb0a26d60e22658dd3230713c6c05a3465dd8ad040ffc991aea318"}, + {file = "coverage-7.5.2-cp38-cp38-win32.whl", hash = "sha256:23f2f16958b16152b43a39a5ecf4705757ddd284b3b17a77da3a62aef9c057ef"}, + {file = "coverage-7.5.2-cp38-cp38-win_amd64.whl", hash = "sha256:9e41c94035e5cdb362beed681b58a707e8dc29ea446ea1713d92afeded9d1ddd"}, + {file = "coverage-7.5.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:06d96b9b19bbe7f049c2be3c4f9e06737ec6d8ef8933c7c3a4c557ef07936e46"}, + {file = "coverage-7.5.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:878243e1206828908a6b4a9ca7b1aa8bee9eb129bf7186fc381d2646f4524ce9"}, + {file = "coverage-7.5.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:482df956b055d3009d10fce81af6ffab28215d7ed6ad4a15e5c8e67cb7c5251c"}, + {file = "coverage-7.5.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a35c97af60a5492e9e89f8b7153fe24eadfd61cb3a2fb600df1a25b5dab34b7e"}, + {file = "coverage-7.5.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24bb4c7859a3f757a116521d4d3a8a82befad56ea1bdacd17d6aafd113b0071e"}, + {file = "coverage-7.5.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e1046aab24c48c694f0793f669ac49ea68acde6a0798ac5388abe0a5615b5ec8"}, + {file = "coverage-7.5.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:448ec61ea9ea7916d5579939362509145caaecf03161f6f13e366aebb692a631"}, + {file = "coverage-7.5.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4a00bd5ba8f1a4114720bef283cf31583d6cb1c510ce890a6da6c4268f0070b7"}, + {file = "coverage-7.5.2-cp39-cp39-win32.whl", hash = "sha256:9f805481d5eff2a96bac4da1570ef662bf970f9a16580dc2c169c8c3183fa02b"}, + {file = "coverage-7.5.2-cp39-cp39-win_amd64.whl", hash = "sha256:2c79f058e7bec26b5295d53b8c39ecb623448c74ccc8378631f5cb5c16a7e02c"}, + {file = "coverage-7.5.2-pp38.pp39.pp310-none-any.whl", hash = "sha256:40dbb8e7727560fe8ab65efcddfec1ae25f30ef02e2f2e5d78cfb52a66781ec5"}, + {file = "coverage-7.5.2.tar.gz", hash = "sha256:13017a63b0e499c59b5ba94a8542fb62864ba3016127d1e4ef30d354fc2b00e9"}, ] [package.dependencies] @@ -554,6 +554,28 @@ wrapt = ">=1.10,<2" [package.extras] dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] +[[package]] +name = "docker" +version = "7.1.0" +description = "A Python library for the Docker Engine API." +optional = false +python-versions = ">=3.8" +files = [ + {file = "docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0"}, + {file = "docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c"}, +] + +[package.dependencies] +pywin32 = {version = ">=304", markers = "sys_platform == \"win32\""} +requests = ">=2.26.0" +urllib3 = ">=1.26.0" + +[package.extras] +dev = ["coverage (==7.2.7)", "pytest (==7.4.2)", "pytest-cov (==4.1.0)", "pytest-timeout (==2.1.0)", "ruff (==0.1.8)"] +docs = ["myst-parser (==0.18.0)", "sphinx (==5.1.1)"] +ssh = ["paramiko (>=2.4.3)"] +websockets = ["websocket-client (>=1.3.0)"] + [[package]] name = "docutils" version = "0.16" @@ -1060,46 +1082,49 @@ files = [ [[package]] name = "moto" -version = "4.2.14" +version = "5.0.8" description = "" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "moto-4.2.14-py2.py3-none-any.whl", hash = "sha256:6d242dbbabe925bb385ddb6958449e5c827670b13b8e153ed63f91dbdb50372c"}, - {file = "moto-4.2.14.tar.gz", hash = "sha256:8f9263ca70b646f091edcc93e97cda864a542e6d16ed04066b1370ed217bd190"}, + {file = "moto-5.0.8-py2.py3-none-any.whl", hash = "sha256:7d1035e366434bfa9fcc0621f07d5aa724b6846408071d540137a0554c46f214"}, + {file = "moto-5.0.8.tar.gz", hash = "sha256:517fb808dc718bcbdda54c6ffeaca0adc34cf6e10821bfb01216ce420a31765c"}, ] [package.dependencies] boto3 = ">=1.9.201" -botocore = ">=1.12.201" +botocore = ">=1.14.0" cryptography = ">=3.3.1" +docker = {version = ">=3.0.0", optional = true, markers = "extra == \"dynamodb\""} Jinja2 = ">=2.10.1" +py-partiql-parser = {version = "0.5.5", optional = true, markers = "extra == \"dynamodb\" or extra == \"s3\""} python-dateutil = ">=2.1,<3.0.0" +PyYAML = {version = ">=5.1", optional = true, markers = "extra == \"s3\""} requests = ">=2.5" -responses = ">=0.13.0" +responses = ">=0.15.0" werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1" xmltodict = "*" [package.extras] -all = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -apigateway = ["PyYAML (>=5.1)", "ecdsa (!=0.15)", "openapi-spec-validator (>=0.5.0)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] -apigatewayv2 = ["PyYAML (>=5.1)"] +all = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "jsonpath-ng", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.5)", "pyparsing (>=3.0.7)", "setuptools"] +apigateway = ["PyYAML (>=5.1)", "joserfc (>=0.9.0)", "openapi-spec-validator (>=0.5.0)"] +apigatewayv2 = ["PyYAML (>=5.1)", "openapi-spec-validator (>=0.5.0)"] appsync = ["graphql-core"] awslambda = ["docker (>=3.0.0)"] batch = ["docker (>=3.0.0)"] -cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -cognitoidp = ["ecdsa (!=0.15)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] -dynamodb = ["docker (>=3.0.0)", "py-partiql-parser (==0.5.0)"] -dynamodbstreams = ["docker (>=3.0.0)", "py-partiql-parser (==0.5.0)"] -ec2 = ["sshpubkeys (>=3.1.0)"] +cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.5)", "pyparsing (>=3.0.7)", "setuptools"] +cognitoidp = ["joserfc (>=0.9.0)"] +dynamodb = ["docker (>=3.0.0)", "py-partiql-parser (==0.5.5)"] +dynamodbstreams = ["docker (>=3.0.0)", "py-partiql-parser (==0.5.5)"] glue = ["pyparsing (>=3.0.7)"] iotdata = ["jsondiff (>=1.1.2)"] -proxy = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -resourcegroupstaggingapi = ["PyYAML (>=5.1)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] -s3 = ["PyYAML (>=5.1)", "py-partiql-parser (==0.5.0)"] -s3crc32c = ["PyYAML (>=5.1)", "crc32c", "py-partiql-parser (==0.5.0)"] -server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +proxy = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "jsonpath-ng", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.5)", "pyparsing (>=3.0.7)", "setuptools"] +resourcegroupstaggingapi = ["PyYAML (>=5.1)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.5)", "pyparsing (>=3.0.7)"] +s3 = ["PyYAML (>=5.1)", "py-partiql-parser (==0.5.5)"] +s3crc32c = ["PyYAML (>=5.1)", "crc32c", "py-partiql-parser (==0.5.5)"] +server = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "jsonpath-ng", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.5)", "pyparsing (>=3.0.7)", "setuptools"] ssm = ["PyYAML (>=5.1)"] +stepfunctions = ["antlr4-python3-runtime", "jsonpath-ng"] xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] [[package]] @@ -1245,13 +1270,13 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "prompt-toolkit" -version = "3.0.43" +version = "3.0.44" description = "Library for building powerful interactive command lines in Python" optional = false python-versions = ">=3.7.0" files = [ - {file = "prompt_toolkit-3.0.43-py3-none-any.whl", hash = "sha256:a11a29cb3bf0a28a387fe5122cdb649816a957cd9261dcedf8c9f1fef33eacf6"}, - {file = "prompt_toolkit-3.0.43.tar.gz", hash = "sha256:3527b7af26106cbc65a040bcc84839a3566ec1b051bb0bfe953631e704b0ff7d"}, + {file = "prompt_toolkit-3.0.44-py3-none-any.whl", hash = "sha256:205a20669633d042d3722a528b8e7cd3f4dbd9e1450935f596c2cc61166762dd"}, + {file = "prompt_toolkit-3.0.44.tar.gz", hash = "sha256:c1dfd082c4259964bc8bcce1f8460d9dbeb5d4a37bfc25b8082bc02cd41c8af6"}, ] [package.dependencies] @@ -1319,6 +1344,20 @@ files = [ [package.extras] tests = ["pytest"] +[[package]] +name = "py-partiql-parser" +version = "0.5.5" +description = "Pure Python PartiQL Parser" +optional = false +python-versions = "*" +files = [ + {file = "py_partiql_parser-0.5.5-py2.py3-none-any.whl", hash = "sha256:90d278818385bd60c602410c953ee78f04ece599d8cd21c656fc5e47399577a1"}, + {file = "py_partiql_parser-0.5.5.tar.gz", hash = "sha256:ed07f8edf4b55e295cab4f5fd3e2ba3196cee48a43fe210d53ddd6ffce1cf1ff"}, +] + +[package.extras] +dev = ["black (==22.6.0)", "flake8", "mypy", "pytest"] + [[package]] name = "pyasn1" version = "0.6.0" @@ -1462,6 +1501,20 @@ files = [ [package.dependencies] pytest = ">=3.2.5" +[[package]] +name = "pytest-socket" +version = "0.7.0" +description = "Pytest Plugin to disable socket calls during tests" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pytest_socket-0.7.0-py3-none-any.whl", hash = "sha256:7e0f4642177d55d317bbd58fc68c6bd9048d6eadb2d46a89307fa9221336ce45"}, + {file = "pytest_socket-0.7.0.tar.gz", hash = "sha256:71ab048cbbcb085c15a4423b73b619a8b35d6a307f46f78ea46be51b1b7e11b3"}, +] + +[package.dependencies] +pytest = ">=6.2.5" + [[package]] name = "pytest-timeout" version = "2.1.0" @@ -1504,6 +1557,29 @@ files = [ [package.dependencies] lark = ">=1,<2" +[[package]] +name = "pywin32" +version = "306" +description = "Python for Window Extensions" +optional = false +python-versions = "*" +files = [ + {file = "pywin32-306-cp310-cp310-win32.whl", hash = "sha256:06d3420a5155ba65f0b72f2699b5bacf3109f36acbe8923765c22938a69dfc8d"}, + {file = "pywin32-306-cp310-cp310-win_amd64.whl", hash = "sha256:84f4471dbca1887ea3803d8848a1616429ac94a4a8d05f4bc9c5dcfd42ca99c8"}, + {file = "pywin32-306-cp311-cp311-win32.whl", hash = "sha256:e65028133d15b64d2ed8f06dd9fbc268352478d4f9289e69c190ecd6818b6407"}, + {file = "pywin32-306-cp311-cp311-win_amd64.whl", hash = "sha256:a7639f51c184c0272e93f244eb24dafca9b1855707d94c192d4a0b4c01e1100e"}, + {file = "pywin32-306-cp311-cp311-win_arm64.whl", hash = "sha256:70dba0c913d19f942a2db25217d9a1b726c278f483a919f1abfed79c9cf64d3a"}, + {file = "pywin32-306-cp312-cp312-win32.whl", hash = "sha256:383229d515657f4e3ed1343da8be101000562bf514591ff383ae940cad65458b"}, + {file = "pywin32-306-cp312-cp312-win_amd64.whl", hash = "sha256:37257794c1ad39ee9be652da0462dc2e394c8159dfd913a8a4e8eb6fd346da0e"}, + {file = "pywin32-306-cp312-cp312-win_arm64.whl", hash = "sha256:5821ec52f6d321aa59e2db7e0a35b997de60c201943557d108af9d4ae1ec7040"}, + {file = "pywin32-306-cp37-cp37m-win32.whl", hash = "sha256:1c73ea9a0d2283d889001998059f5eaaba3b6238f767c9cf2833b13e6a685f65"}, + {file = "pywin32-306-cp37-cp37m-win_amd64.whl", hash = "sha256:72c5f621542d7bdd4fdb716227be0dd3f8565c11b280be6315b06ace35487d36"}, + {file = "pywin32-306-cp38-cp38-win32.whl", hash = "sha256:e4c092e2589b5cf0d365849e73e02c391c1349958c5ac3e9d5ccb9a28e017b3a"}, + {file = "pywin32-306-cp38-cp38-win_amd64.whl", hash = "sha256:e8ac1ae3601bee6ca9f7cb4b5363bf1c0badb935ef243c4733ff9a393b1690c0"}, + {file = "pywin32-306-cp39-cp39-win32.whl", hash = "sha256:e25fd5b485b55ac9c057f67d94bc203f3f6595078d1fb3b458c9c28b7153a802"}, + {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"}, +] + [[package]] name = "pyyaml" version = "6.0.1" @@ -2053,4 +2129,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "42b1e3342bf064466eabcbd15a001be18770e485f70fe8bdb0cb5768170a23e5" +content-hash = "48dd23624a37c26ef765864da8ade5d2bdbd40c3f6b234ed0bc1f1cc511110f8" diff --git a/pyproject.toml b/pyproject.toml index 7c462bf..e4b2281 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,10 +52,11 @@ seed-isort-config = "^2.2.0" flake8 = "^6.0.0" wheel = "^0.40" pytest-depends = "^1.0.1" +pytest-socket = "^0.7.0" pytest-lazy-fixture = "^0.6.3" coverage = "^7.2" pytest-cov = "^4.0.0" -moto = {extras = ["sts"], version = "^4.1.4"} +moto = {extras = ["sts","dynamodb", "s3"], version = "^5.0.8"} deepdiff = "^6.2.0" Sphinx = "5.1.1" diff --git a/tests/authenticators/test_aws_auth.py b/tests/authenticators/test_aws_auth.py index 4098226..04c5ccd 100644 --- a/tests/authenticators/test_aws_auth.py +++ b/tests/authenticators/test_aws_auth.py @@ -16,7 +16,7 @@ import pytest from botocore.credentials import Credentials -from moto import mock_sts +from moto import mock_aws from tfworker.authenticators.aws import AWSAuthenticator, MissingArgumentException from tfworker.commands.root import RootCommand @@ -68,7 +68,7 @@ def test_with_no_backend_bucket(self): AWSAuthenticator(state_args={}, deployment="deployfu") assert "backend_bucket" in str(e.value) - @mock_sts + @mock_aws def test_with_access_key_pair_creds( self, sts_client, state_args, aws_access_key_id, aws_secret_access_key ): @@ -77,7 +77,7 @@ def test_with_access_key_pair_creds( assert auth.secret_access_key == aws_secret_access_key assert auth.session_token is None - @mock_sts + @mock_aws def test_with_access_key_pair_creds_and_role_arn( self, sts_client, state_args_with_role_arn, aws_secret_access_key ): @@ -109,7 +109,7 @@ def test_with_profile( assert auth.secret_access_key == aws_credentials_instance.secret_key assert auth.session_token is None - @mock_sts + @mock_aws def test_with_prefix(self, state_args): auth = AWSAuthenticator(state_args, deployment="deployfu") assert auth.prefix == DEFAULT_BACKEND_PREFIX.format(deployment="deployfu") diff --git a/tests/backends/test_s3.py b/tests/backends/test_s3.py index f518c19..31ae6c2 100644 --- a/tests/backends/test_s3.py +++ b/tests/backends/test_s3.py @@ -11,12 +11,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os import random import string +from unittest.mock import MagicMock, patch import pytest +from botocore.exceptions import ClientError +from moto import mock_aws +from tests.conftest import MockAWSAuth +from tfworker.backends import S3Backend from tfworker.backends.base import BackendError +from tfworker.handlers import HandlerError STATE_BUCKET = "test_bucket" STATE_PREFIX = "terraform" @@ -25,6 +32,7 @@ EMPTY_STATE = f"{STATE_PREFIX}/{STATE_DEPLOYMENT}/empty/terraform.tfstate" OCCUPIED_STATE = f"{STATE_PREFIX}/{STATE_DEPLOYMENT}/occupied/terraform.tfstate" LOCK_DIGEST = "1234123412341234" +NO_SUCH_BUCKET = "no_such_bucket" @pytest.fixture(scope="class") @@ -159,6 +167,191 @@ def test_clean_locking_state(self, basec, state_setup, dynamodb_client): ) +class TestS3BackendInit: + def setup_method(self, method): + self.authenticators = {"aws": MockAWSAuth()} + self.definitions = {} + + def test_no_session(self): + self.authenticators["aws"]._session = None + with pytest.raises(BackendError): + result = S3Backend(self.authenticators, self.definitions) + + def test_no_backend_session(self): + self.authenticators["aws"]._backend_session = None + with pytest.raises(BackendError): + result = S3Backend(self.authenticators, self.definitions) + + @patch("tfworker.backends.S3Backend._ensure_locking_table", return_value=None) + @patch("tfworker.backends.S3Backend._ensure_backend_bucket", return_value=None) + @patch("tfworker.backends.S3Backend._get_bucket_files", return_value={}) + def test_deployment_undefined( + self, + mock_get_bucket_files, + mock_ensure_backend_bucket, + mock_ensure_locking_table, + ): + # arrange + result = S3Backend(self.authenticators, self.definitions) + assert result._deployment == "undefined" + assert mock_get_bucket_files.called + assert mock_ensure_backend_bucket.called + assert mock_ensure_locking_table.called + + @patch("tfworker.backends.S3Backend._ensure_locking_table", return_value=None) + @patch("tfworker.backends.S3Backend._ensure_backend_bucket", return_value=None) + @patch("tfworker.backends.S3Backend._get_bucket_files", return_value={}) + @patch("tfworker.backends.s3.S3Handler", side_effect=HandlerError("message")) + def test_handler_error( + self, + mock_get_bucket_files, + mock_ensure_backend_bucket, + mock_ensure_locking_table, + mock_handler, + ): + with pytest.raises(SystemExit): + result = S3Backend(self.authenticators, self.definitions) + + +class TestS3BackendEnsureBackendBucket: + from botocore.exceptions import ClientError + + @pytest.fixture(autouse=True) + def setup_class(self, state_setup): + pass + + @patch("tfworker.backends.S3Backend._ensure_locking_table", return_value=None) + @patch("tfworker.backends.S3Backend._ensure_backend_bucket", return_value=None) + @patch("tfworker.backends.S3Backend._get_bucket_files", return_value={}) + def setup_method( + self, + method, + mock_get_bucket_files, + mock_ensure_backend_bucket, + mock_ensure_locking_table, + ): + with mock_aws(): + self.authenticators = {"aws": MockAWSAuth()} + self.definitions = {} + self.backend = S3Backend(self.authenticators, self.definitions) + self.backend._authenticator.bucket = STATE_BUCKET + self.backend._authenticator.backend_region = STATE_REGION + + def teardown_method(self, method): + with mock_aws(): + try: + self.backend._s3_client.delete_bucket(Bucket=NO_SUCH_BUCKET) + except Exception: + pass + + @mock_aws + def test_check_bucket_does_not_exist(self): + result = self.backend._check_bucket_exists(NO_SUCH_BUCKET) + assert result is False + + @mock_aws + def test_check_bucket_exists(self): + result = self.backend._check_bucket_exists(STATE_BUCKET) + assert result is True + + @mock_aws + def test_check_bucket_exists_error(self): + self.backend._s3_client = MagicMock() + self.backend._s3_client.head_bucket.side_effect = ClientError( + {"Error": {"Code": "403", "Message": "Unauthorized"}}, "head_bucket" + ) + + with pytest.raises(ClientError): + result = self.backend._check_bucket_exists(STATE_BUCKET) + assert self.backend._s3_client.head_bucket.called + + @mock_aws + def test_bucket_not_exist_no_create(self, capfd): + self.backend._authenticator.create_backend_bucket = False + self.backend._authenticator.bucket = NO_SUCH_BUCKET + with pytest.raises(BackendError): + result = self.backend._ensure_backend_bucket() + assert ( + "Backend bucket not found and --no-create-backend-bucket specified." + in capfd.readouterr().out + ) + + @mock_aws + def test_create_bucket(self): + self.backend._authenticator.create_backend_bucket = True + self.backend._authenticator.bucket = NO_SUCH_BUCKET + assert NO_SUCH_BUCKET not in [ + x["Name"] for x in self.backend._s3_client.list_buckets()["Buckets"] + ] + result = self.backend._ensure_backend_bucket() + assert result is None + assert NO_SUCH_BUCKET in [ + x["Name"] for x in self.backend._s3_client.list_buckets()["Buckets"] + ] + + @mock_aws + def test_create_bucket_invalid_location_constraint(self, capsys): + self.backend._authenticator.create_backend_bucket = True + self.backend._authenticator.bucket = NO_SUCH_BUCKET + self.backend._authenticator.backend_region = "us-west-1" + # moto doesn't properly raise a location constraint when the session doesn't match the region + # so we'll just do it manually + assert self.backend._authenticator.backend_session.region_name != "us-west-1" + assert self.backend._authenticator.backend_region == "us-west-1" + assert NO_SUCH_BUCKET not in [ + x["Name"] for x in self.backend._s3_client.list_buckets()["Buckets"] + ] + self.backend._s3_client = MagicMock() + self.backend._s3_client.create_bucket.side_effect = ClientError( + { + "Error": { + "Code": "InvalidLocationConstraint", + "Message": "InvalidLocationConstraint", + } + }, + "create_bucket", + ) + + with pytest.raises(SystemExit): + result = self.backend._create_bucket(NO_SUCH_BUCKET) + assert "InvalidLocationConstraint" in capsys.readouterr().out + + assert NO_SUCH_BUCKET not in [ + x["Name"] for x in self.backend._s3_client.list_buckets()["Buckets"] + ] + + # This test can not be enabled until several other tests are refactored to not create the bucket needlessly + # as the method itself skips this check when being run through a test, the same also applies to "BucketAlreadyOwnedByYou" + # @mock_aws + # def test_create_bucket_already_exists(self, capsys): + # self.backend._authenticator.create_backend_bucket = True + # self.backend._authenticator.bucket = STATE_BUCKET + # assert STATE_BUCKET in [ x['Name'] for x in self.backend._s3_client.list_buckets()['Buckets'] ] + + # with pytest.raises(SystemExit): + # result = self.backend._create_bucket(STATE_BUCKET) + # assert f"Bucket {STATE_BUCKET} already exists" in capsys.readouterr().out + + def test_create_bucket_error(self): + self.backend._authenticator.create_backend_bucket = True + self.backend._authenticator.bucket = NO_SUCH_BUCKET + self.backend._s3_client = MagicMock() + self.backend._s3_client.create_bucket.side_effect = ClientError( + {"Error": {"Code": "403", "Message": "Unauthorized"}}, "create_bucket" + ) + + with pytest.raises(ClientError): + result = self.backend._create_bucket(NO_SUCH_BUCKET) + assert self.backend._s3_client.create_bucket.called + + +def test_backend_remotes(basec, state_setup): + remotes = basec.backend.remotes() + assert len(remotes) == 2 + assert "empty" in remotes + assert "occupied" in remotes + + def test_backend_clean_all(basec, request, state_setup, dynamodb_client, s3_client): # this function should trigger an exit with pytest.raises(SystemExit): diff --git a/tests/conftest.py b/tests/conftest.py index 04d9d02..ac0052a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,7 +20,7 @@ import boto3 import pytest -from moto import mock_dynamodb, mock_s3, mock_sts +from moto import mock_aws from pytest_lazyfixture import lazy_fixture import tfworker @@ -70,19 +70,19 @@ def aws_credentials(): @pytest.fixture(scope="class") def s3_client(aws_credentials): - with mock_s3(): + with mock_aws(): yield boto3.client("s3", region_name="us-west-2") @pytest.fixture(scope="class") def dynamodb_client(aws_credentials): - with mock_dynamodb(): + with mock_aws(): yield boto3.client("dynamodb", region_name="us-west-2") @pytest.fixture(scope="class") def sts_client(aws_credentials): - with mock_sts(): + with mock_aws(): yield boto3.client("sts", region_name="us-west-2") @@ -93,8 +93,10 @@ class MockAWSAuth: (cross account assumed roles, user identity, etc...) """ + @mock_aws def __init__(self): self._session = boto3.Session() + self._backend_session = self._session self.bucket = "test_bucket" self.prefix = "terraform/test-0001" @@ -102,6 +104,10 @@ def __init__(self): def session(self): return self._session + @property + def backend_session(self): + return self._backend_session + @pytest.fixture() def grootc(): diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 623e612..f2208b4 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -68,6 +68,7 @@ def does_not_raise(): class TestPlugins: + @pytest.mark.enable_socket @pytest.mark.depends(on="get_url") def test_plugin_download(self, rootc): plugins = tfworker.plugins.PluginsCollection( diff --git a/tests/util/test_system.py b/tests/util/test_system.py index f3d061f..f0cda0f 100644 --- a/tests/util/test_system.py +++ b/tests/util/test_system.py @@ -17,7 +17,7 @@ import pytest -from tfworker.util.system import pipe_exec, which, get_version, strip_ansi +from tfworker.util.system import get_version, pipe_exec, strip_ansi, which # context manager to allow testing exceptions in parameterized tests @@ -33,11 +33,13 @@ def mock_pipe_exec(args, stdin=None, cwd=None, env=None): def mock_tf_version(args: str): return (0, args.encode(), "".encode()) + def mock_distribution(*args, **kwargs): Class = mock.MagicMock() Class.version = "1.2.3" return Class + class TestUtilSystem: @pytest.mark.parametrize( "commands, exit_code, cwd, stdin, stdout, stderr, stream_output", @@ -139,6 +141,7 @@ def test_get_version(self): def test_get_version_unknown(self): from pkg_resources import DistributionNotFound + with mock.patch( "tfworker.util.system.get_distribution", side_effect=DistributionNotFound, diff --git a/tfworker/backends/base.py b/tfworker/backends/base.py index 6fa40cc..550de42 100644 --- a/tfworker/backends/base.py +++ b/tfworker/backends/base.py @@ -18,7 +18,16 @@ class BackendError(Exception): - pass + # add custom "help" parameter to the exception + def __init__(self, message, help=None): + super().__init__(message) + self._help = help + + @property + def help(self): + if self._help is None: + return "No help available" + return self._help class BaseBackend(metaclass=ABCMeta): diff --git a/tfworker/backends/s3.py b/tfworker/backends/s3.py index c7f33bb..b1f9564 100644 --- a/tfworker/backends/s3.py +++ b/tfworker/backends/s3.py @@ -34,103 +34,32 @@ class S3Backend(BaseBackend): plan_storage = False def __init__(self, authenticators, definitions, deployment=None): - self._authenticator = authenticators[self.auth_tag] + # print the module name for debugging self._definitions = definitions - self._deployment = "undefined" - self._handlers = None - - if deployment: - self._deployment = deployment - - self._ddb_client = boto3.client( - "dynamodb", - region_name=self._authenticator.backend_region, - aws_access_key_id=self._authenticator.access_key_id, - aws_secret_access_key=self._authenticator.secret_access_key, - aws_session_token=self._authenticator.session_token, - ) - - locking_table_name = f"terraform-{deployment}" - - # Check locking table for aws backend - click.secho( - f"Checking backend locking table: {locking_table_name}", fg="yellow" - ) + self._authenticator = authenticators[self.auth_tag] + if not self._authenticator.session: + raise BackendError( + "AWS session not available", + help="Either provide AWS credentials or a profile, see --help for more information.", + ) + if not self._authenticator.backend_session: + raise BackendError( + "AWS backend session not available", + help="Either provide AWS credentials or a profile, see --help for more information.", + ) - if self._check_table_exists(locking_table_name): - click.secho("DynamoDB lock table found, continuing.", fg="yellow") + if deployment is None: + self._deployment = "undefined" else: - click.secho( - "DynamoDB lock table not found, creating, please wait...", fg="yellow" - ) - self._create_table(locking_table_name) + self._deployment = deployment - # Initialize s3 client and create bucket if necessary. + # Setup AWS clients and ensure backend resources are available + self._ddb_client = self._authenticator.backend_session.client("dynamodb") self._s3_client = self._authenticator.backend_session.client("s3") - try: - self._s3_client.head_bucket(Bucket=self._authenticator.bucket) - except botocore.exceptions.ClientError as err: - err_str = str(err) - if "Not Found" not in err_str: - raise err - if self._authenticator.create_backend_bucket: - try: - self._s3_client.create_bucket( - Bucket=self._authenticator.bucket, - CreateBucketConfiguration={ - "LocationConstraint": self._authenticator.backend_region - }, - ACL="private", - ) - except botocore.exceptions.ClientError as err: - err_str = str(err) - if "InvalidLocationConstraint" in err_str: - click.secho( - "InvalidLocationConstraint raised when trying to create a bucket. " - "Verify the AWS Region passed to the worker matches the AWS region " - "in the profile.", - fg="red", - ) - elif "BucketAlreadyExists" in err_str: - # Ignore when testing - if "PYTEST_CURRENT_TEST" not in os.environ: - click.secho(err_str, fg="red") - sys.exit(4) - elif "BucketAlreadyOwnedByYou" not in err_str: - raise err - - # Block public access - self._s3_client.put_public_access_block( - Bucket=self._authenticator.bucket, - PublicAccessBlockConfiguration={ - "BlockPublicAcls": True, - "IgnorePublicAcls": True, - "BlockPublicPolicy": True, - "RestrictPublicBuckets": True, - }, - ) + self._ensure_locking_table() + self._ensure_backend_bucket() + self._bucket_files = self._get_bucket_files() - # Enable versioning on the bucket - s3_resource = self._authenticator.backend_session.resource("s3") - versioning = s3_resource.BucketVersioning(self._authenticator.bucket) - versioning.enable() - else: - raise BackendError( - "Backend bucket not found and --no-create-backend-bucket specified." - ) - - # Generate a list of all files in the bucket, at the desired prefix for the deployment, used for "--all-remote-states" option and clean - s3_paginator = self._s3_client.get_paginator("list_objects_v2").paginate( - Bucket=self._authenticator.bucket, - Prefix=self._authenticator.prefix, - ) - - self._bucket_files = set() - for page in s3_paginator: - if "Contents" in page: - for key in page["Contents"]: - # just append the last part of the prefix to the list - self._bucket_files.add(key["Key"].split("/")[-2]) try: self._handlers = S3Handler(self._authenticator) self.plan_storage = True @@ -149,12 +78,6 @@ def remotes(self) -> list: """return a list of the remote bucket keys""" return list(self._bucket_files) - def _check_table_exists(self, name: str) -> bool: - """check if a supplied dynamodb table exists""" - if name in self._ddb_client.list_tables()["TableNames"]: - return True - return False - def _clean_bucket_state(self, definition=None): """ clean_state validates all of the terraform states are empty, @@ -210,6 +133,33 @@ def _clean_locking_state(self, deployment, definition=None): fg="yellow", ) + def _ensure_locking_table(self) -> None: + """ + _ensure_locking_table checks for the existence of the locking table, and + creates it if it doesn't exist + """ + # get dynamodb client from backend session + locking_table_name = f"terraform-{self._deployment}" + + # Check locking table for aws backend + click.secho( + f"Checking backend locking table: {locking_table_name}", fg="yellow" + ) + + if self._check_table_exists(locking_table_name): + click.secho("DynamoDB lock table found, continuing.", fg="yellow") + else: + click.secho( + "DynamoDB lock table not found, creating, please wait...", fg="yellow" + ) + self._create_table(locking_table_name) + + def _check_table_exists(self, name: str) -> bool: + """check if a supplied dynamodb table exists""" + if name in self._ddb_client.list_tables()["TableNames"]: + return True + return False + def _create_table( self, name: str, read_capacity: int = 1, write_capacity: int = 1 ) -> None: @@ -233,6 +183,119 @@ def _create_table( TableName=name, WaiterConfig={"Delay": 10, "MaxAttempts": 30} ) + def _ensure_backend_bucket(self) -> None: + """ + _ensure_backend_bucket checks for the existence of the backend bucket, and + creates it if it doesn't exist, along with setting the appropriate bucket + permissions + """ + bucket_present = self._check_bucket_exists(self._authenticator.bucket) + + if bucket_present: + click.secho( + f"Backend bucket: {self._authenticator.bucket} found", fg="yellow" + ) + return + + if not self._authenticator.create_backend_bucket and not bucket_present: + raise BackendError( + "Backend bucket not found and --no-create-backend-bucket specified." + ) + + self._create_bucket(self._authenticator.bucket) + self._create_bucket_versioning(self._authenticator.bucket) + self._create_bucket_public_access_block(self._authenticator.bucket) + + def _create_bucket(self, name: str) -> None: + """ + _create_bucket creates a new s3 bucket + """ + try: + click.secho(f"Creating backend bucket: {name}", fg="yellow") + self._s3_client.create_bucket( + Bucket=name, + CreateBucketConfiguration={ + "LocationConstraint": self._authenticator.backend_region + }, + ACL="private", + ) + except botocore.exceptions.ClientError as err: + err_str = str(err) + if "InvalidLocationConstraint" in err_str: + click.secho( + "InvalidLocationConstraint raised when trying to create a bucket. " + "Verify the AWS backend region passed to the worker matches the " + "backend AWS region in the profile.", + fg="red", + ) + raise SystemExit(1) + elif "BucketAlreadyExists" in err_str: + # Ignore when testing + if "PYTEST_CURRENT_TEST" not in os.environ: + click.secho( + f"Bucket {name} already exists, this is not expected since a moment ago it did not", + fg="red", + ) + raise SystemExit(1) + elif "BucketAlreadyOwnedByYou" not in err_str: + raise err + + def _create_bucket_versioning(self, name: str) -> None: + """ + _create_bucket_versioning enables versioning on the bucket + """ + click.secho(f"Enabling versioning on bucket: {name}", fg="yellow") + self._s3_client.put_bucket_versioning( + Bucket=name, VersioningConfiguration={"Status": "Enabled"} + ) + + def _create_bucket_public_access_block(self, name: str) -> None: + """ + _create_bucket_public_access_block blocks public access to the bucket + """ + click.secho(f"Blocking public access to bucket: {name}", fg="yellow") + self._s3_client.put_public_access_block( + Bucket=name, + PublicAccessBlockConfiguration={ + "BlockPublicAcls": True, + "IgnorePublicAcls": True, + "BlockPublicPolicy": True, + "RestrictPublicBuckets": True, + }, + ) + + def _check_bucket_exists(self, name: str) -> bool: + """ + check if a supplied bucket exists + """ + try: + self._s3_client.head_bucket(Bucket=name) + return True + except botocore.exceptions.ClientError as err: + err_str = str(err) + if "Not Found" in err_str: + return False + raise err + + def _get_bucket_files(self) -> set: + """ + _get_bucket_files returns a set of the keys in the bucket + """ + bucket_files = set() + s3_paginator = self._s3_client.get_paginator("list_objects_v2").paginate( + Bucket=self._authenticator.bucket, + Prefix=self._authenticator.prefix, + ) + + for page in s3_paginator: + if "Contents" in page: + for key in page["Contents"]: + # just append the last part of the prefix to the list, as they + # are relative to the base path, and deployment name + bucket_files.add(key["Key"].split("/")[-2]) + + return bucket_files + def _delete_with_versions(self, key): """ _delete_with_versions should handle object deletions, and all references / versions of the object diff --git a/tfworker/commands/base.py b/tfworker/commands/base.py index 1bd024c..8f72f0c 100644 --- a/tfworker/commands/base.py +++ b/tfworker/commands/base.py @@ -23,7 +23,7 @@ from tfworker.handlers.exceptions import HandlerError, UnknownHandler from tfworker.plugins import PluginsCollection from tfworker.providers import ProvidersCollection -from tfworker.util.system import pipe_exec, which, get_version +from tfworker.util.system import get_version, pipe_exec, which class MissingDependencyException(Exception): @@ -115,6 +115,7 @@ def __init__(self, rootc, deployment="undefined", limit=tuple(), **kwargs): ) except BackendError as e: click.secho(e, fg="red") + click.secho(e.help, fg="red") raise SystemExit(1) # if backend_plans is requested, check if backend supports it diff --git a/tfworker/util/system.py b/tfworker/util/system.py index 8e9ca9a..bc21766 100644 --- a/tfworker/util/system.py +++ b/tfworker/util/system.py @@ -149,6 +149,7 @@ def is_exe(fpath): return exe_file return None + def get_version() -> str: """ Get the version of the current package