From ea9186fae2c4fdaa23463ffe4f6fda6dc81747d6 Mon Sep 17 00:00:00 2001 From: ash Date: Mon, 29 Jan 2024 10:32:29 +0000 Subject: [PATCH] Self-service building (backend pieces) (#39) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add query to get a user's groups so that the frontend can display a list of folders they can put a new environment in * Document LDAP config * Properly format exceptions from backend Converting an exception to a string just extracts the message – you don't get to see what type the exception was, which is critical. * Install watchfiles in dev This is required for the server to efficiently restart on file changes. * Return useful error when builder connection fails * Make builder URL configurable * Send package versions to builder * Make it possible to export the GraphQL schema with `strawberry export-schema softpack_core.graphql:GraphQL.schema` * Eagerly create softpack.yml when build queued * Upgrade jinja2: Closes #34 * Remove requirements.txt This isn't used for anything?? --- README.md | 16 ++++- poetry.lock | 97 +++++++++++++++++++++++++-- pyproject.toml | 1 + requirements.txt | 25 ------- softpack_core/config/conf/config.yml | 6 +- softpack_core/config/models.py | 7 ++ softpack_core/config/settings.py | 2 + softpack_core/graphql.py | 6 +- softpack_core/schemas/environment.py | 71 +++++++++++++++----- softpack_core/schemas/groups.py | 42 ++++++++++++ tests/integration/test_environment.py | 47 ++++++++++--- tests/integration/test_groups.py | 35 ++++++++++ tests/integration/utils.py | 16 +++-- 13 files changed, 309 insertions(+), 62 deletions(-) delete mode 100644 requirements.txt create mode 100644 softpack_core/schemas/groups.py create mode 100644 tests/integration/test_groups.py diff --git a/README.md b/README.md index 5de99fc..e124f07 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,21 @@ $ source spack/share/spack/setup-env.sh ``` To start the service, you will also need to configure a git repository to store -artifacts. +artifacts, and configure details of an LDAP server to query group information: + +```yaml +artifacts: + repo: # see "integration tests" below + +ldap: + server: "ldaps://ldap.example" + base: "ou=group,dc=example,dc=ac,dc=uk" + filter: "memberuid={user}" + group: + attr: "cn" + pattern: ".*" + +``` ### Stable release diff --git a/poetry.lock b/poetry.lock index 3a2eacc..9f7d198 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "aiosqlite" @@ -1272,13 +1272,13 @@ trio = ["async_generator", "trio"] [[package]] name = "jinja2" -version = "3.1.2" +version = "3.1.3" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" files = [ - {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, - {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, ] [package.dependencies] @@ -3746,6 +3746,93 @@ files = [ [package.extras] watchmedo = ["PyYAML (>=3.10)"] +[[package]] +name = "watchfiles" +version = "0.21.0" +description = "Simple, modern and high performance file watching and code reload in python." +optional = false +python-versions = ">=3.8" +files = [ + {file = "watchfiles-0.21.0-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:27b4035013f1ea49c6c0b42d983133b136637a527e48c132d368eb19bf1ac6aa"}, + {file = "watchfiles-0.21.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c81818595eff6e92535ff32825f31c116f867f64ff8cdf6562cd1d6b2e1e8f3e"}, + {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6c107ea3cf2bd07199d66f156e3ea756d1b84dfd43b542b2d870b77868c98c03"}, + {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d9ac347653ebd95839a7c607608703b20bc07e577e870d824fa4801bc1cb124"}, + {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5eb86c6acb498208e7663ca22dbe68ca2cf42ab5bf1c776670a50919a56e64ab"}, + {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f564bf68404144ea6b87a78a3f910cc8de216c6b12a4cf0b27718bf4ec38d303"}, + {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d0f32ebfaa9c6011f8454994f86108c2eb9c79b8b7de00b36d558cadcedaa3d"}, + {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6d45d9b699ecbac6c7bd8e0a2609767491540403610962968d258fd6405c17c"}, + {file = "watchfiles-0.21.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:aff06b2cac3ef4616e26ba17a9c250c1fe9dd8a5d907d0193f84c499b1b6e6a9"}, + {file = "watchfiles-0.21.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d9792dff410f266051025ecfaa927078b94cc7478954b06796a9756ccc7e14a9"}, + {file = "watchfiles-0.21.0-cp310-none-win32.whl", hash = "sha256:214cee7f9e09150d4fb42e24919a1e74d8c9b8a9306ed1474ecaddcd5479c293"}, + {file = "watchfiles-0.21.0-cp310-none-win_amd64.whl", hash = "sha256:1ad7247d79f9f55bb25ab1778fd47f32d70cf36053941f07de0b7c4e96b5d235"}, + {file = "watchfiles-0.21.0-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:668c265d90de8ae914f860d3eeb164534ba2e836811f91fecc7050416ee70aa7"}, + {file = "watchfiles-0.21.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3a23092a992e61c3a6a70f350a56db7197242f3490da9c87b500f389b2d01eef"}, + {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e7941bbcfdded9c26b0bf720cb7e6fd803d95a55d2c14b4bd1f6a2772230c586"}, + {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:11cd0c3100e2233e9c53106265da31d574355c288e15259c0d40a4405cbae317"}, + {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d78f30cbe8b2ce770160d3c08cff01b2ae9306fe66ce899b73f0409dc1846c1b"}, + {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6674b00b9756b0af620aa2a3346b01f8e2a3dc729d25617e1b89cf6af4a54eb1"}, + {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fd7ac678b92b29ba630d8c842d8ad6c555abda1b9ef044d6cc092dacbfc9719d"}, + {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c873345680c1b87f1e09e0eaf8cf6c891b9851d8b4d3645e7efe2ec20a20cc7"}, + {file = "watchfiles-0.21.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:49f56e6ecc2503e7dbe233fa328b2be1a7797d31548e7a193237dcdf1ad0eee0"}, + {file = "watchfiles-0.21.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:02d91cbac553a3ad141db016e3350b03184deaafeba09b9d6439826ee594b365"}, + {file = "watchfiles-0.21.0-cp311-none-win32.whl", hash = "sha256:ebe684d7d26239e23d102a2bad2a358dedf18e462e8808778703427d1f584400"}, + {file = "watchfiles-0.21.0-cp311-none-win_amd64.whl", hash = "sha256:4566006aa44cb0d21b8ab53baf4b9c667a0ed23efe4aaad8c227bfba0bf15cbe"}, + {file = "watchfiles-0.21.0-cp311-none-win_arm64.whl", hash = "sha256:c550a56bf209a3d987d5a975cdf2063b3389a5d16caf29db4bdddeae49f22078"}, + {file = "watchfiles-0.21.0-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:51ddac60b96a42c15d24fbdc7a4bfcd02b5a29c047b7f8bf63d3f6f5a860949a"}, + {file = "watchfiles-0.21.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:511f0b034120cd1989932bf1e9081aa9fb00f1f949fbd2d9cab6264916ae89b1"}, + {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:cfb92d49dbb95ec7a07511bc9efb0faff8fe24ef3805662b8d6808ba8409a71a"}, + {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f92944efc564867bbf841c823c8b71bb0be75e06b8ce45c084b46411475a915"}, + {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:642d66b75eda909fd1112d35c53816d59789a4b38c141a96d62f50a3ef9b3360"}, + {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d23bcd6c8eaa6324fe109d8cac01b41fe9a54b8c498af9ce464c1aeeb99903d6"}, + {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18d5b4da8cf3e41895b34e8c37d13c9ed294954907929aacd95153508d5d89d7"}, + {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1b8d1eae0f65441963d805f766c7e9cd092f91e0c600c820c764a4ff71a0764c"}, + {file = "watchfiles-0.21.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1fd9a5205139f3c6bb60d11f6072e0552f0a20b712c85f43d42342d162be1235"}, + {file = "watchfiles-0.21.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a1e3014a625bcf107fbf38eece0e47fa0190e52e45dc6eee5a8265ddc6dc5ea7"}, + {file = "watchfiles-0.21.0-cp312-none-win32.whl", hash = "sha256:9d09869f2c5a6f2d9df50ce3064b3391d3ecb6dced708ad64467b9e4f2c9bef3"}, + {file = "watchfiles-0.21.0-cp312-none-win_amd64.whl", hash = "sha256:18722b50783b5e30a18a8a5db3006bab146d2b705c92eb9a94f78c72beb94094"}, + {file = "watchfiles-0.21.0-cp312-none-win_arm64.whl", hash = "sha256:a3b9bec9579a15fb3ca2d9878deae789df72f2b0fdaf90ad49ee389cad5edab6"}, + {file = "watchfiles-0.21.0-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:4ea10a29aa5de67de02256a28d1bf53d21322295cb00bd2d57fcd19b850ebd99"}, + {file = "watchfiles-0.21.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:40bca549fdc929b470dd1dbfcb47b3295cb46a6d2c90e50588b0a1b3bd98f429"}, + {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9b37a7ba223b2f26122c148bb8d09a9ff312afca998c48c725ff5a0a632145f7"}, + {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec8c8900dc5c83650a63dd48c4d1d245343f904c4b64b48798c67a3767d7e165"}, + {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8ad3fe0a3567c2f0f629d800409cd528cb6251da12e81a1f765e5c5345fd0137"}, + {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d353c4cfda586db2a176ce42c88f2fc31ec25e50212650c89fdd0f560ee507b"}, + {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:83a696da8922314ff2aec02987eefb03784f473281d740bf9170181829133765"}, + {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a03651352fc20975ee2a707cd2d74a386cd303cc688f407296064ad1e6d1562"}, + {file = "watchfiles-0.21.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:3ad692bc7792be8c32918c699638b660c0de078a6cbe464c46e1340dadb94c19"}, + {file = "watchfiles-0.21.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06247538e8253975bdb328e7683f8515ff5ff041f43be6c40bff62d989b7d0b0"}, + {file = "watchfiles-0.21.0-cp38-none-win32.whl", hash = "sha256:9a0aa47f94ea9a0b39dd30850b0adf2e1cd32a8b4f9c7aa443d852aacf9ca214"}, + {file = "watchfiles-0.21.0-cp38-none-win_amd64.whl", hash = "sha256:8d5f400326840934e3507701f9f7269247f7c026d1b6cfd49477d2be0933cfca"}, + {file = "watchfiles-0.21.0-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:7f762a1a85a12cc3484f77eee7be87b10f8c50b0b787bb02f4e357403cad0c0e"}, + {file = "watchfiles-0.21.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6e9be3ef84e2bb9710f3f777accce25556f4a71e15d2b73223788d528fcc2052"}, + {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4c48a10d17571d1275701e14a601e36959ffada3add8cdbc9e5061a6e3579a5d"}, + {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c889025f59884423428c261f212e04d438de865beda0b1e1babab85ef4c0f01"}, + {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:66fac0c238ab9a2e72d026b5fb91cb902c146202bbd29a9a1a44e8db7b710b6f"}, + {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b4a21f71885aa2744719459951819e7bf5a906a6448a6b2bbce8e9cc9f2c8128"}, + {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1c9198c989f47898b2c22201756f73249de3748e0fc9de44adaf54a8b259cc0c"}, + {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8f57c4461cd24fda22493109c45b3980863c58a25b8bec885ca8bea6b8d4b28"}, + {file = "watchfiles-0.21.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:853853cbf7bf9408b404754b92512ebe3e3a83587503d766d23e6bf83d092ee6"}, + {file = "watchfiles-0.21.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d5b1dc0e708fad9f92c296ab2f948af403bf201db8fb2eb4c8179db143732e49"}, + {file = "watchfiles-0.21.0-cp39-none-win32.whl", hash = "sha256:59137c0c6826bd56c710d1d2bda81553b5e6b7c84d5a676747d80caf0409ad94"}, + {file = "watchfiles-0.21.0-cp39-none-win_amd64.whl", hash = "sha256:6cb8fdc044909e2078c248986f2fc76f911f72b51ea4a4fbbf472e01d14faa58"}, + {file = "watchfiles-0.21.0-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:ab03a90b305d2588e8352168e8c5a1520b721d2d367f31e9332c4235b30b8994"}, + {file = "watchfiles-0.21.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:927c589500f9f41e370b0125c12ac9e7d3a2fd166b89e9ee2828b3dda20bfe6f"}, + {file = "watchfiles-0.21.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bd467213195e76f838caf2c28cd65e58302d0254e636e7c0fca81efa4a2e62c"}, + {file = "watchfiles-0.21.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02b73130687bc3f6bb79d8a170959042eb56eb3a42df3671c79b428cd73f17cc"}, + {file = "watchfiles-0.21.0-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:08dca260e85ffae975448e344834d765983237ad6dc308231aa16e7933db763e"}, + {file = "watchfiles-0.21.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:3ccceb50c611c433145502735e0370877cced72a6c70fd2410238bcbc7fe51d8"}, + {file = "watchfiles-0.21.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57d430f5fb63fea141ab71ca9c064e80de3a20b427ca2febcbfcef70ff0ce895"}, + {file = "watchfiles-0.21.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0dd5fad9b9c0dd89904bbdea978ce89a2b692a7ee8a0ce19b940e538c88a809c"}, + {file = "watchfiles-0.21.0-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:be6dd5d52b73018b21adc1c5d28ac0c68184a64769052dfeb0c5d9998e7f56a2"}, + {file = "watchfiles-0.21.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:b3cab0e06143768499384a8a5efb9c4dc53e19382952859e4802f294214f36ec"}, + {file = "watchfiles-0.21.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c6ed10c2497e5fedadf61e465b3ca12a19f96004c15dcffe4bd442ebadc2d85"}, + {file = "watchfiles-0.21.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43babacef21c519bc6631c5fce2a61eccdfc011b4bcb9047255e9620732c8097"}, + {file = "watchfiles-0.21.0.tar.gz", hash = "sha256:c76c635fabf542bb78524905718c39f736a98e5ab25b23ec6d4abede1a85a6a3"}, +] + +[package.dependencies] +anyio = ">=3.0.0" + [[package]] name = "webencodings" version = "0.5.1" @@ -3881,4 +3968,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0" -content-hash = "275c6e07b55140f0db60185c15cd07c6cc3e6210bb46a879d04aebb528628743" +content-hash = "5274440f6530849e74f8f1c3566649bbe73fd1add4181915662392852447d73d" diff --git a/pyproject.toml b/pyproject.toml index a3cb5d3..fea300c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,7 @@ types-setuptools = "^67.7.0.1" types-PyYAML = "^6.0.12.9" types-requests = "^2.30.0.0" virtualenv = "^20.20.0" +watchfiles = "^0.21.0" [tool.poetry.group.doc] optional = true diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index b93801a..0000000 --- a/requirements.txt +++ /dev/null @@ -1,25 +0,0 @@ -certifi==2023.5.7 -charset-normalizer==3.1.0 -click==8.1.3 -colorama==0.4.6 -ghp-import==2.1.0 -idna==3.4 -Jinja2==3.1.2 -Markdown==3.3.7 -MarkupSafe==2.1.3 -mergedeep==1.3.4 -mkdocs==1.4.3 -mkdocs-include-markdown-plugin==4.0.4 -mkdocs-material==9.1.16 -mkdocs-material-extensions==1.1.1 -packaging==23.1 -Pygments==2.15.1 -pymdown-extensions==10.0.1 -python-dateutil==2.8.2 -PyYAML==6.0 -pyyaml_env_tag==0.1 -regex==2023.6.3 -requests==2.31.0 -six==1.16.0 -urllib3==2.0.7 -watchdog==3.0.0 diff --git a/softpack_core/config/conf/config.yml b/softpack_core/config/conf/config.yml index c593ec6..2a9f50e 100644 --- a/softpack_core/config/conf/config.yml +++ b/softpack_core/config/conf/config.yml @@ -9,6 +9,10 @@ server: - http://localhost:3000 - http://localhost:5173 +builder: + host: 0.0.0.0 + port: 8001 + artifacts: repo: url: https://github.com/mjkw31/softpack-artifacts @@ -35,4 +39,4 @@ spack: # filter: # group: # attr: -# pattern: \ No newline at end of file +# pattern: diff --git a/softpack_core/config/models.py b/softpack_core/config/models.py index 7a67193..fec3c8c 100644 --- a/softpack_core/config/models.py +++ b/softpack_core/config/models.py @@ -23,6 +23,13 @@ class HeaderConfig(BaseModel): port: int +class BuilderConfig(BaseModel): + """Builder config model.""" + + host: str + port: int + + class VaultConfig(BaseModel): """HashiCorp vault config.""" diff --git a/softpack_core/config/settings.py b/softpack_core/config/settings.py index 5421360..4896ee1 100644 --- a/softpack_core/config/settings.py +++ b/softpack_core/config/settings.py @@ -16,6 +16,7 @@ from .models import ( ArtifactsConfig, + BuilderConfig, LDAPConfig, ServerConfig, SpackConfig, @@ -32,6 +33,7 @@ class Settings(BaseSettings): ldap: Optional[LDAPConfig] artifacts: ArtifactsConfig spack: SpackConfig + builder: BuilderConfig class Config: """Configuration loader.""" diff --git a/softpack_core/graphql.py b/softpack_core/graphql.py index cf3bffd..ce9cf9f 100644 --- a/softpack_core/graphql.py +++ b/softpack_core/graphql.py @@ -17,6 +17,7 @@ from .app import app from .schemas.base import BaseSchema from .schemas.environment import EnvironmentSchema +from .schemas.groups import GroupsSchema from .schemas.package_collection import PackageCollectionSchema @@ -24,7 +25,7 @@ class GraphQL(API): """GraphQL API.""" prefix = "/graphql" - schemas = [EnvironmentSchema, PackageCollectionSchema] + schemas = [EnvironmentSchema, PackageCollectionSchema, GroupsSchema] commands = Typer(help="GraphQL commands.") @staticmethod @@ -126,4 +127,5 @@ def __init__(self, schema: strawberry.Schema, prefix: str) -> None: """ super().__init__(schema=schema, path=prefix) - router = Router(schema=Schema(schemas), prefix=prefix) + schema = Schema(schemas) + router = Router(schema=schema, prefix=prefix) diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py index 3498010..d7f76b7 100644 --- a/softpack_core/schemas/environment.py +++ b/softpack_core/schemas/environment.py @@ -7,14 +7,17 @@ import io from dataclasses import dataclass from pathlib import Path +from traceback import format_exception_only from typing import Iterable, List, Optional, Tuple, Union, cast import httpx import starlette.datastructures import strawberry +import yaml from fastapi import UploadFile from strawberry.file_uploads import Upload +from softpack_core.app import app from softpack_core.artifacts import Artifacts, Package, State from softpack_core.module import GenerateEnvReadme, ToSoftpackYML from softpack_core.schemas.base import BaseSchema @@ -80,6 +83,11 @@ class EnvironmentAlreadyExistsError(Error): name: str +@strawberry.type +class BuilderError(Error): + """Unable to connect to builder.""" + + # Unions CreateResponse = strawberry.union( "CreateResponse", @@ -87,6 +95,7 @@ class EnvironmentAlreadyExistsError(Error): CreateEnvironmentSuccess, InvalidInputError, EnvironmentAlreadyExistsError, + BuilderError, ], ) @@ -252,19 +261,33 @@ def create(cls, env: EnvironmentInput) -> CreateResponse: # type: ignore env.name = name - # TODO: remove hard-coding of URL. # Send build request - httpx.post( - "http://0.0.0.0:7080/environments/build", - json={ - "name": f"{env.path}/{env.name}", - "version": str(version), - "model": { - "description": env.description, - "packages": [f"{pkg.name}" for pkg in env.packages], + try: + host = app.settings.builder.host + port = app.settings.builder.port + r = httpx.post( + f"http://{host}:{port}/environments/build", + json={ + "name": f"{env.path}/{env.name}", + "version": str(version), + "model": { + "description": env.description, + "packages": [ + { + "name": pkg.name, + "version": pkg.version, + } + for pkg in env.packages + ], + }, }, - }, - ) + ) + r.raise_for_status() + except Exception as e: + return BuilderError( + message="Connection to builder failed: " + + "".join(format_exception_only(type(e), e)) + ) return CreateEnvironmentSuccess( message="Successfully scheduled environment creation" @@ -310,14 +333,30 @@ def create_new_env( # Create folder with place-holder file new_folder_path = Path(env.path, env.name) try: - tree_oid = cls.artifacts.create_file( - new_folder_path, env_type, "", True + softpack_definition = dict( + description=env.description, + packages=[ + pkg.name + ("@" + pkg.version if pkg.version else "") + for pkg in env.packages + ], + ) + ymlData = yaml.dump(softpack_definition) + + tree_oid = cls.artifacts.create_files( + new_folder_path, + [ + (env_type, ""), # e.g. .built_by_softpack + (cls.artifacts.environments_file, ymlData), # softpack.yml + ], + True, ) cls.artifacts.commit_and_push( tree_oid, "create environment folder" ) except RuntimeError as e: - return InvalidInputError(message=str(e)) + return InvalidInputError( + message="".join(format_exception_only(type(e), e)) + ) return CreateEnvironmentSuccess( message="Successfully created environment in artifacts repo" @@ -535,7 +574,9 @@ async def write_artifacts( ) except Exception as e: - return InvalidInputError(message=str(e)) + return InvalidInputError( + message="".join(format_exception_only(type(e), e)) + ) @classmethod async def update_from_module( diff --git a/softpack_core/schemas/groups.py b/softpack_core/schemas/groups.py new file mode 100644 index 0000000..4f053dd --- /dev/null +++ b/softpack_core/schemas/groups.py @@ -0,0 +1,42 @@ +"""Copyright (c) 2024 Genome Research Ltd. + +This source code is licensed under the MIT license found in the +LICENSE file in the root directory of this source tree. +""" + +from dataclasses import dataclass +from typing import Iterable + +import strawberry + +from ..ldapapi import LDAP + + +@strawberry.type +class Group: + """A Strawberry model representing a single unix group.""" + + name: str + + @classmethod + def from_username(cls, username: str) -> Iterable["Group"]: + """Get the groups the given user belongs to. + + Args: + username: Their usernamer. + + Returns: + Iterable: An iterator over unix group names. + """ + groups = LDAP().groups(username) + return (Group(name=group) for group in groups) + + +class GroupsSchema: + """Group schema.""" + + @dataclass + class Query: + """GraphQL query schema.""" + + groups: list[Group] = Group.from_username # type: ignore diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py index 314bfbc..15a0f23 100644 --- a/tests/integration/test_environment.py +++ b/tests/integration/test_environment.py @@ -10,9 +10,10 @@ import pygit2 import pytest +import yaml from fastapi import UploadFile -from softpack_core.artifacts import Artifacts +from softpack_core.artifacts import Artifacts, app from softpack_core.schemas.environment import ( CreateEnvironmentSuccess, DeleteEnvironmentSuccess, @@ -42,19 +43,44 @@ def test_create(httpx_post, testable_env_input: EnvironmentInput) -> None: name="test_env_create2", path="groups/not_already_in_repo", description="description2", - packages=[Package(name="pkg_test2")], + packages=[ + Package(name="pkg_test2"), + Package(name="pkg_test3", version="3.1"), + ], ) ) assert isinstance(result, CreateEnvironmentSuccess) httpx_post.assert_called() - path = Path( + dir = Path( Environment.artifacts.environments_root, testable_env_input.path, testable_env_input.name + "-1", - Environment.artifacts.built_by_softpack_file, ) - assert file_in_remote(path) + builtPath = dir / Environment.artifacts.built_by_softpack_file + ymlPath = dir / Environment.artifacts.environments_file + assert file_in_remote(builtPath) + ymlFile = file_in_remote(ymlPath) + expected_yaml = { + "description": "description", + "packages": ["pkg_test"], + } + assert yaml.safe_load(ymlFile.data.decode()) == expected_yaml + + dir = Path( + Environment.artifacts.environments_root, + "groups/not_already_in_repo", + "test_env_create2-1", + ) + builtPath = dir / Environment.artifacts.built_by_softpack_file + ymlPath = dir / Environment.artifacts.environments_file + assert file_in_remote(builtPath) + ymlFile = file_in_remote(ymlPath) + expected_yaml = { + "description": "description2", + "packages": ["pkg_test2", "pkg_test3@3.1"], + } + assert yaml.safe_load(ymlFile.data.decode()) == expected_yaml result = Environment.create(testable_env_input) assert isinstance(result, CreateEnvironmentSuccess) @@ -82,16 +108,21 @@ def builder_called_correctly( post_mock, testable_env_input: EnvironmentInput ) -> None: # TODO: don't mock this; actually have a real builder service to test with? - # Also need to not hard-code the url here. + host = app.settings.builder.host + port = app.settings.builder.port post_mock.assert_called_with( - "http://0.0.0.0:7080/environments/build", + f"http://{host}:{port}/environments/build", json={ "name": f"{testable_env_input.path}/{testable_env_input.name}", "version": "1", "model": { "description": testable_env_input.description, "packages": [ - f"{pkg.name}" for pkg in testable_env_input.packages + { + "name": pkg.name, + "version": pkg.version, + } + for pkg in testable_env_input.packages ], }, }, diff --git a/tests/integration/test_groups.py b/tests/integration/test_groups.py new file mode 100644 index 0000000..1827e19 --- /dev/null +++ b/tests/integration/test_groups.py @@ -0,0 +1,35 @@ +"""Copyright (c) 2024 Genome Research Ltd. + +This source code is licensed under the MIT license found in the +LICENSE file in the root directory of this source tree. +""" + +import getpass + +import ldap + +from softpack_core.schemas.groups import Group + + +def test_groups(mocker) -> None: + username = getpass.getuser() + groups = list(Group.from_username(username)) + assert len(groups) + assert groups[0].name + + groups = list(Group.from_username("foo")) + assert not len(groups) + + def search_s(*args): + return [ + ("cn=testteam,ou=group,dc=foo", {"cn": [b"testteam"]}), + ("cn=otherteam,ou=group,dc=foo", {"cn": [b"otherteam"]}), + ] + + mocker.patch.object( + ldap.ldapobject.SimpleLDAPObject, "search_s", new=search_s + ) + + groups = list(Group.from_username("foo")) + group_names = {group.name for group in groups} + assert group_names == {"testteam", "otherteam"} diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 6948588..dd82250 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -171,25 +171,31 @@ def get_user_path_without_environments( return Path(*(artifacts.user_folder(user).parts[1:])) -def file_in_remote(*paths_with_environment: Union[str, Path]) -> bool: +def file_in_remote( + *paths_with_environment: Union[str, Path] +) -> Union[pygit2.Tree, pygit2.Blob]: temp_dir = tempfile.TemporaryDirectory() app.settings.artifacts.path = Path(temp_dir.name) artifacts = Artifacts() + file = None for path_with_environment in paths_with_environment: path = Path(path_with_environment) - if not file_in_repo(artifacts, path): + file = file_in_repo(artifacts, path) + if not file: return False - return True + return file -def file_in_repo(artifacts: Artifacts, path: Path) -> bool: +def file_in_repo( + artifacts: Artifacts, path: Path +) -> Union[pygit2.Tree, pygit2.Blob]: current = artifacts.repo.head.peel(pygit2.Tree) for part in path.parts: if part not in current: return False current = current[part] - return True + return current