diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py index 5c6b8e1..5e0f9ae 100644 --- a/softpack_core/schemas/environment.py +++ b/softpack_core/schemas/environment.py @@ -292,15 +292,13 @@ def iter(cls) -> Iterable["Environment"]: ) for env in environment_objects: + env.avg_wait_secs = avg_wait_secs status = status_map.get(str(Path(env.path, env.name))) if not status: - if env.state == State.queued: - env.state = State.failed continue env.requested = status.requested env.build_start = status.build_start env.build_done = status.build_done - env.avg_wait_secs = avg_wait_secs return environment_objects @@ -360,7 +358,29 @@ def create(cls, env: EnvironmentInput) -> CreateResponse: # type: ignore version += 1 - # Send build request + response = cls.submit_env_to_builder(env) + if response is not None: + return response + + return CreateEnvironmentSuccess( + message="Successfully scheduled environment creation" + ) + + @classmethod + def submit_env_to_builder( + cls, env: EnvironmentInput + ) -> Union[None, BuilderError, InvalidInputError]: + """Submit an environment to the builder.""" + try: + m = re.fullmatch(r"^(.*)-(\d+)$", env.name) + if not m: + raise Exception + versionless_name, version = m.groups() + except Exception: + return InvalidInputError( + message=f"could not parse version from name: {env.name!r}" + ) + try: host = app.settings.builder.host port = app.settings.builder.port @@ -383,15 +403,11 @@ def create(cls, env: EnvironmentInput) -> CreateResponse: # type: ignore ) r.raise_for_status() except Exception as e: - cls.delete(env.name, env.path) return BuilderError( message="Connection to builder failed: " + "".join(format_exception_only(type(e), e)) ) - - return CreateEnvironmentSuccess( - message="Successfully scheduled environment creation" - ) + return None @classmethod def create_new_env( diff --git a/softpack_core/service.py b/softpack_core/service.py index 6ea3829..e9f11ab 100644 --- a/softpack_core/service.py +++ b/softpack_core/service.py @@ -10,14 +10,16 @@ import typer import uvicorn -from fastapi import APIRouter, Request, UploadFile +from fastapi import APIRouter, Request, Response, UploadFile from typer import Typer from typing_extensions import Annotated +from softpack_core.artifacts import State from softpack_core.schemas.environment import ( CreateEnvironmentSuccess, Environment, EnvironmentInput, + PackageInput, WriteArtifactSuccess, ) @@ -92,3 +94,35 @@ async def upload_artifacts( # type: ignore[no-untyped-def] raise Exception(resp) return resp + + @staticmethod + @router.post("/resend-pending-builds") + async def resend_pending_builds( # type: ignore[no-untyped-def] + response: Response, + ): + """Resubmit any pending builds to the builder.""" + successes = 0 + failures = 0 + for env in Environment.iter(): + if env.state != State.queued: + continue + result = Environment.submit_env_to_builder( + EnvironmentInput( + name=env.name, + path=env.path, + description=env.description, + packages=[PackageInput(**vars(p)) for p in env.packages], + ) + ) + if result is None: + successes += 1 + else: + failures += 1 + + if failures == 0: + message = "Successfully triggered resends" + else: + response.status_code = 500 + message = "Failed to trigger all resends" + + return {"message": message, "successes": successes, "failures": failures} diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py index fcd0721..478fd9c 100644 --- a/tests/integration/test_environment.py +++ b/tests/integration/test_environment.py @@ -14,7 +14,7 @@ import yaml from fastapi import UploadFile -from softpack_core.artifacts import Artifacts, app +from softpack_core.artifacts import Artifacts from softpack_core.schemas.environment import ( BuilderError, CreateEnvironmentSuccess, @@ -29,7 +29,7 @@ UpdateEnvironmentSuccess, WriteArtifactSuccess, ) -from tests.integration.utils import file_in_remote +from tests.integration.utils import builder_called_correctly, file_in_remote pytestmark = pytest.mark.repo @@ -142,7 +142,7 @@ def test_create_path_invalid_disallowed(httpx_post, testable_env_input, path): assert isinstance(result, InvalidInputError) -def test_create_cleans_up_after_builder_failure( +def test_create_does_not_clean_up_after_builder_failure( httpx_post, testable_env_input ): httpx_post.side_effect = Exception('could not contact builder') @@ -156,33 +156,8 @@ def test_create_cleans_up_after_builder_failure( ) builtPath = dir / Environment.artifacts.built_by_softpack_file ymlPath = dir / Environment.artifacts.environments_file - assert not file_in_remote(builtPath) - assert not file_in_remote(ymlPath) - - -def builder_called_correctly( - post_mock, testable_env_input: EnvironmentInput -) -> None: - # TODO: don't mock this; actually have a real builder service to test with? - host = app.settings.builder.host - port = app.settings.builder.port - post_mock.assert_called_with( - f"http://{host}:{port}/environments/build", - json={ - "name": f"{testable_env_input.path}/{testable_env_input.name}", - "version": "1", - "model": { - "description": testable_env_input.description, - "packages": [ - { - "name": pkg.name, - "version": pkg.version, - } - for pkg in testable_env_input.packages - ], - }, - }, - ) + assert file_in_remote(builtPath) + assert file_in_remote(ymlPath) def test_delete(httpx_post, testable_env_input) -> None: @@ -307,8 +282,8 @@ def test_iter_no_statuses(testable_env_input, mocker): assert envs[0].build_start is None assert envs[0].build_done is None assert envs[0].avg_wait_secs is None - assert envs[0].state == State.failed - assert envs[1].state == State.failed + assert envs[0].state == State.queued + assert envs[1].state == State.queued @pytest.mark.asyncio diff --git a/tests/integration/test_resend_builds.py b/tests/integration/test_resend_builds.py new file mode 100644 index 0000000..bfe76c5 --- /dev/null +++ b/tests/integration/test_resend_builds.py @@ -0,0 +1,59 @@ +"""Copyright (c) 2024 Genome Research Ltd. + +This source code is licensed under the MIT license found in the +LICENSE file in the root directory of this source tree. +""" + + +import pytest +from fastapi.testclient import TestClient + +from softpack_core.app import app +from softpack_core.schemas.environment import ( + CreateEnvironmentSuccess, + Environment, + EnvironmentInput, +) +from softpack_core.service import ServiceAPI +from tests.integration.utils import builder_called_correctly + +pytestmark = pytest.mark.repo + + +def test_resend_pending_builds( + httpx_post, testable_env_input: EnvironmentInput +): + Environment.delete("test_environment", "users/test_user") + Environment.delete("test_environment", "groups/test_group") + ServiceAPI.register() + client = TestClient(app.router) + + orig_name = testable_env_input.name + testable_env_input.name += "-1" + r = Environment.create_new_env( + testable_env_input, Environment.artifacts.built_by_softpack_file + ) + assert isinstance(r, CreateEnvironmentSuccess) + testable_env_input.name = orig_name + + httpx_post.assert_not_called() + + resp = client.post( + url="/resend-pending-builds", + ) + assert resp.status_code == 200 + assert resp.json().get("message") == "Successfully triggered resends" + assert resp.json().get("successes") == 1 + assert resp.json().get("failures") == 0 + + httpx_post.assert_called_once() + builder_called_correctly(httpx_post, testable_env_input) + + httpx_post.side_effect = Exception('could not contact builder') + resp = client.post( + url="/resend-pending-builds", + ) + assert resp.status_code == 500 + assert resp.json().get("message") == "Failed to trigger all resends" + assert resp.json().get("successes") == 0 + assert resp.json().get("failures") == 1 diff --git a/tests/integration/test_spack.py b/tests/integration/test_spack.py index 3baf616..299a4a6 100644 --- a/tests/integration/test_spack.py +++ b/tests/integration/test_spack.py @@ -45,7 +45,7 @@ def test_spack_packages(): else: assert len(packages) > len(pkgs) - spack = Spack(custom_repo = app.settings.spack.repo) + spack = Spack(custom_repo=app.settings.spack.repo) spack.packages() diff --git a/tests/integration/utils.py b/tests/integration/utils.py index dd82250..31b2e16 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -12,6 +12,7 @@ import pytest from softpack_core.artifacts import Artifacts, app +from softpack_core.schemas.environment import EnvironmentInput artifacts_dict = dict[ str, @@ -199,3 +200,28 @@ def file_in_repo( current = current[part] return current + + +def builder_called_correctly( + post_mock, testable_env_input: EnvironmentInput +) -> None: + # TODO: don't mock this; actually have a real builder service to test with? + host = app.settings.builder.host + port = app.settings.builder.port + post_mock.assert_called_with( + f"http://{host}:{port}/environments/build", + json={ + "name": f"{testable_env_input.path}/{testable_env_input.name}", + "version": "1", + "model": { + "description": testable_env_input.description, + "packages": [ + { + "name": pkg.name, + "version": pkg.version, + } + for pkg in testable_env_input.packages + ], + }, + }, + )