From 865a5c0ea310be4701f1d5a1d2203da16615279c Mon Sep 17 00:00:00 2001 From: Michael Woolnough Date: Mon, 11 Nov 2024 14:08:08 +0000 Subject: [PATCH] Remove GraphQL endpoints and replace with JSON endpoints. --- README.md | 6 - poetry.lock | 48 +-- pyproject.toml | 2 - schema.graphql | 142 -------- softpack_core/artifacts.py | 11 +- softpack_core/graphql.py | 124 ------- softpack_core/main.py | 2 - softpack_core/schemas/base.py | 15 - softpack_core/schemas/environment.py | 344 +++++++++----------- softpack_core/schemas/groups.py | 16 +- softpack_core/schemas/package_collection.py | 23 +- softpack_core/service.py | 91 +++++- tests/integration/test_environment.py | 16 +- tests/{unit => integration}/test_service.py | 145 +++++++++ tests/integration/test_spack.py | 2 +- 15 files changed, 412 insertions(+), 575 deletions(-) delete mode 100644 schema.graphql delete mode 100644 softpack_core/graphql.py delete mode 100644 softpack_core/schemas/base.py rename tests/{unit => integration}/test_service.py (55%) diff --git a/README.md b/README.md index cf58fde..7762514 100644 --- a/README.md +++ b/README.md @@ -171,12 +171,6 @@ Run [MkDocs] server to view documentation: poetry run mkdocs serve ``` -To generate a GraphQL schema file: - -``` -poetry run strawberry export-schema softpack_core.graphql:GraphQL.schema > schema.graphql -``` - [pip]: https://pip.pypa.io [Python installation guide]: http://docs.python-guide.org/en/latest/starting/installation/ diff --git a/poetry.lock b/poetry.lock index 9f7d198..cecb1ce 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. [[package]] name = "aiosqlite" @@ -955,17 +955,6 @@ pyopenssl = ["cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] reauth = ["pyu2f (>=0.1.5)"] requests = ["requests (>=2.20.0,<3.0.0dev)"] -[[package]] -name = "graphql-core" -version = "3.2.3" -description = "GraphQL implementation for Python, a port of GraphQL.js, the JavaScript reference implementation for GraphQL." -optional = false -python-versions = ">=3.6,<4" -files = [ - {file = "graphql-core-3.2.3.tar.gz", hash = "sha256:06d2aad0ac723e35b1cb47885d3e5c45e956a53bc1b209a9fc5369007fe46676"}, - {file = "graphql_core-3.2.3-py3-none-any.whl", hash = "sha256:5766780452bd5ec8ba133f8bf287dc92713e3868ddd83aee4faab9fc3e303dc3"}, -] - [[package]] name = "greenlet" version = "2.0.2" @@ -3389,39 +3378,6 @@ typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\"" [package.extras] full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart", "pyyaml"] -[[package]] -name = "strawberry-graphql" -version = "0.177.1" -description = "A library for creating GraphQL APIs" -optional = false -python-versions = ">=3.7,<4.0" -files = [ - {file = "strawberry_graphql-0.177.1-py3-none-any.whl", hash = "sha256:5f97a28603ca5b2b139cbe1d906f6b2fc6f677b74ec5509caee6de7e8c5760bb"}, - {file = "strawberry_graphql-0.177.1.tar.gz", hash = "sha256:87e8638cc3f87dfb59bd28a22ba0c2dbbf382c078fec7831cee47a3ce761586a"}, -] - -[package.dependencies] -graphql-core = ">=3.2.0,<3.3.0" -python-dateutil = ">=2.7.0,<3.0.0" -typing_extensions = ">=4.0.0,<5.0.0" - -[package.extras] -aiohttp = ["aiohttp (>=3.7.4.post0,<4.0.0)"] -asgi = ["python-multipart (>=0.0.5,<0.0.7)", "starlette (>=0.18.0)"] -chalice = ["chalice (>=1.22,<2.0)"] -channels = ["asgiref (>=3.2,<4.0)", "channels (>=3.0.5)"] -cli = ["click (>=7.0,<9.0)", "libcst (>=0.4.7)", "pygments (>=2.3,<3.0)", "rich (>=12.0.0)"] -debug = ["libcst (>=0.4.7)", "rich (>=12.0.0)"] -debug-server = ["click (>=7.0,<9.0)", "libcst (>=0.4.7)", "pygments (>=2.3,<3.0)", "python-multipart (>=0.0.5,<0.0.7)", "rich (>=12.0.0)", "starlette (>=0.18.0)", "uvicorn (>=0.11.6,<0.22.0)"] -django = ["Django (>=3.2)", "asgiref (>=3.2,<4.0)"] -fastapi = ["fastapi (>=0.65.2)", "python-multipart (>=0.0.5,<0.0.7)"] -flask = ["flask (>=1.1)"] -opentelemetry = ["opentelemetry-api (<2)", "opentelemetry-sdk (<2)"] -pydantic = ["pydantic (<2)"] -pyinstrument = ["pyinstrument (>=4.0.0)"] -sanic = ["sanic (>=20.12.2)"] -starlite = ["starlite (>=1.48.0)"] - [[package]] name = "tblib" version = "1.7.0" @@ -3968,4 +3924,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0" -content-hash = "5274440f6530849e74f8f1c3566649bbe73fd1add4181915662392852447d73d" +content-hash = "90b4d33a0009e969f8f1df940d3a09c8f919f0473367d31e861ebfc39e051f09" diff --git a/pyproject.toml b/pyproject.toml index fea300c..6033448 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,6 @@ ruamel-yaml = "^0.17.21" semver = "^3.0.0" singleton-decorator = "^1.0.0" sqlalchemy = "1.4.45" -strawberry-graphql = "^0.177.1" typer = "^0.9.0" pytest-mock = "^3.11.1" pytest-asyncio = "^0.21.1" @@ -134,7 +133,6 @@ skip_gitignore = true disallow_untyped_calls = true disallow_untyped_defs = true ignore_missing_imports = true -plugins = "strawberry.ext.mypy_plugin" [tool.pytest.ini_options] filterwarnings = [ diff --git a/schema.graphql b/schema.graphql deleted file mode 100644 index 2a79b5a..0000000 --- a/schema.graphql +++ /dev/null @@ -1,142 +0,0 @@ -schema { - query: SchemaQuery - mutation: SchemaMutation -} - -union AddTagResponse = AddTagSuccess | InvalidInputError | EnvironmentNotFoundError - -type AddTagSuccess implements Success { - message: String! -} - -type BuilderError implements Error { - message: String! -} - -type CreateEnvironmentSuccess implements Success { - message: String! -} - -union CreateResponse = CreateEnvironmentSuccess | InvalidInputError | EnvironmentAlreadyExistsError | BuilderError - -type DeleteEnvironmentSuccess implements Success { - message: String! -} - -union DeleteResponse = DeleteEnvironmentSuccess | EnvironmentNotFoundError - -type Environment { - id: String! - name: String! - path: String! - description: String! - readme: String! - type: Type! - packages: [Package!]! - state: State - tags: [String!]! - username: String - failureReason: String - hidden: Boolean! - created: Int! - interpreters: Interpreters! - environments: [Environment!]! -} - -type EnvironmentAlreadyExistsError implements Error { - message: String! - path: String! - name: String! -} - -input EnvironmentInput { - name: String! - path: String! - username: String = "" - description: String! - packages: [PackageInput!]! - tags: [String!] = null -} - -type EnvironmentNotFoundError implements Error { - message: String! - path: String! - name: String! -} - -interface Error { - message: String! -} - -type Group { - name: String! -} - -union HiddenResponse = HiddenSuccess | InvalidInputError | EnvironmentNotFoundError - -type HiddenSuccess implements Success { - message: String! -} - -type Interpreters { - r: String - python: String -} - -type InvalidInputError implements Error { - message: String! -} - -type Package { - name: String! - version: String -} - -input PackageInput { - name: String! - version: String = null -} - -type PackageMultiVersion { - name: String! - versions: [String!]! -} - -type SchemaMutation { - createEnvironment(env: EnvironmentInput!): CreateResponse! - deleteEnvironment(name: String!, path: String!): DeleteResponse! - addTag(name: String!, path: String!, tag: String!): AddTagResponse! - setHidden(name: String!, path: String!, hidden: Boolean!): HiddenResponse! - createFromModule(file: Upload!, modulePath: String!, environmentPath: String!): CreateResponse! - updateFromModule(file: Upload!, modulePath: String!, environmentPath: String!): UpdateResponse! -} - -type SchemaQuery { - environments: [Environment!]! - packageCollections: [PackageMultiVersion!]! - groups(username: String!): [Group!]! -} - -enum State { - ready - queued - failed - waiting -} - -interface Success { - message: String! -} - -enum Type { - softpack - module -} - -type UpdateEnvironmentSuccess implements Success { - message: String! -} - -union UpdateResponse = UpdateEnvironmentSuccess | InvalidInputError | EnvironmentNotFoundError - -scalar Upload diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py index 2ef170c..4717d2c 100644 --- a/softpack_core/artifacts.py +++ b/softpack_core/artifacts.py @@ -14,7 +14,6 @@ from typing import Iterable, Iterator, List, Optional, Tuple, Union, cast import pygit2 -import strawberry import yaml from box import Box from fastapi import UploadFile @@ -25,9 +24,9 @@ from .ldapapi import LDAP -@strawberry.type +@dataclass class Package(PackageBase): - """A Strawberry model representing a package.""" + """A data class representing a package.""" version: Optional[str] = None @@ -50,15 +49,14 @@ def from_name(cls, name: str) -> 'Package': return Package(name=name) -@strawberry.type +@dataclass class Interpreters: - """A Strawberry model representing the interpreters in an environment.""" + """A data class model representing the interpreters in an environment.""" r: Optional[str] = None python: Optional[str] = None -@strawberry.enum class State(Enum): """Environment states.""" @@ -68,7 +66,6 @@ class State(Enum): waiting = 'waiting' -@strawberry.enum class Type(Enum): """Environment types.""" diff --git a/softpack_core/graphql.py b/softpack_core/graphql.py deleted file mode 100644 index 3a846e0..0000000 --- a/softpack_core/graphql.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Copyright (c) 2023 Genome Research Ltd. - -This source code is licensed under the MIT license found in the -LICENSE file in the root directory of this source tree. -""" - -import dataclasses -import itertools -from typing import Any, Callable, Iterable, Tuple, Union, cast - -import strawberry -from strawberry.fastapi import GraphQLRouter -from typer import Typer -from typing_extensions import Type - -from .api import API -from .schemas.base import BaseSchema -from .schemas.environment import EnvironmentSchema -from .schemas.groups import GroupsSchema -from .schemas.package_collection import PackageCollectionSchema - - -class GraphQL(API): - """GraphQL API.""" - - prefix = "/graphql" - schemas = [ - EnvironmentSchema, - PackageCollectionSchema, - GroupsSchema, - ] - commands = Typer(help="GraphQL commands.") - - class Schema(strawberry.Schema): - """GraphQL Schema class.""" - - def __init__(self, schemas: list[type[object]]) -> None: - """Constructor. - - Args: - schemas: List of schema providers. - """ - self.schemas = schemas - super().__init__( - query=self.strawberry_class(BaseSchema.Query), - mutation=self.strawberry_class(BaseSchema.Mutation), - ) - - def strawberry_class(self, obj: Type[Any]) -> Type[Any]: - """Define a new dataclass wrapped in a strawberry.type. - - Args: - obj: The type used for defining a new strawberry class. - - Returns: - Type[Any]: A new dataclass type. - - """ - return strawberry.type( - dataclasses.make_dataclass( - "_".join([self.__class__.__name__, obj.__name__]), - map(self.strawberry_field, self.get_fields(obj)), - ) - ) - - def get_fields(self, obj: Type[Any]) -> Iterable: - """Get fields of a dataclass. - - Args: - obj: A dataclass object or instance. - - Returns: - Iterable: An iterable over a list of dataclass fields. - """ - - def fields(schema: Type[Any]) -> tuple[Any, ...]: - try: - return dataclasses.fields(getattr(schema, obj.__name__)) - except AttributeError: - return () - - return itertools.chain.from_iterable(map(fields, self.schemas)) - - def strawberry_field( - self, field: dataclasses.Field - ) -> Union[Tuple[str, type], Tuple[str, type, Any]]: - """Get strawberry field. - - Get strawberry field as a tuple of dataclass name, type and - dataclass.Field. - - Args: - field: A dataclass field. - - Returns: - tuple: dataclass name, type and dataclass.Field. - """ - spec = (field.name, field.type) - if field.default == dataclasses.MISSING: - return spec - - return ( - *spec, - dataclasses.field( - default=strawberry.field( - resolver=cast(Callable[..., Any], field.default) - ) - ), - ) - - class Router(GraphQLRouter): - """GraphQL router.""" - - def __init__(self, schema: strawberry.Schema, prefix: str) -> None: - """Constructor. - - Args: - schema: GraphQL schema - prefix: Path prefix for the GraphQL route. - """ - super().__init__(schema=schema, path=prefix) - - schema = Schema(schemas) - router = Router(schema=schema, prefix=prefix) diff --git a/softpack_core/main.py b/softpack_core/main.py index 55803c6..539e77e 100644 --- a/softpack_core/main.py +++ b/softpack_core/main.py @@ -7,10 +7,8 @@ from typing import Any from .app import app -from .graphql import GraphQL from .service import ServiceAPI -GraphQL.register() ServiceAPI.register() diff --git a/softpack_core/schemas/base.py b/softpack_core/schemas/base.py deleted file mode 100644 index e2883d7..0000000 --- a/softpack_core/schemas/base.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Copyright (c) 2023 Genome Research Ltd. - -This source code is licensed under the MIT license found in the -LICENSE file in the root directory of this source tree. -""" - - -class BaseSchema: - """A GraphQL base schema class.""" - - class Query: - """GraphQL query schema.""" - - class Mutation: - """GraphQL query mutation.""" diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py index 034a70b..c6974b4 100644 --- a/softpack_core/schemas/environment.py +++ b/softpack_core/schemas/environment.py @@ -15,12 +15,9 @@ from typing import List, Optional, Tuple, Union, cast import httpx -import starlette.datastructures -import strawberry import yaml from box import Box from fastapi import UploadFile -from strawberry.file_uploads import Upload from softpack_core.app import app from softpack_core.artifacts import ( @@ -32,141 +29,127 @@ artifacts, ) from softpack_core.module import GenerateEnvReadme, ToSoftpackYML -from softpack_core.schemas.base import BaseSchema -# Interfaces -@strawberry.interface -class Success: - """Interface for successful results.""" +@dataclass +class CreateEnvironmentSuccess: + """Environment successfully scheduled.""" message: str -@strawberry.interface -class Error: - """Interface for errors.""" +@dataclass +class UpdateEnvironmentSuccess: + """Environment successfully updated.""" message: str -# Success types -@strawberry.type -class CreateEnvironmentSuccess(Success): - """Environment successfully scheduled.""" - - -@strawberry.type -class UpdateEnvironmentSuccess(Success): - """Environment successfully updated.""" - - -@strawberry.type -class AddTagSuccess(Success): +@dataclass +class AddTagSuccess: """Successfully added tag to environment.""" + message: str + -@strawberry.type -class HiddenSuccess(Success): +@dataclass +class HiddenSuccess: """Successfully set hidden status on environment.""" + message: str + -@strawberry.type -class DeleteEnvironmentSuccess(Success): +@dataclass +class DeleteEnvironmentSuccess: """Environment successfully deleted.""" + message: str + -@strawberry.type -class WriteArtifactSuccess(Success): +@dataclass +class WriteArtifactSuccess: """Artifact successfully created.""" + message: str + # Error types -@strawberry.type -class InvalidInputError(Error): +@dataclass +class InvalidInputError: """Invalid input data.""" + error: str + -@strawberry.type -class WriteArtifactFailure(Error): +@dataclass +class WriteArtifactFailure: """Artifact failed to be created.""" + error: str + -@strawberry.type -class EnvironmentNotFoundError(Error): +@dataclass +class EnvironmentNotFoundError: """Environment not found.""" path: str name: str - message: str = "No environment with this path and name found." + error: str = "No environment with this path and name found." -@strawberry.type -class EnvironmentAlreadyExistsError(Error): +@dataclass +class EnvironmentAlreadyExistsError: """Environment name already exists.""" path: str name: str + error: str -@strawberry.type -class BuilderError(Error): +@dataclass +class BuilderError: """Unable to connect to builder.""" + error: str -# Unions -CreateResponse = strawberry.union( - "CreateResponse", - [ - CreateEnvironmentSuccess, - InvalidInputError, - EnvironmentAlreadyExistsError, - BuilderError, - ], -) -UpdateResponse = strawberry.union( - "UpdateResponse", - [ - UpdateEnvironmentSuccess, - InvalidInputError, - EnvironmentNotFoundError, - ], -) - -AddTagResponse = strawberry.union( - "AddTagResponse", - [ - AddTagSuccess, - InvalidInputError, - EnvironmentNotFoundError, - ], -) - -HiddenResponse = strawberry.union( - "HiddenResponse", - [ - HiddenSuccess, - InvalidInputError, - EnvironmentNotFoundError, - ], -) - -DeleteResponse = strawberry.union( - "DeleteResponse", - [ - DeleteEnvironmentSuccess, - EnvironmentNotFoundError, - ], -) - -WriteArtifactResponse = strawberry.union( - "WriteArtifactResponse", - [ - WriteArtifactSuccess, - InvalidInputError, - ], -) +# Unions +CreateResponse = Union[ + CreateEnvironmentSuccess, + InvalidInputError, + EnvironmentAlreadyExistsError, + BuilderError, + EnvironmentNotFoundError, +] + +UpdateResponse = Union[ + UpdateEnvironmentSuccess, + InvalidInputError, + EnvironmentNotFoundError, +] + +AddTagResponse = Union[ + AddTagSuccess, + InvalidInputError, + EnvironmentNotFoundError, + WriteArtifactFailure, +] + +HiddenResponse = Union[ + HiddenSuccess, + InvalidInputError, + EnvironmentNotFoundError, +] + +DeleteResponse = Union[ + DeleteEnvironmentSuccess, + EnvironmentNotFoundError, +] + +WriteArtifactResponse = Union[ + WriteArtifactSuccess, + InvalidInputError, +] def validate_tag(tag: str) -> Union[None, InvalidInputError]: @@ -178,26 +161,25 @@ def validate_tag(tag: str) -> Union[None, InvalidInputError]: """ if tag != tag.strip(): return InvalidInputError( - message="Tags must not contain leading or trailing whitespace" + error="Tags must not contain leading or trailing whitespace" ) if re.fullmatch(r"[a-zA-Z0-9 ._-]+", tag) is None: return InvalidInputError( - message="Tags must contain only alphanumerics, dots, " + error="Tags must contain only alphanumerics, dots, " "underscores, dashes, and spaces" ) if re.search(r"\s\s", tag) is not None: return InvalidInputError( - message="Tags must not contain runs of multiple spaces" + error="Tags must not contain runs of multiple spaces" ) return None -@strawberry.input class PackageInput(Package): - """A Strawberry input model representing a package.""" + """A data class model representing a package.""" def to_package(self) -> Package: """Create a Package object from a PackageInput object. @@ -207,15 +189,15 @@ def to_package(self) -> Package: return Package(**vars(self)) -@strawberry.input +@dataclass class EnvironmentInput: - """A Strawberry input model representing an environment.""" + """A data class model representing an environment.""" name: str path: str - username: Optional[str] = "" description: str packages: list[PackageInput] + username: Optional[str] = "" tags: Optional[list[str]] = None def validate(self) -> Union[None, InvalidInputError]: @@ -230,13 +212,15 @@ def validate(self) -> Union[None, InvalidInputError]: if any( len(value) == 0 for key, value in vars(self).items() - if key != "tags" and key != "username" + if key != "tags" + and key != "username" + and key != "__pydantic_initialised__" ): - return InvalidInputError(message="all fields must be filled in") + return InvalidInputError(error="all fields must be filled in") if not re.fullmatch("^[a-zA-Z0-9_-][a-zA-Z0-9_.-]*$", self.name): return InvalidInputError( - message="name must only contain alphanumerics, " + error="name must only contain alphanumerics, " "dash, and underscore" ) @@ -245,11 +229,11 @@ def validate(self) -> Union[None, InvalidInputError]: Artifacts.groups_folder_name, ] if not any(self.path.startswith(dir + "/") for dir in valid_dirs): - return InvalidInputError(message="Invalid path") + return InvalidInputError(error="Invalid path") if not re.fullmatch(r"^[^/]+/[a-zA-Z0-9_-]+$", self.path): return InvalidInputError( - message="user/group subdirectory must only contain " + error="user/group subdirectory must only contain " "alphanumerics, dash, and underscore" ) @@ -307,7 +291,7 @@ def get_all(cls) -> Union[List["BuildStatus"], BuilderError]: json = r.json() except Exception as e: return BuilderError( - message="Connection to builder failed: " + error="Connection to builder failed: " + "".join(format_exception_only(type(e), e)) ) @@ -326,11 +310,36 @@ def get_all(cls) -> Union[List["BuildStatus"], BuilderError]: ] -@strawberry.type +@dataclass +class DelEnvironmentInput: + """A class representing an input to delete environment.""" + + path: str + name: str + + +@dataclass +class AddTagInput: + """A class representing an input to add a tag.""" + + path: str + name: str + tag: str + + +@dataclass +class SetHiddenInput: + """A class representing an input to set hidden.""" + + path: str + name: str + hidden: bool + + +@dataclass class Environment: - """A Strawberry model representing a single environment.""" + """A data class representing a single environment.""" - id: str name: str path: str description: str @@ -408,7 +417,6 @@ def from_artifact(cls, obj: Artifacts.Object) -> Optional["Environment"]: return None return Environment( - id=obj.oid, name=obj.name, path=str(obj.path.parent), description=spec.description, @@ -440,7 +448,7 @@ def create(cls, env: EnvironmentInput) -> CreateResponse: # type: ignore if not env.name: return InvalidInputError( - message="environment name must not be blank" + error="environment name must not be blank" ) while not isinstance( @@ -479,9 +487,9 @@ def create(cls, env: EnvironmentInput) -> CreateResponse: # type: ignore else: return EnvironmentNotFoundError(path=env.path, name=env.name) - response = cls.submit_env_to_builder(env) - if response is not None: - return response + builder_response = cls.submit_env_to_builder(env) + if builder_response is not None: + return builder_response return CreateEnvironmentSuccess( message="Successfully scheduled environment creation" @@ -506,7 +514,7 @@ def submit_env_to_builder( versionless_name, version = m.groups() except Exception: return InvalidInputError( - message=f"could not parse version from name: {env.name!r}" + error=f"could not parse version from name: {env.name!r}" ) if env.has_requested_recipes(): @@ -535,7 +543,7 @@ def submit_env_to_builder( r.raise_for_status() except Exception as e: return BuilderError( - message="Connection to builder failed: " + error="Connection to builder failed: " + "".join(format_exception_only(type(e), e)) ) return None @@ -569,7 +577,7 @@ def create_new_env( # Check if an env with same name already exists at given path if artifacts.get(Path(env.path), env.name): return EnvironmentAlreadyExistsError( - message="This name is already used in this location", + error="This name is already used in this location", path=env.path, name=env.name, ) @@ -612,7 +620,7 @@ def create_new_env( artifacts.commit_and_push(tree_oid, "create environment folder") except RuntimeError as e: return InvalidInputError( - message="".join(format_exception_only(type(e), e)) + error="".join(format_exception_only(type(e), e)) ) return CreateEnvironmentSuccess( @@ -658,12 +666,14 @@ async def add_tag( A message confirming the success or failure of the operation. """ environment_path = Path(path, name) - response: Optional[Error] = cls.check_env_exists(environment_path) + response: Optional[EnvironmentNotFoundError] = cls.check_env_exists( + environment_path + ) if response is not None: return response - if (response := validate_tag(tag)) is not None: - return response + if (tagResponse := validate_tag(tag)) is not None: + return tagResponse tree = artifacts.get(Path(path), name) if tree is None: @@ -677,12 +687,12 @@ async def add_tag( metadata = cls.read_metadata(path, name) metadata.tags = sorted(tags) - response = await cls.store_metadata(environment_path, metadata) + metadataResponse = await cls.store_metadata(environment_path, metadata) - if isinstance(response, WriteArtifactSuccess): + if isinstance(metadataResponse, WriteArtifactSuccess): return AddTagSuccess(message="Tag successfully added") - return WriteArtifactFailure + return WriteArtifactFailure(error="Failed to add tag") @classmethod def read_metadata(cls, path: str, name: str) -> Box: @@ -719,7 +729,9 @@ async def set_hidden( ) -> HiddenResponse: # type: ignore """This method sets the hidden status for the given environment.""" environment_path = Path(path, name) - response: Optional[Error] = cls.check_env_exists(environment_path) + response: Optional[EnvironmentNotFoundError] = cls.check_env_exists( + environment_path + ) if response is not None: return response @@ -730,12 +742,12 @@ async def set_hidden( metadata.hidden = hidden - response = await cls.store_metadata(environment_path, metadata) + metadataResponse = await cls.store_metadata(environment_path, metadata) - if isinstance(response, WriteArtifactSuccess): + if isinstance(metadataResponse, WriteArtifactSuccess): return HiddenSuccess(message="Hidden metadata set") - return response + return metadataResponse async def update_metadata(cls, key: str, value: str | None) -> None: """Takes a key and sets the value unless value is None.""" @@ -776,14 +788,14 @@ def delete(cls, name: str, path: str) -> DeleteResponse: # type: ignore ) return EnvironmentNotFoundError( - message="No environment with this name found in this location.", + error="No environment with this name found in this location.", path=path, name=name, ) @classmethod async def create_from_module( - cls, file: Upload, module_path: str, environment_path: str + cls, file: bytes, module_path: str, environment_path: str ) -> CreateResponse: # type: ignore """Create an Environment based on an existing module. @@ -824,7 +836,7 @@ async def create_from_module( if not isinstance(result, WriteArtifactSuccess): cls.delete(name=env.name, path=environment_path) return InvalidInputError( - message="Write of module file failed: " + result.message + error="Write of module file failed: " + result.error ) return CreateEnvironmentSuccess( @@ -833,7 +845,7 @@ async def create_from_module( @classmethod async def convert_module_file_to_artifacts( - cls, file: Upload, env_name: str, env_path: str, module_path: str + cls, contents: bytes, env_name: str, env_path: str, module_path: str ) -> WriteArtifactResponse: # type: ignore """convert_module_file_to_artifacts parses a module and writes to repo. @@ -846,7 +858,6 @@ async def convert_module_file_to_artifacts( Returns: WriteArtifactResponse: success or failure indicator. """ - contents = await file.read() yml = ToSoftpackYML(env_name, contents) readme = GenerateEnvReadme(module_path) @@ -870,9 +881,9 @@ async def convert_module_file_to_artifacts( @classmethod async def write_module_artifacts( cls, - module_file: Upload, - softpack_file: Upload, - readme_file: Upload, + module_file: UploadFile, + softpack_file: UploadFile, + readme_file: UploadFile, environment_path: str, ) -> WriteArtifactResponse: # type: ignore """Writes the given module and softpack files to the artifacts repo. @@ -890,9 +901,9 @@ async def write_module_artifacts( WriteArtifactResponse: contains message and commit hash of softpack.yml upload. """ - module_file.name = artifacts.module_file - readme_file.name = artifacts.readme_file - softpack_file.name = artifacts.environments_file + module_file.filename = artifacts.module_file + readme_file.filename = artifacts.readme_file + softpack_file.filename = artifacts.environments_file return await cls.write_artifacts( folder_path=environment_path, @@ -901,7 +912,7 @@ async def write_module_artifacts( @classmethod async def write_artifact( - cls, file: Upload, folder_path: str, file_name: str + cls, file: UploadFile, folder_path: str, file_name: str ) -> WriteArtifactResponse: # type: ignore """Add a file to the Artifacts repo. @@ -910,7 +921,7 @@ async def write_artifact( folder_path: the path to the folder that the file will be added to. file_name: the name of the file to be added. """ - file.name = file_name + file.filename = file_name return await cls.write_artifacts(folder_path, [file]) @@ -918,7 +929,7 @@ async def write_artifact( async def write_artifacts( cls, folder_path: str, - files: list[Union[Upload, UploadFile, Tuple[str, str]]], + files: list[Union[UploadFile, Tuple[str, str]]], commitMsg: str = "write artifact", ) -> WriteArtifactResponse: # type: ignore """Add one or more files to the Artifacts repo. @@ -935,13 +946,9 @@ async def write_artifacts( new_files.append( cast(Tuple[str, Union[str, UploadFile]], file) ) - elif isinstance(file, starlette.datastructures.UploadFile): - new_files.append( - (file.filename or "", cast(UploadFile, file)) - ) else: new_files.append( - (file.name, cast(str, (await file.read()).decode())) + (file.filename or "", cast(UploadFile, file)) ) tree_oid = artifacts.create_files( @@ -968,7 +975,7 @@ async def write_artifacts( ) except Exception as e: return InvalidInputError( - message="".join(format_exception_only(type(e), e)) + error="".join(format_exception_only(type(e), e)) ) @classmethod @@ -985,7 +992,7 @@ def env_index_from_path(cls, folder_path: str) -> Optional[int]: @classmethod async def update_from_module( - cls, file: Upload, module_path: str, environment_path: str + cls, file: bytes, module_path: str, environment_path: str ) -> UpdateResponse: # type: ignore """Update an Environment based on an existing module. @@ -1015,46 +1022,15 @@ async def update_from_module( if result is not None: return result - result = await cls.convert_module_file_to_artifacts( + convertResult = await cls.convert_module_file_to_artifacts( file, env.name, environment_path, module_path ) - if not isinstance(result, WriteArtifactSuccess): + if not isinstance(convertResult, WriteArtifactSuccess): return InvalidInputError( - message="Write of module file failed: " + result.message + error="Write of module file failed: " + convertResult.error ) return UpdateEnvironmentSuccess( message="Successfully updated environment in artifacts repo" ) - - -class EnvironmentSchema(BaseSchema): - """Environment schema.""" - - @dataclass - class Query: - """GraphQL query schema.""" - - environments: list[Environment] = Environment.iter # type: ignore - - @dataclass - class Mutation: - """GraphQL mutation schema.""" - - createEnvironment: CreateResponse = Environment.create # type: ignore - deleteEnvironment: DeleteResponse = Environment.delete # type: ignore - addTag: AddTagResponse = Environment.add_tag # type: ignore - setHidden: HiddenResponse = Environment.set_hidden # type: ignore - # writeArtifact: WriteArtifactResponse = ( # type: ignore - # Environment.write_artifact - # ) - # writeArtifacts: WriteArtifactResponse = ( # type: ignore - # Environment.write_artifacts - # ) - createFromModule: CreateResponse = ( # type: ignore - Environment.create_from_module - ) - updateFromModule: UpdateResponse = ( # type: ignore - Environment.update_from_module - ) diff --git a/softpack_core/schemas/groups.py b/softpack_core/schemas/groups.py index 4f053dd..716aa61 100644 --- a/softpack_core/schemas/groups.py +++ b/softpack_core/schemas/groups.py @@ -7,14 +7,12 @@ from dataclasses import dataclass from typing import Iterable -import strawberry - from ..ldapapi import LDAP -@strawberry.type +@dataclass class Group: - """A Strawberry model representing a single unix group.""" + """A data class representing a single unix group.""" name: str @@ -30,13 +28,3 @@ def from_username(cls, username: str) -> Iterable["Group"]: """ groups = LDAP().groups(username) return (Group(name=group) for group in groups) - - -class GroupsSchema: - """Group schema.""" - - @dataclass - class Query: - """GraphQL query schema.""" - - groups: list[Group] = Group.from_username # type: ignore diff --git a/softpack_core/schemas/package_collection.py b/softpack_core/schemas/package_collection.py index 2338c75..752dae6 100644 --- a/softpack_core/schemas/package_collection.py +++ b/softpack_core/schemas/package_collection.py @@ -4,22 +4,21 @@ LICENSE file in the root directory of this source tree. """ -from dataclasses import dataclass -import strawberry +from dataclasses import dataclass from softpack_core.app import app from softpack_core.spack import Package -@strawberry.type +@dataclass class PackageMultiVersion(Package): - """A Strawberry model representing a package in a collection.""" + """A data class representing a package in a collection.""" -@strawberry.type +@dataclass class PackageCollection: - """A Strawberry model representing a package collection.""" + """A data class representing a package collection.""" name: str packages: list[PackageMultiVersion] @@ -52,15 +51,3 @@ def from_package(cls, package: Package) -> PackageMultiVersion: return PackageMultiVersion( name=package.name, versions=package.versions ) # type: ignore [call-arg] - - -class PackageCollectionSchema: - """Package collection schema.""" - - @dataclass - class Query: - """GraphQL query schema.""" - - packageCollections: list[ - PackageMultiVersion - ] = PackageCollection.iter # type: ignore diff --git a/softpack_core/service.py b/softpack_core/service.py index 538ce87..46d9590 100644 --- a/softpack_core/service.py +++ b/softpack_core/service.py @@ -16,21 +16,25 @@ import uvicorn import yaml from fastapi import APIRouter, Request, Response, UploadFile -from strawberry.file_uploads import Upload from typer import Typer from typing_extensions import Annotated from softpack_core.artifacts import Artifacts, State, artifacts from softpack_core.config.models import EmailConfig from softpack_core.schemas.environment import ( + AddTagInput, BuilderError, BuildStatus, CreateEnvironmentSuccess, + DelEnvironmentInput, Environment, EnvironmentInput, PackageInput, + SetHiddenInput, WriteArtifactSuccess, ) +from softpack_core.schemas.groups import Group +from softpack_core.schemas.package_collection import PackageCollection from .api import API from .app import app @@ -116,7 +120,7 @@ async def upload_artifacts( # type: ignore[no-untyped-def] path = Path(env_path) env = Environment.get_env(path.parent, path.name) newState = State.queued - files = cast(list[Union[Upload, UploadFile, Tuple[str, str]]], file) + files = cast(list[Union[UploadFile, Tuple[str, str]]], file) if env: for i in range(len(file)): @@ -448,6 +452,89 @@ async def buildStatus( # type: ignore[no-untyped-def] ), } + @staticmethod + @router.post("/createEnvironment") + def create_env( # type: ignore[no-untyped-def] + env: EnvironmentInput, + ): + """Endpoint for creating environments.""" + return Environment.create(env) + + @staticmethod + @router.get("/getEnvironments") + def get_envs(): # type: ignore[no-untyped-def] + """Endpoint for creating environments.""" + return Environment.iter() + + @staticmethod + @router.post("/deleteEnvironment") + def delete_env( # type: ignore[no-untyped-def] + env: DelEnvironmentInput, + ): + """Endpoint for deleting environments.""" + return Environment.delete(env.name, env.path) + + @staticmethod + @router.post("/addTag") + async def add_tag_env( # type: ignore[no-untyped-def] + tag: AddTagInput, + ): + """Endpoint for adding a tag.""" + return await Environment.add_tag(tag.name, tag.path, tag.tag) + + @staticmethod + @router.post("/setHidden") + async def set_hidden( # type: ignore[no-untyped-def] + hide: SetHiddenInput, + ): + """Endpoint for setting hidden.""" + return await Environment.set_hidden(hide.name, hide.path, hide.hidden) + + @staticmethod + @router.post("/uploadModule") + async def upload_module( # type: ignore[no-untyped-def] + module_path: str, + environment_path: str, + file: Request, + ): + """Endpoint for uploading a module.""" + data = await file.body() + + return await Environment.create_from_module( + data, module_path, environment_path + ) + + @staticmethod + @router.post("/updateModule") + async def update_module( # type: ignore[no-untyped-def] + module_path: str, + environment_path: str, + file: Request, + ): + """Endpoint for updating a module.""" + data = await file.body() + + return await Environment.update_from_module( + data, module_path, environment_path + ) + + @staticmethod + @router.get("/packageCollection") + def package_collection(): # type: ignore[no-untyped-def] + """Endpoint for returning spack recipes.""" + return PackageCollection.iter() + + @staticmethod + @router.post("/groups") + async def groups(request: Request): # type: ignore[no-untyped-def] + """Endpoint for finding groups from a username.""" + username = await request.json() + + if not isinstance(username, str): + return {"error": "invalid username"} + + return (group.name for group in Group.from_username(username)) + def send_email( emailConfig: EmailConfig, diff --git a/tests/integration/test_environment.py b/tests/integration/test_environment.py index 38dcdfc..b09d526 100644 --- a/tests/integration/test_environment.py +++ b/tests/integration/test_environment.py @@ -632,24 +632,20 @@ async def test_create_from_module(httpx_post, testable_env_input): with open(test_file_path, "rb") as fh: data = fh.read() - upload = UploadFile(filename="shpc.mod", file=io.BytesIO(data)) - env_name = "some-environment" name = "groups/hgi/" + env_name module_path = "HGI/common/some_environment" result = await Environment.create_from_module( - file=upload, + file=data, module_path=module_path, environment_path=name, ) assert isinstance(result, CreateEnvironmentSuccess) - upload = UploadFile(filename="shpc.mod", file=io.BytesIO(data)) - result = await Environment.create_from_module( - file=upload, + file=data, module_path=module_path, environment_path=name, ) @@ -699,12 +695,10 @@ async def test_create_from_module(httpx_post, testable_env_input): with open(test_modifiy_file_path, "rb") as fh: data = fh.read() - upload = UploadFile(filename="all_fields.mod", file=io.BytesIO(data)) - module_path = "HGI/common/all_fields" result = await Environment.update_from_module( - file=upload, + file=data, module_path=module_path, environment_path=name, ) @@ -723,10 +717,8 @@ async def test_create_from_module(httpx_post, testable_env_input): assert env.type == Artifacts.generated_from_module assert env.state == State.ready - upload = UploadFile(filename="all_fields.mod", file=io.BytesIO(data)) - result = await Environment.update_from_module( - file=upload, + file=data, module_path=module_path, environment_path="users/non/existant", ) diff --git a/tests/unit/test_service.py b/tests/integration/test_service.py similarity index 55% rename from tests/unit/test_service.py rename to tests/integration/test_service.py index fe4cc93..2bf506f 100644 --- a/tests/unit/test_service.py +++ b/tests/integration/test_service.py @@ -5,6 +5,7 @@ """ import multiprocessing +from pathlib import Path from time import sleep import httpx @@ -14,6 +15,7 @@ from softpack_core import __version__ from softpack_core.app import app from softpack_core.config.models import EmailConfig +from softpack_core.schemas.environment import EnvironmentInput from softpack_core.service import ServiceAPI, send_email @@ -138,3 +140,146 @@ def test_build_status(mocker): "groups/test_group/test_environment": "2025-01-02T03:04:05+00:00", "users/foo/bar": "2025-01-02T03:04:05+00:00", } + + +def test_create_env(httpx_post, testable_env_input: EnvironmentInput): + client = TestClient(app.router) + input = testable_env_input.__dict__ + input["packages"] = [pkg.__dict__ for pkg in testable_env_input.packages] + + resp = client.post("/createEnvironment", json=input) + + assert resp.status_code == 200 + assert ( + resp.json().get("message") + == "Successfully scheduled environment creation" + ) + + resp = client.post("/createEnvironment", json={"bad": "value"}) + + assert resp.status_code == 422 + + +def test_delete_env(testable_env_input: EnvironmentInput): + client = TestClient(app.router) + + resp = client.post( + "/deleteEnvironment", + json={"path": "users/test_user", "name": "test_environment"}, + ) + + assert resp.status_code == 200 + assert resp.json().get("message") == "Successfully deleted the environment" + + +def test_add_tag(testable_env_input: EnvironmentInput): + client = TestClient(app.router) + + resp = client.post( + "/addTag", + json={ + "name": "test_environment", + "path": "users/test_user", + "tag": "abc", + }, + ) + + assert resp.status_code == 200 + assert resp.json().get("message") == "Tag successfully added" + + +def test_set_hidden(testable_env_input: EnvironmentInput): + client = TestClient(app.router) + + resp = client.post( + "/setHidden", + json={ + "path": "users/test_user", + "name": "test_environment", + "hidden": True, + }, + ) + + assert resp.status_code == 200 + assert resp.json().get("message") == "Hidden metadata set" + + +def test_upload_and_update_module(testable_env_input: EnvironmentInput): + client = TestClient(app.router) + + resp = client.post( + "/uploadModule?module_path=some/module/path&" + + "environment_path=groups/something/env-1", + data="", + ) + + assert resp.status_code == 200 + assert ( + resp.json().get("message") + == "Successfully created environment in artifacts repo" + ) + + test_files_dir = Path(__file__).parent.parent / "files" / "modules" + test_modifiy_file_path = test_files_dir / "all_fields.mod" + + with open(test_modifiy_file_path, "rb") as fh: + data = fh.read() + + resp = client.post( + "/updateModule?module_path=some/module/path&" + + "environment_path=groups/something/env-1", + data=data, + ) + + assert resp.status_code == 200 + assert ( + resp.json().get("message") + == "Successfully updated environment in artifacts repo" + ) + + +def test_package_collection(): + client = TestClient(app.router) + + resp = client.get("/packageCollection") + + pkgs = resp.json() + + assert isinstance(pkgs, list) + assert len(pkgs) > 0 + + +def test_groups(testable_env_input: EnvironmentInput): + client = TestClient(app.router) + + resp = client.post( + "/groups", + json="root", + ) + + groups = resp.json() + + assert resp.status_code == 200 + assert isinstance(groups, list) + assert len(groups) > 0 + assert all(isinstance(group, str) for group in groups) + + resp = client.post( + "/groups", + json=1, + ) + + assert resp.status_code == 200 + assert resp.json().get("error") == "invalid username" + + +def test_get_envs(testable_env_input: EnvironmentInput): + client = TestClient(app.router) + + resp = client.get( + "/getEnvironments", + ) + + assert resp.status_code == 200 + assert isinstance(resp.json(), list) + assert len(resp.json()) > 0 diff --git a/tests/integration/test_spack.py b/tests/integration/test_spack.py index 96b3893..95e3f22 100644 --- a/tests/integration/test_spack.py +++ b/tests/integration/test_spack.py @@ -73,7 +73,7 @@ def test_spack_package_updater(): spack.custom_repo = app.settings.spack.repo - timeout = time.time() + 60 * 3 + timeout = time.time() + 60 * 10 while True: new_pkgs = spack.stored_packages