Self-service building (backend pieces)

README.md

@@ -38,7 +38,21 @@ $ source spack/share/spack/setup-env.sh
 ```
 
 To start the service, you will also need to configure a git repository to store
-artifacts.
+artifacts, and configure details of an LDAP server to query group information:
+
+```yaml
+artifacts:
+  repo: # see "integration tests" below
+
+ldap:
+  server: "ldaps://ldap.example"
+  base: "ou=group,dc=example,dc=ac,dc=uk"
+  filter: "memberuid={user}"
+  group:
+    attr: "cn"
+    pattern: ".*"
+
+```
 
 ### Stable release
 

pyproject.toml

@@ -70,6 +70,7 @@ types-setuptools = "^67.7.0.1"
 types-PyYAML = "^6.0.12.9"
 types-requests = "^2.30.0.0"
 virtualenv = "^20.20.0"
+watchfiles = "^0.21.0"
 
 [tool.poetry.group.doc]
 optional = true

softpack_core/config/conf/config.yml

@@ -9,6 +9,10 @@ server:
       - http://localhost:3000
       - http://localhost:5173
 
+builder:
+  host: 0.0.0.0
+  port: 8001
+
 artifacts:
   repo:
     url: https://github.com/mjkw31/softpack-artifacts
@@ -35,4 +39,4 @@ spack:
 #    filter:
 #    group:
 #      attr:
-#      pattern:
+#      pattern:

softpack_core/config/models.py

@@ -23,6 +23,13 @@ class HeaderConfig(BaseModel):
     port: int
 
 
+class BuilderConfig(BaseModel):
+    """Builder config model."""
+
+    host: str
+    port: int
+
+
 class VaultConfig(BaseModel):
     """HashiCorp vault config."""
 

softpack_core/config/settings.py

@@ -16,6 +16,7 @@
 
 from .models import (
     ArtifactsConfig,
+    BuilderConfig,
     LDAPConfig,
     ServerConfig,
     SpackConfig,
@@ -32,6 +33,7 @@ class Settings(BaseSettings):
     ldap: Optional[LDAPConfig]
     artifacts: ArtifactsConfig
     spack: SpackConfig
+    builder: BuilderConfig
 
     class Config:
         """Configuration loader."""

softpack_core/graphql.py

@@ -17,14 +17,15 @@
 from .app import app
 from .schemas.base import BaseSchema
 from .schemas.environment import EnvironmentSchema
+from .schemas.groups import GroupsSchema
 from .schemas.package_collection import PackageCollectionSchema
 
 
 class GraphQL(API):
     """GraphQL API."""
 
     prefix = "/graphql"
-    schemas = [EnvironmentSchema, PackageCollectionSchema]
+    schemas = [EnvironmentSchema, PackageCollectionSchema, GroupsSchema]
     commands = Typer(help="GraphQL commands.")
 
     @staticmethod
@@ -126,4 +127,5 @@ def __init__(self, schema: strawberry.Schema, prefix: str) -> None:
             """
             super().__init__(schema=schema, path=prefix)
 
-    router = Router(schema=Schema(schemas), prefix=prefix)
+    schema = Schema(schemas)
+    router = Router(schema=schema, prefix=prefix)

softpack_core/schemas/environment.py

@@ -7,14 +7,17 @@
 import io
 from dataclasses import dataclass
 from pathlib import Path
+from traceback import format_exception_only
 from typing import Iterable, List, Optional, Tuple, Union, cast
 
 import httpx
 import starlette.datastructures
 import strawberry
+import yaml
 from fastapi import UploadFile
 from strawberry.file_uploads import Upload
 
+from softpack_core.app import app
 from softpack_core.artifacts import Artifacts, Package, State
 from softpack_core.module import GenerateEnvReadme, ToSoftpackYML
 from softpack_core.schemas.base import BaseSchema
@@ -80,13 +83,19 @@ class EnvironmentAlreadyExistsError(Error):
     name: str
 
 
+@strawberry.type
+class BuilderError(Error):
+    """Unable to connect to builder."""
+
+
 # Unions
 CreateResponse = strawberry.union(
     "CreateResponse",
     [
         CreateEnvironmentSuccess,
         InvalidInputError,
         EnvironmentAlreadyExistsError,
+        BuilderError,
     ],
 )
 
@@ -252,19 +261,33 @@ def create(cls, env: EnvironmentInput) -> CreateResponse:  # type: ignore
 
         env.name = name
 
-        # TODO: remove hard-coding of URL.
         # Send build request
-        httpx.post(
-            "http://0.0.0.0:7080/environments/build",
-            json={
-                "name": f"{env.path}/{env.name}",
-                "version": str(version),
-                "model": {
-                    "description": env.description,
-                    "packages": [f"{pkg.name}" for pkg in env.packages],
+        try:
+            host = app.settings.builder.host
+            port = app.settings.builder.port
+            r = httpx.post(
+                f"http://{host}:{port}/environments/build",
+                json={
+                    "name": f"{env.path}/{env.name}",
+                    "version": str(version),
+                    "model": {
+                        "description": env.description,
+                        "packages": [
+                            {
+                                "name": pkg.name,
+                                "version": pkg.version,
+                            }
+                            for pkg in env.packages
+                        ],
+                    },
                 },
-            },
-        )
+            )
+            r.raise_for_status()
+        except Exception as e:
+            return BuilderError(
+                message="Connection to builder failed: "
+                + "".join(format_exception_only(e))
+            )
 
         return CreateEnvironmentSuccess(
             message="Successfully scheduled environment creation"
@@ -310,14 +333,28 @@ def create_new_env(
         # Create folder with place-holder file
         new_folder_path = Path(env.path, env.name)
         try:
-            tree_oid = cls.artifacts.create_file(
-                new_folder_path, env_type, "", True
+            softpack_definition = dict(
+                description=env.description,
+                packages=[
+                    pkg.name + ("@" + pkg.version if pkg.version else "")
+                    for pkg in env.packages
+                ],
+            )
+            ymlData = yaml.dump(softpack_definition)
+
+            tree_oid = cls.artifacts.create_files(
+                new_folder_path,
+                [
+                    (env_type, ""),  # e.g. .built_by_softpack
+                    (cls.artifacts.environments_file, ymlData),  # softpack.yml
+                ],
+                True,
             )
             cls.artifacts.commit_and_push(
                 tree_oid, "create environment folder"
             )
         except RuntimeError as e:
-            return InvalidInputError(message=str(e))
+            return InvalidInputError(message="".join(format_exception_only(e)))
 
         return CreateEnvironmentSuccess(
             message="Successfully created environment in artifacts repo"
@@ -535,7 +572,7 @@ async def write_artifacts(
             )
 
         except Exception as e:
-            return InvalidInputError(message=str(e))
+            return InvalidInputError(message="".join(format_exception_only(e)))
 
     @classmethod
     async def update_from_module(