testserver: First iteration of the mock Greenlight server

This uses the `gl-testing` library, and builds a standalone server to
test against. We currently expose four interfaces:

 - The scheduler interface as the main entrypoint to the service
 - The GRPC-Web proxy to develop browser apps and extensions against
   Greenlight.
 - The `bitcoind` interface, so you can generate blocks and confirm
   transactions without lengthy wait times
 - The node's grpc interface directly to work against a single user's
   node

All of these will listen to random ports initially. We write a small
file `metadata.json` which contains the URIs and ports for the first
three, while the node's URI can be retrieved from the scheduler, since
these are spawned on demand as users register.

libs/gl-testserver/gltestserver/__main__.py

@@ -0,0 +1,140 @@
+import json
+from dataclasses import dataclass
+
+import time
+
+from rich.console import Console
+from rich.pretty import pprint
+from rich import inspect
+from pathlib import Path
+from gltesting import fixtures
+import gltesting
+from inspect import isgeneratorfunction
+import click
+import logging
+from rich.logging import RichHandler
+from pyln.testing.utils import BitcoinD
+from typing import Any, List
+
+
+console = Console()
+logging.basicConfig(
+    level="DEBUG",
+    format="%(message)s",
+    datefmt="[%X]",
+    handlers=[
+        RichHandler(rich_tracebacks=True, tracebacks_suppress=[click], console=console)
+    ],
+)
+logger = logging.getLogger("gltestserver")
+
+
+@dataclass
+class TestServer:
+    directory: Path
+    bitcoind: BitcoinD
+    scheduler: gltesting.scheduler.Scheduler
+    finalizers: List[Any]
+    clients: gltesting.clients.Clients
+    grpc_web_proxy: gltesting.grpcweb.GrpcWebProxy
+
+    def stop(self):
+        for f in self.finalizers[::-1]:
+            try:
+                f()
+            except StopIteration:
+                continue
+            except Exception as e:
+                logger.warn(f"Unexpected exception tearing down server: {e}")
+
+    def metadata(self):
+        """Construct a dict of config values for this TestServer."""
+        return {
+            "scheduler_grpc_uri": self.scheduler.grpc_addr,
+            "grpc_web_proxy_uri": f"http://localhost:{self.grpc_web_proxy.web_port}",
+            "bitcoind_rpc_uril": f"http://rpcuser:rpcpass@localhost:{self.bitcoind.rpcport}",
+        }
+
+
+def build():
+    # List of teardown functions to call in reverse order.
+    finalizers = []
+
+    def callfixture(f, *args, **kwargs):
+        """Small shim to bypass the pytest decorator."""
+        F = f.__pytest_wrapped__.obj
+
+        if isgeneratorfunction(F):
+            it = F(*args, **kwargs)
+            v = it.__next__()
+            finalizers.append(it.__next__)
+            return v
+        else:
+            return F(*args, **kwargs)
+
+    directory = Path("/tmp/gl-testserver")
+
+    cert_directory = callfixture(fixtures.cert_directory, directory)
+    root_id = callfixture(fixtures.root_id, cert_directory)
+    users_id = callfixture(fixtures.users_id)
+    nobody_id = callfixture(fixtures.nobody_id, cert_directory)
+    scheduler_id = callfixture(fixtures.scheduler_id, cert_directory)
+    paths = callfixture(fixtures.paths)
+    bitcoind = callfixture(
+        fixtures.bitcoind,
+        directory=directory,
+        teardown_checks=None,
+    )
+    scheduler = callfixture(
+        fixtures.scheduler, scheduler_id=scheduler_id, bitcoind=bitcoind
+    )
+
+    clients = callfixture(
+        fixtures.clients, directory=directory, scheduler=scheduler, nobody_id=nobody_id
+    )
+
+    node_grpc_web_server = callfixture(
+        fixtures.node_grpc_web_proxy, scheduler=scheduler
+    )
+
+    return TestServer(
+        directory=directory,
+        bitcoind=bitcoind,
+        finalizers=finalizers,
+        scheduler=scheduler,
+        clients=clients,
+        grpc_web_proxy=node_grpc_web_server,
+    )
+
+
+@click.group()
+def cli():
+    pass
+
+
+@cli.command()
+def run():
+    gl = build()
+    try:
+        meta = gl.metadata()
+        metafile = gl.directory / "metadata.json"
+        logger.debug(f"Writing testserver metadata to {metafile}")
+        with metafile.open(mode="w") as f:
+            json.dump(meta, f)
+
+        pprint(meta)
+        logger.info(
+            f"Server is up and running with the above config values. To stop press Ctrl-C."
+        )
+        time.sleep(1800)
+    except Exception as e:
+        logger.warning(f"Caught exception running testserver: {e}")
+        pass
+    finally:
+        logger.info("Stopping gl-testserver")
+        # Now tear things down again.
+        gl.stop()
+
+
+if __name__ == "__main__":
+    cli()

libs/gl-testserver/pyproject.toml

@@ -0,0 +1,17 @@
+[project]
+name = "gltestserver"
+version = "0.1.0"
+description = "A standalone test server implementing the public Greenlight interfaces"
+readme = "README.md"
+requires-python = ">=3.8"
+dependencies = [
+    "click>=8.1.7",
+    "gltesting",
+    "rich>=13.9.3",
+]
+
+[project.scripts]
+gltestserver = 'gltestserver.__main__:cli'
+
+[tool.uv.sources]
+gltesting = { workspace = true }

libs/gl-testserver/tests/test_server.py

@@ -0,0 +1,107 @@
+# We do not import `gl-testing` or `pyln-testing` since the
+# `gl-testserver` is intended to run tests externally from a python
+# environment. We will use `gl-client-py` to interact with it though.
+# Ok, one exception, `TailableProc` is used to run and tail the
+# `gl-testserver`.
+
+import shutil
+import tempfile
+import os
+import pytest
+from pyln.testing.utils import TailableProc
+import json
+import signal
+from pathlib import Path
+
+
+@pytest.fixture
+def test_name(request):
+    yield request.function.__name__
+
+
+@pytest.fixture(scope="session")
+def test_base_dir():
+    d = os.getenv("TEST_DIR", "/tmp")
+    directory = tempfile.mkdtemp(prefix="ltests-", dir=d)
+    print("Running tests in {}".format(directory))
+
+    yield directory
+
+
+@pytest.fixture
+def directory(request, test_base_dir, test_name):
+    """Return a per-test specific directory.
+
+    This makes a unique test-directory even if a test is rerun multiple times.
+
+    """
+    directory = os.path.join(test_base_dir, test_name)
+    request.node.has_errors = False
+
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+
+    yield directory
+
+    # This uses the status set in conftest.pytest_runtest_makereport to
+    # determine whether we succeeded or failed. Outcome can be None if the
+    # failure occurs during the setup phase, hence the use to getattr instead
+    # of accessing it directly.
+    rep_call = getattr(request.node, "rep_call", None)
+    outcome = "passed" if rep_call is None else rep_call.outcome
+    failed = not outcome or request.node.has_errors or outcome != "passed"
+
+    if not failed:
+        try:
+            shutil.rmtree(directory)
+        except OSError:
+            # Usually, this means that e.g. valgrind is still running.  Wait
+            # a little and retry.
+            files = [
+                os.path.join(dp, f) for dp, dn, fn in os.walk(directory) for f in fn
+            ]
+            print("Directory still contains files: ", files)
+            print("... sleeping then retrying")
+            time.sleep(10)
+            shutil.rmtree(directory)
+    else:
+        logging.debug(
+            "Test execution failed, leaving the test directory {} intact.".format(
+                directory
+            )
+        )
+
+
+class TestServer(TailableProc):
+    def __init__(self, directory):
+        TailableProc.__init__(self, outputDir=directory)
+        self.cmd_line = [
+            "python3",
+            str(Path(__file__).parent / ".." / "gltestserver" / "__main__.py"),
+            "run",
+        ]
+
+    def start(self):
+        TailableProc.start(self)
+        self.wait_for_log(r"Ctrl-C")
+
+    def stop(self):
+        self.proc.send_signal(signal.SIGTERM)
+        self.proc.wait()
+
+
+@pytest.fixture
+def testserver(directory):
+    ts = TestServer(directory=directory)
+    ts.start()
+
+
+    metadata = json.load(open(f'{directory}/metadata.json'))
+    pprint(metadata)
+
+    yield ts
+    ts.stop()
+
+
+def test_start(testserver):
+    print(TailableProc)

pyproject.toml

@@ -40,4 +40,4 @@ pillow = "^9.5.0"
 python-lsp-server = "^1.10.0"
 
 [tool.uv.workspace]
-members = ["libs/gl-testing"]
+members = ["libs/gl-testing", "libs/gl-testserver"]