testserver: Add a couple of usability improvements

Makefile

@@ -191,3 +191,22 @@ docs-publish: docs
 	  --push \
 	  --branch gh-pages \
 	  --remote origin
+
+gltestserver-image: docker/gl-testserver/Dockerfile
+	docker build \
+	  --build-arg DOCKER_USER=$(shell whoami) \
+	  --build-arg UID=$(shell id -u) \
+	  --build-arg GID=$(shell id -g) \
+	  -t gltestserver \
+	  -f docker/gl-testserver/Dockerfile \
+	  .
+
+gltestserver: gltestserver-image
+	mkdir -p /tmp/gltestserver
+	docker run \
+	  --user $(shell id -u):$(shell id -g) \
+	  -e DOCKER_USER=$(shell whoami) \
+	  --net=host \
+	  -ti \
+	  -v $(shell pwd)/.testserver:/tmp/gltestserver \
+	  gltestserver

docker/gl-testserver/Dockerfile

@@ -0,0 +1,71 @@
+FROM ubuntu:22.04 AS python-builder
+
+ENV RUST_VERSION=1.74
+ENV PATH=$CARGO_HOME/bin:$PATH
+ENV PROTOC_VERSION=3.19.3
+ENV CFSSL_VERSION=1.6.5
+ENV GL_TESTING_IGNORE_HASH=true
+ARG BITCOIN_VERSION=24.0
+ARG GID=0
+ARG UID=0
+ARG DOCKER_USER=dev
+ENV PATH=$PATH:/home/$DOCKER_USER/.local/bin/:/opt/bitcoin/bin:/home/$DOCKER_USER/.cargo/bin
+#ENV VIRTUAL_ENV=/tmp/venv
+ENV CARGO_TARGET_DIR=/tmp/cargo
+
+RUN apt update && apt install -qqy \
+    curl \
+    python3 \
+    python3-pip \
+    python3-venv \
+    libpq-dev \
+    unzip \
+    sudo \
+    git \
+    build-essential \
+    wget
+
+RUN groupadd -g $GID -o $DOCKER_USER &&\
+    useradd -m -u $UID -g $GID -G sudo -o -s /bin/bash $DOCKER_USER && \
+    echo '%sudo   ALL=(ALL:ALL) ALL' >> /etc/sudoers
+
+RUN wget -q https://github.com/cloudflare/cfssl/releases/download/v${CFSSL_VERSION}/cfssl_${CFSSL_VERSION}_linux_amd64 -O /usr/bin/cfssl && \
+    chmod a+x /usr/bin/cfssl
+RUN wget -q https://github.com/cloudflare/cfssl/releases/download/v${CFSSL_VERSION}/cfssljson_${CFSSL_VERSION}_linux_amd64 -O /usr/bin/cfssljson && \
+    chmod a+x /usr/bin/cfssljson
+
+RUN mkdir /tmp/protoc && \
+    cd /tmp/protoc && \
+    wget --quiet \
+      -O protoc.zip \
+      https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip && \
+    unzip protoc.zip && \
+    sudo mv /tmp/protoc/bin/protoc /usr/local/bin && \
+    chmod a+x /usr/local/bin/protoc && \
+    rm -rf /tmp/protoc
+
+RUN cd /tmp/ && \
+    wget "https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/bitcoin-${BITCOIN_VERSION}-x86_64-linux-gnu.tar.gz" -O bitcoin.tar.gz && \
+    tar -xvzf bitcoin.tar.gz && \
+    mv /tmp/bitcoin-$BITCOIN_VERSION/ /opt/bitcoin && \
+    rm -rf bitcoin.tar.gz /tmp/bitcoin-$BITCOIN_VERSION
+
+ADD ../ /repo/libs
+RUN chown $DOCKER_USER:users -R /repo
+
+USER $DOCKER_USER
+
+RUN curl \
+    --proto '=https' \
+    --tlsv1.2 \
+    -sSf https://sh.rustup.rs | sh \
+    -s -- -y --default-toolchain ${RUST_VERSION}
+RUN rustup default stable
+
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh
+
+WORKDIR /repo/libs/gl-testserver/
+RUN echo $PATH
+RUN uv sync --locked -v
+RUN uv run clnvm get-all
+CMD uv run gltestserver run --metadata /tmp/gltestserver

libs/gl-testing/pyproject.toml

@@ -10,7 +10,7 @@ dependencies = [
     "gl-client",
     "grpcio-tools>=1.66",
     "grpcio>=1.66.0",
-    "httpx>=0.27.2",
+    "httpx[http2]==0.27.2",
     "purerpc>=0.8.0",
     "pyln-client==24.2",
     "pyln-testing==24.2",

libs/gl-testserver/gltestserver/__main__.py

@@ -10,6 +10,7 @@
 import click
 import gltesting
 import json
+import os
 import logging
 import tempfile
 import time
@@ -47,10 +48,15 @@ def stop(self):
 
     def metadata(self):
         """Construct a dict of config values for this TestServer."""
+        cert_path = Path(os.environ.get('GL_CERT_PATH'))
         return {
             "scheduler_grpc_uri": self.scheduler.grpc_addr,
             "grpc_web_proxy_uri": f"http://localhost:{self.grpc_web_proxy.web_port}",
             "bitcoind_rpc_uri": f"http://rpcuser:rpcpass@localhost:{self.bitcoind.rpcport}",
+            "cert_path": str(cert_path),
+            "ca_crt_path": str(cert_path / "ca.crt"),
+            "nobody_crt_path": str(cert_path / "users" / "nobody.crt"),
+            "nobody_key_path": str(cert_path / "users" / "nobody-key.pem"),
         }
 
 
@@ -120,27 +126,47 @@ def cli():
       different top-level directory. Defaults to '/tmp/'
     """,
 )
-def run(directory):
+@click.option(
+    '--metadata',
+    type=click.Path(),
+    help="Where to store the metadata.json and .envrc files"
+)
+def run(directory, metadata=None):
     """Start a gl-testserver instance to test against."""
     if not directory:
         directory = Path(tempfile.gettempdir())
     else:
         directory = Path(directory)
 
+    metadata = Path(metadata) if metadata else directory
+
     gl = build(base_dir=directory)
     try:
         meta = gl.metadata()
-        metafile = gl.directory / "metadata.json"
+        metafile = metadata / "metadata.json"
         metafile.parent.mkdir(parents=True, exist_ok=True)
         logger.debug(f"Writing testserver metadata to {metafile}")
         with metafile.open(mode="w") as f:
             json.dump(meta, f)
 
+        envfile = metadata / ".env"
+        logger.info(f"Writing .env file to {envfile}")
+        with envfile.open(mode="w") as f:
+            f.write(f"""\
+            export GL_SCHEDULER_GRPC_URI={meta['scheduler_grpc_uri']}
+            export GL_CERT_PATH={meta['cert_path']}
+            export GL_CA_CRT={meta['ca_crt_path']}
+            export GL_NOBODY_CRT={meta['nobody_crt_path']}
+            export GL_NOBODY_KEY={meta['nobody_key_path']}
+            export RUST_LOG=glclient=debub,info
+            """)
+
         pprint(meta)
         logger.info(
             "Server is up and running with the above config values. To stop press Ctrl-C."
         )
-        time.sleep(1800)
+        while True:
+            time.sleep(1800)
     except Exception as e:
         logger.warning(f"Caught exception running testserver: {e}")
         pass