From 3d053d0f9c69f5d249807571506758dbda49624a Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Fri, 29 Nov 2024 16:03:15 +0100 Subject: [PATCH 1/2] testserver: Add a couple of usability improvements The `.envrc` file can make it simpler to configure your shell. --- Makefile | 19 ++++++ docker/gl-testserver/Dockerfile | 71 +++++++++++++++++++++ libs/gl-testing/pyproject.toml | 2 +- libs/gl-testserver/gltestserver/__main__.py | 32 +++++++++- 4 files changed, 120 insertions(+), 4 deletions(-) create mode 100644 docker/gl-testserver/Dockerfile diff --git a/Makefile b/Makefile index aa77e979a..0899c73ed 100644 --- a/Makefile +++ b/Makefile @@ -191,3 +191,22 @@ docs-publish: docs --push \ --branch gh-pages \ --remote origin + +gltestserver-image: docker/gl-testserver/Dockerfile + docker build \ + --build-arg DOCKER_USER=$(shell whoami) \ + --build-arg UID=$(shell id -u) \ + --build-arg GID=$(shell id -g) \ + -t gltestserver \ + -f docker/gl-testserver/Dockerfile \ + . + +gltestserver: gltestserver-image + mkdir -p /tmp/gltestserver + docker run \ + --user $(shell id -u):$(shell id -g) \ + -e DOCKER_USER=$(shell whoami) \ + --net=host \ + -ti \ + -v $(shell pwd)/.testserver:/tmp/gltestserver \ + gltestserver diff --git a/docker/gl-testserver/Dockerfile b/docker/gl-testserver/Dockerfile new file mode 100644 index 000000000..a19c557b5 --- /dev/null +++ b/docker/gl-testserver/Dockerfile @@ -0,0 +1,71 @@ +FROM ubuntu:22.04 AS python-builder + +ENV RUST_VERSION=1.74 +ENV PATH=$CARGO_HOME/bin:$PATH +ENV PROTOC_VERSION=3.19.3 +ENV CFSSL_VERSION=1.6.5 +ENV GL_TESTING_IGNORE_HASH=true +ARG BITCOIN_VERSION=24.0 +ARG GID=0 +ARG UID=0 +ARG DOCKER_USER=dev +ENV PATH=$PATH:/home/$DOCKER_USER/.local/bin/:/opt/bitcoin/bin:/home/$DOCKER_USER/.cargo/bin +#ENV VIRTUAL_ENV=/tmp/venv +ENV CARGO_TARGET_DIR=/tmp/cargo + +RUN apt update && apt install -qqy \ + curl \ + python3 \ + python3-pip \ + python3-venv \ + libpq-dev \ + unzip \ + sudo \ + git \ + build-essential \ + wget + +RUN groupadd -g $GID -o $DOCKER_USER &&\ + useradd -m -u $UID -g $GID -G sudo -o -s /bin/bash $DOCKER_USER && \ + echo '%sudo ALL=(ALL:ALL) ALL' >> /etc/sudoers + +RUN wget -q https://github.com/cloudflare/cfssl/releases/download/v${CFSSL_VERSION}/cfssl_${CFSSL_VERSION}_linux_amd64 -O /usr/bin/cfssl && \ + chmod a+x /usr/bin/cfssl +RUN wget -q https://github.com/cloudflare/cfssl/releases/download/v${CFSSL_VERSION}/cfssljson_${CFSSL_VERSION}_linux_amd64 -O /usr/bin/cfssljson && \ + chmod a+x /usr/bin/cfssljson + +RUN mkdir /tmp/protoc && \ + cd /tmp/protoc && \ + wget --quiet \ + -O protoc.zip \ + https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip && \ + unzip protoc.zip && \ + sudo mv /tmp/protoc/bin/protoc /usr/local/bin && \ + chmod a+x /usr/local/bin/protoc && \ + rm -rf /tmp/protoc + +RUN cd /tmp/ && \ + wget "https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/bitcoin-${BITCOIN_VERSION}-x86_64-linux-gnu.tar.gz" -O bitcoin.tar.gz && \ + tar -xvzf bitcoin.tar.gz && \ + mv /tmp/bitcoin-$BITCOIN_VERSION/ /opt/bitcoin && \ + rm -rf bitcoin.tar.gz /tmp/bitcoin-$BITCOIN_VERSION + +ADD ../ /repo/libs +RUN chown $DOCKER_USER:users -R /repo + +USER $DOCKER_USER + +RUN curl \ + --proto '=https' \ + --tlsv1.2 \ + -sSf https://sh.rustup.rs | sh \ + -s -- -y --default-toolchain ${RUST_VERSION} +RUN rustup default stable + +RUN curl -LsSf https://astral.sh/uv/install.sh | sh + +WORKDIR /repo/libs/gl-testserver/ +RUN echo $PATH +RUN uv sync --locked -v +RUN uv run clnvm get-all +CMD uv run gltestserver run --metadata /tmp/gltestserver diff --git a/libs/gl-testing/pyproject.toml b/libs/gl-testing/pyproject.toml index b0df7cc16..850bdd326 100644 --- a/libs/gl-testing/pyproject.toml +++ b/libs/gl-testing/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ "gl-client", "grpcio-tools>=1.66", "grpcio>=1.66.0", - "httpx>=0.27.2", + "httpx[http2]>=0.27.2", "purerpc>=0.8.0", "pyln-client==24.2", "pyln-testing==24.2", diff --git a/libs/gl-testserver/gltestserver/__main__.py b/libs/gl-testserver/gltestserver/__main__.py index 5c7af870e..d7b07efde 100644 --- a/libs/gl-testserver/gltestserver/__main__.py +++ b/libs/gl-testserver/gltestserver/__main__.py @@ -10,6 +10,7 @@ import click import gltesting import json +import os import logging import tempfile import time @@ -47,10 +48,15 @@ def stop(self): def metadata(self): """Construct a dict of config values for this TestServer.""" + cert_path = Path(os.environ.get('GL_CERT_PATH')) return { "scheduler_grpc_uri": self.scheduler.grpc_addr, "grpc_web_proxy_uri": f"http://localhost:{self.grpc_web_proxy.web_port}", "bitcoind_rpc_uri": f"http://rpcuser:rpcpass@localhost:{self.bitcoind.rpcport}", + "cert_path": str(cert_path), + "ca_crt_path": str(cert_path / "ca.crt"), + "nobody_crt_path": str(cert_path / "users" / "nobody.crt"), + "nobody_key_path": str(cert_path / "users" / "nobody-key.pem"), } @@ -120,27 +126,47 @@ def cli(): different top-level directory. Defaults to '/tmp/' """, ) -def run(directory): +@click.option( + '--metadata', + type=click.Path(), + help="Where to store the metadata.json and .envrc files" +) +def run(directory, metadata=None): """Start a gl-testserver instance to test against.""" if not directory: directory = Path(tempfile.gettempdir()) else: directory = Path(directory) + metadata = Path(metadata) if metadata else directory + gl = build(base_dir=directory) try: meta = gl.metadata() - metafile = gl.directory / "metadata.json" + metafile = metadata / "metadata.json" metafile.parent.mkdir(parents=True, exist_ok=True) logger.debug(f"Writing testserver metadata to {metafile}") with metafile.open(mode="w") as f: json.dump(meta, f) + envfile = metadata / ".env" + logger.info(f"Writing .env file to {envfile}") + with envfile.open(mode="w") as f: + f.write(f"""\ + export GL_SCHEDULER_GRPC_URI={meta['scheduler_grpc_uri']} + export GL_CERT_PATH={meta['cert_path']} + export GL_CA_CRT={meta['ca_crt_path']} + export GL_NOBODY_CRT={meta['nobody_crt_path']} + export GL_NOBODY_KEY={meta['nobody_key_path']} + export RUST_LOG=glclient=debub,info + """) + pprint(meta) logger.info( "Server is up and running with the above config values. To stop press Ctrl-C." ) - time.sleep(1800) + while True: + time.sleep(1800) except Exception as e: logger.warning(f"Caught exception running testserver: {e}") pass From 6365106dabd2159d0b107f10e1802376a13d4b68 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Fri, 29 Nov 2024 19:15:16 +0100 Subject: [PATCH 2/2] fixup! testserver: Add a couple of usability improvements --- libs/gl-testing/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/gl-testing/pyproject.toml b/libs/gl-testing/pyproject.toml index 850bdd326..17868427e 100644 --- a/libs/gl-testing/pyproject.toml +++ b/libs/gl-testing/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ "gl-client", "grpcio-tools>=1.66", "grpcio>=1.66.0", - "httpx[http2]>=0.27.2", + "httpx[http2]==0.27.2", "purerpc>=0.8.0", "pyln-client==24.2", "pyln-testing==24.2",