use cln version manager

Makefile

@@ -31,16 +31,6 @@ CHANGED_RUST_SOURCES=$(shell git diff --name-only origin/main | grep '\.rs')
 # rebuild them easily.
 GENALL =
 
-CLN_VERSIONS = \
-	v0.10.1 \
-	v0.10.2 \
-	v0.11.0.1 \
-	v0.11.2gl2 \
-	v0.11.2 \
-	v22.11gl1 \
-	v23.05gl1 \
-	v23.08gl1
-
 DOCKER_OPTIONS= \
 	--rm \
 	--user $(shell id -u):$(shell id -g) \
@@ -51,8 +41,6 @@ DOCKER_OPTIONS= \
 	-v /tmp/gltesting/cargo/registry:/opt/cargo/registry \
 	-v ${REPO_ROOT}:/repo
 
-CLN_TARGETS = $(foreach VERSION,$(CLN_VERSIONS),cln-versions/$(VERSION)/usr/local/bin/lightningd)
-
 .PHONY: ensure-docker build-self check-self docker-image docs wheels
 
 include libs/gl-client/Makefile
@@ -138,16 +126,6 @@ docker-check-py: docker-volumes
 		${DOCKER_OPTIONS} \
 		gltesting make build-self check-py
 
-cln-versions/%/usr/local/bin/lightningd: cln-versions/lightningd-%.tar.bz2
-	@echo "Extracting $* from tarball $< into cln-versions/$*/"
-	mkdir -p "cln-versions/$*"
-	tar -xjf $< -C "cln-versions/$*/"
-
-cln-versions/lightningd-%.tar.bz2:
-	mkdir -p cln-versions
-	wget -q "https://storage.googleapis.com/greenlight-artifacts/cln/lightningd-$*.tar.bz2" -O $@
-
-
 cln: ${CLN_TARGETS}
 
 docs:

docker/gl-testing/Dockerfile

@@ -110,20 +110,15 @@ RUN chmod a+rwx -R $CARGO_HOME
 # -----------------------------------------
 # gl-testing requires multiple versions of Core Lightning
 # Download all versions so they can be copied to our dev-container
-FROM ubuntu:20.04 as cln-downloader
-RUN apt update && apt install wget make git -qqy
+FROM python-builder as cln-downloader
 
-# Add all the makefiles
-# Althoug we only need the main one here we have to include all of them
-ADD Makefile /repo/Makefile
-ADD libs/gl-testing/Makefile /repo/libs/gl-testing/Makefile
-ADD libs/gl-client-py/Makefile /repo/libs/gl-client-py/Makefile
-ADD libs/gl-client/Makefile /repo/libs/gl-client/Makefile
+RUN mkdir -p /opt/cln/
+ENV CLNVM_CACHE_DIR=/opt/cln
 
-WORKDIR /repo
+ADD libs/cln-version-manager /repo/libs/cln-version-manager
+RUN cd /repo/libs/cln-version-manager; python -m pip install -e .
 
-RUN git init && make cln
-RUN chmod a+x -R /repo/cln-versions/
+RUN python -m clnvm get-all
 
 # -------------------------------------
 # STAGE: bitcoin-downloader
@@ -225,12 +220,8 @@ ENV RUST_LOG=gl_client=trace,tracing=warn,gl_signerproxy=trace,gl_plugin=trace,l
 ENV GRPC_ENABLE_FORK_SUPPORT=0
 
 # Install cln-versions
-COPY --from=cln-downloader /repo/cln-versions/ /opt/cln
-RUN ln -s /opt/cln/v23.08gl1 /opt/cln-latest
-ENV PATH=/opt/cln-latest/usr/local/bin:$PATH
-
-# Enumerate all versions that gl-testing should find
-ENV CLN_PATH=/opt/cln/v0.10.1/usr/local/bin/:/opt/cln/v0.10.2/usr/local/bin/:/opt/cln/v0.11.0.1/usr/local/bin/:/opt/cln/v0.11.2gl2/usr/local/bin/:/opt/cln/v22.11gl1/usr/local/bin/:/opt/cln/v23.05gl1/usr/local/bin/:/opt/cln/v23.08gl1/usr/local/bin/
+COPY --from=cln-downloader /opt/cln /opt/cln
+ENV CLNVM_CACHE_DIR=/opt/cln
 
 # Install bitcoin-core
 COPY --from=bitcoin-downloader /opt/bitcoin/bin /opt/bitcoin/bin
@@ -259,10 +250,6 @@ RUN groupadd -g $GID -o $DOCKER_USER &&\
     useradd -m -u $UID -g $GID -G sudo -o -s /bin/bash $DOCKER_USER && \
     echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 
-# Configure clnvm
-RUN mkdir -p /opt/clnvm; chmod a+rw /opt/clnvm
-ENV CLNVM_CACHE_DIR=/opt/clnvm
-
 # Create the required tmp-dicts
 RUN chmod a+rw /tmp
 RUN mkdir -p /tmp/gltesting/cargo && mkdir -p /tmp/gltesting/tmp
@@ -274,6 +261,9 @@ WORKDIR /repo
 RUN poetry install
 RUN chmod -R a+rw /tmp/venv
 
+# Create a symlink to the latest cln-version and add it to the path
+RUN ln -s $(clnvm latest --root-path) /opt/cln-latest
+ENV PATH=/opt/cln-latest/usr/local/bin:$PATH
 
 
 

libs/cln-version-manager/clnvm/cli.py

@@ -1,25 +1,35 @@
 import importlib
+import logging
+import logging.config
+from pathlib import Path
+
 import os
 import sys
 from typing import Optional
 
 import clnvm
 from clnvm.cln_version_manager import ClnVersionManager, VersionDescriptor
 
+def configure_logging() -> None:
+    dirname, _ = os.path.split(__file__)
+    logging.config.fileConfig(Path(dirname) / "logging.conf")
+
 # Handle the optional import and provide a nice error message if it fails
 _click = importlib.util.find_spec("click")
 if _click is None:
     print("To use clnvm the `cli` feature must be installed")
     print("You can install the feature using")
     print("> pip install gltesting[cli]")
-    exit()
+    sys.exit(1)
 
 import click
 
 
 @click.group()
-def cli() -> None:
-    pass
+@click.option("--verbose", is_flag=True)
+def cli(verbose: bool) -> None:
+    if verbose:
+        configure_logging()
 
 
 @cli.command()
@@ -33,6 +43,7 @@ def get_all(force: bool) -> None:
             click.echo(result.lightningd)
         except Exception as e:
             click.echo(click.style(str(e), fg="red"), err=True)
+            sys.exit(1)
 
 
 @cli.command()

libs/cln-version-manager/clnvm/cln_version_manager.py

@@ -67,7 +67,7 @@ class VersionDescriptor:
     VersionDescriptor(
         tag="v23.08gl1",
         url="https://storage.googleapis.com/greenlight-artifacts/cln/lightningd-v23.08gl1.tar.bz2",
-        checksum="700e2f4765ba2f8d83ea26e0dc5b47c7176b665c7d6945bdfed2207c90ed9ebd",
+        checksum="0e392c5117e14dc37cf72393f47657a09821f69ab8b45937d7e79ca8d91d17e9",
     ),
 ]
 
@@ -188,7 +188,7 @@ def get(self, cln_version: VersionDescriptor, force: bool = False) -> NodeVersio
             root_path=target_path,
         )
 
-    def _download(self, cln_version: VersionDescriptor, target_path: Path) -> None:
+    def _download(self, cln_version: VersionDescriptor, target_path: Path, verify_tag:bool = False) -> None:
         """Downloads the provided cln_version"""
         tag = cln_version.tag
         logger.info("Downloading version %s to %s", tag, target_path)
@@ -249,22 +249,23 @@ def _download(self, cln_version: VersionDescriptor, target_path: Path) -> None:
         else:
             tf.extractall(path=target_path)
 
-        try:
-            # We verify if the path matches the version
-            lightningd_path = target_path / _LIGHTNINGD_REL_PATH
-            version = (
-                subprocess.check_output([lightningd_path, "--version"])
-                .strip()
-                .decode("ASCII")
-            )
-        except Exception as e:
-            # Clean-up the bad version
-            shutil.rmtree(target_path)
-            raise UnrunnableVersion(tag=cln_version.tag) from e
-
-        if version != tag:
-            # Clean-up teh bad version
-            shutil.rmtree(target_path)
-            raise VersionMismatch(expected=tag, actual=version)
+        if verify_tag:
+            try:
+                # We verify if the path matches the version
+                lightningd_path = target_path / _LIGHTNINGD_REL_PATH
+                version = (
+                    subprocess.check_output([lightningd_path, "--version"])
+                    .strip()
+                    .decode("ASCII")
+                )
+            except Exception as e:
+                # Clean-up the bad version
+                shutil.rmtree(target_path)
+                raise UnrunnableVersion(tag=cln_version.tag) from e
+
+            if version != tag:
+                # Clean-up the bad version
+                shutil.rmtree(target_path)
+                raise VersionMismatch(expected=tag, actual=version)
 
         return

libs/cln-version-manager/clnvm/logging.conf

@@ -0,0 +1,27 @@
+[loggers]
+keys=root,clnvm
+
+[handlers]
+keys=consoleHandler
+
+[formatters]
+keys=simpleFormatter
+
+[logger_root]
+level=DEBUG
+handlers=consoleHandler
+
+[logger_clnvm]
+level=DEBUG
+handlers=consoleHandler
+qualname=clnvm
+propagate=0
+
+[handler_consoleHandler]
+class=StreamHandler
+level=DEBUG
+formatter=simpleFormatter
+args=(sys.stdout,)
+
+[formatter_simpleFormatter]
+format=%(asctime)s - %(name)s - %(levelname)s - %(message)s

libs/gl-testing/gltesting/scheduler.py

@@ -11,7 +11,7 @@
 from dataclasses import dataclass
 from pathlib import Path
 from threading import Condition
-from typing import List, Optional
+from typing import Dict, List, Optional
 
 import anyio
 import purerpc
@@ -26,6 +26,7 @@
 from gltesting.node import NodeProcess
 from gltesting.utils import Network, NodeVersion, SignerVersion
 
+from clnvm import ClnVersionManager
 
 @dataclass
 class Node:
@@ -59,35 +60,11 @@ class InviteCode:
     is_redeemed: bool
 
 
-def enumerate_cln_versions():
+def enumerate_cln_versions() -> Dict[str, NodeVersion]:
     """Search `$PATH` and `$CLN_PATH` for CLN versions."""
-    path = os.environ["PATH"].split(":")
-    path += os.environ.get("CLN_PATH", "").split(":")
-    path = [p for p in path if p != ""]
-
-    logging.debug(f"Looking for CLN versions in {path}")
-
-    version_paths = [shutil.which("lightningd", path=p) for p in path]
-    version_paths = [p for p in version_paths if p is not None]
-
-    versions = {}
-    for v in version_paths:
-        lightningd = Path(v).resolve()
-        path_parts = lightningd.parts
-        bin_path = Path(*path_parts[:-1])
-        root_path = Path(*path_parts[:-4])
-
-        logging.debug(f"Detecting version of lightningd at {v}")
-        vs = subprocess.check_output([v, "--version"]).strip().decode("ASCII")
-        versions[vs] = NodeVersion(
-            lightningd=lightningd,
-            bin_path=bin_path,
-            root_path=root_path,
-            name=vs)
-        logging.debug(f"Determined version {vs} for executable {v}")
-
-    logging.info(f"Found {len(versions)} versions: {versions}")
-    return versions
+    manager = ClnVersionManager()
+    return manager.get_all()
+
 
 def generate_secret(len=5):
     return "".join(random.choices(string.ascii_uppercase, k=len))