From 417e73cef12cd159115358258fb8be4e8a22c5b6 Mon Sep 17 00:00:00 2001 From: Marc Richter Date: Wed, 8 Apr 2020 22:58:34 +0200 Subject: [PATCH 01/19] Updated installer with latest versions. --- jupyterhub_traefik_proxy/install.py | 238 ++++++++++++++-------------- 1 file changed, 119 insertions(+), 119 deletions(-) diff --git a/jupyterhub_traefik_proxy/install.py b/jupyterhub_traefik_proxy/install.py index ef2b4dcc..81cc3c7a 100644 --- a/jupyterhub_traefik_proxy/install.py +++ b/jupyterhub_traefik_proxy/install.py @@ -10,37 +10,30 @@ import warnings checksums_traefik = { - "https://github.com/traefik/traefik/releases/download/v1.7.29/traefik_linux-arm64": "d27c220bdcc8bae33436adce309fd856c2ee295bd3dd5416428d3b4a173b8310", - "https://github.com/traefik/traefik/releases/download/v1.7.29/traefik_linux-amd64": "70cd8847354326fb17acd10251c44450cf5d6c4fd8df130f2c6f86dd7b1b52d1", - "https://github.com/traefik/traefik/releases/download/v1.7.29/traefik_darwin-amd64": "bbe30c8e7aa5e76442187be409c07e6b798e7ba67d7d3d60856e0a7664654c46", - "https://github.com/containous/traefik/releases/download/v1.7.28/traefik_linux-amd64": "b70284ac72b4f9a119be92f206fc0c6dbc0db18ff7295d4df6701c0b292ecbf0", - "https://github.com/containous/traefik/releases/download/v1.7.28/traefik_darwin-amd64": "3e4bb0146bed06c842ae7a91e711e5ba98339f529b84aa80c766a01dd39d9731", - "https://github.com/containous/traefik/releases/download/v1.7.18/traefik_linux-amd64": "3c2d153d80890b6fc8875af9f8ced32c4d684e1eb5a46d9815337cb343dfd92e", - "https://github.com/containous/traefik/releases/download/v1.7.18/traefik_darwin-amd64": "84e07a184c31b7fb86417ba3a237ad334a26bcb1ed53fd56f0774afaa34074d9", - "https://github.com/containous/traefik/releases/download/v1.7.5/traefik_linux-amd64": "4417a9d83753e1ad6bdd64bbbeaeb4b279bcc71542e779b7bcb3b027c6e3356e", - "https://github.com/containous/traefik/releases/download/v1.7.5/traefik_darwin-amd64": "379d4af242743a3fe44b44a1ee6df68ea8332578d85de35f264e062c19fd20a0", - "https://github.com/containous/traefik/releases/download/v1.7.0/traefik_linux-amd64": "b84cb03e8a175b8b7d1a30246d19705f607c6ae5ee89f2dca7a1adccab919135", - "https://github.com/containous/traefik/releases/download/v1.7.0/traefik_darwin-amd64": "3000cb9f8ed567e9bc567cce33107f6877f2017c69fae8ac235b51a7a94229bf", + "https://github.com/containous/traefik/releases/download/v2.2.0/traefik_v2.2.0_linux_amd64.tar.gz": + "eddea0507ad715c723662e7c10fdab554eb64379748278cd2d09403063e3e32f", + "https://github.com/containous/traefik/releases/download/v2.2.0/traefik_v2.2.0_darwin_amd64.tar.gz": + "8bfa2393b265ef01aca12be94d67080961299968bd602f3708480eed273b95e0", + "https://github.com/containous/traefik/releases/download/v2.2.0/traefik_v2.2.0_windows_amd64.zip": + "9a794e395b7eba8d44118c4a1fb358fbf14abff3f5f5d264f46b1d6c243b9a5e", } checksums_etcd = { - "https://github.com/etcd-io/etcd/releases/download/v3.4.15/etcd-v3.4.15-linux-arm64.tar.gz": "fcc522275300cf90d42377106d47a2e384d1d2083af205cbb7833a79ef5a49d1", - "https://github.com/etcd-io/etcd/releases/download/v3.4.15/etcd-v3.4.15-linux-amd64.tar.gz": "3bd00836ea328db89ecba3ed2155293934c0d09e64b53d6c9dfc0a256e724b81", - "https://github.com/etcd-io/etcd/releases/download/v3.4.15/etcd-v3.4.15-darwin-amd64.tar.gz": "c596709069193bffc639a22558bdea4d801128e635909ea01a6fd5b5c85da729", - "https://github.com/etcd-io/etcd/releases/download/v3.3.10/etcd-v3.3.10-linux-amd64.tar.gz": "1620a59150ec0a0124a65540e23891243feb2d9a628092fb1edcc23974724a45", - "https://github.com/etcd-io/etcd/releases/download/v3.3.10/etcd-v3.3.10-darwin-amd64.tar.gz": "fac4091c7ba6f032830fad7809a115909d0f0cae5cbf5b34044540def743577b", - "https://github.com/etcd-io/etcd/releases/download/v3.2.25/etcd-v3.3.10-linux-amd64.tar.gz": "8a509ffb1443088d501f19e339a0d9c0058ce20599752c3baef83c1c68790ef7", - "https://github.com/etcd-io/etcd/releases/download/v3.2.25/etcd-v3.3.10-darwin-amd64.tar.gz": "9950684a01d7431bc12c3dba014f222d55a862c6f8af64c09c42d7a59ed6790d", + "https://github.com/etcd-io/etcd/releases/download/v3.4.7/etcd-v3.4.7-linux-amd64.tar.gz": + "4ad86e663b63feb4855e1f3a647e719d6d79cf6306410c52b7f280fa56f8eb6b", + "https://github.com/etcd-io/etcd/releases/download/v3.4.7/etcd-v3.4.7-darwin-amd64.zip": + "ffe3237fcb70b7ce91c16518c2f62f3fa9ff74ddc10f7b6ca83a3b5b29ade19a", + "https://github.com/etcd-io/etcd/releases/download/v3.4.7/etcd-v3.4.7-windows-amd64.zip": + "3863ea59bcb407113524b51406810e33d58daff11ca10d1192f289185ae94ffe", } checksums_consul = { - "https://releases.hashicorp.com/consul/1.9.4/consul_1.9.4_darwin.zip": "c168240d52f67c71b30ef51b3594673cad77d0dbbf38c412b2ee30b39ef30843", - "https://releases.hashicorp.com/consul/1.9.4/consul_1.9.4_linux_amd64.zip": "da3919197ef33c4205bb7df3cc5992ccaae01d46753a72fe029778d7f52fb610", - "https://releases.hashicorp.com/consul/1.9.4/consul_1.9.4_linux_arm64.zip": "012c552aff502f907416c9a119d2dfed88b92e981f9b160eb4fe292676afdaeb", - "https://releases.hashicorp.com/consul/1.6.1/consul_1.6.1_linux_amd64.zip": "a8568ca7b6797030b2c32615b4786d4cc75ce7aee2ed9025996fe92b07b31f7e", - "https://releases.hashicorp.com/consul/1.6.1/consul_1.6.1_darwin_amd64.zip": "4bc205e06b2921f998cb6ddbe70de57f8e558e226e44aba3f337f2f245678b85", - "https://releases.hashicorp.com/consul/1.5.0/consul_1.5.0_linux_amd64.zip": "1399064050019db05d3378f757e058ec4426a917dd2d240336b51532065880b6", - "https://releases.hashicorp.com/consul/1.5.0/consul_1.5.0_darwin_amd64.zip": "b4033ea6871fe6136ee5d940c834be2248463c3ec248dc22370e6d5360931325", + "https://releases.hashicorp.com/consul/1.7.2/consul_1.7.2_linux_amd64.zip": + "5ab689cad175c08a226a5c41d16392bc7dd30ceaaf90788411542a756773e698", + "https://releases.hashicorp.com/consul/1.7.2/consul_1.7.2_darwin_amd64.zip": + "c474f00b022cae38acae2d19b2a707a4fcb08dfdd22875efeefdf052ce19c90b", + "https://releases.hashicorp.com/consul/1.7.2/consul_1.7.2_windows_amd64.zip": + "e9b9355f77f80b2c0940888cb0d27c44a5879c31e379ef21ffcfd36c91d202c1", } @@ -54,54 +47,75 @@ def checksum_file(path): def install_traefik(prefix, plat, traefik_version): - traefik_bin = os.path.join(prefix, "traefik") + plat = plat.replace("-", "_") + if "windows" in plat: + traefik_archive_extension = "zip" + traefik_bin = os.path.join(prefix, "traefik.exe") + else: + traefik_archive_extension = "tar.gz" + traefik_bin = os.path.join(prefix, "traefik") + + traefik_archive = "traefik_v" + traefik_version + "_" + plat + "." + traefik_archive_extension + traefik_archive_path = os.path.join(prefix, traefik_archive) + + traefik_archive = "traefik_v" + traefik_version + "_" + plat + "." + traefik_archive_extension + traefik_archive_path = os.path.join(prefix, traefik_archive) + + traefik_archive = "traefik_v" + traefik_version + "_" + plat + "." + traefik_archive_extension + traefik_archive_path = os.path.join(prefix, traefik_archive) traefik_url = ( "https://github.com/containous/traefik/releases" - f"/download/v{traefik_version}/traefik_{plat}" + f"/download/v{traefik_version}/{traefik_archive}" ) - if os.path.exists(traefik_bin): + if os.path.exists(traefik_bin) and os.path.exists(traefik_archive_path): print(f"Traefik already exists") if traefik_url not in checksums_traefik: warnings.warn( - f"Couldn't verify checksum for traefik-v{traefik_version}-{plat}", + f"Traefik {traefik_version} not supported !", stacklevel=2, ) os.chmod(traefik_bin, 0o755) print("--- Done ---") return else: - checksum = checksum_file(traefik_bin) - if checksum == checksums_traefik[traefik_url]: + if checksum_file(traefik_archive_path) == checksums_traefik[traefik_url]: os.chmod(traefik_bin, 0o755) print("--- Done ---") return else: - print(f"checksum mismatch on {traefik_bin}") + print(f"checksum mismatch on {traefik_archive_path}") + os.remove(traefik_archive_path) os.remove(traefik_bin) - print(f"Downloading traefik {traefik_version}...") - urlretrieve(traefik_url, traefik_bin) - if traefik_url in checksums_traefik: - checksum = checksum_file(traefik_bin) - if checksum != checksums_traefik[traefik_url]: + print(f"Downloading traefik {traefik_version}...") + urlretrieve(traefik_url, traefik_archive_path) + + if checksum_file(traefik_archive_path) != checksums_traefik[traefik_url]: raise IOError("Checksum failed") + + print("Extracting the archive...") + if traefik_archive_extension == "tar.gz": + with tarfile.open(traefik_archive_path, "r") as tar_ref: + tar_ref.extract("traefik", prefix) + else: + with zipfile.ZipFile(traefik_archive_path, "r") as zip_ref: + zip_ref.extract("traefik.exe", prefix) + + os.chmod(traefik_bin, 0o755) else: warnings.warn( - f"Couldn't verify checksum for traefik-v{traefik_version}-{plat}", + f"Traefik {traefik_version} not supported !", stacklevel=2, ) - os.chmod(traefik_bin, 0o755) - print("--- Done ---") def install_etcd(prefix, plat, etcd_version): etcd_downloaded_dir_name = f"etcd-v{etcd_version}-{plat}" - etcd_archive_extension = ".tar.gz" if "linux" in plat: etcd_archive_extension = "tar.gz" else: @@ -115,7 +129,7 @@ def install_etcd(prefix, plat, etcd_version): etcdctl_bin = os.path.join(prefix, "etcdctl") etcd_url = ( - "https://github.com/etcd-io/etcd/releases/" + "https://github.com/etcd-io/etcd/releases" f"/download/v{etcd_version}/etcd-v{etcd_version}-{plat}.{etcd_archive_extension}" ) @@ -123,7 +137,7 @@ def install_etcd(prefix, plat, etcd_version): print(f"Etcd and etcdctl already exist") if etcd_url not in checksums_etcd: warnings.warn( - f"Couldn't verify checksum for etcd-v{etcd_version}-{plat}", + f"Etcd {etcd_version} not supported !", stacklevel=2, ) os.chmod(etcd_bin, 0o755) @@ -131,68 +145,65 @@ def install_etcd(prefix, plat, etcd_version): print("--- Done ---") return else: - checksum_etcd_archive = checksum_file(etcd_downloaded_archive) - if checksum_etcd_archive == checksums_etcd[etcd_url]: + if checksum_file(etcd_downloaded_archive) == checksums_etcd[etcd_url]: os.chmod(etcd_bin, 0o755) os.chmod(etcdctl_bin, 0o755) print("--- Done ---") return else: - print(f"checksum mismatch on etcd") + print(f"checksum mismatch on {etcd_downloaded_archive}") os.remove(etcd_bin) os.remove(etcdctl_bin) os.remove(etcd_downloaded_archive) - if not os.path.exists(etcd_downloaded_archive): - print(f"Downloading {etcd_downloaded_dir_name} archive...") - urlretrieve(etcd_url, etcd_downloaded_archive) - else: - print(f"Archive {etcd_downloaded_dir_name} already exists") - - if etcd_archive_extension == "zip": - with zipfile.ZipFile(etcd_downloaded_archive, "r") as zip_ref: - zip_ref.extract(etcd_downloaded_dir_name + "/etcd", etcd_binaries) - zip_ref.extract(etcd_downloaded_dir_name + "/etcdctl", etcd_binaries) - else: - with (tarfile.open(etcd_downloaded_archive, "r")) as tar_ref: - print("Extracting the archive...") - tar_ref.extract(etcd_downloaded_dir_name + "/etcd", etcd_binaries) - tar_ref.extract(etcd_downloaded_dir_name + "/etcdctl", etcd_binaries) - - shutil.copy(os.path.join(etcd_binaries, etcd_downloaded_dir_name, "etcd"), etcd_bin) - shutil.copy( - os.path.join(etcd_binaries, etcd_downloaded_dir_name, "etcdctl"), etcdctl_bin - ) - if etcd_url in checksums_etcd: - checksum_etcd_archive = checksum_file(etcd_downloaded_archive) - if checksum_etcd_archive != checksums_etcd[etcd_url]: + if not os.path.exists(etcd_downloaded_archive): + print(f"Downloading {etcd_downloaded_dir_name} archive...") + urlretrieve(etcd_url, etcd_downloaded_archive) + else: + print(f"Archive {etcd_downloaded_dir_name} already exists") + + if checksum_file(etcd_downloaded_archive) != checksums_etcd[etcd_url]: raise IOError("Checksum failed") + + print("Extracting the archive...") + + if etcd_archive_extension == "zip": + with zipfile.ZipFile(etcd_downloaded_archive, "r") as zip_ref: + zip_ref.extract(etcd_downloaded_dir_name + "/etcd", etcd_binaries) + zip_ref.extract(etcd_downloaded_dir_name + "/etcdctl", etcd_binaries) + else: + with (tarfile.open(etcd_downloaded_archive, "r")) as tar_ref: + tar_ref.extract(etcd_downloaded_dir_name + "/etcd", etcd_binaries) + tar_ref.extract(etcd_downloaded_dir_name + "/etcdctl", etcd_binaries) + + shutil.copy(os.path.join(etcd_binaries, etcd_downloaded_dir_name, "etcd"), etcd_bin) + shutil.copy( + os.path.join(etcd_binaries, etcd_downloaded_dir_name, "etcdctl"), etcdctl_bin) + + os.chmod(etcd_bin, 0o755) + os.chmod(etcdctl_bin, 0o755) + + # Cleanup + shutil.rmtree(etcd_binaries) else: warnings.warn( - f"Couldn't verify checksum for etcd-v{etcd_version}-{plat}", stacklevel=2 + f"Etcd {etcd_version} not supported !", + stacklevel=2 ) - os.chmod(etcd_bin, 0o755) - os.chmod(etcdctl_bin, 0o755) - - # Cleanup - shutil.rmtree(etcd_binaries) - print("--- Done ---") def install_consul(prefix, plat, consul_version): plat = plat.replace("-", "_") consul_downloaded_dir_name = f"consul_v{consul_version}_{plat}" - consul_archive_extension = ".tar.gz" consul_archive_extension = "zip" consul_downloaded_archive = os.path.join( prefix, consul_downloaded_dir_name + "." + consul_archive_extension ) consul_binaries = os.path.join(prefix, "consul_binaries") - consul_bin = os.path.join(prefix, "consul") consul_url = ( @@ -204,49 +215,45 @@ def install_consul(prefix, plat, consul_version): print(f"Consul already exists") if consul_url not in checksums_consul: warnings.warn( - f"Couldn't verify checksum for consul_v{consul_version}_{plat}", + f"Consul {consul_version} not supported !", stacklevel=2, ) os.chmod(consul_bin, 0o755) print("--- Done ---") return else: - checksum_consul_archive = checksum_file(consul_downloaded_archive) - if checksum_consul_archive == checksums_consul[consul_url]: + if checksum_file(consul_downloaded_archive) == checksums_consul[consul_url]: os.chmod(consul_bin, 0o755) print("--- Done ---") return else: - print(f"checksum mismatch on consul") + print(f"checksum mismatch on {consul_downloaded_archive}") os.remove(consul_bin) os.remove(consul_downloaded_archive) - if not os.path.exists(consul_downloaded_archive): - print(f"Downloading {consul_downloaded_dir_name} archive...") - urlretrieve(consul_url, consul_downloaded_archive) - else: - print(f"Archive {consul_downloaded_dir_name} already exists") + if consul_url in checksums_consul: + if not os.path.exists(consul_downloaded_archive): + print(f"Downloading {consul_downloaded_dir_name} archive...") + urlretrieve(consul_url, consul_downloaded_archive) + else: + print(f"Archive {consul_downloaded_dir_name} already exists") - with zipfile.ZipFile(consul_downloaded_archive, "r") as zip_ref: - zip_ref.extract("consul", consul_binaries) + if checksum_file(consul_downloaded_archive) != checksums_consul[consul_url]: + raise IOError("Checksum failed") - shutil.copy(os.path.join(consul_binaries, "consul"), consul_bin) + with zipfile.ZipFile(consul_downloaded_archive, "r") as zip_ref: + zip_ref.extract("consul", consul_binaries) - if consul_url in checksums_consul: - checksum_consul_archive = checksum_file(consul_downloaded_archive) - if checksum_consul_archive != checksums_consul[consul_url]: - raise IOError("Checksum failed") + shutil.copy(os.path.join(consul_binaries, "consul"), consul_bin) + os.chmod(consul_bin, 0o755) + # Cleanup + shutil.rmtree(consul_binaries) else: warnings.warn( - f"Couldn't verify checksum for consul_v{consul_version}_{plat}", + f"Consul {consul_version} not supported !", stacklevel=2, ) - os.chmod(consul_bin, 0o755) - - # Cleanup - shutil.rmtree(consul_binaries) - print("--- Done ---") @@ -258,24 +265,17 @@ def main(): """\ Checksums available for: - traefik: - - v1.7.28-linux-amd64 - - v1.7.28-darwin-amd64 - - v1.7.18-linux-amd64 - - v1.7.18-darwin-amd64 - - v1.7.5-linux-amd64 - - v1.7.5-darwin-amd64 - - v1.7.0-linux-amd64 - - v1.7.0-darwin-amd64 + - v2.2.0-linux-amd64 + - v2.2.0-darwin-amd64 + - v2.2.0-windows-amd64 - etcd: - - v3.3.10-linux-amd64 - - v3.3.10-darwin-amd64 - - v3.2.25-linux-amd64 - - v3.2.25-darwin-amd64 + - v3.4.7-linux-amd64 + - v3.4.7-darwin-amd64 + - v3.4.7-windows-amd64 - consul: - - v1.5.0_linux_amd64 - - v1.5.0_darwin_amd64 - - v1.6.1_linux_amd64 - - v1.6.1_darwin_amd64 + - v1.7.2_linux_amd64 + - v1.7.2_darwin_amd64 + - v1.7.2_windows_amd64 """ ), formatter_class=argparse.RawTextHelpFormatter, @@ -324,7 +324,7 @@ def main(): parser.add_argument( "--traefik-version", dest="traefik_version", - default="1.7.28", + default="2.2.0", help=textwrap.dedent( """\ The version of traefik to download. @@ -348,7 +348,7 @@ def main(): parser.add_argument( "--etcd-version", dest="etcd_version", - default="3.3.10", + default="3.4.7", help=textwrap.dedent( """\ The version of etcd to download. @@ -372,7 +372,7 @@ def main(): parser.add_argument( "--consul-version", dest="consul_version", - default="1.6.1", + default="1.7.2", help=textwrap.dedent( """\ The version of consul to download. From f36b2f0bd48b0c5b5dc33118866f8ecb00a3b7cb Mon Sep 17 00:00:00 2001 From: Alex Leach Date: Fri, 11 Jun 2021 15:02:45 +0000 Subject: [PATCH 02/19] Upgrade the jupyterhub_traefik_proxy to work with the Traefik v2 API. - Renamed frontends and backedns to routers and services, respectively. - Traefik API paths at /api/providers/{provider_name} no longer work, so search /api/http/{services|routers} instead - Traefik file provider doesn't like arbitrary data (any more?), so have put JupyterHub's 'data' object into the dynamic configuration file in separate root keys. To Do: - Haven't touched the consul or etcd providers, other than to rename 'frontends' and 'backends', as above. - Test, test, test. Additionally, have renamed TraefikTomlProxy to TraefikFileProviderProxy, and edited relevant traefik_utils functions so the proxy provider should now work with either yaml or toml files. (need to test). - jupyterhub_config.py will now need to reference the proxy class 'traefik_file', e.g.:- c.JupyterHub.proxy_class = 'traefik_file' c.TraefikFileProviderProxy = '/path/to/rules.toml' --- jupyterhub_traefik_proxy/__init__.py | 2 +- .../{toml.py => fileprovider.py} | 127 +++++++++------ jupyterhub_traefik_proxy/kv_proxy.py | 34 ++-- jupyterhub_traefik_proxy/proxy.py | 35 +++-- jupyterhub_traefik_proxy/traefik_utils.py | 147 ++++++++++-------- setup.py | 2 +- 6 files changed, 207 insertions(+), 140 deletions(-) rename jupyterhub_traefik_proxy/{toml.py => fileprovider.py} (60%) diff --git a/jupyterhub_traefik_proxy/__init__.py b/jupyterhub_traefik_proxy/__init__.py index 45346b81..9f1e608a 100644 --- a/jupyterhub_traefik_proxy/__init__.py +++ b/jupyterhub_traefik_proxy/__init__.py @@ -2,7 +2,7 @@ from .proxy import TraefikProxy # noqa from .kv_proxy import TKvProxy # noqa -from .toml import TraefikTomlProxy +from .fileprovider import TraefikFileProviderProxy from ._version import get_versions diff --git a/jupyterhub_traefik_proxy/toml.py b/jupyterhub_traefik_proxy/fileprovider.py similarity index 60% rename from jupyterhub_traefik_proxy/toml.py rename to jupyterhub_traefik_proxy/fileprovider.py index e20e8a38..ca383823 100644 --- a/jupyterhub_traefik_proxy/toml.py +++ b/jupyterhub_traefik_proxy/fileprovider.py @@ -31,8 +31,8 @@ from jupyterhub_traefik_proxy import TraefikProxy -class TraefikTomlProxy(TraefikProxy): - """JupyterHub Proxy implementation using traefik and toml config file""" +class TraefikFileProviderProxy(TraefikProxy): + """JupyterHub Proxy implementation using traefik and toml or yaml config file""" mutex = Any() @@ -40,7 +40,7 @@ class TraefikTomlProxy(TraefikProxy): def _default_mutex(self): return asyncio.Lock() - toml_dynamic_config_file = Unicode( + dynamic_config_file = Unicode( "rules.toml", config=True, help="""traefik's dynamic configuration file""" ) @@ -48,30 +48,34 @@ def __init__(self, **kwargs): super().__init__(**kwargs) try: # Load initial routing table from disk - self.routes_cache = traefik_utils.load_routes(self.toml_dynamic_config_file) + self.routes_cache = traefik_utils.load_routes(self.dynamic_config_file) except FileNotFoundError: self.routes_cache = {} if not self.routes_cache: - self.routes_cache = {"backends": {}, "frontends": {}} + self.routes_cache = { + "http" : {"services": {}, "routers": {}}, + "jupyter": {"routers" : {} } + } async def _setup_traefik_static_config(self): await super()._setup_traefik_static_config() + # Is this not the same as the dynamic config file? self.static_config["file"] = {"filename": "rules.toml", "watch": True} try: traefik_utils.persist_static_conf( - self.toml_static_config_file, self.static_config + self.static_config_file, self.static_config ) try: - os.stat(self.toml_dynamic_config_file) + os.stat(self.dynamic_config_file) except FileNotFoundError: # Make sure that the dynamic configuration file exists self.log.info( - f"Creating the toml dynamic configuration file: {self.toml_dynamic_config_file}" + f"Creating the dynamic configuration file: {self.dynamic_config_file}" ) - open(self.toml_dynamic_config_file, "a").close() + open(self.dynamic_config_file, "a").close() except IOError: self.log.exception("Couldn't set up traefik's static config.") raise @@ -82,7 +86,7 @@ async def _setup_traefik_static_config(self): def _start_traefik(self): self.log.info("Starting traefik...") try: - self._launch_traefik(config_type="toml") + self._launch_traefik(config_type="fileprovider") except FileNotFoundError as e: self.log.error( "Failed to find traefik \n" @@ -93,24 +97,24 @@ def _start_traefik(self): def _clean_resources(self): try: if self.should_start: - os.remove(self.toml_static_config_file) - os.remove(self.toml_dynamic_config_file) + os.remove(self.static_config_file) + os.remove(self.dynamic_config_file) except: self.log.error("Failed to remove traefik's configuration files") raise def _get_route_unsafe(self, traefik_routespec): - backend_alias = traefik_utils.generate_alias(traefik_routespec, "backend") - frontend_alias = traefik_utils.generate_alias(traefik_routespec, "frontend") + service_alias = traefik_utils.generate_alias(traefik_routespec, "service") + router_alias = traefik_utils.generate_alias(traefik_routespec, "router") routespec = self._routespec_from_traefik_path(traefik_routespec) - result = {"data": "", "target": "", "routespec": routespec} + result = {"data": None, "target": None, "routespec": routespec} def get_target_data(d, to_find): if to_find == "url": key = "target" else: key = to_find - if result[key]: + if result[key] is not None: return for k, v in d.items(): if k == to_find: @@ -118,17 +122,35 @@ def get_target_data(d, to_find): if isinstance(v, dict): get_target_data(v, to_find) - if backend_alias in self.routes_cache["backends"]: - get_target_data(self.routes_cache["backends"][backend_alias], "url") + service_node = self.routes_cache["http"]["services"].get(service_alias, None) + if service_node is not None: + get_target_data(service_node, "url") - if frontend_alias in self.routes_cache["frontends"]: - get_target_data(self.routes_cache["frontends"][frontend_alias], "data") + router_node = self.routes_cache["jupyter"]["routers"].get(router_alias, None) + if router_node is not None: + get_target_data(router_node, "data") - if not result["data"] and not result["target"]: + if result["data"] is None and result["target"] is None: self.log.info("No route for {} found!".format(routespec)) result = None - else: - result["data"] = json.loads(result["data"]) + self.log.debug("treefik routespec: {0}".format(traefik_routespec)) + self.log.debug("result for routespec {0}:-\n{1}".format(routespec, result)) + + # No longer bother converting `data` to/from JSON + #else: + # result["data"] = json.loads(result["data"]) + + #if service_alias in self.routes_cache["services"]: + # get_target_data(self.routes_cache["services"][service_alias], "url") + + #if router_alias in self.routes_cache["routers"]: + # get_target_data(self.routes_cache["routers"][router_alias], "data") + + #if not result["data"] and not result["target"]: + # self.log.info("No route for {} found!".format(routespec)) + # result = None + #else: + # result["data"] = json.loads(result["data"]) return result async def start(self): @@ -164,6 +186,8 @@ async def add_route(self, routespec, target, data): target (str): A full URL that will be the target of this route. data (dict): A JSONable dict that will be associated with this route, and will be returned when retrieving information about this route. + FIXME: Why do we need to pass data back and forth to traefik? + Traefik v2 doesn't seem to allow a data key... Will raise an appropriate Exception (FIXME: find what?) if the route could not be added. @@ -171,26 +195,38 @@ async def add_route(self, routespec, target, data): The proxy implementation should also have a way to associate the fact that a route came from JupyterHub. """ - routespec = self._routespec_to_traefik_path(routespec) - backend_alias = traefik_utils.generate_alias(routespec, "backend") - frontend_alias = traefik_utils.generate_alias(routespec, "frontend") - data = json.dumps(data) - rule = traefik_utils.generate_rule(routespec) + traefik_routespec = self._routespec_to_traefik_path(routespec) + service_alias = traefik_utils.generate_alias(traefik_routespec, "service") + router_alias = traefik_utils.generate_alias(traefik_routespec, "router") + #data = json.dumps(data) + rule = traefik_utils.generate_rule(traefik_routespec) async with self.mutex: - self.routes_cache["frontends"][frontend_alias] = { - "backend": backend_alias, - "passHostHeader": True, - "routes": {"test": {"rule": rule, "data": data}}, + self.routes_cache["http"]["routers"][router_alias] = { + "service": service_alias, + "rule": rule, + # The data node is passed by JupyterHub. We can store its data in our routes_cache, + # but giving it to Traefik causes issues... + #"data" : data + #"routes": {"test": {"rule": rule, "data": data}}, } - self.routes_cache["backends"][backend_alias] = { - "servers": {"server1": {"url": target, "weight": 1}} + # Add the data node to a separate top-level node + self.routes_cache["jupyter"]["routers"][router_alias] = {"data": data} + + self.routes_cache["http"]["services"][service_alias] = { + "loadBalancer" : { + "servers": {"server1": {"url": target} }, + "passHostHeader": True + } } traefik_utils.persist_routes( - self.toml_dynamic_config_file, self.routes_cache + self.dynamic_config_file, self.routes_cache ) + self.log.debug("treefik routespec: {0}".format(traefik_routespec)) + self.log.debug("data for routespec {0}:-\n{1}".format(routespec, data)) + if self.should_start: try: # Check if traefik was launched @@ -201,10 +237,10 @@ async def add_route(self, routespec, target, data): ) raise try: - await self._wait_for_route(routespec, provider="file") + await self._wait_for_route(traefik_routespec) except TimeoutError: self.log.error( - f"Is Traefik configured to watch {self.toml_dynamic_config_file}?" + f"Is Traefik configured to watch {self.dynamic_config_file}?" ) raise @@ -214,14 +250,14 @@ async def delete_route(self, routespec): **Subclasses must define this method** """ routespec = self._routespec_to_traefik_path(routespec) - backend_alias = traefik_utils.generate_alias(routespec, "backend") - frontend_alias = traefik_utils.generate_alias(routespec, "frontend") + service_alias = traefik_utils.generate_alias(routespec, "service") + router_alias = traefik_utils.generate_alias(routespec, "router") async with self.mutex: - self.routes_cache["frontends"].pop(frontend_alias, None) - self.routes_cache["backends"].pop(backend_alias, None) + self.routes_cache["http"]["routers"].pop(router_alias, None) + self.routes_cache["http"]["services"].pop(service_alias, None) - traefik_utils.persist_routes(self.toml_dynamic_config_file, self.routes_cache) + traefik_utils.persist_routes(self.dynamic_config_file, self.routes_cache) async def get_all_routes(self): """Fetch and return all the routes associated by JupyterHub from the @@ -241,11 +277,13 @@ async def get_all_routes(self): all_routes = {} async with self.mutex: - for key, value in self.routes_cache["frontends"].items(): + for key, value in self.routes_cache["http"]["routers"].items(): escaped_routespec = "".join(key.split("_", 1)[1:]) traefik_routespec = escapism.unescape(escaped_routespec) routespec = self._routespec_from_traefik_path(traefik_routespec) - all_routes[routespec] = self._get_route_unsafe(traefik_routespec) + all_routes.update({ + routespec : self._get_route_unsafe(traefik_routespec) + }) return all_routes @@ -272,3 +310,4 @@ async def get_route(self, routespec): routespec = self._routespec_to_traefik_path(routespec) async with self.mutex: return self._get_route_unsafe(routespec) + diff --git a/jupyterhub_traefik_proxy/kv_proxy.py b/jupyterhub_traefik_proxy/kv_proxy.py index b1dc4fb3..32f56b3c 100644 --- a/jupyterhub_traefik_proxy/kv_proxy.py +++ b/jupyterhub_traefik_proxy/kv_proxy.py @@ -87,12 +87,12 @@ async def _kv_atomic_add_route_parts( are expected to have the following structure: [ key: jupyterhub_routespec , value: target ] [ key: target , value: data ] - [ key: route_keys.backend_url_path , value: target ] - [ key: route_keys.frontend_rule_path , value: rule ] - [ key: route_keys.frontend_backend_path, value: - route_keys.backend_alias] - [ key: route_keys.backend_weight_path , value: w(int) ] - (where `w` is the weight of the backend to be used during load balancing) + [ key: route_keys.service_url_path , value: target ] + [ key: route_keys.router_rule_path , value: rule ] + [ key: route_keys.router_service_path, value: + route_keys.service_alias] + [ key: route_keys.service_weight_path , value: w(int) ] + (where `w` is the weight of the service to be used during load balancing) Returns: result (tuple): @@ -113,10 +113,10 @@ async def _kv_atomic_delete_route_parts(self, jupyterhub_routespec, route_keys): The keys associated with a route are: jupyterhub_routespec, target, - route_keys.backend_url_path, - route_keys.frontend_rule_path, - route_keys.frontend_backend_path, - route_keys.backend_weight_path, + route_keys.service_url_path, + route_keys.router_rule_path, + route_keys.router_service_path, + route_keys.service_weight_path, Returns: result (tuple): @@ -184,7 +184,7 @@ async def _kv_get_jupyterhub_prefixed_entries(self): def _clean_resources(self): try: if self.should_start: - os.remove(self.toml_static_config_file) + os.remove(self.static_config_file) except: self.log.error("Failed to remove traefik's configuration files") raise @@ -205,7 +205,7 @@ async def _setup_traefik_static_config(self): self._define_kv_specific_static_config() try: traefik_utils.persist_static_conf( - self.toml_static_config_file, self.static_config + self.static_config_file, self.static_config ) except IOError: self.log.exception("Couldn't set up traefik's static config.") @@ -273,12 +273,12 @@ async def add_route(self, routespec, target, data): raise if status: self.log.info( - "Added backend %s with the alias %s.", target, route_keys.backend_alias + "Added service %s with the alias %s.", target, route_keys.service_alias ) self.log.info( - "Added frontend %s for backend %s with the following routing rule %s.", - route_keys.frontend_alias, - route_keys.backend_alias, + "Added router %s for service %s with the following routing rule %s.", + route_keys.router_alias, + route_keys.service_alias, routespec, ) else: @@ -286,7 +286,7 @@ async def add_route(self, routespec, target, data): "Couldn't add route for %s. Response: %s", routespec, response ) - await self._wait_for_route(routespec, provider=self.kv_name) + await self._wait_for_route(routespec) async def delete_route(self, routespec): """Delete a route and all the traefik related info associated given a routespec, diff --git a/jupyterhub_traefik_proxy/proxy.py b/jupyterhub_traefik_proxy/proxy.py index d2421c8f..ecb1fd77 100644 --- a/jupyterhub_traefik_proxy/proxy.py +++ b/jupyterhub_traefik_proxy/proxy.py @@ -36,7 +36,7 @@ class TraefikProxy(Proxy): traefik_process = Any() - toml_static_config_file = Unicode( + static_config_file = Unicode( "traefik.toml", config=True, help="""traefik's static configuration file""" ) @@ -108,40 +108,42 @@ def _generate_htpassword(self): ht.set_password(self.traefik_api_username, self.traefik_api_password) self.traefik_api_hashed_password = str(ht.to_string()).split(":")[1][:-3] - async def _check_for_traefik_endpoint(self, routespec, kind, provider): - """Check for an expected frontend or backend + async def _check_for_traefik_service(self, routespec, kind): + """Check for an expected router or service This is used to wait for traefik to load configuration from a provider """ - expected = traefik_utils.generate_alias(routespec, kind) - path = "/api/providers/{}/{}s".format(provider, kind) + # expected e.g. 'service' + '_' + routespec @ file + expected = traefik_utils.generate_alias(routespec, kind) + "@file" + path = "/api/http/{0}s".format(kind) try: resp = await self._traefik_api_request(path) - data = json.loads(resp.body) + json_data = json.loads(resp.body) except Exception: self.log.exception("Error checking traefik api for %s %s", kind, routespec) return False - if expected not in data: - self.log.debug("traefik %s not yet in %ss", expected, kind) - self.log.debug("Current traefik %ss: %s", kind, data) + service_names = [service['name'] for service in json_data] + if expected not in service_names: + self.log.debug("traefik %s not yet in %s", expected, kind) + self.log.debug("Current traefik %ss: %s", kind, json_data) return False # found the expected endpoint return True - async def _wait_for_route(self, routespec, provider): + async def _wait_for_route(self, routespec): self.log.info("Waiting for %s to register with traefik", routespec) async def _check_traefik_dynamic_conf_ready(): """Check if traefik loaded its dynamic configuration yet""" - if not await self._check_for_traefik_endpoint( - routespec, "backend", provider + if not await self._check_for_traefik_service( + routespec, "service" ): return False - if not await self._check_for_traefik_endpoint( - routespec, "frontend", provider + if not await self._check_for_traefik_service( + routespec, "router" ): return False @@ -173,7 +175,8 @@ async def _wait_for_static_config(self, provider): async def _check_traefik_static_conf_ready(): """Check if traefik loaded its static configuration yet""" try: - resp = await self._traefik_api_request("/api/providers/" + provider) + #resp = await self._traefik_api_request("/api/overview/providers/" + provider) + resp = await self._traefik_api_request("/api/overview") except Exception: self.log.exception("Error checking for traefik static configuration") return False @@ -199,7 +202,7 @@ def _stop_traefik(self): self.traefik_process.wait() def _launch_traefik(self, config_type): - if config_type == "toml" or config_type == "etcdv3" or config_type == "consul": + if config_type == "fileprovider" or config_type == "etcdv3" or config_type == "consul": config_file_path = abspath(join(dirname(__file__), "traefik.toml")) self.traefik_process = Popen( ["traefik", "-c", config_file_path], stdout=None diff --git a/jupyterhub_traefik_proxy/traefik_utils.py b/jupyterhub_traefik_proxy/traefik_utils.py index 76484f3e..c4cb7794 100644 --- a/jupyterhub_traefik_proxy/traefik_utils.py +++ b/jupyterhub_traefik_proxy/traefik_utils.py @@ -5,7 +5,6 @@ from urllib.parse import unquote import escapism -import toml from contextlib import contextmanager from collections import namedtuple @@ -28,12 +27,12 @@ def generate_rule(routespec): routespec = unquote(routespec) if routespec.startswith("/"): # Path-based route, e.g. /proxy/path/ - rule = "PathPrefix:" + routespec + rule = "PathPrefix(`{0}`)".format(routespec) else: # Host-based routing, e.g. host.tld/proxy/path/ host, path_prefix = routespec.split("/", 1) path_prefix = "/" + path_prefix - rule = "Host:" + host + ";PathPrefix:" + path_prefix + rule = "Host(`{0}`) && PathPrefix(`{1}`)".format(host, path_prefix) return rule @@ -42,75 +41,74 @@ def generate_alias(routespec, server_type=""): return server_type + "_" + escapism.escape(routespec, safe=safe) -def generate_backend_entry( - proxy, backend_alias, separator="/", url=False, weight=False -): - backend_entry = "" +def generate_service_entry( proxy, service_alias, separator="/", url=False, + weight=False): + service_entry = "" if separator == "/": - backend_entry = proxy.kv_traefik_prefix - backend_entry += separator.join(["backends", backend_alias, "servers", "server1"]) + service_entry = proxy.kv_traefik_prefix + service_entry += separator.join(["services", service_alias, "servers", "server1"]) if url: - backend_entry += separator + "url" + service_entry += separator + "url" elif weight: - backend_entry += separator + "weight" + service_entry += separator + "weight" - return backend_entry + return service_entry -def generate_frontend_backend_entry(proxy, frontend_alias): - return proxy.kv_traefik_prefix + "frontends/" + frontend_alias + "/backend" +def generate_router_service_entry(proxy, router_alias): + return proxy.kv_traefik_prefix + "routers/" + router_alias + "/service" -def generate_frontend_rule_entry(proxy, frontend_alias, separator="/"): - frontend_rule_entry = separator.join( - ["frontends", frontend_alias, "routes", "test"] +def generate_router_rule_entry(proxy, router_alias, separator="/"): + router_rule_entry = separator.join( + ["routers", router_alias, "routes", "test"] ) if separator == "/": - frontend_rule_entry = ( - proxy.kv_traefik_prefix + frontend_rule_entry + separator + "rule" + router_rule_entry = ( + proxy.kv_traefik_prefix + router_rule_entry + separator + "rule" ) - return frontend_rule_entry + return router_rule_entry def generate_route_keys(proxy, routespec, separator="/"): - backend_alias = generate_alias(routespec, "backend") - frontend_alias = generate_alias(routespec, "frontend") + service_alias = generate_alias(routespec, "service") + router_alias = generate_alias(routespec, "router") RouteKeys = namedtuple( "RouteKeys", [ - "backend_alias", - "backend_url_path", - "backend_weight_path", - "frontend_alias", - "frontend_backend_path", - "frontend_rule_path", + "service_alias", + "service_url_path", + "service_weight_path", + "router_alias", + "router_service_path", + "router_rule_path", ], ) if separator != ".": - backend_url_path = generate_backend_entry(proxy, backend_alias, url=True) - frontend_rule_path = generate_frontend_rule_entry(proxy, frontend_alias) - backend_weight_path = generate_backend_entry(proxy, backend_alias, weight=True) - frontend_backend_path = generate_frontend_backend_entry(proxy, frontend_alias) + service_url_path = generate_service_entry(proxy, service_alias, url=True) + router_rule_path = generate_router_rule_entry(proxy, router_alias) + service_weight_path = generate_service_entry(proxy, service_alias, weight=True) + router_service_path = generate_router_service_entry(proxy, router_alias) else: - backend_url_path = generate_backend_entry( - proxy, backend_alias, separator=separator + service_url_path = generate_service_entry( + proxy, service_alias, separator=separator ) - frontend_rule_path = generate_frontend_rule_entry( - proxy, frontend_alias, separator=separator + router_rule_path = generate_router_rule_entry( + proxy, router_alias, separator=separator ) - backend_weight_path = "" - frontend_backend_path = "" + service_weight_path = "" + router_service_path = "" return RouteKeys( - backend_alias, - backend_url_path, - backend_weight_path, - frontend_alias, - frontend_backend_path, - frontend_rule_path, + service_alias, + service_url_path, + service_weight_path, + router_alias, + router_service_path, + router_rule_path, ) @@ -142,20 +140,47 @@ def atomic_writing(path): # already deleted by os.replace above pass - -def persist_static_conf(file, static_conf_dict): - with open(file, "w") as f: - toml.dump(static_conf_dict, f) - - -def persist_routes(file, routes_dict): - with atomic_writing(file) as config_fd: - toml.dump(routes_dict, config_fd) - - -def load_routes(file): - try: - with open(file, "r") as config_fd: - return toml.load(config_fd) - except: - raise +class TraefikConfigFileHandler(object): + """Handles reading and writing Traefik config files. Can operate + on both toml and yaml files""" + def __init__(self, file_path): + file_ext = file_path.rsplit('.', 1)[-1] + if file_ext == 'yaml': + import yaml as config_handler + elif file_ext == 'toml': + import toml as config_handler + else: + raise TypeError("type should be either 'toml' or 'yaml'") + + self.file_path = file_path + # Redefined to either yaml.dump or toml.dump + self._dump = config_handler.dump + #self._dumps = config_handler.dumps + # Redefined by __init__, to either yaml.load or toml.load + self._load = config_handler.load + + def load(self): + """Depending on self.file_path, call either yaml.load or toml.load""" + return self._load(self.file_path) + + def dump(self, data): + with open(self.file_path, "w") as f: + self._dump(data, f) + + def atomic_dump(self, data): + """Save data to self.file_path after opening self.file_path with + :func:`atomic_writing`""" + with atomic_writing(self.file_path) as f: + self._dump(data, f) + +def persist_static_conf(file_path, static_conf_dict): + handler = TraefikConfigFileHandler(file_path) + handler.dump(static_conf_dict) + +def persist_routes(file_path, routes_dict): + handler = TraefikConfigFileHandler(file_path) + handler.atomic_dump(routes_dict) + +def load_routes(file_path): + handler = TraefikConfigFileHandler(file_path) + return handler.load() diff --git a/setup.py b/setup.py index a4a48cc1..3475f1a6 100644 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ def run(self): cmdclass=cmdclass, entry_points={ "jupyterhub.proxies": [ - "traefik_toml = jupyterhub_traefik_proxy:TraefikTomlProxy", + "traefik_file = jupyterhub_traefik_proxy:TraefikFileProviderProxy", ] }, ) From d75ae3feb343ed282afa9bbe34bb188204a679c8 Mon Sep 17 00:00:00 2001 From: Alex Leach Date: Fri, 18 Jun 2021 13:32:53 +0000 Subject: [PATCH 03/19] Remove any refererence to `traefik_toml` or `TraefikTomlProxy` from the documentation, examples and performance test suite. These are replaced by `traefik_file` and `TraefikFileProviderProxy`, respectively. --- README.md | 4 ++-- docs/source/api/index.rst | 4 ++-- docs/source/index.rst | 4 ++-- docs/source/install.md | 14 +++++++------- ...ub_config_toml.py => jupyterhub_config_file.py} | 10 +++++----- 5 files changed, 18 insertions(+), 18 deletions(-) rename examples/{jupyterhub_config_toml.py => jupyterhub_config_file.py} (55%) diff --git a/README.md b/README.md index bcf22e45..570a8971 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ depending on how traefik store its routing configuration. For **smaller**, single-node deployments: -* TraefikTomlProxy +* TraefikFileProviderProxy For **distributed** setups: @@ -32,7 +32,7 @@ The [documentation](https://jupyterhub-traefik-proxy.readthedocs.io) contains a guide](https://jupyterhub-traefik-proxy.readthedocs.io/en/latest/install.html) with examples for the three different implementations. -* [For TraefikTomlProxy](https://jupyterhub-traefik-proxy.readthedocs.io/en/latest/toml.html#example-setup) +* [For TraefikFileProviderProxy](https://jupyterhub-traefik-proxy.readthedocs.io/en/latest/file.html#example-setup) * [For TraefikEtcdProxy](https://jupyterhub-traefik-proxy.readthedocs.io/en/latest/etcd.html#example-setup) * [For TraefikConsulProxy](https://jupyterhub-traefik-proxy.readthedocs.io/en/latest/consul.html#example-setup) diff --git a/docs/source/api/index.rst b/docs/source/api/index.rst index 95481165..410c9a6c 100644 --- a/docs/source/api/index.rst +++ b/docs/source/api/index.rst @@ -15,10 +15,10 @@ Module: :mod:`jupyterhub_traefik_proxy` .. autoconfigurable:: TraefikProxy :members: -:class:`TraefikTomlProxy` +:class:`TraefikFileProviderProxy` ------------------------- -.. autoconfigurable:: TraefikTomlProxy +.. autoconfigurable:: TraefikFileProviderProxy :members: :class:`TKvProxy` diff --git a/docs/source/index.rst b/docs/source/index.rst index d4750855..0f73a31e 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -20,7 +20,7 @@ Moreover it offers *HTTPS* support through a straight-forward `ACME (Let's Encry There are three versions for the proxy, depending on how traefik stores the routes: * *for* **smaller**, *single-node deployments*: - * TraefikTomlProxy + * TraefikFileProviderProxy * *for* **distributed** *setups*: * TraefikEtcdProxy * TraefikConsulProxy @@ -39,7 +39,7 @@ Getting Started .. toctree:: :maxdepth: 1 - toml + fileprovider etcd consul diff --git a/docs/source/install.md b/docs/source/install.md index af1dd013..d5ea9858 100644 --- a/docs/source/install.md +++ b/docs/source/install.md @@ -69,7 +69,7 @@ ## Enabling traefik-proxy in JupyterHub -[TraefikTomlProxy](https://github.com/jupyterhub/traefik-proxy/blob/master/jupyterhub_traefik_proxy/toml.py), [TraefikEtcdProxy](https://github.com/jupyterhub/traefik-proxy/blob/master/jupyterhub_traefik_proxy/etcd.py) and [TraefikConsulProxy](https://github.com/jupyterhub/traefik-proxy/blob/master/jupyterhub_traefik_proxy/consul.py) are custom proxy implementations that subclass [Proxy](https://github.com/jupyterhub/jupyterhub/blob/master/jupyterhub/proxy.py) and can register in JupyterHub config using `c.JupyterHub.proxy_class` entrypoint. +[TraefikFileProviderProxy](https://github.com/jupyterhub/traefik-proxy/blob/Traefik_v2/jupyterhub_traefik_proxy/fileprovider.py), [TraefikEtcdProxy](https://github.com/jupyterhub/traefik-proxy/blob/Traefik_v2/jupyterhub_traefik_proxy/etcd.py) and [TraefikConsulProxy](https://github.com/jupyterhub/traefik-proxy/blob/Traefik_v2/jupyterhub_traefik_proxy/consul.py) are custom proxy implementations that subclass [Proxy](https://github.com/jupyterhub/jupyterhub/blob/Traefik_v2/jupyterhub/proxy.py) and can register in JupyterHub config using `c.JupyterHub.proxy_class` entrypoint. On startup, JupyterHub will look by default for a configuration file, *jupyterhub_config.py*, in the current working directory. If the configuration file is not in the current working directory, you can load a specific config file and start JupyterHub using: @@ -78,13 +78,13 @@ you can load a specific config file and start JupyterHub using: $ jupyterhub -f /path/to/jupyterhub_config.py ``` -There is an example configuration file [here](https://github.com/jupyterhub/traefik-proxy/blob/master/examples/jupyterhub_config.py) that configures JupyterHub to run with *TraefikEtcdProxy* as the proxy and uses dummyauthenticator and simplespawner to enable testing without administrative privileges. +There is an example configuration file [here](https://github.com/jupyterhub/traefik-proxy/blob/Traefik_v2/examples/jupyterhub_config.py) that configures JupyterHub to run with *TraefikEtcdProxy* as the proxy and uses dummyauthenticator and simplespawner to enable testing without administrative privileges. In *jupyterhub_config.py*: ``` -c.JupyterHub.proxy_class = "traefik_toml" -# will configure JupyterHub to run with TraefikTomlProxy +c.JupyterHub.proxy_class = "traefik_file" +# will configure JupyterHub to run with TraefikFileProviderProxy ``` ``` @@ -110,9 +110,9 @@ c.JupyterHub.proxy_class = "traefik_consul" The port on which traefik-proxy's api will run, as well as the username and password used for authenticating, can be passed to the proxy through `jupyterhub_config.py`, e.g.: ``` - c.TraefikTomlProxy.traefik_api_url = "http://127.0.0.1:8099" - c.TraefikTomlProxy.traefik_api_password = "admin" - c.TraefikTomlProxy.traefik_api_username = "admin" + c.TraefikFileProviderProxy.traefik_api_url = "http://127.0.0.1:8099" + c.TraefikFileProviderProxy.traefik_api_password = "admin" + c.TraefikFileProviderProxy.traefik_api_username = "admin" ``` Check out TraefikProxy's **API Reference** for more configuration options.

diff --git a/examples/jupyterhub_config_toml.py b/examples/jupyterhub_config_file.py similarity index 55% rename from examples/jupyterhub_config_toml.py rename to examples/jupyterhub_config_file.py index b17ca1f3..927ea54d 100644 --- a/examples/jupyterhub_config_toml.py +++ b/examples/jupyterhub_config_file.py @@ -1,6 +1,6 @@ """sample jupyterhub config file for testing -configures jupyterhub to run with traefik_toml. +configures jupyterhub to run with traefik_file. configures jupyterhub with dummyauthenticator and simplespawner to enable testing without administrative privileges. @@ -8,10 +8,10 @@ requires jupyterhub 1.0.dev """ -c.JupyterHub.proxy_class = "traefik_toml" -c.TraefikTomlProxy.traefik_api_username = "admin" -c.TraefikTomlProxy.traefik_api_password = "admin" -c.TraefikTomlProxy.traefik_log_level = "INFO" +c.JupyterHub.proxy_class = "traefik_file" +c.TraefikFileProviderProxy.traefik_api_username = "admin" +c.TraefikFileProviderProxy.traefik_api_password = "admin" +c.TraefikFileProviderProxy.traefik_log_level = "INFO" # use dummy and simple auth/spawner for testing c.JupyterHub.authenticator_class = "dummy" From c9ffae4fac150a53fcdcaba9dc1c1a7bd36b11f8 Mon Sep 17 00:00:00 2001 From: Alex Leach Date: Mon, 5 Jul 2021 21:09:45 +0000 Subject: [PATCH 04/19] Major work to support the traefik v2 file provider. This is a re-work of original commit faa2832437bf9a2bcd0023a24c34fea150d5d4c1 to try and make it a bit more manageable. In this commit, work specifically on the file provider proxy. - Relevant documentation has been updated in README.md, docs/source/file.md (renamed from toml.md) - requirements.txt removes toml a forced dependency as ruamel.yaml could be used instead. This latter module is required by jupyterhub anyway, so should already be present on a system running JupyterHub. - The external file provider now watches a directory instead of a file, so have added a pre-baked dynamic_config directory (and file), where the rules.toml file will be saved and managed by the TraefikFileProviderProxy. - Have duplicated the toml_proxy and external_toml_proxy pytest fixtures to test both toml and yaml files. (Would have preferred to parametrize the existing fixtures to avoid duplicating them, but couldn't figure out how). - `tests/test_traefik_api_auth.py` - Have had to make the test give traefik more of a chance to read its dynamic config before running the test. Previously, in traefik v1, the api authentication would be configured in the static config. Now the api 'middleware' is configured in the dynamic config, the previous wait condition of just waiting for the port to come up didn't give traefik enough time to set up the API authentication on that entrypoint / router. --- docs/source/toml.md | 127 ++++++----- jupyterhub_traefik_proxy/fileprovider.py | 179 +++++++-------- jupyterhub_traefik_proxy/install.py | 94 ++++---- jupyterhub_traefik_proxy/proxy.py | 203 +++++++++++++----- jupyterhub_traefik_proxy/traefik_utils.py | 52 +++-- requirements.txt | 4 +- .../dynamic_config/dynamic_conf.toml | 11 + tests/config_files/rules.toml | 0 tests/config_files/traefik.toml | 27 ++- tests/conftest.py | 78 +++++-- tests/dummy_http_server.py | 2 +- tests/proxytest.py | 10 +- tests/test_proxy.py | 10 + tests/test_traefik_api_auth.py | 57 +++-- tests/test_traefik_utils.py | 3 +- 15 files changed, 538 insertions(+), 319 deletions(-) create mode 100644 tests/config_files/dynamic_config/dynamic_conf.toml delete mode 100644 tests/config_files/rules.toml diff --git a/docs/source/toml.md b/docs/source/toml.md index 7c88503f..0f4178d9 100644 --- a/docs/source/toml.md +++ b/docs/source/toml.md @@ -1,9 +1,9 @@ -# Using TraefikTomlProxy +# Using TraefikFileProviderProxy -**jupyterhub-traefik-proxy** can be used with simple toml configuration files, for smaller, single-node deployments such as +**jupyterhub-traefik-proxy** can be used with simple toml or yaml configuration files, for smaller, single-node deployments such as [The Littlest JupyterHub](https://tljh.jupyter.org). -## How-To install TraefikTomlProxy +## How-To install TraefikFileProviderProxy 1. Install **jupyterhub** 2. Install **jupyterhub-traefik-proxy** @@ -11,23 +11,23 @@ * You can find the full installation guide and examples in the [Introduction section](install.html#traefik-proxy-installation) -## How-To enable TraefikTomlProxy +## How-To enable TraefikFileProviderProxy -You can enable JupyterHub to work with `TraefikTomlProxy` in jupyterhub_config.py, using the `proxy_class` configuration option. +You can enable JupyterHub to work with `TraefikFileProviderProxy` in jupyterhub_config.py, using the `proxy_class` configuration option. You can choose to: -* use the `traefik_toml` entrypoint, new in JupyterHub 1.0, e.g.: +* use the `traefik_file` entrypoint, new in JupyterHub 1.0, e.g.: ```python - c.JupyterHub.proxy_class = "traefik_toml" + c.JupyterHub.proxy_class = "traefik_file" ``` -* use the TraefikTomlProxy object, in which case, you have to import the module, e.g.: +* use the TraefikFileProviderProxy object, in which case, you have to import the module, e.g.: ```python - from jupyterhub_traefik_proxy import TraefikTomlProxy - c.JupyterHub.proxy_class = TraefikTomlProxy + from jupyterhub_traefik_proxy import TraefikFileProviderProxy + c.JupyterHub.proxy_class = TraefikFileProviderProxy ``` @@ -39,10 +39,10 @@ Traefik's configuration is divided into two parts: * The **dynamic** configuration (can be hot-reloaded, without restarting the proxy), where the routing table will be updated continuously. -Traefik allows us to have one file for the static configuration (the `traefik.toml`) and one or several files for the routes, that traefik would watch. +Traefik allows us to have one file for the static configuration file (`traefik.toml` or `traefik.yaml`) and one or several files for the routes, that traefik would watch. ```{note} - **TraefikTomlProxy**, uses two configuration files: one file for the routes (**rules.toml**), and one for the static configuration (**traefik.toml**). + **TraefikFileProviderProxy**, uses two configuration files: one file for the routes (**rules.toml** or **rules.yaml**), and one for the static configuration (**traefik.toml** or **traefik.yaml**). ``` @@ -52,39 +52,45 @@ By **default**, Traefik will search for `traefik.toml` and `rules.toml` in the f * $HOME/.traefik/ * . the working directory -You can override this in TraefikTomlProxy, by modifying the **toml_static_config_file** argument: +You can override this in TraefikFileProviderProxy, by modifying the **toml_static_config_file** argument: ```python -c.TraefikTomlProxy.toml_static_config_file="/path/to/static_config_filename.toml" +c.TraefikFileProviderProxy.static_config_file="/path/to/static_config_filename.toml" ``` -Similarly, you can override the dynamic configuration file by modifying the **toml_dynamic_config_file** argument: +Similarly, you can override the dynamic configuration file by modifying the **dynamic_config_file** argument: ```python -c.TraefikTomlProxy.toml_dynamic_config_file="/path/to/dynamic_config_filename.toml" +c.TraefikFileProviderProxy.dynamic_config_file="/path/to/dynamic_config_filename.toml" ``` ```{note} -* **When JupyterHub starts the proxy**, it writes the static config once, then only edits the routes config file. +* **When JupyterHub starts the proxy**, it writes the static config once, then only edits the dynamic config file. * **When JupyterHub does not start the proxy**, the user is totally responsible for the static config and JupyterHub is responsible exclusively for the routes. + +* **When JupyterHub does not start the proxy**, the user should tell `traefik` to get its dynamic configuration +from a directory. Then, one (or more) dynamic configuration file(s) can be managed externally, and `dynamic_config_file` +will be managed by JupyterHub. This allows e.g., the administrator to configure traefik's API outside of JupyterHub. + ``` -## Externally managed TraefikTomlProxy +## Externally managed TraefikFileProviderProxy -When TraefikTomlProxy is externally managed, service managers like [systemd](https://www.freedesktop.org/wiki/Software/systemd/) +When TraefikFileProviderProxy is externally managed, service managers like [systemd](https://www.freedesktop.org/wiki/Software/systemd/) or [docker](https://www.docker.com/) will be responsible for starting and stopping the proxy. -If TraefikTomlProxy is used as an externally managed service, then make sure you follow the steps enumerated below: +If TraefikFileProviderProxy is used as an externally managed service, then make sure you follow the steps enumerated below: -1. Let JupyterHub know that the proxy being used is TraefikTomlProxy, using the *proxy_class* configuration option: +1. Let JupyterHub know that the proxy being used is TraefikFileProviderProxy, using the *proxy_class* configuration option: ```python - c.JupyterHub.proxy_class = "traefik_toml" + from jupyterhub_traefik_proxy import TraefikFileProviderProxy + c.JupyterHub.proxy_class = TraefikFileProviderProxy ``` -2. Configure `TraefikTomlProxy` in **jupyterhub_config.py** +2. Configure `TraefikFileProviderProxy` in **jupyterhub_config.py** JupyterHub configuration file, *jupyterhub_config.py* must specify at least: * That the proxy is externally managed @@ -96,84 +102,93 @@ If TraefikTomlProxy is used as an externally managed service, then make sure you Example configuration: ```python # JupyterHub shouldn't start the proxy, it's already running - c.TraefikTomlProxy.should_start = False + c.TraefikFileProviderProxy.should_start = False # if not the default: - c.TraefikTomlProxy.toml_dynamic_config_file = "somefile.toml" + c.TraefikFileProviderProxy.dynamic_config_file = "/path/to/somefile.toml" # traefik api credentials - c.TraefikTomlProxy.traefik_api_username = "abc" - c.TraefikTomlProxy.traefik_api_password = "xxx" + c.TraefikFileProviderProxy.traefik_api_username = "abc" + c.TraefikFileProviderProxy.traefik_api_password = "xxx" ``` -3. Ensure **traefik.toml** +3. Ensure **traefik.toml** / **traefik.yaml** - The static configuration file, *traefik.toml* must configure at least: + The static configuration file, *traefik.toml* (or **traefik.yaml**) must configure at least: * The default entrypoint - * The api entrypoint (*and authenticate it*) + * The api entrypoint (*and authenticate it in a user-managed dynamic configuration file*) * The websockets protocol - * The dynamic configuration file to watch - (*make sure this configuration file exists, even if empty before the proxy is launched*) + * The dynamic configuration directory to watch + (*make sure this configuration directory exists, even if empty before the proxy is launched*) + * Check `tests/config_files/traefik.toml` for an example. ## Example setup -This is an example setup for using JupyterHub and TraefikTomlProxy managed by another service than JupyterHub. +This is an example setup for using JupyterHub and TraefikFileProviderProxy managed by another service than JupyterHub. 1. Configure the proxy through the JupyterHub configuration file, *jupyterhub_config.py*, e.g.: ```python - from jupyterhub_traefik_proxy import TraefikTomlProxy + from jupyterhub_traefik_proxy import TraefikFileProviderProxy # mark the proxy as externally managed - c.TraefikTomlProxy.should_start = False + c.TraefikFileProviderProxy.should_start = False # traefik api endpoint login password - c.TraefikTomlProxy.traefik_api_password = "admin" + c.TraefikFileProviderProxy.traefik_api_password = "admin" # traefik api endpoint login username - c.TraefikTomlProxy.traefik_api_username = "api_admin" + c.TraefikFileProviderProxy.traefik_api_username = "api_admin" - # traefik's dynamic configuration file - c.TraefikTomlProxy.toml_dynamic_config_file = "path/to/rules.toml" + # traefik's dynamic configuration file, which will be managed by JupyterHub + c.TraefikFileProviderProxy.dynamic_config_file = "/var/run/traefik/rules.toml" - # configure JupyterHub to use TraefikTomlProxy - c.JupyterHub.proxy_class = TraefikTomlProxy + # configure JupyterHub to use TraefikFileProviderProxy + c.JupyterHub.proxy_class = TraefikFileProviderProxy ``` 2. Create a traefik static configuration file, *traefik.toml*, e.g.: ``` - # the default entrypoint - defaultentrypoints = ["http"] - # the api entrypoint [api] dashboard = true - entrypoint = "auth_api" # websockets protocol [wss] protocol = "http" # the port on localhost where traefik accepts http requests - [entryPoints.http] + [entryPoints.web] address = ":8000" # the port on localhost where the traefik api and dashboard can be found - [entryPoints.auth_api] + [entryPoints.enter_api] address = ":8099" - # authenticate the traefik api entrypoint - [entryPoints.auth_api.auth.basic] - users = [ "api_admin:$apr1$eS/j3kum$q/X2khsIEG/bBGsteP.x./",] - - # the dynamic configuration file - [file] - filename = "rules.toml" + # the dynamic configuration directory + # This must match the directory provided in Step 1. above. + [providers.file] + directory = "/var/run/traefik" watch = true ``` -3. Start traefik with the configuration specified above, e.g.: +3. Create a traefik dynamic configuration file in the directory provided in the dynamic configuration above, to provide the api authentication parameters, e.g. + + ``` + # Router configuration for the api service + [http.routers.router-api] + rule = "Host(`localhost`) && PathPrefix(`/api`)" + entryPoints = ["enter_api"] + service = "api@internal" + middlewares = ["auth_api"] + + # authenticate the traefik api entrypoint + [http.middlewares.auth_api.basicAuth] + users = [ "api_admin:$apr1$eS/j3kum$q/X2khsIEG/bBGsteP.x./",] + ``` + +4. Start traefik with the configuration specified above, e.g.: ```bash - $ traefik -c traefik.toml + $ traefik --configfile traefik.toml ``` diff --git a/jupyterhub_traefik_proxy/fileprovider.py b/jupyterhub_traefik_proxy/fileprovider.py index ca383823..020eb9fc 100644 --- a/jupyterhub_traefik_proxy/fileprovider.py +++ b/jupyterhub_traefik_proxy/fileprovider.py @@ -24,7 +24,7 @@ import string import escapism -from traitlets import Any, default, Unicode +from traitlets import Any, default, Unicode, observe from . import traefik_utils from jupyterhub.proxy import Proxy @@ -40,59 +40,58 @@ class TraefikFileProviderProxy(TraefikProxy): def _default_mutex(self): return asyncio.Lock() + @default("provider_name") + def _provider_name(self): + return "file" + dynamic_config_file = Unicode( "rules.toml", config=True, help="""traefik's dynamic configuration file""" ) + dynamic_config_handler = Any() + + @default("dynamic_config_handler") + def _default_handler(self): + return traefik_utils.TraefikConfigFileHandler(self.dynamic_config_file) + + # If dynamic_config_file is changed, then update the dynamic config file handler + @observe("dynamic_config_file") + def _set_dynamic_config_file(self, change): + self.dynamic_config_handler = traefik_utils.TraefikConfigFileHandler(self.dynamic_config_file) + def __init__(self, **kwargs): super().__init__(**kwargs) try: - # Load initial routing table from disk - self.routes_cache = traefik_utils.load_routes(self.dynamic_config_file) + # Load initial dynamic config from disk + self.dynamic_config = self.dynamic_config_handler.load() except FileNotFoundError: - self.routes_cache = {} + self.dynamic_config = {} - if not self.routes_cache: - self.routes_cache = { + if not self.dynamic_config: + self.dynamic_config = { "http" : {"services": {}, "routers": {}}, "jupyter": {"routers" : {} } } - async def _setup_traefik_static_config(self): - await super()._setup_traefik_static_config() + def persist_dynamic_config(self): + """Save the dynamic config file with the current dynamic_config""" + self.dynamic_config_handler.atomic_dump(self.dynamic_config) - # Is this not the same as the dynamic config file? - self.static_config["file"] = {"filename": "rules.toml", "watch": True} - - try: - traefik_utils.persist_static_conf( - self.static_config_file, self.static_config - ) - try: - os.stat(self.dynamic_config_file) - except FileNotFoundError: - # Make sure that the dynamic configuration file exists - self.log.info( - f"Creating the dynamic configuration file: {self.dynamic_config_file}" - ) - open(self.dynamic_config_file, "a").close() - except IOError: - self.log.exception("Couldn't set up traefik's static config.") - raise - except: - self.log.error("Couldn't set up traefik's static config. Unexpected error:") - raise + async def _setup_traefik_dynamic_config(self): + await super()._setup_traefik_dynamic_config() + self.log.info( + f"Creating the dynamic configuration file: {self.dynamic_config_file}" + ) + self.persist_dynamic_config() - def _start_traefik(self): - self.log.info("Starting traefik...") - try: - self._launch_traefik(config_type="fileprovider") - except FileNotFoundError as e: - self.log.error( - "Failed to find traefik \n" - "The proxy can be downloaded from https://github.com/containous/traefik/releases/download." - ) - raise + async def _setup_traefik_static_config(self): + self.static_config["providers"] = { + "file" : { + "filename": self.dynamic_config_file, + "watch": True + } + } + await super()._setup_traefik_static_config() def _clean_resources(self): try: @@ -122,35 +121,19 @@ def get_target_data(d, to_find): if isinstance(v, dict): get_target_data(v, to_find) - service_node = self.routes_cache["http"]["services"].get(service_alias, None) + service_node = self.dynamic_config["http"]["services"].get(service_alias, None) if service_node is not None: get_target_data(service_node, "url") - router_node = self.routes_cache["jupyter"]["routers"].get(router_alias, None) - if router_node is not None: - get_target_data(router_node, "data") + jupyter_routers = self.dynamic_config["jupyter"]["routers"].get(router_alias, None) + if jupyter_routers is not None: + get_target_data(jupyter_routers, "data") if result["data"] is None and result["target"] is None: - self.log.info("No route for {} found!".format(routespec)) + self.log.info(f"No route for {routespec} found!") result = None - self.log.debug("treefik routespec: {0}".format(traefik_routespec)) - self.log.debug("result for routespec {0}:-\n{1}".format(routespec, result)) - - # No longer bother converting `data` to/from JSON - #else: - # result["data"] = json.loads(result["data"]) - - #if service_alias in self.routes_cache["services"]: - # get_target_data(self.routes_cache["services"][service_alias], "url") - - #if router_alias in self.routes_cache["routers"]: - # get_target_data(self.routes_cache["routers"][router_alias], "data") - - #if not result["data"] and not result["target"]: - # self.log.info("No route for {} found!".format(routespec)) - # result = None - #else: - # result["data"] = json.loads(result["data"]) + self.log.debug(f"traefik routespec: {traefik_routespec}") + self.log.debug(f"result for routespec {routespec}:-\n{result}") return result async def start(self): @@ -162,7 +145,7 @@ async def start(self): if the proxy is to be started by the Hub """ await super().start() - await self._wait_for_static_config(provider="file") + await self._wait_for_static_config() async def stop(self): """Stop the proxy. @@ -195,37 +178,50 @@ async def add_route(self, routespec, target, data): The proxy implementation should also have a way to associate the fact that a route came from JupyterHub. """ + self.log.debug(f"\tTraefikFileProviderProxy.add_route: Adding {routespec} for {target}") traefik_routespec = self._routespec_to_traefik_path(routespec) service_alias = traefik_utils.generate_alias(traefik_routespec, "service") router_alias = traefik_utils.generate_alias(traefik_routespec, "router") - #data = json.dumps(data) rule = traefik_utils.generate_rule(traefik_routespec) async with self.mutex: - self.routes_cache["http"]["routers"][router_alias] = { + # If we've emptied the http and/or routers section, create it. + if "http" not in self.dynamic_config: + self.dynamic_config["http"] = { + "routers": {}, + } + self.dynamic_config["jupyter"] = { + "routers": {} + } + + elif "routers" not in self.dynamic_config["http"]: + self.dynamic_config["http"]["routers"] = {} + self.dynamic_config["jupyter"]["routers"] = {} + + # Is it worth querying the api for all entrypoints? + # Otherwise we just bind to all of them ... + self.dynamic_config["http"]["routers"][router_alias] = { "service": service_alias, "rule": rule, - # The data node is passed by JupyterHub. We can store its data in our routes_cache, - # but giving it to Traefik causes issues... - #"data" : data - #"routes": {"test": {"rule": rule, "data": data}}, + } + # Add the data node to a separate top-level node, so traefik doesn't complain. + self.dynamic_config["jupyter"]["routers"][router_alias] = { + "data": data } - # Add the data node to a separate top-level node - self.routes_cache["jupyter"]["routers"][router_alias] = {"data": data} + if "services" not in self.dynamic_config["http"]: + self.dynamic_config["http"]["services"] = {} - self.routes_cache["http"]["services"][service_alias] = { - "loadBalancer" : { + self.dynamic_config["http"]["services"][service_alias] = { + "loadBalancer": { "servers": {"server1": {"url": target} }, "passHostHeader": True } } - traefik_utils.persist_routes( - self.dynamic_config_file, self.routes_cache - ) + self.persist_dynamic_config() - self.log.debug("treefik routespec: {0}".format(traefik_routespec)) - self.log.debug("data for routespec {0}:-\n{1}".format(routespec, data)) + self.log.debug(f"traefik routespec: {traefik_routespec}") + self.log.debug(f"data for routespec {routespec}:-\n{data}") if self.should_start: try: @@ -254,10 +250,22 @@ async def delete_route(self, routespec): router_alias = traefik_utils.generate_alias(routespec, "router") async with self.mutex: - self.routes_cache["http"]["routers"].pop(router_alias, None) - self.routes_cache["http"]["services"].pop(service_alias, None) - - traefik_utils.persist_routes(self.dynamic_config_file, self.routes_cache) + + self.dynamic_config["http"]["routers"].pop(router_alias, None) + self.dynamic_config["http"]["services"].pop(service_alias, None) + self.dynamic_config["jupyter"]["routers"].pop(router_alias, None) + + # If empty, delete the keys + if not self.dynamic_config["http"]["routers"]: + self.dynamic_config["http"].pop("routers") + if not self.dynamic_config["http"]["services"]: + self.dynamic_config["http"].pop("services") + if not self.dynamic_config["http"]: + self.dynamic_config.pop("http") + if not self.dynamic_config["jupyter"]["routers"]: + self.dynamic_config["jupyter"].pop("routers") + + self.persist_dynamic_config() async def get_all_routes(self): """Fetch and return all the routes associated by JupyterHub from the @@ -277,8 +285,11 @@ async def get_all_routes(self): all_routes = {} async with self.mutex: - for key, value in self.routes_cache["http"]["routers"].items(): - escaped_routespec = "".join(key.split("_", 1)[1:]) + for router, value in self.dynamic_config["http"]["routers"].items(): + if router not in self.dynamic_config["jupyter"]["routers"]: + # Only check routers defined in jupyter node + continue + escaped_routespec = "".join(router.split("_", 1)[1:]) traefik_routespec = escapism.unescape(escaped_routespec) routespec = self._routespec_from_traefik_path(traefik_routespec) all_routes.update({ diff --git a/jupyterhub_traefik_proxy/install.py b/jupyterhub_traefik_proxy/install.py index 81cc3c7a..23b5724d 100644 --- a/jupyterhub_traefik_proxy/install.py +++ b/jupyterhub_traefik_proxy/install.py @@ -10,30 +10,36 @@ import warnings checksums_traefik = { - "https://github.com/containous/traefik/releases/download/v2.2.0/traefik_v2.2.0_linux_amd64.tar.gz": - "eddea0507ad715c723662e7c10fdab554eb64379748278cd2d09403063e3e32f", - "https://github.com/containous/traefik/releases/download/v2.2.0/traefik_v2.2.0_darwin_amd64.tar.gz": - "8bfa2393b265ef01aca12be94d67080961299968bd602f3708480eed273b95e0", - "https://github.com/containous/traefik/releases/download/v2.2.0/traefik_v2.2.0_windows_amd64.zip": - "9a794e395b7eba8d44118c4a1fb358fbf14abff3f5f5d264f46b1d6c243b9a5e", + "https://github.com/traefik/traefik/releases/download/v2.4.8/traefik_v2.4.8_linux_arm64.tar.gz": "0931fdd9c855fcafd38eba7568a1d287200fad5afd1aef7d112fb3a48d822fcc", + "https://github.com/traefik/traefik/releases/download/v2.4.8/traefik_v2.4.8_linux_amd64.tar.gz": "de8d56f6777c5098834d4f8d9ed419b7353a3afe913393a55b6fd14779564129", + "https://github.com/traefik/traefik/releases/download/v2.4.8/traefik_v2.4.8_darwin_amd64.tar.gz": "7d946baa422acfcf166e19779052c005722db03de3ab4d7aff586c4b4873a0f3", + "https://github.com/traefik/traefik/releases/download/v2.4.8/traefik_v2.4.8_windows_amd64.zip": "4203443cb1e91d76f81d1e2a41fb70e66452d951b1ffd8964218a7bc211c377d", + "https://github.com/traefik/traefik/releases/download/v2.3.7/traefik_v2.3.7_linux_amd64.tar.gz": "a357d40bc9b81ae76070a2bc0334dfd15e77143f41415a93f83bb53af1756909", + "https://github.com/traefik/traefik/releases/download/v2.3.7/traefik_v2.3.7_darwin_amd64.tar.gz": "c84fc21b8ee34bba8a66f0f9e71c6c2ea69684ac6330916551f1f111826b9bb3", + "https://github.com/traefik/traefik/releases/download/v2.3.7/traefik_v2.3.7_windows_amd64.zip": "eb54b1c9c752a6eaf48d28ff8409c17379a29b9d58390107411762ab6e4edfb4", + "https://github.com/traefik/traefik/releases/download/v2.2.11/traefik_v2.2.11_linux_amd64.tar.gz": "b677386423403c63fb9ac9667d39591be587a1a4928afc2e59449c78343bad9c", + "https://github.com/traefik/traefik/releases/download/v2.2.11/traefik_v2.2.11_darwin_amd64.tar.gz": "efb1c2bc23e16a9083e5a210594186d026cdec0b522a6b4754ceff43b07d8031", + "https://github.com/traefik/traefik/releases/download/v2.2.11/traefik_v2.2.11_windows_amd64.zip": "ee867133e00b2d8395c239d8fed04a26b362e650b371dc0b653f0ee9d52471e6", } checksums_etcd = { - "https://github.com/etcd-io/etcd/releases/download/v3.4.7/etcd-v3.4.7-linux-amd64.tar.gz": - "4ad86e663b63feb4855e1f3a647e719d6d79cf6306410c52b7f280fa56f8eb6b", - "https://github.com/etcd-io/etcd/releases/download/v3.4.7/etcd-v3.4.7-darwin-amd64.zip": - "ffe3237fcb70b7ce91c16518c2f62f3fa9ff74ddc10f7b6ca83a3b5b29ade19a", - "https://github.com/etcd-io/etcd/releases/download/v3.4.7/etcd-v3.4.7-windows-amd64.zip": - "3863ea59bcb407113524b51406810e33d58daff11ca10d1192f289185ae94ffe", + "https://github.com/etcd-io/etcd/releases/download/v3.4.15/etcd-v3.4.15-linux-arm64.tar.gz": "fcc522275300cf90d42377106d47a2e384d1d2083af205cbb7833a79ef5a49d1", + "https://github.com/etcd-io/etcd/releases/download/v3.4.15/etcd-v3.4.15-linux-amd64.tar.gz": "3bd00836ea328db89ecba3ed2155293934c0d09e64b53d6c9dfc0a256e724b81", + "https://github.com/etcd-io/etcd/releases/download/v3.4.15/etcd-v3.4.15-darwin-amd64.tar.gz": "c596709069193bffc639a22558bdea4d801128e635909ea01a6fd5b5c85da729", + "https://github.com/etcd-io/etcd/releases/download/v3.3.10/etcd-v3.3.10-linux-amd64.tar.gz": "1620a59150ec0a0124a65540e23891243feb2d9a628092fb1edcc23974724a45", + "https://github.com/etcd-io/etcd/releases/download/v3.3.10/etcd-v3.3.10-darwin-amd64.tar.gz": "fac4091c7ba6f032830fad7809a115909d0f0cae5cbf5b34044540def743577b", + "https://github.com/etcd-io/etcd/releases/download/v3.2.25/etcd-v3.2.25-linux-amd64.tar.gz": "8a509ffb1443088d501f19e339a0d9c0058ce20599752c3baef83c1c68790ef7", + "https://github.com/etcd-io/etcd/releases/download/v3.2.25/etcd-v3.2.25-darwin-amd64.tar.gz": "9950684a01d7431bc12c3dba014f222d55a862c6f8af64c09c42d7a59ed6790d", } checksums_consul = { - "https://releases.hashicorp.com/consul/1.7.2/consul_1.7.2_linux_amd64.zip": - "5ab689cad175c08a226a5c41d16392bc7dd30ceaaf90788411542a756773e698", - "https://releases.hashicorp.com/consul/1.7.2/consul_1.7.2_darwin_amd64.zip": - "c474f00b022cae38acae2d19b2a707a4fcb08dfdd22875efeefdf052ce19c90b", - "https://releases.hashicorp.com/consul/1.7.2/consul_1.7.2_windows_amd64.zip": - "e9b9355f77f80b2c0940888cb0d27c44a5879c31e379ef21ffcfd36c91d202c1", + "https://releases.hashicorp.com/consul/1.9.4/consul_1.9.4_linux_amd64.zip": "da3919197ef33c4205bb7df3cc5992ccaae01d46753a72fe029778d7f52fb610", + "https://releases.hashicorp.com/consul/1.9.4/consul_1.9.4_linux_arm64.zip": "012c552aff502f907416c9a119d2dfed88b92e981f9b160eb4fe292676afdaeb", + "https://releases.hashicorp.com/consul/1.9.4/consul_1.9.4_darwin.zip": "c168240d52f67c71b30ef51b3594673cad77d0dbbf38c412b2ee30b39ef30843", + "https://releases.hashicorp.com/consul/1.6.1/consul_1.6.1_linux_amd64.zip": "a8568ca7b6797030b2c32615b4786d4cc75ce7aee2ed9025996fe92b07b31f7e", + "https://releases.hashicorp.com/consul/1.6.1/consul_1.6.1_darwin_amd64.zip": "4bc205e06b2921f998cb6ddbe70de57f8e558e226e44aba3f337f2f245678b85", + "https://releases.hashicorp.com/consul/1.5.0/consul_1.5.0_linux_amd64.zip": "1399064050019db05d3378f757e058ec4426a917dd2d240336b51532065880b6", + "https://releases.hashicorp.com/consul/1.5.0/consul_1.5.0_darwin_amd64.zip": "b4033ea6871fe6136ee5d940c834be2248463c3ec248dc22370e6d5360931325", } @@ -58,14 +64,8 @@ def install_traefik(prefix, plat, traefik_version): traefik_archive = "traefik_v" + traefik_version + "_" + plat + "." + traefik_archive_extension traefik_archive_path = os.path.join(prefix, traefik_archive) - traefik_archive = "traefik_v" + traefik_version + "_" + plat + "." + traefik_archive_extension - traefik_archive_path = os.path.join(prefix, traefik_archive) - - traefik_archive = "traefik_v" + traefik_version + "_" + plat + "." + traefik_archive_extension - traefik_archive_path = os.path.join(prefix, traefik_archive) - traefik_url = ( - "https://github.com/containous/traefik/releases" + "https://github.com/traefik/traefik/releases" f"/download/v{traefik_version}/{traefik_archive}" ) @@ -73,7 +73,7 @@ def install_traefik(prefix, plat, traefik_version): print(f"Traefik already exists") if traefik_url not in checksums_traefik: warnings.warn( - f"Traefik {traefik_version} not supported !", + f"Traefik {traefik_version} not tested !", stacklevel=2, ) os.chmod(traefik_bin, 0o755) @@ -89,28 +89,27 @@ def install_traefik(prefix, plat, traefik_version): os.remove(traefik_archive_path) os.remove(traefik_bin) - if traefik_url in checksums_traefik: - print(f"Downloading traefik {traefik_version}...") - urlretrieve(traefik_url, traefik_archive_path) + print(f"Downloading traefik {traefik_version} from {traefik_url}...") + urlretrieve(traefik_url, traefik_archive_path) + if traefik_url in checksums_traefik: if checksum_file(traefik_archive_path) != checksums_traefik[traefik_url]: raise IOError("Checksum failed") - - print("Extracting the archive...") - if traefik_archive_extension == "tar.gz": - with tarfile.open(traefik_archive_path, "r") as tar_ref: - tar_ref.extract("traefik", prefix) - else: - with zipfile.ZipFile(traefik_archive_path, "r") as zip_ref: - zip_ref.extract("traefik.exe", prefix) - - os.chmod(traefik_bin, 0o755) else: warnings.warn( - f"Traefik {traefik_version} not supported !", + f"Traefik {traefik_version} not tested !", stacklevel=2, ) + print("Extracting the archive...") + if traefik_archive_extension == "tar.gz": + with tarfile.open(traefik_archive_path, "r") as tar_ref: + tar_ref.extract("traefik", prefix) + else: + with zipfile.ZipFile(traefik_archive_path, "r") as zip_ref: + zip_ref.extract("traefik.exe", prefix) + + os.chmod(traefik_bin, 0o755) print("--- Done ---") @@ -137,8 +136,8 @@ def install_etcd(prefix, plat, etcd_version): print(f"Etcd and etcdctl already exist") if etcd_url not in checksums_etcd: warnings.warn( - f"Etcd {etcd_version} not supported !", - stacklevel=2, + f"Etcd {etcd_version} not supported ! Or, at least, we don't " + f"recognise {etcd_url} in our checksums", stacklevel=2, ) os.chmod(etcd_bin, 0o755) os.chmod(etcdctl_bin, 0o755) @@ -188,7 +187,8 @@ def install_etcd(prefix, plat, etcd_version): shutil.rmtree(etcd_binaries) else: warnings.warn( - f"Etcd {etcd_version} not supported !", + f"Etcd {etcd_version} not supported ! Or, at least, we don't " + f"recognise {etcd_url} in our checksums", stacklevel=2 ) @@ -215,7 +215,8 @@ def install_consul(prefix, plat, consul_version): print(f"Consul already exists") if consul_url not in checksums_consul: warnings.warn( - f"Consul {consul_version} not supported !", + f"Consul {consul_version} not supported ! Or, at least we don't have " + f"it {consul_url} in our checksums", stacklevel=2, ) os.chmod(consul_bin, 0o755) @@ -250,7 +251,8 @@ def install_consul(prefix, plat, consul_version): shutil.rmtree(consul_binaries) else: warnings.warn( - f"Consul {consul_version} not supported !", + f"Consul {consul_version} not supported ! Or, at least we don't have " + f"it {consul_url} in our checksums", stacklevel=2, ) @@ -348,7 +350,7 @@ def main(): parser.add_argument( "--etcd-version", dest="etcd_version", - default="3.4.7", + default="3.4.15", help=textwrap.dedent( """\ The version of etcd to download. diff --git a/jupyterhub_traefik_proxy/proxy.py b/jupyterhub_traefik_proxy/proxy.py index ecb1fd77..7994750b 100644 --- a/jupyterhub_traefik_proxy/proxy.py +++ b/jupyterhub_traefik_proxy/proxy.py @@ -20,7 +20,8 @@ import json from os.path import abspath, dirname, join -from subprocess import Popen +from subprocess import Popen, TimeoutExpired +import asyncio.subprocess from urllib.parse import urlparse from traitlets import Any, Bool, Dict, Integer, Unicode, default @@ -41,7 +42,7 @@ class TraefikProxy(Proxy): ) traefik_api_url = Unicode( - "http://127.0.0.1:8099", + "http://localhost:8099", config=True, help="""traefik authenticated api endpoint url""", ) @@ -52,12 +53,36 @@ class TraefikProxy(Proxy): help="""validate SSL certificate of traefik api endpoint""", ) - traefik_log_level = Unicode("ERROR", config=True, help="""traefik's log level""") + debug = Bool(False, config=True, help="""Debug the proxy class?""") + + traefik_log_level = Unicode("DEBUG", config=True, help="""traefik's log level""") traefik_api_password = Unicode( config=True, help="""The password for traefik api login""" ) + provider_name = Unicode( + config=True, help="""The provider name that Traefik expects, e.g. file, consul, etcd""" + ) + + def __init__(self, **kwargs): + super().__init__(**kwargs) + if kwargs.get('debug', self.debug) == True: + import sys, logging + # Check we don't already have a StreamHandler + addHandler = True + for handler in self.log.handlers: + if isinstance(handler, logging.StreamHandler): + addHandler = False + if addHandler: + self.log.setLevel("DEBUG") + handler = logging.StreamHandler(sys.stdout) + handler.setLevel("DEBUG") + self.log.addHandler(handler) + self.log.debug(f"Initialising {type(self).__name__}") + + #if kwargs.get('debug', self.debug) is True: + @default("traefik_api_password") def _warn_empty_password(self): self.log.warning("Traefik API password was not set.") @@ -100,41 +125,38 @@ def _warn_empty_username(self): ) static_config = Dict() + dynamic_config = Dict() def _generate_htpassword(self): - from passlib.apache import HtpasswdFile - - ht = HtpasswdFile() - ht.set_password(self.traefik_api_username, self.traefik_api_password) - self.traefik_api_hashed_password = str(ht.to_string()).split(":")[1][:-3] + from passlib.hash import apr_md5_crypt + self.traefik_api_hashed_password = apr_md5_crypt.hash(self.traefik_api_password) async def _check_for_traefik_service(self, routespec, kind): - """Check for an expected router or service + """Check for an expected router or service in the Traefik API. This is used to wait for traefik to load configuration from a provider """ # expected e.g. 'service' + '_' + routespec @ file - expected = traefik_utils.generate_alias(routespec, kind) + "@file" - path = "/api/http/{0}s".format(kind) + expected = traefik_utils.generate_alias(routespec, kind) + "@" + self.provider_name + path = f"/api/http/{kind}s" try: resp = await self._traefik_api_request(path) json_data = json.loads(resp.body) except Exception: - self.log.exception("Error checking traefik api for %s %s", kind, routespec) + self.log.exception(f"Error checking traefik api for {kind} {routespec}") return False service_names = [service['name'] for service in json_data] if expected not in service_names: - self.log.debug("traefik %s not yet in %s", expected, kind) - self.log.debug("Current traefik %ss: %s", kind, json_data) + self.log.debug(f"traefik {expected} not yet in {kind}") return False # found the expected endpoint return True async def _wait_for_route(self, routespec): - self.log.info("Waiting for %s to register with traefik", routespec) + self.log.info(f"Waiting for {routespec} to register with traefik") async def _check_traefik_dynamic_conf_ready(): """Check if traefik loaded its dynamic configuration yet""" @@ -151,7 +173,7 @@ async def _check_traefik_dynamic_conf_ready(): await exponential_backoff( _check_traefik_dynamic_conf_ready, - "Traefik route for %s configuration not available" % routespec, + f"Traefik route for {routespec} configuration not available", timeout=self.check_route_timeout, ) @@ -169,13 +191,14 @@ async def _traefik_api_request(self, path): self.log.warning("%s GET %s", resp.code, url) else: self.log.debug("%s GET %s", resp.code, url) + + self.log.debug(f"Succesfully received data from {path}: {resp.body}") return resp - async def _wait_for_static_config(self, provider): + async def _wait_for_static_config(self): async def _check_traefik_static_conf_ready(): """Check if traefik loaded its static configuration yet""" try: - #resp = await self._traefik_api_request("/api/overview/providers/" + provider) resp = await self._traefik_api_request("/api/overview") except Exception: self.log.exception("Error checking for traefik static configuration") @@ -198,58 +221,118 @@ async def _check_traefik_static_conf_ready(): def _stop_traefik(self): self.log.info("Cleaning up proxy[%i]...", self.traefik_process.pid) - self.traefik_process.kill() - self.traefik_process.wait() - - def _launch_traefik(self, config_type): - if config_type == "fileprovider" or config_type == "etcdv3" or config_type == "consul": - config_file_path = abspath(join(dirname(__file__), "traefik.toml")) - self.traefik_process = Popen( - ["traefik", "-c", config_file_path], stdout=None - ) - else: + self.traefik_process.terminate() + try: + self.traefik_process.communicate(timeout=10) + except TimeoutExpired: + self.traefik_process.kill() + self.traefik_process.communicate() + finally: + self.traefik_process.wait() + + def _start_traefik(self): + if self.provider_name not in ("file", "etcd", "consul"): raise ValueError( "Configuration mode not supported \n.\ - The proxy can only be configured through toml, etcd and consul" + The proxy can only be configured through fileprovider, etcd and consul" ) + try: + self.traefik_process = Popen([ + "traefik", "--configfile", abspath(self.static_config_file) + ]) + except FileNotFoundError as e: + self.log.error( + "Failed to find traefik \n" + "The proxy can be downloaded from https://github.com/containous/traefik/releases/download." + ) + raise + except Exception as e: + self.log.exception(e) + raise async def _setup_traefik_static_config(self): + """When should_start=True, we are in control of traefik's static configuration + file. This sets up the entrypoints and api handler in self.static_config, and + then saves it to :attrib:`self.static_config_file`. + + Subclasses should specify any traefik providers themselves, in + :attrib:`self.static_config["providers"]` + """ self.log.info("Setting up traefik's static config...") - self._generate_htpassword() - self.static_config = {} - self.static_config["debug"] = True - self.static_config["logLevel"] = self.traefik_log_level + self.static_config["log"] = { "level": self.traefik_log_level } + entryPoints = {} if self.ssl_cert and self.ssl_key: - self.static_config["defaultentrypoints"] = ["https"] - entryPoints["https"] = { + entryPoints["websecure"] = { "address": ":" + str(urlparse(self.public_url).port), - "tls": { - "certificates": [ - {"certFile": self.ssl_cert, "keyFile": self.ssl_key} - ] - }, + "tls": {}, } else: - self.static_config["defaultentrypoints"] = ["http"] - entryPoints["http"] = {"address": ":" + str(urlparse(self.public_url).port)} - - auth = { - "basic": { - "users": [ - self.traefik_api_username + ":" + self.traefik_api_hashed_password - ] - } - } - entryPoints["auth_api"] = { + entryPoints["web"] = {"address": ":" + str(urlparse(self.public_url).port)} + + entryPoints["enter_api"] = { "address": ":" + str(urlparse(self.traefik_api_url).port), - "auth": auth, } self.static_config["entryPoints"] = entryPoints - self.static_config["api"] = {"dashboard": True, "entrypoint": "auth_api"} - self.static_config["wss"] = {"protocol": "http"} + self.static_config["api"] = {"dashboard": True} + + handler = traefik_utils.TraefikConfigFileHandler(self.static_config_file) + try: + self.log.debug(f"Persisting the static config: {self.static_config}") + handler.atomic_dump(self.static_config) + except IOError: + self.log.exception("Couldn't set up traefik's static config.") + raise + except: + self.log.error("Couldn't set up traefik's static config. Unexpected error:") + raise + + async def _setup_traefik_dynamic_config(self): + self.log.info("Setting up traefik's dynamic config...") + self._generate_htpassword() + api_url = urlparse(self.traefik_api_url) + api_path = api_url.path if api_url.path else '/api' + api_credentials = "{0}:{1}".format( + self.traefik_api_username, + self.traefik_api_hashed_password + ) + self.dynamic_config.update({ + "http": { + "routers": { + "route_api": { + "rule": f"Host(`{api_url.hostname}`) && (PathPrefix(`{api_path}`) || PathPrefix(`/dashboard`))", + "entryPoints": ["enter_api"], + "service": "api@internal", + "middlewares": ["auth_api"] + }, + }, + "middlewares": { + "auth_api": { + "basicAuth": { + "users": [ + api_credentials + ] + } + } + } + } + }) + if self.ssl_cert and self.ssl_key: + self.dynamic_config.update({ + "tls": { + "stores": { + "default": { + "defaultCertificate": { + "certFile": self.ssl_cert, + "keyFile": self.ssl_key + } + } + } + } + }) + def _routespec_to_traefik_path(self, routespec): path = self.validate_routespec(routespec) @@ -271,6 +354,7 @@ async def start(self): if the proxy is to be started by the Hub """ await self._setup_traefik_static_config() + await self._setup_traefik_dynamic_config() self._start_traefik() async def stop(self): @@ -348,3 +432,14 @@ async def get_route(self, routespec): None: if there are no routes matching the given routespec """ raise NotImplementedError() + + async def persist_dynamic_config(self): + """Update the Traefik dynamic configuration, depending on the backend + provider in use. This is used to e.g. set up the api endpoint's + authentication (username and password), as well as default tls + certificates to use. + + :arg:`settings` is a Dict containing the traefik settings, which will + be updated on the Traefik provider depending on the subclass in use. + """ + raise NotImplementedError() diff --git a/jupyterhub_traefik_proxy/traefik_utils.py b/jupyterhub_traefik_proxy/traefik_utils.py index c4cb7794..29bafcaa 100644 --- a/jupyterhub_traefik_proxy/traefik_utils.py +++ b/jupyterhub_traefik_proxy/traefik_utils.py @@ -13,8 +13,10 @@ class KVStorePrefix(Unicode): def validate(self, obj, value): u = super().validate(obj, value) - if not u.endswith("/"): - u = u + "/" + # We'll join the prefix with e.g. prefix.join(pathspec), + # therefore always strip the trailing "/" from any prefix + if u.endswith("/"): + u = u.rstrip("/") proxy_class = type(obj).__name__ if "Consul" in proxy_class and u.startswith("/"): @@ -41,31 +43,36 @@ def generate_alias(routespec, server_type=""): return server_type + "_" + escapism.escape(routespec, safe=safe) -def generate_service_entry( proxy, service_alias, separator="/", url=False, - weight=False): - service_entry = "" +def generate_service_entry( proxy, service_alias, separator="/", url=False): + service_entry = separator.join( + ["http", "services", service_alias, "loadBalancer", "servers", "server1"] + ) if separator == "/": - service_entry = proxy.kv_traefik_prefix - service_entry += separator.join(["services", service_alias, "servers", "server1"]) + service_entry = proxy.kv_traefik_prefix + separator + service_entry if url: service_entry += separator + "url" - elif weight: - service_entry += separator + "weight" - return service_entry +def generate_service_weight_entry( proxy, service_alias, separator="/"): + return separator.join( + [proxy.kv_traefik_prefix, "http", "services", service_alias, + "weighted", "services", "0", "weight"] + ) + def generate_router_service_entry(proxy, router_alias): - return proxy.kv_traefik_prefix + "routers/" + router_alias + "/service" + return "/".join( + [proxy.kv_traefik_prefix, "http", "routers", router_alias, "service"] + ) def generate_router_rule_entry(proxy, router_alias, separator="/"): router_rule_entry = separator.join( - ["routers", router_alias, "routes", "test"] + ["http", "routers", router_alias] ) if separator == "/": - router_rule_entry = ( - proxy.kv_traefik_prefix + router_rule_entry + separator + "rule" + router_rule_entry = separator.join( + [proxy.kv_traefik_prefix, router_rule_entry, "rule"] ) return router_rule_entry @@ -80,7 +87,6 @@ def generate_route_keys(proxy, routespec, separator="/"): [ "service_alias", "service_url_path", - "service_weight_path", "router_alias", "router_service_path", "router_rule_path", @@ -90,7 +96,6 @@ def generate_route_keys(proxy, routespec, separator="/"): if separator != ".": service_url_path = generate_service_entry(proxy, service_alias, url=True) router_rule_path = generate_router_rule_entry(proxy, router_alias) - service_weight_path = generate_service_entry(proxy, service_alias, weight=True) router_service_path = generate_router_service_entry(proxy, router_alias) else: service_url_path = generate_service_entry( @@ -99,13 +104,11 @@ def generate_route_keys(proxy, routespec, separator="/"): router_rule_path = generate_router_rule_entry( proxy, router_alias, separator=separator ) - service_weight_path = "" router_service_path = "" return RouteKeys( service_alias, service_url_path, - service_weight_path, router_alias, router_service_path, router_rule_path, @@ -146,7 +149,8 @@ class TraefikConfigFileHandler(object): def __init__(self, file_path): file_ext = file_path.rsplit('.', 1)[-1] if file_ext == 'yaml': - import yaml as config_handler + from ruamel.yaml import YAML + config_handler = YAML(typ="safe") elif file_ext == 'toml': import toml as config_handler else: @@ -161,7 +165,8 @@ def __init__(self, file_path): def load(self): """Depending on self.file_path, call either yaml.load or toml.load""" - return self._load(self.file_path) + with open(self.file_path, "r") as fd: + return self._load(fd) def dump(self, data): with open(self.file_path, "w") as f: @@ -177,10 +182,3 @@ def persist_static_conf(file_path, static_conf_dict): handler = TraefikConfigFileHandler(file_path) handler.dump(static_conf_dict) -def persist_routes(file_path, routes_dict): - handler = TraefikConfigFileHandler(file_path) - handler.atomic_dump(routes_dict) - -def load_routes(file_path): - handler = TraefikConfigFileHandler(file_path) - return handler.load() diff --git a/requirements.txt b/requirements.txt index ccb6dc38..b9540e50 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,6 @@ aiohttp escapism jupyterhub>=0.9 passlib -toml +# toml is now optional, as can use yaml configuration files instead now... +#toml +#ruamel.yaml diff --git a/tests/config_files/dynamic_config/dynamic_conf.toml b/tests/config_files/dynamic_config/dynamic_conf.toml new file mode 100644 index 00000000..e309690e --- /dev/null +++ b/tests/config_files/dynamic_config/dynamic_conf.toml @@ -0,0 +1,11 @@ +# Example dynamic configuration file for an external file provider proxy. +# Defines the API listener and its authentication + +[http.routers.router-api] + rule = "Host(`localhost`) && (PathPrefix(`/api`) || PathPrefix(`/dashboard`))" + entryPoints = ["enter_api"] + service = "api@internal" + middlewares = ["auth_api"] + +[http.middlewares.auth_api.basicAuth] + users = [ "api_admin:$apr1$eS/j3kum$q/X2khsIEG/bBGsteP.x./",] diff --git a/tests/config_files/rules.toml b/tests/config_files/rules.toml deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/config_files/traefik.toml b/tests/config_files/traefik.toml index 93013389..5f57109c 100644 --- a/tests/config_files/traefik.toml +++ b/tests/config_files/traefik.toml @@ -1,23 +1,20 @@ -defaultentrypoints = [ "http",] -debug = true -logLevel = "ERROR" +[log] + level = "debug" [api] -dashboard = true -entrypoint = "auth_api" + dashboard = true [wss] -protocol = "http" + protocol = "http" -[file] -filename = "./tests/config_files/rules.toml" -watch = true +[providers.file] + directory = "./tests/config_files/dynamic_config" + watch = true -[entryPoints.http] -address = "127.0.0.1:8000" +[entryPoints] + [entryPoints.my_web_api] + address = "127.0.0.1:8000" -[entryPoints.auth_api] -address = "127.0.0.1:8099" + [entryPoints.enter_api] + address = "127.0.0.1:8099" -[entryPoints.auth_api.auth.basic] -users = [ "api_admin:$apr1$eS/j3kum$q/X2khsIEG/bBGsteP.x./",] diff --git a/tests/conftest.py b/tests/conftest.py index e1ca34e5..6776c529 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,7 @@ import pytest from _pytest.mark import Mark -from jupyterhub_traefik_proxy import TraefikTomlProxy +from jupyterhub_traefik_proxy import TraefikFileProviderProxy # Define a "slow" test marker so that we can run the slow tests at the end @@ -36,14 +36,40 @@ def pytest_configure(config): @pytest.fixture +# There must be a way to parameterise this to run on both yaml and toml files? async def toml_proxy(): - """Fixture returning a configured TraefikTomlProxy""" - proxy = TraefikTomlProxy( + """Fixture returning a configured TraefikFileProviderProxy""" + dynamic_config_file = os.path.join( + os.getcwd(), "tests", "config_files", "dynamic_config", "rules.toml" + ) + proxy = TraefikFileProviderProxy( + public_url="http://127.0.0.1:8000", + traefik_api_password="admin", + traefik_api_username="api_admin", + check_route_timeout=180, + should_start=True, + dynamic_config_file=dynamic_config_file, + static_config_file="traefik.toml" + ) + + await proxy.start() + yield proxy + await proxy.stop() + + +@pytest.fixture +async def yaml_proxy(): + dynamic_config_file = os.path.join( + os.getcwd(), "tests", "config_files", "dynamic_config", "rules.yaml" + ) + proxy = TraefikFileProviderProxy( public_url="http://127.0.0.1:8000", traefik_api_password="admin", traefik_api_username="api_admin", check_route_timeout=180, should_start=True, + dynamic_config_file=dynamic_config_file, + static_config_file="traefik.yaml" ) await proxy.start() @@ -52,23 +78,49 @@ async def toml_proxy(): @pytest.fixture -def external_toml_proxy(): - proxy = TraefikTomlProxy( +async def external_toml_proxy(launch_traefik_file): + dynamic_config_file = os.path.join( + os.getcwd(), "tests", "config_files", "dynamic_config", "rules.toml" + ) + proxy = TraefikFileProviderProxy( public_url="http://127.0.0.1:8000", traefik_api_password="admin", traefik_api_username="api_admin", check_route_timeout=45, + should_start=False, + dynamic_config_file=dynamic_config_file, ) - proxy.should_start = False - proxy.toml_dynamic_config_file = "./tests/config_files/rules.toml" - # Start traefik manually - traefik_process = subprocess.Popen( - ["traefik", "-c", "./tests/config_files/traefik.toml"], stdout=None + + yield proxy + os.remove(dynamic_config_file) + + +@pytest.fixture +async def external_yaml_proxy(launch_traefik_file): + dynamic_config_file = os.path.join( + os.getcwd(), "tests", "config_files", "dynamic_config", "rules.yaml" + ) + proxy = TraefikFileProviderProxy( + public_url="http://127.0.0.1:8000", + traefik_api_password="admin", + traefik_api_username="api_admin", + check_route_timeout=180, + should_start=False, + dynamic_config_file=dynamic_config_file, ) + yield proxy - open("./tests/config_files/rules.toml", "w").close() - traefik_process.kill() - traefik_process.wait() + os.remove(dynamic_config_file) + + +@pytest.fixture +def launch_traefik_file(): + proc = subprocess.Popen( + ["traefik", "--configfile", "./tests/config_files/traefik.toml"] + ) + yield proc + proc.kill() + proc.wait() @pytest.fixture(scope="session", autouse=False) diff --git a/tests/dummy_http_server.py b/tests/dummy_http_server.py index 73941b9b..11f572a7 100644 --- a/tests/dummy_http_server.py +++ b/tests/dummy_http_server.py @@ -42,7 +42,7 @@ def run(port=80): run(port=int(argv[1])) else: asyncio.get_event_loop().run_until_complete( - websockets.serve(send_port, "localhost", int(argv[1])) + websockets.serve(send_port, "127.0.0.1", int(argv[1])) ) asyncio.get_event_loop().run_forever() else: diff --git a/tests/proxytest.py b/tests/proxytest.py index 90c8ab17..13d9c4d8 100644 --- a/tests/proxytest.py +++ b/tests/proxytest.py @@ -89,6 +89,8 @@ def _launch_backend(port, proto="http"): for proc in running_backends: proc.kill() + for proc in running_backends: + proc.communicate() for proc in running_backends: proc.wait() @@ -145,7 +147,7 @@ async def wait_for_services(urls): ), ], ) -async def test_add_get_delete( +async def _test_add_get_delete( request, proxy, launch_backend, routespec, existing_routes, event_loop ): default_target = "http://127.0.0.1:9000" @@ -282,7 +284,7 @@ async def _wait_for_deletion(): await test_route_exist(spec, extra_backends[i]) -async def test_get_all_routes(proxy, launch_backend): +async def _test_get_all_routes(proxy, launch_backend): routespecs = ["/proxy/path1", "/proxy/path2/", "/proxy/path3/"] targets = [ "http://127.0.0.1:9900", @@ -328,7 +330,7 @@ async def test_get_all_routes(proxy, launch_backend): assert routes == expected_output -async def test_host_origin_headers(proxy, launch_backend): +async def _test_host_origin_headers(proxy, launch_backend): routespec = "/user/username/" target = "http://127.0.0.1:9000" data = {} @@ -372,7 +374,7 @@ async def test_host_origin_headers(proxy, launch_backend): @pytest.mark.parametrize("username", ["zoe", "50fia", "秀樹", "~TestJH", "has@"]) -async def test_check_routes(proxy, username): +async def _test_check_routes(proxy, username): # fill out necessary attributes for check_routes proxy.app = MockApp() proxy.hub = proxy.app.hub diff --git a/tests/test_proxy.py b/tests/test_proxy.py index 6cf311e2..a202b691 100644 --- a/tests/test_proxy.py +++ b/tests/test_proxy.py @@ -10,8 +10,18 @@ @pytest.fixture( params=[ + #"no_auth_consul_proxy", + #"auth_consul_proxy", + #"no_auth_etcd_proxy", + #"auth_etcd_proxy", "toml_proxy", + "yaml_proxy", + #"external_consul_proxy", + #"auth_external_consul_proxy", + #"external_etcd_proxy", + #"auth_external_etcd_proxy", "external_toml_proxy", + "external_yaml_proxy", ] ) def proxy(request): diff --git a/tests/test_traefik_api_auth.py b/tests/test_traefik_api_auth.py index edf8f50b..ece2064a 100644 --- a/tests/test_traefik_api_auth.py +++ b/tests/test_traefik_api_auth.py @@ -13,6 +13,11 @@ @pytest.fixture( params=[ "toml_proxy", + "yaml_proxy", + "no_auth_etcd_proxy", + "auth_etcd_proxy", + "no_auth_consul_proxy", + "auth_consul_proxy", ] ) def proxy(request): @@ -24,25 +29,43 @@ def proxy(request): [("api_admin", "admin", 200), ("api_admin", "1234", 401), ("", "", 401)], ) async def test_traefik_api_auth(proxy, username, password, expected_rc): - traefik_port = urlparse(proxy.public_url).port + traefik_api_url = proxy.traefik_api_url + "/api" - await exponential_backoff( - utils.check_host_up, "Traefik not reacheable", ip="localhost", port=traefik_port - ) + # Must have a trailing slash! + dashboard_url = proxy.traefik_api_url + "/dashboard/" + + # There is now a delay between traefik's public ports + # being reachable and the API being accessible. So, give traefik + # a chance to load its dynamic configuration and configure the + # API handler + async def api_login(): + try: + if not username and not password: + resp = await AsyncHTTPClient().fetch(traefik_api_url) + else: + resp = await AsyncHTTPClient().fetch( + dashboard_url, + auth_username=username, + auth_password=password, + ) + except ConnectionRefusedError: + rc = None + except Exception as e: + rc = e.response.code + else: + rc = resp.code + return rc - try: - if not username and not password: - resp = await AsyncHTTPClient().fetch(proxy.traefik_api_url + "/dashboard") + async def cmp_api_login(): + rc = await api_login() + if rc == expected_rc: + return True else: - resp = await AsyncHTTPClient().fetch( - proxy.traefik_api_url + "/dashboard/", - auth_username=username, - auth_password=password, - ) - rc = resp.code - except ConnectionRefusedError: - rc = None - except Exception as e: - rc = e.response.code + return False + + await exponential_backoff( + cmp_api_login, "Traefik API not reachable" + ) + rc = await api_login() assert rc == expected_rc diff --git a/tests/test_traefik_utils.py b/tests/test_traefik_utils.py index edfd4013..a6e4238c 100644 --- a/tests/test_traefik_utils.py +++ b/tests/test_traefik_utils.py @@ -29,7 +29,8 @@ def test_roundtrip_routes(): file = "test_roudtrip.toml" open(file, "a").close() traefik_utils.persist_routes(file, routes) - reloaded = traefik_utils.load_routes(file) + handler = traefik_utils.TraefikConfigFileHandler(file) + reloaded = handler.load() os.remove(file) assert reloaded == routes From 3f5b6732207f93fd1c0ca1ec84a04f0012ccb774 Mon Sep 17 00:00:00 2001 From: Alex Leach Date: Mon, 5 Jul 2021 21:11:38 +0000 Subject: [PATCH 05/19] Rename docs/source/toml.md to docs/source/yaml.md --- docs/source/{toml.md => yaml.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/source/{toml.md => yaml.md} (100%) diff --git a/docs/source/toml.md b/docs/source/yaml.md similarity index 100% rename from docs/source/toml.md rename to docs/source/yaml.md From 176a6ae9e8ca04e01d0e51bf8d7918e12f97b95b Mon Sep 17 00:00:00 2001 From: Alex Leach Date: Tue, 6 Jul 2021 14:31:18 +0000 Subject: [PATCH 06/19] Revert some breaking changes and tidy for PR - fileprovider.py - Reintroduce _start_traefik, removed in commit 05e7c070ca113fe56cd3f419109867b423957826 - install.py - Revert some changes made in commit 05e7c070ca113fe56cd3f419109867b423957826. - proxy.py - Revert some API changes on the method signatures, which broke compatibility with the TKvProxy subclasses. - test_install.py - Make sure download traefik v2, as the download URLs have changed since v1 (and is reflected in install.py). - test_traefik_api_auth.py - No longer test TKvProxy subclasses, they don't work - test_traefik_utils.py - Remove reference to traefik_utils.persist_routes, which no longer exists. --- jupyterhub_traefik_proxy/fileprovider.py | 11 ++ jupyterhub_traefik_proxy/install.py | 143 +++++++++++------------ jupyterhub_traefik_proxy/proxy.py | 12 +- tests/test_installer.py | 4 +- tests/test_traefik_api_auth.py | 8 +- tests/test_traefik_utils.py | 7 +- 6 files changed, 101 insertions(+), 84 deletions(-) diff --git a/jupyterhub_traefik_proxy/fileprovider.py b/jupyterhub_traefik_proxy/fileprovider.py index 020eb9fc..42acb013 100644 --- a/jupyterhub_traefik_proxy/fileprovider.py +++ b/jupyterhub_traefik_proxy/fileprovider.py @@ -93,6 +93,17 @@ async def _setup_traefik_static_config(self): } await super()._setup_traefik_static_config() + def _start_traefik(self): + self.log.info("Starting traefik...") + try: + self._launch_traefik() + except FileNotFoundError as e: + self.log.error( + "Failed to find traefik \n" + "The proxy can be downloaded from https://github.com/containous/traefik/releases/download." + ) + raise + def _clean_resources(self): try: if self.should_start: diff --git a/jupyterhub_traefik_proxy/install.py b/jupyterhub_traefik_proxy/install.py index 23b5724d..f5017cef 100644 --- a/jupyterhub_traefik_proxy/install.py +++ b/jupyterhub_traefik_proxy/install.py @@ -69,52 +69,47 @@ def install_traefik(prefix, plat, traefik_version): f"/download/v{traefik_version}/{traefik_archive}" ) - if os.path.exists(traefik_bin) and os.path.exists(traefik_archive_path): + if os.path.exists(traefik_bin): print(f"Traefik already exists") if traefik_url not in checksums_traefik: warnings.warn( - f"Traefik {traefik_version} not tested !", + f"Couldn't verify checksum for traefik-v{traefik_version}-{plat}", stacklevel=2, ) os.chmod(traefik_bin, 0o755) print("--- Done ---") return else: - if checksum_file(traefik_archive_path) == checksums_traefik[traefik_url]: + checksum = checksum_file(traefik_bin) + if checksum == checksums_traefik[traefik_url]: os.chmod(traefik_bin, 0o755) print("--- Done ---") return else: - print(f"checksum mismatch on {traefik_archive_path}") - os.remove(traefik_archive_path) + print(f"checksum mismatch on {traefik_bin}") os.remove(traefik_bin) - print(f"Downloading traefik {traefik_version} from {traefik_url}...") - urlretrieve(traefik_url, traefik_archive_path) + print(f"Downloading traefik {traefik_version}...") + urlretrieve(traefik_url, traefik_bin) if traefik_url in checksums_traefik: - if checksum_file(traefik_archive_path) != checksums_traefik[traefik_url]: + checksum = checksum_file(traefik_bin) + if checksum != checksums_traefik[traefik_url]: raise IOError("Checksum failed") else: warnings.warn( - f"Traefik {traefik_version} not tested !", + f"Couldn't verify checksum for traefik-v{traefik_version}-{plat}", stacklevel=2, ) - print("Extracting the archive...") - if traefik_archive_extension == "tar.gz": - with tarfile.open(traefik_archive_path, "r") as tar_ref: - tar_ref.extract("traefik", prefix) - else: - with zipfile.ZipFile(traefik_archive_path, "r") as zip_ref: - zip_ref.extract("traefik.exe", prefix) - os.chmod(traefik_bin, 0o755) + print("--- Done ---") def install_etcd(prefix, plat, etcd_version): etcd_downloaded_dir_name = f"etcd-v{etcd_version}-{plat}" + etcd_archive_extension = ".tar.gz" if "linux" in plat: etcd_archive_extension = "tar.gz" else: @@ -136,74 +131,76 @@ def install_etcd(prefix, plat, etcd_version): print(f"Etcd and etcdctl already exist") if etcd_url not in checksums_etcd: warnings.warn( - f"Etcd {etcd_version} not supported ! Or, at least, we don't " - f"recognise {etcd_url} in our checksums", stacklevel=2, + f"Couldn't verify checksum for etcd-v{etcd_version}-{plat}", + stacklevel=2, ) os.chmod(etcd_bin, 0o755) os.chmod(etcdctl_bin, 0o755) print("--- Done ---") return else: - if checksum_file(etcd_downloaded_archive) == checksums_etcd[etcd_url]: + checksum_etcd_archive = checksum_file(etcd_downloaded_archive) + if checksum_etcd_archive == checksums_etcd[etcd_url]: os.chmod(etcd_bin, 0o755) os.chmod(etcdctl_bin, 0o755) print("--- Done ---") return else: - print(f"checksum mismatch on {etcd_downloaded_archive}") + print(f"checksum mismatch on etcd") os.remove(etcd_bin) os.remove(etcdctl_bin) os.remove(etcd_downloaded_archive) - if etcd_url in checksums_etcd: - if not os.path.exists(etcd_downloaded_archive): - print(f"Downloading {etcd_downloaded_dir_name} archive...") - urlretrieve(etcd_url, etcd_downloaded_archive) - else: - print(f"Archive {etcd_downloaded_dir_name} already exists") - - if checksum_file(etcd_downloaded_archive) != checksums_etcd[etcd_url]: - raise IOError("Checksum failed") - - print("Extracting the archive...") - - if etcd_archive_extension == "zip": - with zipfile.ZipFile(etcd_downloaded_archive, "r") as zip_ref: - zip_ref.extract(etcd_downloaded_dir_name + "/etcd", etcd_binaries) - zip_ref.extract(etcd_downloaded_dir_name + "/etcdctl", etcd_binaries) - else: - with (tarfile.open(etcd_downloaded_archive, "r")) as tar_ref: - tar_ref.extract(etcd_downloaded_dir_name + "/etcd", etcd_binaries) - tar_ref.extract(etcd_downloaded_dir_name + "/etcdctl", etcd_binaries) - - shutil.copy(os.path.join(etcd_binaries, etcd_downloaded_dir_name, "etcd"), etcd_bin) - shutil.copy( - os.path.join(etcd_binaries, etcd_downloaded_dir_name, "etcdctl"), etcdctl_bin) + if not os.path.exists(etcd_downloaded_archive): + print(f"Downloading {etcd_downloaded_dir_name} archive...") + urlretrieve(etcd_url, etcd_downloaded_archive) + else: + print(f"Archive {etcd_downloaded_dir_name} already exists") - os.chmod(etcd_bin, 0o755) - os.chmod(etcdctl_bin, 0o755) + if etcd_archive_extension == "zip": + with zipfile.ZipFile(etcd_downloaded_archive, "r") as zip_ref: + zip_ref.extract(etcd_downloaded_dir_name + "/etcd", etcd_binaries) + zip_ref.extract(etcd_downloaded_dir_name + "/etcdctl", etcd_binaries) + else: + with (tarfile.open(etcd_downloaded_archive, "r")) as tar_ref: + print("Extracting the archive...") + tar_ref.extract(etcd_downloaded_dir_name + "/etcd", etcd_binaries) + tar_ref.extract(etcd_downloaded_dir_name + "/etcdctl", etcd_binaries) + + shutil.copy(os.path.join(etcd_binaries, etcd_downloaded_dir_name, "etcd"), etcd_bin) + shutil.copy( + os.path.join(etcd_binaries, etcd_downloaded_dir_name, "etcdctl"), etcdctl_bin + ) - # Cleanup - shutil.rmtree(etcd_binaries) + if etcd_url in checksums_etcd: + checksum_etcd_archive = checksum_file(etcd_downloaded_archive) + if checksum_etcd_archive != checksums_etcd[etcd_url]: + raise IOError("Checksum failed") else: warnings.warn( - f"Etcd {etcd_version} not supported ! Or, at least, we don't " - f"recognise {etcd_url} in our checksums", - stacklevel=2 + f"Couldn't verify checksum for etcd-v{etcd_version}-{plat}", stacklevel=2 ) + os.chmod(etcd_bin, 0o755) + os.chmod(etcdctl_bin, 0o755) + + # Cleanup + shutil.rmtree(etcd_binaries) + print("--- Done ---") def install_consul(prefix, plat, consul_version): plat = plat.replace("-", "_") consul_downloaded_dir_name = f"consul_v{consul_version}_{plat}" + consul_archive_extension = ".tar.gz" consul_archive_extension = "zip" consul_downloaded_archive = os.path.join( prefix, consul_downloaded_dir_name + "." + consul_archive_extension ) consul_binaries = os.path.join(prefix, "consul_binaries") + consul_bin = os.path.join(prefix, "consul") consul_url = ( @@ -215,47 +212,49 @@ def install_consul(prefix, plat, consul_version): print(f"Consul already exists") if consul_url not in checksums_consul: warnings.warn( - f"Consul {consul_version} not supported ! Or, at least we don't have " - f"it {consul_url} in our checksums", + f"Couldn't verify checksum for consul_v{consul_version}_{plat}", stacklevel=2, ) os.chmod(consul_bin, 0o755) print("--- Done ---") return else: - if checksum_file(consul_downloaded_archive) == checksums_consul[consul_url]: + checksum_consul_archive = checksum_file(consul_downloaded_archive) + if checksum_consul_archive == checksums_consul[consul_url]: os.chmod(consul_bin, 0o755) print("--- Done ---") return else: - print(f"checksum mismatch on {consul_downloaded_archive}") + print(f"checksum mismatch on consul") os.remove(consul_bin) os.remove(consul_downloaded_archive) - if consul_url in checksums_consul: - if not os.path.exists(consul_downloaded_archive): - print(f"Downloading {consul_downloaded_dir_name} archive...") - urlretrieve(consul_url, consul_downloaded_archive) - else: - print(f"Archive {consul_downloaded_dir_name} already exists") + if not os.path.exists(consul_downloaded_archive): + print(f"Downloading {consul_downloaded_dir_name} archive...") + urlretrieve(consul_url, consul_downloaded_archive) + else: + print(f"Archive {consul_downloaded_dir_name} already exists") - if checksum_file(consul_downloaded_archive) != checksums_consul[consul_url]: - raise IOError("Checksum failed") + with zipfile.ZipFile(consul_downloaded_archive, "r") as zip_ref: + zip_ref.extract("consul", consul_binaries) - with zipfile.ZipFile(consul_downloaded_archive, "r") as zip_ref: - zip_ref.extract("consul", consul_binaries) + shutil.copy(os.path.join(consul_binaries, "consul"), consul_bin) - shutil.copy(os.path.join(consul_binaries, "consul"), consul_bin) - os.chmod(consul_bin, 0o755) - # Cleanup - shutil.rmtree(consul_binaries) + if consul_url in checksums_consul: + checksum_consul_archive = checksum_file(consul_downloaded_archive) + if checksum_consul_archive != checksums_consul[consul_url]: + raise IOError("Checksum failed") else: warnings.warn( - f"Consul {consul_version} not supported ! Or, at least we don't have " - f"it {consul_url} in our checksums", + f"Couldn't verify checksum for consul_v{consul_version}_{plat}", stacklevel=2, ) + os.chmod(consul_bin, 0o755) + + # Cleanup + shutil.rmtree(consul_binaries) + print("--- Done ---") diff --git a/jupyterhub_traefik_proxy/proxy.py b/jupyterhub_traefik_proxy/proxy.py index 7994750b..3a83be69 100644 --- a/jupyterhub_traefik_proxy/proxy.py +++ b/jupyterhub_traefik_proxy/proxy.py @@ -195,7 +195,8 @@ async def _traefik_api_request(self, path): self.log.debug(f"Succesfully received data from {path}: {resp.body}") return resp - async def _wait_for_static_config(self): + async def _wait_for_static_config(self, provider=None): + # TODO: Remove provider argument after refactoring kv_proxy and subclasses async def _check_traefik_static_conf_ready(): """Check if traefik loaded its static configuration yet""" try: @@ -230,8 +231,13 @@ def _stop_traefik(self): finally: self.traefik_process.wait() - def _start_traefik(self): - if self.provider_name not in ("file", "etcd", "consul"): + def _launch_traefik(self, config_type=None): + # Keep the _launch_traefik API backwards-compatible, while otherwise + # getting the provider from self.provider_name + # TODO: Make the breaking change! + if config_type is None: + config_type = self.provider_name + if config_type not in ("file", "etcdv3", "consul"): raise ValueError( "Configuration mode not supported \n.\ The proxy can only be configured through fileprovider, etcd and consul" diff --git a/tests/test_installer.py b/tests/test_installer.py index 149a4ad5..157a336b 100644 --- a/tests/test_installer.py +++ b/tests/test_installer.py @@ -104,7 +104,7 @@ def test_version(tmpdir): installer_module, f"--output={deps_dir}", "--traefik", - "--traefik-version=1.7.0", + "--traefik-version=2.4.8", "--etcd", "--etcd-version=3.2.25", "--consul", @@ -176,7 +176,7 @@ def test_warning(tmpdir): installer_module, f"--output={deps_dir}", "--traefik", - "--traefik-version=1.6.6", + "--traefik-version=2.4.1", ], stderr=subprocess.STDOUT, ) diff --git a/tests/test_traefik_api_auth.py b/tests/test_traefik_api_auth.py index ece2064a..407a0ff9 100644 --- a/tests/test_traefik_api_auth.py +++ b/tests/test_traefik_api_auth.py @@ -14,10 +14,10 @@ params=[ "toml_proxy", "yaml_proxy", - "no_auth_etcd_proxy", - "auth_etcd_proxy", - "no_auth_consul_proxy", - "auth_consul_proxy", + #"no_auth_etcd_proxy", + #"auth_etcd_proxy", + #"no_auth_consul_proxy", + #"auth_consul_proxy", ] ) def proxy(request): diff --git a/tests/test_traefik_utils.py b/tests/test_traefik_utils.py index a6e4238c..05311cc8 100644 --- a/tests/test_traefik_utils.py +++ b/tests/test_traefik_utils.py @@ -28,9 +28,10 @@ def test_roundtrip_routes(): file = "test_roudtrip.toml" open(file, "a").close() - traefik_utils.persist_routes(file, routes) - handler = traefik_utils.TraefikConfigFileHandler(file) - reloaded = handler.load() + save_handler = traefik_utils.TraefikConfigFileHandler(file) + save_handler.atomic_dump(routes) + load_handler = traefik_utils.TraefikConfigFileHandler(file) + reloaded = load_handler.load() os.remove(file) assert reloaded == routes From 1bf59d3850e268c932de6ff7ca119ee317bad651 Mon Sep 17 00:00:00 2001 From: Alex Leach Date: Tue, 6 Jul 2021 15:24:19 +0000 Subject: [PATCH 07/19] Fix breaking reversions made to install.py Traefik v2 is now distributed as a compressed archive, not the raw binary, so needs to be uncompressed before it can be used. --- jupyterhub_traefik_proxy/install.py | 48 +++++++++++++---------------- 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/jupyterhub_traefik_proxy/install.py b/jupyterhub_traefik_proxy/install.py index f5017cef..20b8f454 100644 --- a/jupyterhub_traefik_proxy/install.py +++ b/jupyterhub_traefik_proxy/install.py @@ -69,31 +69,12 @@ def install_traefik(prefix, plat, traefik_version): f"/download/v{traefik_version}/{traefik_archive}" ) - if os.path.exists(traefik_bin): - print(f"Traefik already exists") - if traefik_url not in checksums_traefik: - warnings.warn( - f"Couldn't verify checksum for traefik-v{traefik_version}-{plat}", - stacklevel=2, - ) - os.chmod(traefik_bin, 0o755) - print("--- Done ---") - return - else: - checksum = checksum_file(traefik_bin) - if checksum == checksums_traefik[traefik_url]: - os.chmod(traefik_bin, 0o755) - print("--- Done ---") - return - else: - print(f"checksum mismatch on {traefik_bin}") - os.remove(traefik_bin) - - print(f"Downloading traefik {traefik_version}...") - urlretrieve(traefik_url, traefik_bin) + if not os.path.exists(traefik_archive_path): + print(f"Downloading traefik {traefik_archive}...") + urlretrieve(traefik_url, traefik_archive_path) if traefik_url in checksums_traefik: - checksum = checksum_file(traefik_bin) + checksum = checksum_file(traefik_archive_path) if checksum != checksums_traefik[traefik_url]: raise IOError("Checksum failed") else: @@ -102,6 +83,14 @@ def install_traefik(prefix, plat, traefik_version): stacklevel=2, ) + print("Extracting the archive...") + if traefik_archive_extension == "tar.gz": + with tarfile.open(traefik_archive_path, "r") as tar_ref: + tar_ref.extract("traefik", prefix) + else: + with zipfile.ZipFile(traefik_archive_path, "r") as zip_ref: + zip_ref.extract("traefik.exe", prefix) + os.chmod(traefik_bin, 0o755) print("--- Done ---") @@ -266,9 +255,16 @@ def main(): """\ Checksums available for: - traefik: - - v2.2.0-linux-amd64 - - v2.2.0-darwin-amd64 - - v2.2.0-windows-amd64 + - v2.4.8-linux-arm64 + - v2.4.8-linux-amd64 + - v2.4.8-darwin-amd64 + - v2.4.8-windows-amd64 + - v2.3.7-linux-amd64 + - v2.3.7-darwin-amd64 + - v2.3.7-windows-amd64 + - v2.2.11-linux-amd64 + - v2.2.11-darwin-amd64 + - v2.2.11-windows-amd64 - etcd: - v3.4.7-linux-amd64 - v3.4.7-darwin-amd64 From 53f53e86b5caac02010ea788237902f2281498f1 Mon Sep 17 00:00:00 2001 From: Alex Leach Date: Sat, 17 Jul 2021 11:37:42 +0000 Subject: [PATCH 08/19] Tidy up a couple of files. Correct traefik-v2 documentation URLs in install.md. Reduce line-lengths of f-strings in proxy.py and traefik_utils.py. Co-authored-by: GeorgianaElena --- docs/source/install.md | 4 ++-- jupyterhub_traefik_proxy/proxy.py | 11 +++++------ jupyterhub_traefik_proxy/traefik_utils.py | 7 +++---- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/docs/source/install.md b/docs/source/install.md index d5ea9858..7d9050b4 100644 --- a/docs/source/install.md +++ b/docs/source/install.md @@ -69,7 +69,7 @@ ## Enabling traefik-proxy in JupyterHub -[TraefikFileProviderProxy](https://github.com/jupyterhub/traefik-proxy/blob/Traefik_v2/jupyterhub_traefik_proxy/fileprovider.py), [TraefikEtcdProxy](https://github.com/jupyterhub/traefik-proxy/blob/Traefik_v2/jupyterhub_traefik_proxy/etcd.py) and [TraefikConsulProxy](https://github.com/jupyterhub/traefik-proxy/blob/Traefik_v2/jupyterhub_traefik_proxy/consul.py) are custom proxy implementations that subclass [Proxy](https://github.com/jupyterhub/jupyterhub/blob/Traefik_v2/jupyterhub/proxy.py) and can register in JupyterHub config using `c.JupyterHub.proxy_class` entrypoint. +[TraefikFileProviderProxy](https://github.com/jupyterhub/traefik-proxy/blob/traefik-v2/jupyterhub_traefik_proxy/fileprovider.py), [TraefikEtcdProxy](https://github.com/jupyterhub/traefik-proxy/blob/traefik-v2/jupyterhub_traefik_proxy/etcd.py) and [TraefikConsulProxy](https://github.com/jupyterhub/traefik-proxy/blob/traefik-v2/jupyterhub_traefik_proxy/consul.py) are custom proxy implementations that subclass [Proxy](https://github.com/jupyterhub/jupyterhub/blob/traefik-v2/jupyterhub/proxy.py) and can register in JupyterHub config using `c.JupyterHub.proxy_class` entrypoint. On startup, JupyterHub will look by default for a configuration file, *jupyterhub_config.py*, in the current working directory. If the configuration file is not in the current working directory, you can load a specific config file and start JupyterHub using: @@ -78,7 +78,7 @@ you can load a specific config file and start JupyterHub using: $ jupyterhub -f /path/to/jupyterhub_config.py ``` -There is an example configuration file [here](https://github.com/jupyterhub/traefik-proxy/blob/Traefik_v2/examples/jupyterhub_config.py) that configures JupyterHub to run with *TraefikEtcdProxy* as the proxy and uses dummyauthenticator and simplespawner to enable testing without administrative privileges. +There is an example configuration file [here](https://github.com/jupyterhub/traefik-proxy/blob/traefik-v2/examples/jupyterhub_config.py) that configures JupyterHub to run with *TraefikEtcdProxy* as the proxy and uses dummyauthenticator and simplespawner to enable testing without administrative privileges. In *jupyterhub_config.py*: diff --git a/jupyterhub_traefik_proxy/proxy.py b/jupyterhub_traefik_proxy/proxy.py index 3a83be69..0a785e61 100644 --- a/jupyterhub_traefik_proxy/proxy.py +++ b/jupyterhub_traefik_proxy/proxy.py @@ -299,16 +299,15 @@ async def _setup_traefik_dynamic_config(self): self.log.info("Setting up traefik's dynamic config...") self._generate_htpassword() api_url = urlparse(self.traefik_api_url) - api_path = api_url.path if api_url.path else '/api' - api_credentials = "{0}:{1}".format( - self.traefik_api_username, - self.traefik_api_hashed_password - ) + api_path = api_url.path if api_url.path else "/api" + api_credentials = f"{self.traefik_api_username}:" \ + f"{self.traefik_api_hashed_password}" self.dynamic_config.update({ "http": { "routers": { "route_api": { - "rule": f"Host(`{api_url.hostname}`) && (PathPrefix(`{api_path}`) || PathPrefix(`/dashboard`))", + "rule": f"Host(`{api_url.hostname}`) && " \ + f"(PathPrefix(`{api_path}`) || PathPrefix(`/dashboard`))", "entryPoints": ["enter_api"], "service": "api@internal", "middlewares": ["auth_api"] diff --git a/jupyterhub_traefik_proxy/traefik_utils.py b/jupyterhub_traefik_proxy/traefik_utils.py index 29bafcaa..297f1b94 100644 --- a/jupyterhub_traefik_proxy/traefik_utils.py +++ b/jupyterhub_traefik_proxy/traefik_utils.py @@ -13,7 +13,7 @@ class KVStorePrefix(Unicode): def validate(self, obj, value): u = super().validate(obj, value) - # We'll join the prefix with e.g. prefix.join(pathspec), + # We'll join the prefix with e.g. "/".join(pathspec), # therefore always strip the trailing "/" from any prefix if u.endswith("/"): u = u.rstrip("/") @@ -29,12 +29,11 @@ def generate_rule(routespec): routespec = unquote(routespec) if routespec.startswith("/"): # Path-based route, e.g. /proxy/path/ - rule = "PathPrefix(`{0}`)".format(routespec) + rule = f"PathPrefix(`{routespec}`)" else: # Host-based routing, e.g. host.tld/proxy/path/ host, path_prefix = routespec.split("/", 1) - path_prefix = "/" + path_prefix - rule = "Host(`{0}`) && PathPrefix(`{1}`)".format(host, path_prefix) + rule = f"Host(`{host}`) && PathPrefix(`/{path_prefix}`)" return rule From c6d3baaec968b3c43f06d0319d485c98bfdc934c Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Tue, 6 Jul 2021 13:06:58 +0300 Subject: [PATCH 09/19] Remove consul and etcd proxies from pkg and stop testing them --- tests/test_proxy.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/test_proxy.py b/tests/test_proxy.py index a202b691..bf1428f2 100644 --- a/tests/test_proxy.py +++ b/tests/test_proxy.py @@ -10,16 +10,8 @@ @pytest.fixture( params=[ - #"no_auth_consul_proxy", - #"auth_consul_proxy", - #"no_auth_etcd_proxy", - #"auth_etcd_proxy", "toml_proxy", "yaml_proxy", - #"external_consul_proxy", - #"auth_external_consul_proxy", - #"external_etcd_proxy", - #"auth_external_etcd_proxy", "external_toml_proxy", "external_yaml_proxy", ] From cfe956a0fa354138a544746389162ef0862fb879 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Tue, 6 Jul 2021 13:10:27 +0300 Subject: [PATCH 10/19] Remove consul and etcd proxy ref from tests --- tests/test_traefik_api_auth.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_traefik_api_auth.py b/tests/test_traefik_api_auth.py index 407a0ff9..0199f5b0 100644 --- a/tests/test_traefik_api_auth.py +++ b/tests/test_traefik_api_auth.py @@ -14,10 +14,6 @@ params=[ "toml_proxy", "yaml_proxy", - #"no_auth_etcd_proxy", - #"auth_etcd_proxy", - #"no_auth_consul_proxy", - #"auth_consul_proxy", ] ) def proxy(request): From e03a463b59bd57d89513460afa5bbd1f2a515790 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Mon, 16 Aug 2021 17:08:30 +0300 Subject: [PATCH 11/19] Update the default traefik and consul versions in the installer --- jupyterhub_traefik_proxy/install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jupyterhub_traefik_proxy/install.py b/jupyterhub_traefik_proxy/install.py index 20b8f454..e5568756 100644 --- a/jupyterhub_traefik_proxy/install.py +++ b/jupyterhub_traefik_proxy/install.py @@ -321,7 +321,7 @@ def main(): parser.add_argument( "--traefik-version", dest="traefik_version", - default="2.2.0", + default="2.4.8", help=textwrap.dedent( """\ The version of traefik to download. @@ -369,7 +369,7 @@ def main(): parser.add_argument( "--consul-version", dest="consul_version", - default="1.7.2", + default="1.9.4", help=textwrap.dedent( """\ The version of consul to download. From cadd54dd99552dece3bdb2ff541af4771ca0bb0d Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Mon, 16 Aug 2021 17:09:30 +0300 Subject: [PATCH 12/19] Rename the file provider proxy main doc page --- docs/source/{yaml.md => file.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/source/{yaml.md => file.md} (100%) diff --git a/docs/source/yaml.md b/docs/source/file.md similarity index 100% rename from docs/source/yaml.md rename to docs/source/file.md From 8d5e2906501b1c49da35fdf852ee72b8f39b1284 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Tue, 17 Aug 2021 10:14:14 +0300 Subject: [PATCH 13/19] Rename the file provider proxy --- README.md | 4 +- docs/source/api/index.rst | 4 +- docs/source/file.md | 60 ++++++++++++------------ docs/source/index.rst | 2 +- docs/source/install.md | 10 ++-- examples/jupyterhub_config_file.py | 6 +-- jupyterhub_traefik_proxy/__init__.py | 2 +- jupyterhub_traefik_proxy/fileprovider.py | 4 +- setup.py | 2 +- tests/conftest.py | 12 ++--- 10 files changed, 53 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 570a8971..354ca42c 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ depending on how traefik store its routing configuration. For **smaller**, single-node deployments: -* TraefikFileProviderProxy +* TraefikFileProxy For **distributed** setups: @@ -32,7 +32,7 @@ The [documentation](https://jupyterhub-traefik-proxy.readthedocs.io) contains a guide](https://jupyterhub-traefik-proxy.readthedocs.io/en/latest/install.html) with examples for the three different implementations. -* [For TraefikFileProviderProxy](https://jupyterhub-traefik-proxy.readthedocs.io/en/latest/file.html#example-setup) +* [For TraefikFileProxy](https://jupyterhub-traefik-proxy.readthedocs.io/en/latest/file.html#example-setup) * [For TraefikEtcdProxy](https://jupyterhub-traefik-proxy.readthedocs.io/en/latest/etcd.html#example-setup) * [For TraefikConsulProxy](https://jupyterhub-traefik-proxy.readthedocs.io/en/latest/consul.html#example-setup) diff --git a/docs/source/api/index.rst b/docs/source/api/index.rst index 410c9a6c..3cc18cd7 100644 --- a/docs/source/api/index.rst +++ b/docs/source/api/index.rst @@ -15,10 +15,10 @@ Module: :mod:`jupyterhub_traefik_proxy` .. autoconfigurable:: TraefikProxy :members: -:class:`TraefikFileProviderProxy` +:class:`TraefikFileProxy` ------------------------- -.. autoconfigurable:: TraefikFileProviderProxy +.. autoconfigurable:: TraefikFileProxy :members: :class:`TKvProxy` diff --git a/docs/source/file.md b/docs/source/file.md index 0f4178d9..e49eabbd 100644 --- a/docs/source/file.md +++ b/docs/source/file.md @@ -1,9 +1,9 @@ -# Using TraefikFileProviderProxy +# Using TraefikFileProxy **jupyterhub-traefik-proxy** can be used with simple toml or yaml configuration files, for smaller, single-node deployments such as [The Littlest JupyterHub](https://tljh.jupyter.org). -## How-To install TraefikFileProviderProxy +## How-To install TraefikFileProxy 1. Install **jupyterhub** 2. Install **jupyterhub-traefik-proxy** @@ -11,9 +11,9 @@ * You can find the full installation guide and examples in the [Introduction section](install.html#traefik-proxy-installation) -## How-To enable TraefikFileProviderProxy +## How-To enable TraefikFileProxy -You can enable JupyterHub to work with `TraefikFileProviderProxy` in jupyterhub_config.py, using the `proxy_class` configuration option. +You can enable JupyterHub to work with `TraefikFileProxy` in jupyterhub_config.py, using the `proxy_class` configuration option. You can choose to: @@ -23,11 +23,11 @@ You can choose to: c.JupyterHub.proxy_class = "traefik_file" ``` -* use the TraefikFileProviderProxy object, in which case, you have to import the module, e.g.: +* use the TraefikFileProxy object, in which case, you have to import the module, e.g.: ```python - from jupyterhub_traefik_proxy import TraefikFileProviderProxy - c.JupyterHub.proxy_class = TraefikFileProviderProxy + from jupyterhub_traefik_proxy import TraefikFileProxy + c.JupyterHub.proxy_class = TraefikFileProxy ``` @@ -42,7 +42,7 @@ where the routing table will be updated continuously. Traefik allows us to have one file for the static configuration file (`traefik.toml` or `traefik.yaml`) and one or several files for the routes, that traefik would watch. ```{note} - **TraefikFileProviderProxy**, uses two configuration files: one file for the routes (**rules.toml** or **rules.yaml**), and one for the static configuration (**traefik.toml** or **traefik.yaml**). + **TraefikFileProxy**, uses two configuration files: one file for the routes (**rules.toml** or **rules.yaml**), and one for the static configuration (**traefik.toml** or **traefik.yaml**). ``` @@ -52,16 +52,16 @@ By **default**, Traefik will search for `traefik.toml` and `rules.toml` in the f * $HOME/.traefik/ * . the working directory -You can override this in TraefikFileProviderProxy, by modifying the **toml_static_config_file** argument: +You can override this in TraefikFileProxy, by modifying the **toml_static_config_file** argument: ```python -c.TraefikFileProviderProxy.static_config_file="/path/to/static_config_filename.toml" +c.TraefikFileProxy.static_config_file="/path/to/static_config_filename.toml" ``` Similarly, you can override the dynamic configuration file by modifying the **dynamic_config_file** argument: ```python -c.TraefikFileProviderProxy.dynamic_config_file="/path/to/dynamic_config_filename.toml" +c.TraefikFileProxy.dynamic_config_file="/path/to/dynamic_config_filename.toml" ``` ```{note} @@ -77,20 +77,20 @@ will be managed by JupyterHub. This allows e.g., the administrator to configure ``` -## Externally managed TraefikFileProviderProxy +## Externally managed TraefikFileProxy -When TraefikFileProviderProxy is externally managed, service managers like [systemd](https://www.freedesktop.org/wiki/Software/systemd/) +When TraefikFileProxy is externally managed, service managers like [systemd](https://www.freedesktop.org/wiki/Software/systemd/) or [docker](https://www.docker.com/) will be responsible for starting and stopping the proxy. -If TraefikFileProviderProxy is used as an externally managed service, then make sure you follow the steps enumerated below: +If TraefikFileProxy is used as an externally managed service, then make sure you follow the steps enumerated below: -1. Let JupyterHub know that the proxy being used is TraefikFileProviderProxy, using the *proxy_class* configuration option: +1. Let JupyterHub know that the proxy being used is TraefikFileProxy, using the *proxy_class* configuration option: ```python - from jupyterhub_traefik_proxy import TraefikFileProviderProxy - c.JupyterHub.proxy_class = TraefikFileProviderProxy + from jupyterhub_traefik_proxy import TraefikFileProxy + c.JupyterHub.proxy_class = TraefikFileProxy ``` -2. Configure `TraefikFileProviderProxy` in **jupyterhub_config.py** +2. Configure `TraefikFileProxy` in **jupyterhub_config.py** JupyterHub configuration file, *jupyterhub_config.py* must specify at least: * That the proxy is externally managed @@ -102,14 +102,14 @@ If TraefikFileProviderProxy is used as an externally managed service, then make Example configuration: ```python # JupyterHub shouldn't start the proxy, it's already running - c.TraefikFileProviderProxy.should_start = False + c.TraefikFileProxy.should_start = False # if not the default: - c.TraefikFileProviderProxy.dynamic_config_file = "/path/to/somefile.toml" + c.TraefikFileProxy.dynamic_config_file = "/path/to/somefile.toml" # traefik api credentials - c.TraefikFileProviderProxy.traefik_api_username = "abc" - c.TraefikFileProviderProxy.traefik_api_password = "xxx" + c.TraefikFileProxy.traefik_api_username = "abc" + c.TraefikFileProxy.traefik_api_password = "xxx" ``` 3. Ensure **traefik.toml** / **traefik.yaml** @@ -124,27 +124,27 @@ If TraefikFileProviderProxy is used as an externally managed service, then make ## Example setup -This is an example setup for using JupyterHub and TraefikFileProviderProxy managed by another service than JupyterHub. +This is an example setup for using JupyterHub and TraefikFileProxy managed by another service than JupyterHub. 1. Configure the proxy through the JupyterHub configuration file, *jupyterhub_config.py*, e.g.: ```python - from jupyterhub_traefik_proxy import TraefikFileProviderProxy + from jupyterhub_traefik_proxy import TraefikFileProxy # mark the proxy as externally managed - c.TraefikFileProviderProxy.should_start = False + c.TraefikFileProxy.should_start = False # traefik api endpoint login password - c.TraefikFileProviderProxy.traefik_api_password = "admin" + c.TraefikFileProxy.traefik_api_password = "admin" # traefik api endpoint login username - c.TraefikFileProviderProxy.traefik_api_username = "api_admin" + c.TraefikFileProxy.traefik_api_username = "api_admin" # traefik's dynamic configuration file, which will be managed by JupyterHub - c.TraefikFileProviderProxy.dynamic_config_file = "/var/run/traefik/rules.toml" + c.TraefikFileProxy.dynamic_config_file = "/var/run/traefik/rules.toml" - # configure JupyterHub to use TraefikFileProviderProxy - c.JupyterHub.proxy_class = TraefikFileProviderProxy + # configure JupyterHub to use TraefikFileProxy + c.JupyterHub.proxy_class = TraefikFileProxy ``` 2. Create a traefik static configuration file, *traefik.toml*, e.g.: diff --git a/docs/source/index.rst b/docs/source/index.rst index 0f73a31e..1b3ad8ce 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -20,7 +20,7 @@ Moreover it offers *HTTPS* support through a straight-forward `ACME (Let's Encry There are three versions for the proxy, depending on how traefik stores the routes: * *for* **smaller**, *single-node deployments*: - * TraefikFileProviderProxy + * TraefikFileProxy * *for* **distributed** *setups*: * TraefikEtcdProxy * TraefikConsulProxy diff --git a/docs/source/install.md b/docs/source/install.md index 7d9050b4..f8369713 100644 --- a/docs/source/install.md +++ b/docs/source/install.md @@ -69,7 +69,7 @@ ## Enabling traefik-proxy in JupyterHub -[TraefikFileProviderProxy](https://github.com/jupyterhub/traefik-proxy/blob/traefik-v2/jupyterhub_traefik_proxy/fileprovider.py), [TraefikEtcdProxy](https://github.com/jupyterhub/traefik-proxy/blob/traefik-v2/jupyterhub_traefik_proxy/etcd.py) and [TraefikConsulProxy](https://github.com/jupyterhub/traefik-proxy/blob/traefik-v2/jupyterhub_traefik_proxy/consul.py) are custom proxy implementations that subclass [Proxy](https://github.com/jupyterhub/jupyterhub/blob/traefik-v2/jupyterhub/proxy.py) and can register in JupyterHub config using `c.JupyterHub.proxy_class` entrypoint. +[TraefikFileProxy](https://github.com/jupyterhub/traefik-proxy/blob/traefik-v2/jupyterhub_traefik_proxy/fileprovider.py), [TraefikEtcdProxy](https://github.com/jupyterhub/traefik-proxy/blob/traefik-v2/jupyterhub_traefik_proxy/etcd.py) and [TraefikConsulProxy](https://github.com/jupyterhub/traefik-proxy/blob/traefik-v2/jupyterhub_traefik_proxy/consul.py) are custom proxy implementations that subclass [Proxy](https://github.com/jupyterhub/jupyterhub/blob/traefik-v2/jupyterhub/proxy.py) and can register in JupyterHub config using `c.JupyterHub.proxy_class` entrypoint. On startup, JupyterHub will look by default for a configuration file, *jupyterhub_config.py*, in the current working directory. If the configuration file is not in the current working directory, you can load a specific config file and start JupyterHub using: @@ -84,7 +84,7 @@ In *jupyterhub_config.py*: ``` c.JupyterHub.proxy_class = "traefik_file" -# will configure JupyterHub to run with TraefikFileProviderProxy +# will configure JupyterHub to run with TraefikFileProxy ``` ``` @@ -110,9 +110,9 @@ c.JupyterHub.proxy_class = "traefik_consul" The port on which traefik-proxy's api will run, as well as the username and password used for authenticating, can be passed to the proxy through `jupyterhub_config.py`, e.g.: ``` - c.TraefikFileProviderProxy.traefik_api_url = "http://127.0.0.1:8099" - c.TraefikFileProviderProxy.traefik_api_password = "admin" - c.TraefikFileProviderProxy.traefik_api_username = "admin" + c.TraefikFileProxy.traefik_api_url = "http://127.0.0.1:8099" + c.TraefikFileProxy.traefik_api_password = "admin" + c.TraefikFileProxy.traefik_api_username = "admin" ``` Check out TraefikProxy's **API Reference** for more configuration options.

diff --git a/examples/jupyterhub_config_file.py b/examples/jupyterhub_config_file.py index 927ea54d..236a6bf7 100644 --- a/examples/jupyterhub_config_file.py +++ b/examples/jupyterhub_config_file.py @@ -9,9 +9,9 @@ """ c.JupyterHub.proxy_class = "traefik_file" -c.TraefikFileProviderProxy.traefik_api_username = "admin" -c.TraefikFileProviderProxy.traefik_api_password = "admin" -c.TraefikFileProviderProxy.traefik_log_level = "INFO" +c.TraefikFileProxy.traefik_api_username = "admin" +c.TraefikFileProxy.traefik_api_password = "admin" +c.TraefikFileProxy.traefik_log_level = "INFO" # use dummy and simple auth/spawner for testing c.JupyterHub.authenticator_class = "dummy" diff --git a/jupyterhub_traefik_proxy/__init__.py b/jupyterhub_traefik_proxy/__init__.py index 9f1e608a..1bcc0501 100644 --- a/jupyterhub_traefik_proxy/__init__.py +++ b/jupyterhub_traefik_proxy/__init__.py @@ -2,7 +2,7 @@ from .proxy import TraefikProxy # noqa from .kv_proxy import TKvProxy # noqa -from .fileprovider import TraefikFileProviderProxy +from .fileprovider import TraefikFileProxy from ._version import get_versions diff --git a/jupyterhub_traefik_proxy/fileprovider.py b/jupyterhub_traefik_proxy/fileprovider.py index 42acb013..e7b98053 100644 --- a/jupyterhub_traefik_proxy/fileprovider.py +++ b/jupyterhub_traefik_proxy/fileprovider.py @@ -31,7 +31,7 @@ from jupyterhub_traefik_proxy import TraefikProxy -class TraefikFileProviderProxy(TraefikProxy): +class TraefikFileProxy(TraefikProxy): """JupyterHub Proxy implementation using traefik and toml or yaml config file""" mutex = Any() @@ -189,7 +189,7 @@ async def add_route(self, routespec, target, data): The proxy implementation should also have a way to associate the fact that a route came from JupyterHub. """ - self.log.debug(f"\tTraefikFileProviderProxy.add_route: Adding {routespec} for {target}") + self.log.debug(f"\tTraefikFileProxy.add_route: Adding {routespec} for {target}") traefik_routespec = self._routespec_to_traefik_path(routespec) service_alias = traefik_utils.generate_alias(traefik_routespec, "service") router_alias = traefik_utils.generate_alias(traefik_routespec, "router") diff --git a/setup.py b/setup.py index 3475f1a6..cc5bef5b 100644 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ def run(self): cmdclass=cmdclass, entry_points={ "jupyterhub.proxies": [ - "traefik_file = jupyterhub_traefik_proxy:TraefikFileProviderProxy", + "traefik_file = jupyterhub_traefik_proxy:TraefikFileProxy", ] }, ) diff --git a/tests/conftest.py b/tests/conftest.py index 6776c529..a690a964 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,7 @@ import pytest from _pytest.mark import Mark -from jupyterhub_traefik_proxy import TraefikFileProviderProxy +from jupyterhub_traefik_proxy import TraefikFileProxy # Define a "slow" test marker so that we can run the slow tests at the end @@ -38,11 +38,11 @@ def pytest_configure(config): @pytest.fixture # There must be a way to parameterise this to run on both yaml and toml files? async def toml_proxy(): - """Fixture returning a configured TraefikFileProviderProxy""" + """Fixture returning a configured TraefikFileProxy""" dynamic_config_file = os.path.join( os.getcwd(), "tests", "config_files", "dynamic_config", "rules.toml" ) - proxy = TraefikFileProviderProxy( + proxy = TraefikFileProxy( public_url="http://127.0.0.1:8000", traefik_api_password="admin", traefik_api_username="api_admin", @@ -62,7 +62,7 @@ async def yaml_proxy(): dynamic_config_file = os.path.join( os.getcwd(), "tests", "config_files", "dynamic_config", "rules.yaml" ) - proxy = TraefikFileProviderProxy( + proxy = TraefikFileProxy( public_url="http://127.0.0.1:8000", traefik_api_password="admin", traefik_api_username="api_admin", @@ -82,7 +82,7 @@ async def external_toml_proxy(launch_traefik_file): dynamic_config_file = os.path.join( os.getcwd(), "tests", "config_files", "dynamic_config", "rules.toml" ) - proxy = TraefikFileProviderProxy( + proxy = TraefikFileProxy( public_url="http://127.0.0.1:8000", traefik_api_password="admin", traefik_api_username="api_admin", @@ -100,7 +100,7 @@ async def external_yaml_proxy(launch_traefik_file): dynamic_config_file = os.path.join( os.getcwd(), "tests", "config_files", "dynamic_config", "rules.yaml" ) - proxy = TraefikFileProviderProxy( + proxy = TraefikFileProxy( public_url="http://127.0.0.1:8000", traefik_api_password="admin", traefik_api_username="api_admin", From 4108d151c0b148171a4e0c262de9732b76d46f5b Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Tue, 17 Aug 2021 11:51:10 +0300 Subject: [PATCH 14/19] Remove underscore from test func name --- tests/proxytest.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/proxytest.py b/tests/proxytest.py index 13d9c4d8..3e389704 100644 --- a/tests/proxytest.py +++ b/tests/proxytest.py @@ -147,7 +147,7 @@ async def wait_for_services(urls): ), ], ) -async def _test_add_get_delete( +async def test_add_get_delete( request, proxy, launch_backend, routespec, existing_routes, event_loop ): default_target = "http://127.0.0.1:9000" @@ -284,7 +284,7 @@ async def _wait_for_deletion(): await test_route_exist(spec, extra_backends[i]) -async def _test_get_all_routes(proxy, launch_backend): +async def test_get_all_routes(proxy, launch_backend): routespecs = ["/proxy/path1", "/proxy/path2/", "/proxy/path3/"] targets = [ "http://127.0.0.1:9900", @@ -330,7 +330,7 @@ async def _test_get_all_routes(proxy, launch_backend): assert routes == expected_output -async def _test_host_origin_headers(proxy, launch_backend): +async def test_host_origin_headers(proxy, launch_backend): routespec = "/user/username/" target = "http://127.0.0.1:9000" data = {} @@ -374,7 +374,7 @@ async def _test_host_origin_headers(proxy, launch_backend): @pytest.mark.parametrize("username", ["zoe", "50fia", "秀樹", "~TestJH", "has@"]) -async def _test_check_routes(proxy, username): +async def test_check_routes(proxy, username): # fill out necessary attributes for check_routes proxy.app = MockApp() proxy.hub = proxy.app.hub From 87ff82f5834fc11fa8104f85cc247766e9373b9f Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Thu, 21 Oct 2021 13:08:17 +0300 Subject: [PATCH 15/19] Temporary rename back the proxy file to visualize better the changes --- jupyterhub_traefik_proxy/{fileprovider.py => toml.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename jupyterhub_traefik_proxy/{fileprovider.py => toml.py} (100%) diff --git a/jupyterhub_traefik_proxy/fileprovider.py b/jupyterhub_traefik_proxy/toml.py similarity index 100% rename from jupyterhub_traefik_proxy/fileprovider.py rename to jupyterhub_traefik_proxy/toml.py From f143a8995b4f8ffa2f595d23dbf28cf28dccbe84 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Mon, 25 Oct 2021 16:25:25 +0300 Subject: [PATCH 16/19] Remove more of the etcd and consul mentions --- jupyterhub_traefik_proxy/install.py | 236 +------------- jupyterhub_traefik_proxy/kv_proxy.py | 369 ---------------------- jupyterhub_traefik_proxy/traefik_utils.py | 38 --- 3 files changed, 1 insertion(+), 642 deletions(-) delete mode 100644 jupyterhub_traefik_proxy/kv_proxy.py diff --git a/jupyterhub_traefik_proxy/install.py b/jupyterhub_traefik_proxy/install.py index e5568756..e636a91f 100644 --- a/jupyterhub_traefik_proxy/install.py +++ b/jupyterhub_traefik_proxy/install.py @@ -3,7 +3,6 @@ from urllib.request import urlretrieve import tarfile import zipfile -import shutil import argparse import textwrap import hashlib @@ -22,26 +21,6 @@ "https://github.com/traefik/traefik/releases/download/v2.2.11/traefik_v2.2.11_windows_amd64.zip": "ee867133e00b2d8395c239d8fed04a26b362e650b371dc0b653f0ee9d52471e6", } -checksums_etcd = { - "https://github.com/etcd-io/etcd/releases/download/v3.4.15/etcd-v3.4.15-linux-arm64.tar.gz": "fcc522275300cf90d42377106d47a2e384d1d2083af205cbb7833a79ef5a49d1", - "https://github.com/etcd-io/etcd/releases/download/v3.4.15/etcd-v3.4.15-linux-amd64.tar.gz": "3bd00836ea328db89ecba3ed2155293934c0d09e64b53d6c9dfc0a256e724b81", - "https://github.com/etcd-io/etcd/releases/download/v3.4.15/etcd-v3.4.15-darwin-amd64.tar.gz": "c596709069193bffc639a22558bdea4d801128e635909ea01a6fd5b5c85da729", - "https://github.com/etcd-io/etcd/releases/download/v3.3.10/etcd-v3.3.10-linux-amd64.tar.gz": "1620a59150ec0a0124a65540e23891243feb2d9a628092fb1edcc23974724a45", - "https://github.com/etcd-io/etcd/releases/download/v3.3.10/etcd-v3.3.10-darwin-amd64.tar.gz": "fac4091c7ba6f032830fad7809a115909d0f0cae5cbf5b34044540def743577b", - "https://github.com/etcd-io/etcd/releases/download/v3.2.25/etcd-v3.2.25-linux-amd64.tar.gz": "8a509ffb1443088d501f19e339a0d9c0058ce20599752c3baef83c1c68790ef7", - "https://github.com/etcd-io/etcd/releases/download/v3.2.25/etcd-v3.2.25-darwin-amd64.tar.gz": "9950684a01d7431bc12c3dba014f222d55a862c6f8af64c09c42d7a59ed6790d", -} - -checksums_consul = { - "https://releases.hashicorp.com/consul/1.9.4/consul_1.9.4_linux_amd64.zip": "da3919197ef33c4205bb7df3cc5992ccaae01d46753a72fe029778d7f52fb610", - "https://releases.hashicorp.com/consul/1.9.4/consul_1.9.4_linux_arm64.zip": "012c552aff502f907416c9a119d2dfed88b92e981f9b160eb4fe292676afdaeb", - "https://releases.hashicorp.com/consul/1.9.4/consul_1.9.4_darwin.zip": "c168240d52f67c71b30ef51b3594673cad77d0dbbf38c412b2ee30b39ef30843", - "https://releases.hashicorp.com/consul/1.6.1/consul_1.6.1_linux_amd64.zip": "a8568ca7b6797030b2c32615b4786d4cc75ce7aee2ed9025996fe92b07b31f7e", - "https://releases.hashicorp.com/consul/1.6.1/consul_1.6.1_darwin_amd64.zip": "4bc205e06b2921f998cb6ddbe70de57f8e558e226e44aba3f337f2f245678b85", - "https://releases.hashicorp.com/consul/1.5.0/consul_1.5.0_linux_amd64.zip": "1399064050019db05d3378f757e058ec4426a917dd2d240336b51532065880b6", - "https://releases.hashicorp.com/consul/1.5.0/consul_1.5.0_darwin_amd64.zip": "b4033ea6871fe6136ee5d940c834be2248463c3ec248dc22370e6d5360931325", -} - def checksum_file(path): """Compute the sha256 checksum of a path""" @@ -96,157 +75,6 @@ def install_traefik(prefix, plat, traefik_version): print("--- Done ---") -def install_etcd(prefix, plat, etcd_version): - etcd_downloaded_dir_name = f"etcd-v{etcd_version}-{plat}" - etcd_archive_extension = ".tar.gz" - if "linux" in plat: - etcd_archive_extension = "tar.gz" - else: - etcd_archive_extension = "zip" - etcd_downloaded_archive = os.path.join( - prefix, etcd_downloaded_dir_name + "." + etcd_archive_extension - ) - etcd_binaries = os.path.join(prefix, "etcd_binaries") - - etcd_bin = os.path.join(prefix, "etcd") - etcdctl_bin = os.path.join(prefix, "etcdctl") - - etcd_url = ( - "https://github.com/etcd-io/etcd/releases" - f"/download/v{etcd_version}/etcd-v{etcd_version}-{plat}.{etcd_archive_extension}" - ) - - if os.path.exists(etcd_bin) and os.path.exists(etcdctl_bin): - print(f"Etcd and etcdctl already exist") - if etcd_url not in checksums_etcd: - warnings.warn( - f"Couldn't verify checksum for etcd-v{etcd_version}-{plat}", - stacklevel=2, - ) - os.chmod(etcd_bin, 0o755) - os.chmod(etcdctl_bin, 0o755) - print("--- Done ---") - return - else: - checksum_etcd_archive = checksum_file(etcd_downloaded_archive) - if checksum_etcd_archive == checksums_etcd[etcd_url]: - os.chmod(etcd_bin, 0o755) - os.chmod(etcdctl_bin, 0o755) - print("--- Done ---") - return - else: - print(f"checksum mismatch on etcd") - os.remove(etcd_bin) - os.remove(etcdctl_bin) - os.remove(etcd_downloaded_archive) - - if not os.path.exists(etcd_downloaded_archive): - print(f"Downloading {etcd_downloaded_dir_name} archive...") - urlretrieve(etcd_url, etcd_downloaded_archive) - else: - print(f"Archive {etcd_downloaded_dir_name} already exists") - - if etcd_archive_extension == "zip": - with zipfile.ZipFile(etcd_downloaded_archive, "r") as zip_ref: - zip_ref.extract(etcd_downloaded_dir_name + "/etcd", etcd_binaries) - zip_ref.extract(etcd_downloaded_dir_name + "/etcdctl", etcd_binaries) - else: - with (tarfile.open(etcd_downloaded_archive, "r")) as tar_ref: - print("Extracting the archive...") - tar_ref.extract(etcd_downloaded_dir_name + "/etcd", etcd_binaries) - tar_ref.extract(etcd_downloaded_dir_name + "/etcdctl", etcd_binaries) - - shutil.copy(os.path.join(etcd_binaries, etcd_downloaded_dir_name, "etcd"), etcd_bin) - shutil.copy( - os.path.join(etcd_binaries, etcd_downloaded_dir_name, "etcdctl"), etcdctl_bin - ) - - if etcd_url in checksums_etcd: - checksum_etcd_archive = checksum_file(etcd_downloaded_archive) - if checksum_etcd_archive != checksums_etcd[etcd_url]: - raise IOError("Checksum failed") - else: - warnings.warn( - f"Couldn't verify checksum for etcd-v{etcd_version}-{plat}", stacklevel=2 - ) - - os.chmod(etcd_bin, 0o755) - os.chmod(etcdctl_bin, 0o755) - - # Cleanup - shutil.rmtree(etcd_binaries) - - print("--- Done ---") - - -def install_consul(prefix, plat, consul_version): - plat = plat.replace("-", "_") - consul_downloaded_dir_name = f"consul_v{consul_version}_{plat}" - consul_archive_extension = ".tar.gz" - consul_archive_extension = "zip" - - consul_downloaded_archive = os.path.join( - prefix, consul_downloaded_dir_name + "." + consul_archive_extension - ) - consul_binaries = os.path.join(prefix, "consul_binaries") - - consul_bin = os.path.join(prefix, "consul") - - consul_url = ( - "https://releases.hashicorp.com/consul/" - f"{consul_version}/consul_{consul_version}_{plat}.{consul_archive_extension}" - ) - - if os.path.exists(consul_bin): - print(f"Consul already exists") - if consul_url not in checksums_consul: - warnings.warn( - f"Couldn't verify checksum for consul_v{consul_version}_{plat}", - stacklevel=2, - ) - os.chmod(consul_bin, 0o755) - print("--- Done ---") - return - else: - checksum_consul_archive = checksum_file(consul_downloaded_archive) - if checksum_consul_archive == checksums_consul[consul_url]: - os.chmod(consul_bin, 0o755) - print("--- Done ---") - return - else: - print(f"checksum mismatch on consul") - os.remove(consul_bin) - os.remove(consul_downloaded_archive) - - if not os.path.exists(consul_downloaded_archive): - print(f"Downloading {consul_downloaded_dir_name} archive...") - urlretrieve(consul_url, consul_downloaded_archive) - else: - print(f"Archive {consul_downloaded_dir_name} already exists") - - with zipfile.ZipFile(consul_downloaded_archive, "r") as zip_ref: - zip_ref.extract("consul", consul_binaries) - - shutil.copy(os.path.join(consul_binaries, "consul"), consul_bin) - - if consul_url in checksums_consul: - checksum_consul_archive = checksum_file(consul_downloaded_archive) - if checksum_consul_archive != checksums_consul[consul_url]: - raise IOError("Checksum failed") - else: - warnings.warn( - f"Couldn't verify checksum for consul_v{consul_version}_{plat}", - stacklevel=2, - ) - - os.chmod(consul_bin, 0o755) - - # Cleanup - shutil.rmtree(consul_binaries) - - print("--- Done ---") - - def main(): parser = argparse.ArgumentParser( @@ -265,14 +93,6 @@ def main(): - v2.2.11-linux-amd64 - v2.2.11-darwin-amd64 - v2.2.11-windows-amd64 - - etcd: - - v3.4.7-linux-amd64 - - v3.4.7-darwin-amd64 - - v3.4.7-windows-amd64 - - consul: - - v1.7.2_linux_amd64 - - v1.7.2_darwin_amd64 - - v1.7.2_windows_amd64 """ ), formatter_class=argparse.RawTextHelpFormatter, @@ -331,62 +151,13 @@ def main(): ), ) - parser.add_argument( - "--etcd", - action="store_true", - help=textwrap.dedent( - """\ - Whether or not to install etcd. - By default etcd is NOT going to be installed. - """ - ), - ) - - parser.add_argument( - "--etcd-version", - dest="etcd_version", - default="3.4.15", - help=textwrap.dedent( - """\ - The version of etcd to download. - If no version is provided, it defaults to: - --- %(default)s --- - """ - ), - ) - - parser.add_argument( - "--consul", - action="store_true", - help=textwrap.dedent( - """\ - Whether or not to install consul. - By default consul is NOT going to be installed: - """ - ), - ) - - parser.add_argument( - "--consul-version", - dest="consul_version", - default="1.9.4", - help=textwrap.dedent( - """\ - The version of consul to download. - If no version is provided, it defaults to: - --- %(default)s --- - """ - ), - ) args = parser.parse_args() deps_dir = args.installation_dir plat = args.plat traefik_version = args.traefik_version - etcd_version = args.etcd_version - consul_version = args.consul_version - if not args.traefik and not args.etcd and not args.consul: + if not args.traefik: print( """Please specify what binary to install. Tip: python3 -m jupyterhub_traefik_proxy.install --help @@ -402,11 +173,6 @@ def main(): if args.traefik: install_traefik(deps_dir, plat, traefik_version) - if args.etcd: - install_etcd(deps_dir, plat, etcd_version) - if args.consul: - install_consul(deps_dir, plat, consul_version) - if __name__ == "__main__": main() diff --git a/jupyterhub_traefik_proxy/kv_proxy.py b/jupyterhub_traefik_proxy/kv_proxy.py deleted file mode 100644 index 32f56b3c..00000000 --- a/jupyterhub_traefik_proxy/kv_proxy.py +++ /dev/null @@ -1,369 +0,0 @@ -"""Traefik implementation - -Custom proxy implementations can subclass :class:`Proxy` -and register in JupyterHub config: - -.. sourcecode:: python - - from mymodule import MyProxy - c.JupyterHub.proxy_class = MyProxy - -Route Specification: - -- A routespec is a URL prefix ([host]/path/), e.g. - 'host.tld/path/' for host-based routing or '/path/' for default routing. -- Route paths should be normalized to always start and end with '/' -""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -import json -import os - -from traitlets import Any, Unicode - -from . import traefik_utils -from jupyterhub_traefik_proxy import TraefikProxy - - -class TKvProxy(TraefikProxy): - """ - JupyterHub Proxy implementation using traefik and a key-value store. - Custom proxy implementations based on trafik and a key-value store - can sublass :class:`TKvProxy`. - """ - - kv_client = Any() - # Key-value store client - - kv_name = Unicode(config=False, help="""The name of the key value store""") - - kv_username = Unicode( - config=True, help="""The username for key value store login""" - ) - - kv_password = Unicode( - config=True, help="""The password for key value store login""" - ) - - kv_url = Unicode(config=True, help="""The URL of the key value store server""") - - kv_traefik_prefix = traefik_utils.KVStorePrefix( - config=True, - help="""The key value store key prefix for traefik static configuration""", - ) - - kv_jupyterhub_prefix = Unicode( - config=True, - help="""The key value store key prefix for traefik dynamic configuration""", - ) - - def _define_kv_specific_static_config(self): - """Define the traefik static configuration that configures - traefik's communication with the key-value store. - - Will be called during startup if should_start is True. - - **Subclasses must define this method** - if the proxy is to be started by the Hub. - - In order to be picked up by the proxy, the static configuration - must be stored into `proxy.static_config` dict under the `kv_name` key. - """ - raise NotImplementedError() - - async def _kv_atomic_add_route_parts( - self, jupyterhub_routespec, target, data, route_keys, rule - ): - """Add the key-value pairs associated with a route within a - key-value store transaction. - - **Subclasses must define this method** - - Will be called during add_route. - - When retrieving or deleting a route, the parts of a route - are expected to have the following structure: - [ key: jupyterhub_routespec , value: target ] - [ key: target , value: data ] - [ key: route_keys.service_url_path , value: target ] - [ key: route_keys.router_rule_path , value: rule ] - [ key: route_keys.router_service_path, value: - route_keys.service_alias] - [ key: route_keys.service_weight_path , value: w(int) ] - (where `w` is the weight of the service to be used during load balancing) - - Returns: - result (tuple): - 'status'(int): The transaction status - (0: failure, positive: success) - 'response'(str): The transaction response - """ - raise NotImplementedError() - - async def _kv_atomic_delete_route_parts(self, jupyterhub_routespec, route_keys): - """Delete the key-value pairs associated with a route within a - key-value store transaction (if the route exists). - - **Subclasses must define this method** - - Will be called during delete_route. - - The keys associated with a route are: - jupyterhub_routespec, - target, - route_keys.service_url_path, - route_keys.router_rule_path, - route_keys.router_service_path, - route_keys.service_weight_path, - - Returns: - result (tuple): - 'status'(int): The transaction status - (0: failure, positive: success). - 'response'(str): The transaction response. - """ - raise NotImplementedError() - - async def _kv_get_target(self, jupyterhub_routespec): - """Retrive the target from the key-value store. - The target is the value associated with `jupyterhub_routespec` key. - - **Subclasses must define this method** - - Returns: - target (str): The full URL associated with this route. - """ - raise NotImplementedError() - - async def _kv_get_data(self, target): - """Retrive the data associated with the `target` from the key-value store. - - **Subclasses must define this method** - - Returns: - data (dict): A JSONable dict that holds extra info about the route - """ - raise NotImplementedError() - - async def _kv_get_route_parts(self, kv_entry): - """Retrive all the parts that make up a route (i.e. routespec, target, data) - from the key-value store given a `kv_entry`. - - A `kv_entry` is a key-value store entry where the key starts with - `proxy.jupyterhub_prefix`. It is expected that only the routespecs - will be prefixed with `proxy.jupyterhub_prefix` when added to the kv store. - - **Subclasses must define this method** - - Returns: - 'routespec': The normalized route specification passed in to add_route - ([host]/path/) - 'target': The target host for this route (proto://host) - 'data': The arbitrary data dict that was passed in by JupyterHub when adding this - route. - """ - raise NotImplementedError() - - async def _kv_get_jupyterhub_prefixed_entries(self): - """Retrive from the kv store all the key-value pairs where the key starts with - `proxy.jupyterhub_prefix`. - It is expected that only the routespecs will be prefixed with `proxy.jupyterhub_prefix` - when added to the kv store. - - **Subclasses must define this method** - - Returns: - 'routes': A list of key-value store entries where the keys start - with `proxy.jupyterhub_prefix`. - """ - - raise NotImplementedError() - - def _clean_resources(self): - try: - if self.should_start: - os.remove(self.static_config_file) - except: - self.log.error("Failed to remove traefik's configuration files") - raise - - def _start_traefik(self): - self.log.info("Starting traefik...") - try: - self._launch_traefik(config_type=self.kv_name) - except FileNotFoundError as e: - self.log.error( - "Failed to find traefik \n" - "The proxy can be downloaded from https://github.com/containous/traefik/releases/download." - ) - raise - - async def _setup_traefik_static_config(self): - await super()._setup_traefik_static_config() - self._define_kv_specific_static_config() - try: - traefik_utils.persist_static_conf( - self.static_config_file, self.static_config - ) - except IOError: - self.log.exception("Couldn't set up traefik's static config.") - raise - except: - self.log.error("Couldn't set up traefik's static config. Unexpected error:") - raise - - async def start(self): - """Start the proxy. - Will be called during startup if should_start is True. - """ - await super().start() - await self._wait_for_static_config(provider=self.kv_name) - - async def stop(self): - """Stop the proxy. - Will be called during teardown if should_start is True. - """ - await super().stop() - self._clean_resources() - - async def add_route(self, routespec, target, data): - """Add a route to the proxy. - - Args: - routespec (str): A URL prefix ([host]/path/) for which this route will be matched, - e.g. host.name/path/ - target (str): A full URL that will be the target of this route. - data (dict): A JSONable dict that will be associated with this route, and will - be returned when retrieving information about this route. - - Will raise an appropriate Exception (FIXME: find what?) if the route could - not be added. - - This proxy implementation prefixes the routespec with `proxy.jupyterhub_prefix` when - adding it to the kv store in orde to associate the fact that the route came from JupyterHub. - Everything traefik related is prefixed with `proxy.traefik_prefix`. - """ - self.log.info("Adding route for %s to %s.", routespec, target) - - routespec = self._routespec_to_traefik_path(routespec) - route_keys = traefik_utils.generate_route_keys(self, routespec) - - # Store the data dict passed in by JupyterHub - data = json.dumps(data) - # Generate the routing rule - rule = traefik_utils.generate_rule(routespec) - - # To be able to delete the route when only routespec is provided - jupyterhub_routespec = self.kv_jupyterhub_prefix + routespec - - status, response = await self._kv_atomic_add_route_parts( - jupyterhub_routespec, target, data, route_keys, rule - ) - - if self.should_start: - try: - # Check if traefik was launched - pid = self.traefik_process.pid - except AttributeError: - self.log.error( - "You cannot add routes if the proxy isn't running! Please start the proxy: proxy.start()" - ) - raise - if status: - self.log.info( - "Added service %s with the alias %s.", target, route_keys.service_alias - ) - self.log.info( - "Added router %s for service %s with the following routing rule %s.", - route_keys.router_alias, - route_keys.service_alias, - routespec, - ) - else: - self.log.error( - "Couldn't add route for %s. Response: %s", routespec, response - ) - - await self._wait_for_route(routespec) - - async def delete_route(self, routespec): - """Delete a route and all the traefik related info associated given a routespec, - (if it exists). - """ - routespec = self._routespec_to_traefik_path(routespec) - jupyterhub_routespec = self.kv_jupyterhub_prefix + routespec - route_keys = traefik_utils.generate_route_keys(self, routespec) - - status, response = await self._kv_atomic_delete_route_parts( - jupyterhub_routespec, route_keys - ) - if status: - self.log.info("Routespec %s was deleted.", routespec) - else: - self.log.error( - "Couldn't delete route %s. Response: %s", routespec, response - ) - - async def get_all_routes(self): - """Fetch and return all the routes associated by JupyterHub from the - proxy. - - Returns a dictionary of routes, where the keys are - routespecs and each value is a dict of the form:: - - { - 'routespec': the route specification ([host]/path/) - 'target': the target host URL (proto://host) for this route - 'data': the attached data dict for this route (as specified in add_route) - } - """ - all_routes = {} - routes = await self._kv_get_jupyterhub_prefixed_entries() - - for kv_entry in routes: - traefik_routespec, target, data = await self._kv_get_route_parts(kv_entry) - routespec = self._routespec_from_traefik_path(traefik_routespec) - all_routes[routespec] = { - "routespec": routespec, - "target": target, - "data": None if data is None else json.loads(data), - } - - return all_routes - - async def get_route(self, routespec): - """Return the route info for a given routespec. - - Args: - routespec (str): - A URI that was used to add this route, - e.g. `host.tld/path/` - - Returns: - result (dict): - dict with the following keys:: - - 'routespec': The normalized route specification passed in to add_route - ([host]/path/) - 'target': The target host for this route (proto://host) - 'data': The arbitrary data dict that was passed in by JupyterHub when adding this - route. - - None: if there are no routes matching the given routespec - """ - routespec = self.validate_routespec(routespec) - traefik_routespec = self._routespec_to_traefik_path(routespec) - jupyterhub_routespec = self.kv_jupyterhub_prefix + traefik_routespec - - target = await self._kv_get_target(jupyterhub_routespec) - if target is None: - return None - data = await self._kv_get_data(target) - - return { - "routespec": routespec, - "target": target, - "data": None if data is None else json.loads(data), - } diff --git a/jupyterhub_traefik_proxy/traefik_utils.py b/jupyterhub_traefik_proxy/traefik_utils.py index 297f1b94..ab5d4c7f 100644 --- a/jupyterhub_traefik_proxy/traefik_utils.py +++ b/jupyterhub_traefik_proxy/traefik_utils.py @@ -76,44 +76,6 @@ def generate_router_rule_entry(proxy, router_alias, separator="/"): return router_rule_entry - -def generate_route_keys(proxy, routespec, separator="/"): - service_alias = generate_alias(routespec, "service") - router_alias = generate_alias(routespec, "router") - - RouteKeys = namedtuple( - "RouteKeys", - [ - "service_alias", - "service_url_path", - "router_alias", - "router_service_path", - "router_rule_path", - ], - ) - - if separator != ".": - service_url_path = generate_service_entry(proxy, service_alias, url=True) - router_rule_path = generate_router_rule_entry(proxy, router_alias) - router_service_path = generate_router_service_entry(proxy, router_alias) - else: - service_url_path = generate_service_entry( - proxy, service_alias, separator=separator - ) - router_rule_path = generate_router_rule_entry( - proxy, router_alias, separator=separator - ) - router_service_path = "" - - return RouteKeys( - service_alias, - service_url_path, - router_alias, - router_service_path, - router_rule_path, - ) - - # atomic writing adapted from jupyter/notebook 5.7 # unlike atomic writing there, which writes the canonical path # and only use the temp file for recovery, From 56b652824540d9dd376513fc8cbcfd7d56ca9f70 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Mon, 25 Oct 2021 16:31:49 +0300 Subject: [PATCH 17/19] Remove more of the etcd and consul mentions --- jupyterhub_traefik_proxy/__init__.py | 1 - jupyterhub_traefik_proxy/consul.py | 242 ---------------------- jupyterhub_traefik_proxy/etcd.py | 196 ------------------ jupyterhub_traefik_proxy/traefik_utils.py | 19 -- 4 files changed, 458 deletions(-) delete mode 100644 jupyterhub_traefik_proxy/consul.py delete mode 100644 jupyterhub_traefik_proxy/etcd.py diff --git a/jupyterhub_traefik_proxy/__init__.py b/jupyterhub_traefik_proxy/__init__.py index 1bcc0501..39dda657 100644 --- a/jupyterhub_traefik_proxy/__init__.py +++ b/jupyterhub_traefik_proxy/__init__.py @@ -1,7 +1,6 @@ """Traefik implementation of the JupyterHub proxy API""" from .proxy import TraefikProxy # noqa -from .kv_proxy import TKvProxy # noqa from .fileprovider import TraefikFileProxy from ._version import get_versions diff --git a/jupyterhub_traefik_proxy/consul.py b/jupyterhub_traefik_proxy/consul.py deleted file mode 100644 index 3459aab0..00000000 --- a/jupyterhub_traefik_proxy/consul.py +++ /dev/null @@ -1,242 +0,0 @@ -"""Traefik implementation - -Custom proxy implementations can subclass :class:`Proxy` -and register in JupyterHub config: - -.. sourcecode:: python - - from mymodule import MyProxy - c.JupyterHub.proxy_class = MyProxy - -Route Specification: - -- A routespec is a URL prefix ([host]/path/), e.g. - 'host.tld/path/' for host-based routing or '/path/' for default routing. -- Route paths should be normalized to always start and end with '/' -""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -import json -import os -from urllib.parse import urlparse -import string -import base64 - -import asyncio -import escapism -from tornado.concurrent import run_on_executor -from traitlets import Any, default, Unicode - -from . import traefik_utils -from jupyterhub_traefik_proxy import TKvProxy -import time - - -class TraefikConsulProxy(TKvProxy): - """JupyterHub Proxy implementation using traefik and Consul""" - - # Consul doesn't accept keys containing // or starting with / so we have to escape them - key_safe_chars = string.ascii_letters + string.digits + "!@#$%^&*();<>-.+?:" - - kv_name = "consul" - - consul_client_ca_cert = Unicode( - config=True, - allow_none=True, - default_value=None, - help="""Consul client root certificates""", - ) - - @default("kv_url") - def _default_kv_url(self): - return "http://127.0.0.1:8500" - - @default("kv_client") - def _default_client(self): - try: - import consul.aio - except ImportError: - raise ImportError("Please install python-consul2 package to use traefik-proxy with consul") - consul_service = urlparse(self.kv_url) - if self.kv_password: - client = consul.aio.Consul( - host=str(consul_service.hostname), - port=consul_service.port, - token=self.kv_password, - cert=self.consul_client_ca_cert, - ) - return client - - return consul.aio.Consul( - host=str(consul_service.hostname), - port=consul_service.port, - cert=self.consul_client_ca_cert, - ) - - @default("kv_traefik_prefix") - def _default_kv_traefik_prefix(self): - return "traefik/" - - @default("kv_jupyterhub_prefix") - def _default_kv_jupyterhub_prefix(self): - return "jupyterhub/" - - def _define_kv_specific_static_config(self): - self.static_config["consul"] = { - "endpoint": str(urlparse(self.kv_url).hostname) - + ":" - + str(urlparse(self.kv_url).port), - "prefix": self.kv_traefik_prefix, - "watch": True, - } - - def _launch_traefik(self, config_type): - os.environ["CONSUL_HTTP_TOKEN"] = self.kv_password - super()._launch_traefik(config_type) - - async def _kv_atomic_add_route_parts( - self, jupyterhub_routespec, target, data, route_keys, rule - ): - escaped_target = escapism.escape(target, safe=self.key_safe_chars) - escaped_jupyterhub_routespec = escapism.escape( - jupyterhub_routespec, safe=self.key_safe_chars - ) - - try: - results = await self.kv_client.txn.put( - payload=[ - { - "KV": { - "Verb": "set", - "Key": escaped_jupyterhub_routespec, - "Value": base64.b64encode(target.encode()).decode(), - } - }, - { - "KV": { - "Verb": "set", - "Key": escaped_target, - "Value": base64.b64encode(data.encode()).decode(), - } - }, - { - "KV": { - "Verb": "set", - "Key": route_keys.backend_url_path, - "Value": base64.b64encode(target.encode()).decode(), - } - }, - { - "KV": { - "Verb": "set", - "Key": route_keys.backend_weight_path, - "Value": base64.b64encode(b"1").decode(), - } - }, - { - "KV": { - "Verb": "set", - "Key": route_keys.frontend_backend_path, - "Value": base64.b64encode( - route_keys.backend_alias.encode() - ).decode(), - } - }, - { - "KV": { - "Verb": "set", - "Key": route_keys.frontend_rule_path, - "Value": base64.b64encode(rule.encode()).decode(), - } - }, - ] - ) - status = 1 - response = "" - except Exception as e: - status = 0 - response = str(e) - - return status, response - - async def _kv_atomic_delete_route_parts(self, jupyterhub_routespec, route_keys): - escaped_jupyterhub_routespec = escapism.escape( - jupyterhub_routespec, safe=self.key_safe_chars - ) - - index, v = await self.kv_client.kv.get(escaped_jupyterhub_routespec) - if v is None: - self.log.warning( - "Route %s doesn't exist. Nothing to delete", jupyterhub_routespec - ) - return True, None - target = v["Value"] - escaped_target = escapism.escape(target, safe=self.key_safe_chars) - - try: - status, response = await self.kv_client.txn.put( - payload=[ - {"KV": {"Verb": "delete", "Key": escaped_jupyterhub_routespec}}, - {"KV": {"Verb": "delete", "Key": escaped_target}}, - {"KV": {"Verb": "delete", "Key": route_keys.backend_url_path}}, - {"KV": {"Verb": "delete", "Key": route_keys.backend_weight_path}}, - {"KV": {"Verb": "delete", "Key": route_keys.frontend_backend_path}}, - {"KV": {"Verb": "delete", "Key": route_keys.frontend_rule_path}}, - ] - ) - status = 1 - response = "" - except Exception as e: - status = 0 - response = str(e) - - return status, response - - async def _kv_get_target(self, jupyterhub_routespec): - escaped_jupyterhub_routespec = escapism.escape( - jupyterhub_routespec, safe=self.key_safe_chars - ) - _, res = await self.kv_client.kv.get(escaped_jupyterhub_routespec) - if res is None: - return None - return res["Value"].decode() - - async def _kv_get_data(self, target): - escaped_target = escapism.escape(target, safe=self.key_safe_chars) - _, res = await self.kv_client.kv.get(escaped_target) - - if res is None: - return None - return res["Value"].decode() - - async def _kv_get_route_parts(self, kv_entry): - key = escapism.unescape(kv_entry["KV"]["Key"]) - value = kv_entry["KV"]["Value"] - - # Strip the "/jupyterhub" prefix from the routespec - routespec = key.replace(self.kv_jupyterhub_prefix, "") - target = base64.b64decode(value.encode()).decode() - data = await self._kv_get_data(target) - - return routespec, target, data - - async def _kv_get_jupyterhub_prefixed_entries(self): - routes = await self.kv_client.txn.put( - payload=[ - { - "KV": { - "Verb": "get-tree", - "Key": escapism.escape( - self.kv_jupyterhub_prefix, safe=self.key_safe_chars - ), - } - } - ] - ) - - return routes["Results"] - - async def stop(self): - await super().stop() diff --git a/jupyterhub_traefik_proxy/etcd.py b/jupyterhub_traefik_proxy/etcd.py deleted file mode 100644 index 9b0e0d15..00000000 --- a/jupyterhub_traefik_proxy/etcd.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Traefik implementation - -Custom proxy implementations can subclass :class:`Proxy` -and register in JupyterHub config: - -.. sourcecode:: python - - from mymodule import MyProxy - c.JupyterHub.proxy_class = MyProxy - -Route Specification: - -- A routespec is a URL prefix ([host]/path/), e.g. - 'host.tld/path/' for host-based routing or '/path/' for default routing. -- Route paths should be normalized to always start and end with '/' -""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -from concurrent.futures import ThreadPoolExecutor -import json -import os -from urllib.parse import urlparse - -from tornado.concurrent import run_on_executor -from traitlets import Any, default, Unicode - -from jupyterhub.utils import maybe_future -from . import traefik_utils -from jupyterhub_traefik_proxy import TKvProxy - - -class TraefikEtcdProxy(TKvProxy): - """JupyterHub Proxy implementation using traefik and etcd""" - - executor = Any() - - kv_name = "etcdv3" - - etcd_client_ca_cert = Unicode( - config=True, - allow_none=True, - default_value=None, - help="""Etcd client root certificates""", - ) - - etcd_client_cert_crt = Unicode( - config=True, - allow_none=True, - default_value=None, - help="""Etcd client certificate chain - (etcd_client_cert_key must also be specified)""", - ) - - etcd_client_cert_key = Unicode( - config=True, - allow_none=True, - default_value=None, - help="""Etcd client private key - (etcd_client_cert_crt must also be specified)""", - ) - - @default("executor") - def _default_executor(self): - return ThreadPoolExecutor(1) - - @default("kv_url") - def _default_kv_url(self): - return "http://127.0.0.1:2379" - - @default("kv_client") - def _default_client(self): - etcd_service = urlparse(self.kv_url) - try: - import etcd3 - except ImportError: - raise ImportError("Please install etcd3 package to use traefik-proxy with etcd3") - if self.kv_password: - return etcd3.client( - host=str(etcd_service.hostname), - port=etcd_service.port, - user=self.kv_username, - password=self.kv_password, - ca_cert=self.etcd_client_ca_cert, - cert_cert=self.etcd_client_cert_crt, - cert_key=self.etcd_client_cert_key, - ) - return etcd3.client( - host=str(etcd_service.hostname), - port=etcd_service.port, - ca_cert=self.etcd_client_ca_cert, - cert_cert=self.etcd_client_cert_crt, - cert_key=self.etcd_client_cert_key, - ) - - @default("kv_traefik_prefix") - def _default_kv_traefik_prefix(self): - return "/traefik/" - - @default("kv_jupyterhub_prefix") - def _default_kv_jupyterhub_prefix(self): - return "/jupyterhub/" - - @run_on_executor - def _etcd_transaction(self, success_actions): - status, response = self.kv_client.transaction( - compare=[], success=success_actions, failure=[] - ) - return status, response - - @run_on_executor - def _etcd_get(self, key): - value, _ = self.kv_client.get(key) - return value - - @run_on_executor - def _etcd_get_prefix(self, prefix): - routes = self.kv_client.get_prefix(prefix) - return routes - - def _define_kv_specific_static_config(self): - self.static_config["etcd"] = { - "username": self.kv_username, - "password": self.kv_password, - "endpoint": str(urlparse(self.kv_url).hostname) - + ":" - + str(urlparse(self.kv_url).port), - "prefix": self.kv_traefik_prefix, - "useapiv3": True, - "watch": True, - } - - async def _kv_atomic_add_route_parts( - self, jupyterhub_routespec, target, data, route_keys, rule - ): - success = [ - self.kv_client.transactions.put(jupyterhub_routespec, target), - self.kv_client.transactions.put(target, data), - self.kv_client.transactions.put(route_keys.backend_url_path, target), - self.kv_client.transactions.put(route_keys.backend_weight_path, "1"), - self.kv_client.transactions.put( - route_keys.frontend_backend_path, route_keys.backend_alias - ), - self.kv_client.transactions.put(route_keys.frontend_rule_path, rule), - ] - status, response = await maybe_future(self._etcd_transaction(success)) - return status, response - - async def _kv_atomic_delete_route_parts(self, jupyterhub_routespec, route_keys): - value = await maybe_future(self._etcd_get(jupyterhub_routespec)) - if value is None: - self.log.warning( - "Route %s doesn't exist. Nothing to delete", jupyterhub_routespec - ) - return True, None - - target = value.decode() - - success = [ - self.kv_client.transactions.delete(jupyterhub_routespec), - self.kv_client.transactions.delete(target), - self.kv_client.transactions.delete(route_keys.backend_url_path), - self.kv_client.transactions.delete(route_keys.backend_weight_path), - self.kv_client.transactions.delete(route_keys.frontend_backend_path), - self.kv_client.transactions.delete(route_keys.frontend_rule_path), - ] - status, response = await maybe_future(self._etcd_transaction(success)) - return status, response - - async def _kv_get_target(self, jupyterhub_routespec): - value = await maybe_future(self._etcd_get(jupyterhub_routespec)) - if value == None: - return None - return value.decode() - - async def _kv_get_data(self, target): - value = await maybe_future(self._etcd_get(target)) - if value is None: - return None - return value - - async def _kv_get_route_parts(self, kv_entry): - key = kv_entry[1].key.decode() - value = kv_entry[0] - - # Strip the "/jupyterhub" prefix from the routespec - routespec = key.replace(self.kv_jupyterhub_prefix, "") - target = value.decode() - data = await self._kv_get_data(target) - - return routespec, target, data - - async def _kv_get_jupyterhub_prefixed_entries(self): - routes = await maybe_future(self._etcd_get_prefix(self.kv_jupyterhub_prefix)) - return routes diff --git a/jupyterhub_traefik_proxy/traefik_utils.py b/jupyterhub_traefik_proxy/traefik_utils.py index ab5d4c7f..99cac53d 100644 --- a/jupyterhub_traefik_proxy/traefik_utils.py +++ b/jupyterhub_traefik_proxy/traefik_utils.py @@ -10,21 +10,6 @@ from collections import namedtuple -class KVStorePrefix(Unicode): - def validate(self, obj, value): - u = super().validate(obj, value) - # We'll join the prefix with e.g. "/".join(pathspec), - # therefore always strip the trailing "/" from any prefix - if u.endswith("/"): - u = u.rstrip("/") - - proxy_class = type(obj).__name__ - if "Consul" in proxy_class and u.startswith("/"): - u = u[1:] - - return u - - def generate_rule(routespec): routespec = unquote(routespec) if routespec.startswith("/"): @@ -139,7 +124,3 @@ def atomic_dump(self, data): with atomic_writing(self.file_path) as f: self._dump(data, f) -def persist_static_conf(file_path, static_conf_dict): - handler = TraefikConfigFileHandler(file_path) - handler.dump(static_conf_dict) - From cd85379de062a123af81f055d8cd1793311535bc Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Tue, 26 Oct 2021 20:18:25 +0300 Subject: [PATCH 18/19] Remove unused imports --- jupyterhub_traefik_proxy/proxy.py | 1 - jupyterhub_traefik_proxy/toml.py | 3 --- jupyterhub_traefik_proxy/traefik_utils.py | 2 -- 3 files changed, 6 deletions(-) diff --git a/jupyterhub_traefik_proxy/proxy.py b/jupyterhub_traefik_proxy/proxy.py index 0a785e61..893b53b0 100644 --- a/jupyterhub_traefik_proxy/proxy.py +++ b/jupyterhub_traefik_proxy/proxy.py @@ -21,7 +21,6 @@ import json from os.path import abspath, dirname, join from subprocess import Popen, TimeoutExpired -import asyncio.subprocess from urllib.parse import urlparse from traitlets import Any, Bool, Dict, Integer, Unicode, default diff --git a/jupyterhub_traefik_proxy/toml.py b/jupyterhub_traefik_proxy/toml.py index e7b98053..856b670b 100644 --- a/jupyterhub_traefik_proxy/toml.py +++ b/jupyterhub_traefik_proxy/toml.py @@ -18,16 +18,13 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -import json import os import asyncio -import string import escapism from traitlets import Any, default, Unicode, observe from . import traefik_utils -from jupyterhub.proxy import Proxy from jupyterhub_traefik_proxy import TraefikProxy diff --git a/jupyterhub_traefik_proxy/traefik_utils.py b/jupyterhub_traefik_proxy/traefik_utils.py index 99cac53d..cdab12bc 100644 --- a/jupyterhub_traefik_proxy/traefik_utils.py +++ b/jupyterhub_traefik_proxy/traefik_utils.py @@ -1,13 +1,11 @@ import os import string from tempfile import NamedTemporaryFile -from traitlets import Unicode from urllib.parse import unquote import escapism from contextlib import contextmanager -from collections import namedtuple def generate_rule(routespec): From 0b083ce3622bd86fe6765cdf9b375eab3418f6c2 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Tue, 26 Oct 2021 20:18:36 +0300 Subject: [PATCH 19/19] Fix installer tests --- tests/test_installer.py | 46 ++++++++++------------------------------- 1 file changed, 11 insertions(+), 35 deletions(-) diff --git a/tests/test_installer.py b/tests/test_installer.py index 157a336b..8e9ad1bd 100644 --- a/tests/test_installer.py +++ b/tests/test_installer.py @@ -21,29 +21,9 @@ def assert_deps_dir_empty(deps_dir): assert not os.listdir(deps_dir) -def assert_only_traefik_existence(deps_dir): +def assert_traefik_existence(deps_dir): traefik_bin = os.path.join(deps_dir, "traefik") - etcd_bin = os.path.join(deps_dir, "etcd") - etcdctl_bin = os.path.join(deps_dir, "etcdctl") - consul_bin = os.path.join(deps_dir, "consul") - assert os.path.exists(traefik_bin) - assert not os.path.exists(etcd_bin) - assert not os.path.exists(etcdctl_bin) - assert not os.path.exists(consul_bin) - - -def assert_binaries_existence(deps_dir): - traefik_bin = os.path.join(deps_dir, "traefik") - etcd_bin = os.path.join(deps_dir, "etcd") - etcdctl_bin = os.path.join(deps_dir, "etcdctl") - consul_bin = os.path.join(deps_dir, "consul") - - assert os.path.exists(traefik_bin) - assert os.path.exists(etcd_bin) - assert os.path.exists(etcdctl_bin) - assert os.path.exists(consul_bin) - def test_default_conf(): parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -59,7 +39,7 @@ def test_install_only_traefik_default_version(): try: subprocess.run([sys.executable, "-m", installer_module, "--traefik"]) - assert_only_traefik_existence(default_deps_dir) + assert_traefik_existence(default_deps_dir) finally: cleanup(default_deps_dir) @@ -70,9 +50,9 @@ def test_install_all_binaries_default_version(): try: subprocess.run( - [sys.executable, "-m", installer_module, "--traefik", "--etcd", "--consul"] + [sys.executable, "-m", installer_module, "--traefik"] ) - assert_binaries_existence(default_deps_dir) + assert_traefik_existence(default_deps_dir) finally: cleanup(default_deps_dir) @@ -84,7 +64,7 @@ def test_output_arg_new_dir(tmpdir): ) assert os.path.exists(deps_dir) - assert_only_traefik_existence(deps_dir) + assert_traefik_existence(deps_dir) def test_output_arg_existing_dir(tmpdir): @@ -92,7 +72,7 @@ def test_output_arg_existing_dir(tmpdir): subprocess.run( [sys.executable, "-m", installer_module, "--traefik", f"--output={deps_dir}"] ) - assert_only_traefik_existence(deps_dir) + assert_traefik_existence(deps_dir) def test_version(tmpdir): @@ -105,15 +85,11 @@ def test_version(tmpdir): f"--output={deps_dir}", "--traefik", "--traefik-version=2.4.8", - "--etcd", - "--etcd-version=3.2.25", - "--consul", - "--consul-version=1.5.0", ] ) assert os.path.exists(deps_dir) - assert_binaries_existence(deps_dir) + assert_traefik_existence(deps_dir) def test_linux_arm_platform(tmpdir): @@ -130,7 +106,7 @@ def test_linux_arm_platform(tmpdir): ) assert os.path.exists(deps_dir) - assert_only_traefik_existence(deps_dir) + assert_traefik_existence(deps_dir) def test_linux_amd64_platform(tmpdir): @@ -147,7 +123,7 @@ def test_linux_amd64_platform(tmpdir): ) assert os.path.exists(deps_dir) - assert_only_traefik_existence(deps_dir) + assert_traefik_existence(deps_dir) def test_mac_platform(tmpdir): @@ -164,7 +140,7 @@ def test_mac_platform(tmpdir): ) assert os.path.exists(deps_dir) - assert_only_traefik_existence(deps_dir) + assert_traefik_existence(deps_dir) def test_warning(tmpdir): @@ -181,5 +157,5 @@ def test_warning(tmpdir): stderr=subprocess.STDOUT, ) assert os.path.exists(deps_dir) - assert_only_traefik_existence(deps_dir) + assert_traefik_existence(deps_dir) assert output.decode().count("UserWarning") == 1