From 8f480461e9a4ddadf86ecf27befe58ff5b81d4f0 Mon Sep 17 00:00:00 2001 From: katelyn martin Date: Mon, 22 Jan 2024 18:45:21 -0500 Subject: [PATCH] =?UTF-8?q?pd:=20=F0=9F=94=A8=20rework=20RootCommand::star?= =?UTF-8?q?t=20auto-https=20logic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` /!\ this is a work in progress and will /!\ /!\ be force pushed until ready for review /!\ ``` fixes #3627. this reorganizes the logic in pd's startup code related to automatically managed https functionality. in particular, this is done in such a manner as to expose an axum `Router` that will permit later work (#3646, #3522) to serve other additional routes from our grpc endpoint. Refs: #3627 Refs: #3646 Refs: #3522 --- Cargo.lock | 304 +++++++++++++++++++++++++------- crates/bin/pd/Cargo.toml | 9 +- crates/bin/pd/src/auto_https.rs | 161 ++++++++++++----- crates/bin/pd/src/main.rs | 64 ++----- 4 files changed, 378 insertions(+), 160 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fe88e71f6d..985489fbab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -602,7 +602,7 @@ checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" dependencies = [ "async-channel 2.1.1", "async-executor", - "async-io 2.2.2", + "async-io 2.3.0", "async-lock 3.3.0", "blocking", "futures-lite 2.2.0", @@ -648,9 +648,9 @@ dependencies = [ [[package]] name = "async-io" -version = "2.2.2" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6afaa937395a620e33dc6a742c593c01aced20aa376ffb0f628121198578ccc7" +checksum = "fb41eb19024a91746eba0773aa5e16036045bbf45733766661099e182ea6a744" dependencies = [ "async-lock 3.3.0", "cfg-if", @@ -658,7 +658,7 @@ dependencies = [ "futures-io", "futures-lite 2.2.0", "parking", - "polling 3.3.1", + "polling 3.3.2", "rustix 0.38.28", "slab", "tracing", @@ -719,7 +719,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e47d90f65a225c4527103a8d747001fc56e375203592b25ad103e1ca13124c5" dependencies = [ - "async-io 2.2.2", + "async-io 2.3.0", "async-lock 2.8.0", "atomic-waker", "cfg-if", @@ -870,9 +870,9 @@ dependencies = [ "bytes", "futures-util", "headers", - "http", - "http-body", - "hyper", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", "itoa", "matchit 0.5.0", "memchr", @@ -901,9 +901,9 @@ dependencies = [ "bitflags 1.3.2", "bytes", "futures-util", - "http", - "http-body", - "hyper", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", "itoa", "matchit 0.7.3", "memchr", @@ -931,8 +931,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.6", "mime", "tower-layer", "tower-service", @@ -947,8 +947,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.6", "mime", "rustversion", "tower-layer", @@ -964,17 +964,40 @@ dependencies = [ "arc-swap", "bytes", "futures-util", - "http", - "http-body", - "hyper", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", "pin-project-lite", "rustls 0.20.9", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "tokio", "tokio-rustls 0.23.4", "tower-service", ] +[[package]] +name = "axum-server" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ad46c3ec4e12f4a4b6835e173ba21c25e484c9d02b49770bf006ce5367c036" +dependencies = [ + "arc-swap", + "bytes", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.1.0", + "hyper-util", + "pin-project-lite", + "rustls 0.21.10", + "rustls-pemfile 2.0.0", + "tokio", + "tokio-rustls 0.24.1", + "tower", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.69" @@ -1893,9 +1916,9 @@ dependencies = [ [[package]] name = "crypto-mac" -version = "0.10.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bff07008ec701e8028e2ceb8f83f0e4274ee62bd2dbdc4fefff2e9a91824081a" +checksum = "4857fd85a0c34b3c3297875b747c1e02e06b6a0ea32dd892d8192b9ce0813ea6" dependencies = [ "generic-array", "subtle", @@ -2893,7 +2916,26 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.11", + "indexmap 2.1.0", + "slab", + "tokio", + "tokio-util 0.7.10", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31d030e59af851932b72ceebadf4a2b5986dba4c3b99dd2493f8273a0f151943" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 1.0.0", "indexmap 2.1.0", "slab", "tokio", @@ -2987,7 +3029,7 @@ dependencies = [ "base64 0.21.6", "bytes", "headers-core", - "http", + "http 0.2.11", "httpdate", "mime", "sha1 0.10.6", @@ -2999,7 +3041,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" dependencies = [ - "http", + "http 0.2.11", ] [[package]] @@ -3101,6 +3143,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.6" @@ -3108,7 +3161,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http", + "http 0.2.11", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.0.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840" +dependencies = [ + "bytes", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", "pin-project-lite", ] @@ -3174,9 +3250,9 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.3.22", + "http 0.2.11", + "http-body 0.4.6", "httparse", "httpdate", "itoa", @@ -3188,6 +3264,25 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5aa53871fc917b1a9ed87b683a5d86db645e23acb32c2e0785a353e522fb75" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.2", + "http 1.0.0", + "http-body 1.0.0", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "tokio", +] + [[package]] name = "hyper-rustls" version = "0.24.2" @@ -3195,8 +3290,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http", - "hyper", + "http 0.2.11", + "hyper 0.14.28", "rustls 0.21.10", "tokio", "tokio-rustls 0.24.1", @@ -3208,7 +3303,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper", + "hyper 0.14.28", "pin-project-lite", "tokio", "tokio-io-timeout", @@ -3221,12 +3316,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "hyper", + "hyper 0.14.28", "native-tls", "tokio", "tokio-native-tls", ] +[[package]] +name = "hyper-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdea9aac0dbe5a9240d68cfd9501e2db94222c6dc06843e06640b9e07f0fdc67" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "hyper 1.1.0", + "pin-project-lite", + "socket2 0.5.5", + "tokio", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.59" @@ -4093,7 +4206,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "953cbbb6f9ba4b9304f4df79b98cdc9d14071ed93065a9fca11c00c5d9181b66" dependencies = [ - "hyper", + "hyper 0.14.28", "indexmap 1.9.3", "ipnet", "metrics", @@ -4646,7 +4759,7 @@ dependencies = [ "ed25519-consensus", "futures", "hex", - "http-body", + "http-body 0.4.6", "ibc-proto", "ibc-types", "indicatif", @@ -4718,8 +4831,8 @@ dependencies = [ "ed25519-consensus", "futures", "hex", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.6", "ibc-proto", "ibc-types", "metrics", @@ -4764,6 +4877,8 @@ dependencies = [ "async-stream 0.2.1", "async-trait", "atty", + "axum 0.6.20", + "axum-server 0.6.0", "base64 0.20.0", "bincode", "blake2b_simd 0.5.11", @@ -4780,7 +4895,7 @@ dependencies = [ "fs_extra", "futures", "hex", - "http", + "http 0.2.11", "ibc-proto", "ibc-types", "ics23", @@ -4818,6 +4933,7 @@ dependencies = [ "regex", "reqwest", "rocksdb", + "rustls 0.20.9", "rustls-acme", "serde", "serde_json", @@ -4830,6 +4946,7 @@ dependencies = [ "tendermint-proto", "tendermint-rpc", "tokio", + "tokio-rustls 0.25.0", "tokio-stream", "tokio-util 0.7.10", "toml 0.5.11", @@ -5781,7 +5898,7 @@ version = "0.64.1" dependencies = [ "anyhow", "axum 0.5.17", - "axum-server", + "axum-server 0.4.7", "bytes", "clap 3.2.25", "decaf377 0.5.0", @@ -5813,7 +5930,7 @@ dependencies = [ "chrono", "futures", "hex", - "http", + "http 0.2.11", "metrics", "pbjson-types", "penumbra-proto", @@ -5842,7 +5959,7 @@ version = "0.64.1" dependencies = [ "futures", "hex", - "http", + "http 0.2.11", "pin-project", "pin-project-lite", "sha2 0.9.9", @@ -6179,9 +6296,9 @@ dependencies = [ [[package]] name = "polling" -version = "3.3.1" +version = "3.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf63fa624ab313c11656b4cda960bfc46c410187ad493c41f6ba2d8c1e991c9e" +checksum = "545c980a3880efd47b2e262f6a4bb6daad6555cf3367aa9c4e52895f69537a41" dependencies = [ "cfg-if", "concurrent-queue", @@ -6852,10 +6969,10 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.22", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", "hyper-rustls", "hyper-tls", "ipnet", @@ -6868,7 +6985,7 @@ dependencies = [ "pin-project-lite", "rustls 0.21.10", "rustls-native-certs", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", @@ -7075,10 +7192,24 @@ checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", "ring 0.17.7", - "rustls-webpki", + "rustls-webpki 0.101.7", "sct", ] +[[package]] +name = "rustls" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e87c9956bd9807afa1f77e0f7594af32566e830e088a5576d27c5b6f30f49d41" +dependencies = [ + "log", + "ring 0.17.7", + "rustls-pki-types", + "rustls-webpki 0.102.1", + "subtle", + "zeroize", +] + [[package]] name = "rustls-acme" version = "0.6.0" @@ -7088,6 +7219,7 @@ dependencies = [ "async-h1", "async-io 1.13.0", "async-trait", + "axum-server 0.4.7", "base64 0.13.1", "chrono", "futures", @@ -7103,6 +7235,8 @@ dependencies = [ "serde_json", "smol", "thiserror", + "tokio", + "tokio-util 0.7.10", "url", "webpki-roots 0.21.1", "x509-parser", @@ -7115,7 +7249,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "schannel", "security-framework", ] @@ -7129,6 +7263,22 @@ dependencies = [ "base64 0.21.6", ] +[[package]] +name = "rustls-pemfile" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35e4980fa29e4c4b212ffb3db068a564cbf560e51d3944b7c88bd8bf5bec64f4" +dependencies = [ + "base64 0.21.6", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9d979b3ce68192e42760c7810125eb6cf2ea10efae545a156063e61f314e2a" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -7139,6 +7289,17 @@ dependencies = [ "untrusted 0.9.0", ] +[[package]] +name = "rustls-webpki" +version = "0.102.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef4ca26037c909dedb327b48c3327d0ba91d3dd3c4e05dad328f210ffb68e95b" +dependencies = [ + "ring 0.17.7", + "rustls-pki-types", + "untrusted 0.9.0", +] + [[package]] name = "rustversion" version = "1.0.14" @@ -7828,9 +7989,9 @@ dependencies = [ [[package]] name = "subtle" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "subtle-encoding" @@ -7866,7 +8027,7 @@ dependencies = [ "decaf377 0.5.0", "futures", "hex", - "http-body", + "http-body 0.4.6", "metrics-tracing-context", "penumbra-asset", "penumbra-keys", @@ -8319,6 +8480,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +dependencies = [ + "rustls 0.22.2", + "rustls-pki-types", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.14" @@ -8429,16 +8601,16 @@ dependencies = [ "axum 0.6.20", "base64 0.21.6", "bytes", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.22", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", "hyper-timeout", "percent-encoding", "pin-project", "prost", "rustls 0.21.10", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "tokio", "tokio-rustls 0.24.1", "tokio-stream", @@ -8470,9 +8642,9 @@ checksum = "0fddb2a37b247e6adcb9f239f4e5cefdcc5ed526141a416b943929f13aea2cce" dependencies = [ "base64 0.21.6", "bytes", - "http", - "http-body", - "hyper", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", "pin-project", "tokio-stream", "tonic", @@ -8547,8 +8719,8 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.6", "http-range-header", "pin-project-lite", "tower", @@ -8567,8 +8739,8 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.6", "http-range-header", "pin-project-lite", "tower-layer", @@ -8778,9 +8950,9 @@ checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" [[package]] name = "universal-hash" -version = "0.4.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f214e8f697e925001e66ec2c6e37a4ef93f0f78c2eed7814394e10c62025b05" +checksum = "8326b2c654932e3e4f9196e69d08fdf7cfd718e1dc6f66b347e6024a0c961402" dependencies = [ "generic-array", "subtle", diff --git a/crates/bin/pd/Cargo.toml b/crates/bin/pd/Cargo.toml index 925dd7b4a5..0d018b7897 100644 --- a/crates/bin/pd/Cargo.toml +++ b/crates/bin/pd/Cargo.toml @@ -122,10 +122,17 @@ console-subscriber = "0.2" metrics-tracing-context = "0.11.0" metrics-util = "0.13" clap = { version = "3", features = ["derive", "env"] } -rustls-acme = "0.6" atty = "0.2" fs_extra = "1.3.0" +axum = { version = "0.6.20", features = ["tokio", "http2"] } +# tokio-rustls-acme = { version = "0.2.0", features = ["axum"] } +rustls = "0.20.9" +rustls-acme = { version = "0.6.0", features = ["axum"] } +tokio-rustls = "0.25.0" +# futures-rustls +axum-server = { version = "0.6.0", features = ["tls-rustls"] } + [dev-dependencies] penumbra-proof-params = { path = "../../crypto/proof-params", features = [ "bundled-proving-keys", diff --git a/crates/bin/pd/src/auto_https.rs b/crates/bin/pd/src/auto_https.rs index 6f3ed67e61..cbc2110d71 100644 --- a/crates/bin/pd/src/auto_https.rs +++ b/crates/bin/pd/src/auto_https.rs @@ -1,54 +1,133 @@ -use std::{ - pin::Pin, - task::{Context, Poll}, -}; +use std::net::SocketAddr; + +use axum_server::accept::DefaultAcceptor; +use futures::Future; +use rustls_acme::futures_rustls::TlsAcceptor; +use tracing::{error_span, Instrument}; -use pin_project::pin_project; -use rustls_acme::futures_rustls::server::TlsStream; -use tokio::{ - io::{AsyncRead, AsyncWrite, ReadBuf}, - net::TcpStream, +use { + anyhow::Error, + rustls::ServerConfig, + std::fmt::Debug, + std::path::PathBuf, + std::sync::Arc, + tokio::task::JoinHandle, + rustls_acme::{axum::AxumAcceptor, caches::DirCache, AcmeConfig, AcmeState}, }; -use tokio_util::compat::Compat; -use tonic::transport::server::Connected; - -/// Wrapper type needed to convert between futures_io and tokio traits -#[pin_project] -pub struct Wrapper { - #[pin] - pub inner: Compat>>, + +/// Protocols supported by this server, in order of preference. +/// +/// See [rfc7301] for more info on ALPN. +/// +/// [rfc7301]: https://datatracker.ietf.org/doc/html/rfc7301 +// +// We also permit HTTP1.1 for backwards-compatibility, specifically for grpc-web. +const ALPN_PROTOCOLS: [&'static [u8]; 2] = [b"h2", b"http/1.1"]; + +/// The location of the file-based certificate cache. +// NB: this must not be an absolute path see [Path::join]. +const CACHE_DIR: &'static str = "tokio_rustls_acme_cache"; + +/// If true, use the production Let's Encrypt environment. +/// +/// If false, the ACME resolver will use the [staging environment]. +/// +/// [staging environment]: https://letsencrypt.org/docs/staging-environment/ +const PRODUCTION_LETS_ENCRYPT: bool = true; + +pub struct AutoHttps { + pub home: PathBuf, + pub domain: Option, } -impl Connected for Wrapper { - type ConnectInfo = ::ConnectInfo; +pub enum AutoHttpsAcceptor { + Acme { acceptor: AxumAcceptor }, + DefaultAcceptor, +} - fn connect_info(&self) -> Self::ConnectInfo { - self.inner.get_ref().get_ref().0.get_ref().connect_info() - } +impl From for AutoHttpsAcceptor { + fn from(acceptor: AxumAcceptor) -> Self { + Self::Acme { acceptor } } } -impl AsyncRead for Wrapper { - fn poll_read( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut ReadBuf<'_>, - ) -> Poll> { - self.project().inner.poll_read(cx, buf) +impl AutoHttps { + + // XXX(kate): decide later if this should be a method or a free-standing function. + pub fn start_server(self, address: SocketAddr, router: axum::Router) -> impl Future> { + // let (_acceptor, acme_worker) = self.acceptor(); + let (rustls_config, acme_worker) = self.acceptor(); + let _ = acme_worker.map(tokio::spawn); + + // let _server = axum::Server::bind(&address) + // .serve(router.into_make_service()); + // let _server = axum_server::bind_rustls(address, rustls_config); + // use futures::TryFutureExt; + // _server.map_err(Error::from) + futures::future::ready(Ok(())) // XXX } -} -impl AsyncWrite for Wrapper { - fn poll_write( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &[u8], - ) -> Poll> { - self.project().inner.poll_write(cx, buf) + /// Enable auto-https. + /// + /// Returns a handle to the worker task, and an [`axum::Acceptor`] + fn acceptor(self) -> (ServerConfig, Option>>) { + let Self { home, domain } = self; + + let domains = match domain { + Some(d) => vec![d], + None => return (todo!(), None), + }; + + let cache = home.join(CACHE_DIR); + let cache = DirCache::new(cache); + + let state = AcmeConfig::new(domains) + .cache(cache) + .directory_lets_encrypt(PRODUCTION_LETS_ENCRYPT) // Use the production LE environment + .state(); + + let mut rustls_config = ServerConfig::builder() + .with_safe_defaults() + .with_no_client_auth() // XXX(kate): came from example, is this what we want? + .with_cert_resolver(state.resolver()); + rustls_config.alpn_protocols = Self::alpn_protocols(); + + // let acceptor = state.axum_acceptor(Arc::new(rustls_config)).into(); + let worker = Self::acme_worker(state); + (rustls_config, Some(worker)) } - fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - self.project().inner.poll_flush(cx) + + /// This function defines the task responsible for handling ACME events. + /// + /// This function will never return, unless an error is encountered. + #[tracing::instrument(level = "error", skip_all)] + async fn acme_worker( + mut state: AcmeState, + ) -> Result<(), anyhow::Error> + where + EC: Debug + 'static, + EA: Debug + 'static, + { + use futures::StreamExt; + loop { + match state.next().await { + Some(Ok(ok)) => tracing::debug!("received acme event: {:?}", ok), + Some(Err(err)) => tracing::error!("acme error: {:?}", err), + None => { + debug_assert!(false, "acme worker unexpectedly reached end-of-stream"); + tracing::error!("acme worker unexpectedly reached end-of-stream"); + anyhow::bail!("unexpected end-of-stream"); + } + } + } } - fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - self.project().inner.poll_shutdown(cx) + + /// Returns a vector of the protocols supported by this server. + /// + /// This is a convenience method to retrieve an owned copy of [`ALPN_PROTOCOLS`]. + fn alpn_protocols() -> Vec> { + ALPN_PROTOCOLS + .into_iter() + .map(<[u8]>::to_vec) + .collect() } } diff --git a/crates/bin/pd/src/main.rs b/crates/bin/pd/src/main.rs index 8315d70efb..c62f40310b 100644 --- a/crates/bin/pd/src/main.rs +++ b/crates/bin/pd/src/main.rs @@ -3,6 +3,7 @@ #![recursion_limit = "512"] use std::{error::Error, net::SocketAddr, path::PathBuf}; +use axum_server::accept::{Accept, DefaultAcceptor}; use console_subscriber::ConsoleLayer; use metrics_tracing_context::{MetricsLayer, TracingContextLayer}; use metrics_util::layers::Stack; @@ -478,40 +479,15 @@ async fn main() -> anyhow::Result<()> { ))); } - let grpc_server = if let Some(domain) = grpc_auto_https { - use pd::auto_https::Wrapper; - use rustls_acme::{caches::DirCache, AcmeConfig}; - use tokio_stream::wrappers::TcpListenerStream; - use tokio_util::compat::{FuturesAsyncReadCompatExt, TokioAsyncReadCompatExt}; - - let mut acme_cache = pd_home.clone(); - acme_cache.push("rustls_acme_cache"); - - let bound_listener = TcpListener::bind(grpc_bind) - .await - .context(format!("Failed to bind HTTPS listener on {}", grpc_bind))?; - let listener = TcpListenerStream::new(bound_listener); - // Configure HTTP2 support for the TLS negotiation; we also permit HTTP1.1 - // for backwards-compatibility, specifically for grpc-web. - let alpn_config = vec!["h2".into(), "http/1.1".into()]; - let tls_incoming = AcmeConfig::new([domain.as_str()]) - .cache(DirCache::new(acme_cache)) - .directory_lets_encrypt(true) // Use the production LE environment - .incoming(listener.map_ok(|conn| conn.compat()), alpn_config) - .map_ok(|incoming| Wrapper { - inner: incoming.compat(), - }); - - tokio::task::Builder::new() - .name("grpc_server") - .spawn(grpc_server.serve_with_incoming(tls_incoming)) - .expect("failed to spawn grpc server") - } else { - tokio::task::Builder::new() - .name("grpc_server") - .spawn(grpc_server.serve(grpc_bind)) - .expect("failed to spawn grpc server") - }; + // Now we must drop down a layer of abstraction, from tonic to axum. + // + // Use auto-https, if enabled. + // + // TODO(katie): this is where we may hang additional routes upon this endpoint + // in the future. see #3646 for more information. + let router = grpc_server.into_router(); + let auto_https = pd::auto_https::AutoHttps { home: pd_home, domain: grpc_auto_https }; + let grpc_server = auto_https.start_server(grpc_bind, router); // Configure a Prometheus recorder and exporter. let (recorder, exporter) = PrometheusBuilder::new() @@ -559,24 +535,8 @@ async fn main() -> anyhow::Result<()> { } )?, - x = grpc_server => x?.map_err(|e| { - let mut msg = format!("grpc server on {} failed: {}", grpc_bind, e); - // Detect if we have a bind error. We need to unpack nested errors, from - // tonic -> hyper -> std. Otherwise, only "transport error" is reported, - // which isn't informative enough to take action. - if let Some(e) = e.source() { - if let Some(e) = e.source() { - if let Some(e) = e.downcast_ref::() { - if e.kind().to_string().contains("address in use") { - msg = format!("grpc bind socket already in use: {}", grpc_bind); - } - } - } - } - tracing::error!("{}", msg); - anyhow::anyhow!(msg) - } - )?, + // TODO(kate): restore friendly logging for bind errors. + x = grpc_server => x?, }; }