diff --git a/.travis.yml b/.travis.yml index 2be4a02..b28d565 100644 --- a/.travis.yml +++ b/.travis.yml @@ -118,10 +118,12 @@ deploy: provider: releases skip_cleanup: true -cache: cargo -before_cache: - # Travis can't cache files that are not readable by "others" - - chmod -R a+r $HOME/.cargo +# we are on nightly and the cache isn’t really helping us. +# we will re-add this once we are back on stable +# cache: cargo +# before_cache: +# # Travis can't cache files that are not readable by "others" +# - chmod -R a+r $HOME/.cargo branches: only: diff --git a/Cargo.lock b/Cargo.lock index 72442b7..b2e0307 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -315,6 +315,14 @@ dependencies = [ "walkdir 2.2.9 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "dogstatsd" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "chrono 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "dtoa" version = "0.4.4" @@ -536,6 +544,7 @@ version = "0.4.3" dependencies = [ "assert_cmd 0.11.1 (registry+https://github.com/rust-lang/crates.io-index)", "dir-diff 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "dogstatsd 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "git2 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2191,6 +2200,7 @@ dependencies = [ "checksum difference 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" "checksum digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" "checksum dir-diff 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1cce6e50ca36311e494793f7629014dc78cd963ba85cd05968ae06a63b867f0b" +"checksum dogstatsd 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "392f3b1b2d7244e5ff4195e475868569b39193015b88a03920b5e69fea37a0b1" "checksum dtoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ea57b42383d091c85abcc2706240b94ab2a8fa1fc81c10ff23c4de06e2a90b5e" "checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" "checksum escargot 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ceb9adbf9874d5d028b5e4c5739d22b71988252b25c9c98fe7cf9738bee84597" diff --git a/Cargo.toml b/Cargo.toml index 05d1e8e..b861902 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ zip = "0.5" rocket = { version = "0.4", default-features = false } rocket_contrib = "0.4" rocket_lamb = "0.6" +dogstatsd = "0.6" [dev-dependencies] assert_cmd = "0.11" diff --git a/src/datadogstatsd.rs b/src/datadogstatsd.rs new file mode 100644 index 0000000..f11d895 --- /dev/null +++ b/src/datadogstatsd.rs @@ -0,0 +1,87 @@ +use dogstatsd::{Client, Options}; +use std::env; + +pub struct DdMetrics { + default_tags: [String; 2], + client: Client, +} +impl Default for DdMetrics { + fn default() -> Self { + let dd_options = Options::default(); + DdMetrics { + default_tags: [String::from("service:hogan"), "env:unknown".to_string()], + client: Client::new(dd_options).unwrap(), + } + } +} +impl DdMetrics { + pub fn new() -> Self { + let dd_options = Options::default(); + let mut env_tag = String::from("env: "); + let key = "ENV"; + match env::var(key) { + Ok(val) => { + info!("{}: {}", key, val); + env_tag.push_str(&val); + } + Err(e) => { + info!("couldn't interpret {}: {}", key, e); + env_tag.push_str("unknown"); + } + } + + let dd_tags = [String::from("service:hogan"), env_tag]; + DdMetrics { + default_tags: dd_tags, + client: Client::new(dd_options).unwrap(), + } + } + pub fn incr(&self, name: &str, url: &str) { + self.client + .incr(name, self.append_url_tag(url).iter()) + .unwrap_or_else(|err| self.error_msg(name, &err.to_string())); + } + + pub fn decr(&self, name: &str, url: &str) { + self.client + .incr(name, self.append_url_tag(url).iter()) + .unwrap_or_else(|err| self.error_msg(name, &err.to_string())); + } + + pub fn gauge(&self, name: &str, url: &str, value: &str) { + self.client + .gauge(name, value, self.append_url_tag(url).iter()) + .unwrap_or_else(|err| self.error_msg(name, &err.to_string())); + } + + fn append_url_tag(&self, url: &str) -> Vec { + let mut dd_tags = Vec::new(); + dd_tags.extend_from_slice(&self.default_tags); + + let mut url_tag = String::from("request_url: "); + url_tag.push_str(url); + + dd_tags.push(url_tag); + dd_tags + } + + fn error_msg(&self, name: &str, err: &str) { + info!("{} dd metrics failed with error {}", name, err) + } +} + +pub enum CustomMetrics { + CacheMiss, + CacheHit, + RequestTime, +} + +impl CustomMetrics { + pub fn metrics_name(self) -> &'static str { + match self { + CustomMetrics::CacheMiss => "hogan.cache_miss.counter", + CustomMetrics::CacheHit => "hogan.cache_hit.counter", + CustomMetrics::RequestTime => "hogan.request_time.gauge", + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 02d5845..d3c8958 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,6 +9,7 @@ pub mod config; pub mod git; pub mod template; pub mod transform; +pub mod datadogstatsd; use regex::Regex; use std::path::{Path, PathBuf}; diff --git a/src/main.rs b/src/main.rs index a9ffcec..31b9991 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,12 +13,17 @@ use failure::Error; use hogan; use hogan::config::ConfigDir; use hogan::config::ConfigUrl; +use hogan::datadogstatsd::{CustomMetrics, DdMetrics}; use hogan::template::{Template, TemplateDir}; use lru_time_cache::LruCache; use regex::{Regex, RegexBuilder}; use rocket::config::Config; +use rocket::fairing::{Fairing, Info, Kind}; use rocket::http::Status; +use rocket::request::{self, FromRequest}; +use rocket::Outcome; use rocket::{Data, State}; +use rocket::{Request, Response}; use rocket_contrib::json::{Json, JsonValue}; use rocket_lamb::RocketExt; use serde::Serialize; @@ -30,10 +35,75 @@ use std::io::ErrorKind::AlreadyExists; use std::io::{Read, Write}; use std::path::PathBuf; use std::sync::Mutex; +use std::time::SystemTime; use stderrlog; use structopt; use structopt::StructOpt; +// static CustomMetrics: DdMetrics = DdMetrics::new(); + +/// Fairing for timing requests. +pub struct RequestTimer; + +/// Value stored in request-local state. +#[derive(Copy, Clone)] +struct TimerStart(Option); + +impl Fairing for RequestTimer { + fn info(&self) -> Info { + Info { + name: "Request Timer", + kind: Kind::Request | Kind::Response, + } + } + + /// Stores the start time of the request in request-local state. + fn on_request(&self, request: &mut Request, _: &Data) { + // Store a `TimerStart` instead of directly storing a `SystemTime` + // to ensure that this usage doesn't conflict with anything else + // that might store a `SystemTime` in request-local cache. + if request.uri().path() != "/ok" { + request.local_cache(|| TimerStart(Some(SystemTime::now()))); + } + } + + /// Adds a header to the response indicating how long the server took to + /// process the request. + fn on_response(&self, request: &Request, response: &mut Response) { + info!("request uri: {}", request.uri().path()); + if request.uri().path() != "/ok" { + let start_time = request.local_cache(|| TimerStart(None)); + if let Some(Ok(duration)) = start_time.0.map(|st| st.elapsed()) { + let ms = duration.as_secs() * 1000 + duration.subsec_millis() as u64; + info!("Request duration: {} ms", ms); + let metrics = DdMetrics::new(); + metrics.gauge( + CustomMetrics::RequestTime.metrics_name(), + request.uri().path(), + &ms.to_string(), + ); + response.set_raw_header("X-Response-Time", format!("{} ms", ms)); + } + } + } +} + +/// Request guard used to retrieve the start time of a request. +#[derive(Copy, Clone)] +pub struct StartTime(pub SystemTime); + +// Allows a route to access the time a request was initiated. +impl<'a, 'r> FromRequest<'a, 'r> for StartTime { + type Error = (); + + fn from_request(request: &'a Request<'r>) -> request::Outcome { + match *request.local_cache(|| TimerStart(None)) { + TimerStart(Some(time)) => Outcome::Success(StartTime(time)), + TimerStart(None) => Outcome::Failure((Status::InternalServerError, ())), + } + } +} + /// Transform templates with handlebars #[derive(StructOpt, Debug)] #[structopt(raw(setting = "structopt::clap::AppSettings::InferSubcommands"))] @@ -130,6 +200,10 @@ enum AppCommand { value_name = "REGEX" )] environments_regex: Regex, + + /// If datadog monitoring is enabled + #[structopt(short = "d", long = "datadog")] + datadog: bool, }, } @@ -171,7 +245,6 @@ impl App { PathBuf::from(shellexpand::tilde(src).into_owned()) } } - fn main() -> Result<(), Error> { let opt = App::from_args(); @@ -238,6 +311,7 @@ fn main() -> Result<(), Error> { cache_size, lambda, environments_regex, + datadog, } => { let config_dir = ConfigDir::new(common.configs_url, &common.ssh_key)?; @@ -249,13 +323,20 @@ fn main() -> Result<(), Error> { let config_dir = Mutex::new(config_dir); info!("Starting server on {}:{}", address, port); + info!("datadog monitoring is setting: {}", datadog); + let dd_metrics = if datadog { + Some(DdMetrics::new()) + } else { + None + }; let state = ServerState { environments, config_dir, environments_regex, strict: common.strict, + dd_metrics, }; - start_server(address, port, lambda, state)?; + start_server(address, port, lambda, state, datadog)?; } } @@ -267,25 +348,35 @@ struct ServerState { config_dir: Mutex, environments_regex: Regex, strict: bool, + dd_metrics: Option, } -fn start_server(address: String, port: u16, lambda: bool, state: ServerState) -> Result<(), Error> { +fn start_server( + address: String, + port: u16, + lambda: bool, + state: ServerState, + dd_enabled: bool, +) -> Result<(), Error> { let mut config = Config::development(); config.set_port(port); config.set_address(address)?; - let server = rocket::custom(config) - .mount( - "/", - routes![ - health_check, - get_envs, - get_config_by_env, - transform_env, - transform_all_envs, - get_branch_sha, - ], - ) - .manage(state); + let routes = routes![ + health_check, + get_envs, + get_config_by_env, + transform_env, + transform_all_envs, + get_branch_sha, + ]; + let server = if dd_enabled { + rocket::custom(config) + .mount("/", routes) + .attach(RequestTimer) + } else { + rocket::custom(config).mount("/", routes) + } + .manage(state); if lambda { server.lambda().launch(); } else { @@ -296,7 +387,7 @@ fn start_server(address: String, port: u16, lambda: bool, state: ServerState) -> #[get("/ok")] fn health_check() -> Status { - Status::NoContent + Status::Ok } #[post("/transform//", data = "")] @@ -307,12 +398,15 @@ fn transform_env( state: State, ) -> Result { let sha = format_sha(&sha); + let uri = format!("/transform/{}/{}", &sha, &env); match get_env( &state.environments, &state.config_dir, None, sha, &state.environments_regex, + &uri, + state.dd_metrics.as_ref(), ) { Some(environments) => match environments.iter().find(|e| e.environment == env) { Some(env) => { @@ -340,12 +434,15 @@ fn transform_all_envs( state: State, ) -> Result, Status> { let sha = format_sha(&sha); + let uri = format!("/transform/{}?{}", &sha, &filename); match get_env( &state.environments, &state.config_dir, None, &sha, &state.environments_regex, + &uri, + state.dd_metrics.as_ref(), ) { Some(environments) => { let handlebars = hogan::transform::handlebars(state.strict); @@ -384,10 +481,21 @@ fn get_envs(sha: String, state: State) -> Result return Err(Status::NotFound); } }; + let uri = format!("/envs/{}", &sha); + info!("uri: {}", uri); if let Some(envs) = cache.get(&sha) { + if let Some(custom_metrics) = &state.dd_metrics { + custom_metrics.incr(CustomMetrics::CacheHit.metrics_name(), &uri); + } + info!("Cache hit"); let env_list = format_envs(envs); Ok(json!(env_list)) } else { + info!("Cache miss"); + // state.dd_metrics.incr(CustomMetrics::CacheMiss.metrics_name(), &uri); + if let Some(custom_metrics) = &state.dd_metrics { + custom_metrics.incr(CustomMetrics::CacheMiss.metrics_name(), &uri); + } match state.config_dir.lock() { Ok(repo) => { if let Some(sha) = repo.refresh(None, Some(&sha)) { @@ -415,12 +523,15 @@ fn get_config_by_env( state: State, ) -> Result { let sha = format_sha(&sha); + let uri = format!("/config/{}/{}", &sha, &env); match get_env( &state.environments, &state.config_dir, None, sha, &state.environments_regex, + &uri, + state.dd_metrics.as_ref(), ) { Some(environments) => match environments.iter().find(|e| e.environment == env) { Some(env) => Ok(json!(env)), @@ -473,6 +584,7 @@ fn init_cache( ConfigDir::Git { head_sha, .. } => { let mut cache = cache.lock().unwrap(); info!("Initializing cache to: {}", head_sha); + cache.insert(head_sha.clone(), repo.find(environments_regex.clone())); Ok(()) } @@ -486,6 +598,8 @@ fn get_env( remote: Option<&str>, sha: &str, environments_regex: &Regex, + request_url: &str, + dd_metrics: Option<&DdMetrics>, ) -> Option> { let mut cache = match cache.lock() { Ok(cache) => cache, @@ -495,8 +609,16 @@ fn get_env( } }; if let Some(envs) = cache.get(sha) { + info!("Cache Hit"); + if let Some(custom_metrics) = dd_metrics { + custom_metrics.incr(CustomMetrics::CacheHit.metrics_name(), request_url); + } Some(envs.clone()) } else { + info!("Cache Miss"); + if let Some(custom_metrics) = dd_metrics { + custom_metrics.incr(CustomMetrics::CacheMiss.metrics_name(), request_url); + } match repo.lock() { Ok(repo) => { if let Some(sha) = repo.refresh(remote, Some(sha)) {