diff --git a/Cargo.lock b/Cargo.lock index b6ff84c..269b79a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -174,7 +174,7 @@ dependencies = [ "lazy_static 1.4.0", "log", "num_cpus", - "parking_lot", + "parking_lot 0.10.2", "threadpool", ] @@ -276,9 +276,9 @@ checksum = "567b077b825e468cc974f0020d4082ee6e03132512f207ef1a02fd5d00d1f32d" [[package]] name = "aho-corasick" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c259a748ac706ba73d609b73fc13469e128337f9a6b2fb3cc82d100f8dd8d511" +checksum = "043164d8ba5c4c3035fec9bbee8647c0261d788f3474306f93bb65901cae0e86" dependencies = [ "memchr", ] @@ -396,9 +396,9 @@ checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" [[package]] name = "bincode" -version = "1.2.1" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5753e2a71534719bf3f4e57006c3a4f0d2c672a4b676eec84161f763eca87dbf" +checksum = "f30d3a39baa26f9651f17b375061f3233dde33424a8b72b0dbe93a68a0bc896d" dependencies = [ "byteorder", "serde", @@ -563,6 +563,15 @@ dependencies = [ "bitflags", ] +[[package]] +name = "cloudabi" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467" +dependencies = [ + "bitflags", +] + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -1015,9 +1024,10 @@ dependencies = [ "handlebars", "itertools", "json-patch", + "lazy_static 1.4.0", "log", "lru_time_cache", - "parking_lot", + "parking_lot 0.11.0", "predicates", "regex", "rusqlite", @@ -1081,6 +1091,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "instant" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7777a24a1ce5de49fcdde84ec46efa487c3af49d5b6e6e0a50367cc5c1096182" + [[package]] name = "iovec" version = "0.1.4" @@ -1237,6 +1253,15 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "lock_api" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de302ce1fe7482db13738fbaf2e21cfb06a986b89c0bf38d88abf16681aada4e" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.8" @@ -1458,8 +1483,19 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" dependencies = [ - "lock_api", - "parking_lot_core", + "lock_api 0.3.4", + "parking_lot_core 0.7.2", +] + +[[package]] +name = "parking_lot" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4893845fa2ca272e647da5d0e46660a314ead9c2fdd9a883aabc32e481a8733" +dependencies = [ + "instant", + "lock_api 0.4.0", + "parking_lot_core 0.8.0", ] [[package]] @@ -1469,7 +1505,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3" dependencies = [ "cfg-if", - "cloudabi", + "cloudabi 0.0.3", + "libc", + "redox_syscall", + "smallvec", + "winapi 0.3.8", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b" +dependencies = [ + "cfg-if", + "cloudabi 0.1.0", + "instant", "libc", "redox_syscall", "smallvec", @@ -2270,9 +2321,9 @@ checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" [[package]] name = "unicode-xid" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" [[package]] name = "unreachable" diff --git a/Cargo.toml b/Cargo.toml index d091efb..1c4ff6e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,8 +33,9 @@ actix-web = '2' actix-rt = '1' actix-service = '1' futures = '0.3' -parking_lot = '0.10' -bincode = '1.2.1' +parking_lot = '0.11' +bincode = '1.3' +lazy_static = '1' [dependencies.rusqlite] version = '0.23' diff --git a/src/app/db.rs b/src/app/db.rs index 0328167..d92a713 100644 --- a/src/app/db.rs +++ b/src/app/db.rs @@ -60,6 +60,8 @@ pub fn write_sql_env( let env_data: WritableEnvironment = data.into(); let data = bincode::serialize(&env_data)?; + info!("Writing to DB. Key: {} Size: {}", key, data.len()); + conn.execute( "INSERT INTO hogan (key, data) VALUES (?1, ?2)", params![key, data], diff --git a/src/app/server.rs b/src/app/server.rs index 65d0aec..9ad1a15 100644 --- a/src/app/server.rs +++ b/src/app/server.rs @@ -153,6 +153,7 @@ async fn start_server( .service(transform_branch_head) .service(get_envs) .service(get_config_by_env) + .service(get_config_by_env_branch) .service(get_branch_sha) .route("/ok", web::to(|| HttpResponse::Ok().finish())) }) @@ -169,19 +170,32 @@ struct TransformEnvParams { env: String, } +lazy_static! { + static ref HEX_REGEX: Regex = Regex::new(r"^[a-f0-9]+$").unwrap(); +} + #[post("transform/{sha}/{env}")] fn transform_route_sha_env( data: String, params: web::Path, state: web::Data, ) -> HttpResponse { - match transform_from_sha(data, ¶ms.sha, ¶ms.env, &state) { + //We keep running into folks that are passing in branch name here and it throws off the caching layer and gives inconsistent results + //This won't catch branch names with all hex values, but would catch the common case like 'master' + let sha = if !HEX_REGEX.is_match(¶ms.sha) { + if let Some(head_sha) = find_branch_head(¶ms.sha, &state) { + head_sha + } else { + return HttpResponse::NotFound().finish(); + } + } else { + params.sha.to_owned() + }; + + match transform_from_sha(data, &sha, ¶ms.env, &state) { Ok(result) => HttpResponse::Ok().body(result), Err(e) => { - warn!( - "Error templating request {} {} {}", - e, params.sha, params.env - ); + warn!("Error templating request {} {} {}", e, sha, params.env); HttpResponse::BadRequest().finish() } } @@ -236,6 +250,28 @@ fn get_config_by_env( } } +#[derive(Deserialize)] +struct ConfigByEnvBranchState { + branch_name: String, + env: String, +} + +#[get("branch/{branch_name:.*}/configs/{env}")] +fn get_config_by_env_branch( + params: web::Path, + state: web::Data, +) -> HttpResponse { + if let Some(head_sha) = find_branch_head(¶ms.branch_name, &state) { + let sha = format_sha(&head_sha); + match get_env(&state, None, sha, ¶ms.env) { + Some(env) => HttpResponse::Ok().json(env), + None => HttpResponse::NotFound().finish(), + } + } else { + HttpResponse::NotFound().finish() + } +} + #[derive(Serialize)] #[serde(rename_all = "camelCase")] struct ShaResponse { @@ -309,19 +345,26 @@ fn format_key(sha: &str, env: &str) -> String { format!("{}::{}", sha, env) } +fn register_cache_hit(state: &ServerState, key: &str) { + info!("Cache Hit {}", key); + if let Some(custom_metrics) = &state.dd_metrics { + custom_metrics.incr(CustomMetrics::CacheHit.metrics_name(), None); + } +} + +fn register_cache_miss(state: &ServerState, key: &str) { + info!("Cache Miss {}", key); + if let Some(custom_metrics) = &state.dd_metrics { + custom_metrics.incr(CustomMetrics::CacheMiss.metrics_name(), None); + } +} + fn get_env_from_cache(state: &ServerState, key: &str) -> Option> { let mut cache = state.environments.lock(); if let Some(env) = cache.get(key) { - info!("Cache Hit {}", key); - if let Some(custom_metrics) = &state.dd_metrics { - custom_metrics.incr(CustomMetrics::CacheHit.metrics_name(), None); - } + register_cache_hit(state, key); Some(env.clone()) } else { - info!("Cache Miss {}", key); - if let Some(custom_metrics) = &state.dd_metrics { - custom_metrics.incr(CustomMetrics::CacheMiss.metrics_name(), None); - } None } } @@ -354,6 +397,15 @@ fn get_env( Some(insert_into_env_cache(state, &key, environment)) } else { let _write_lock = state.write_lock.lock(); + + //Double check if the cache now contains the env we are looking for + if let Some(environment) = db::read_sql_env(&state.db_path, env, sha).unwrap_or(None) { + register_cache_hit(state, &key); + info!("Avoided git lock for config lookup: {}", key); + return Some(Arc::new(environment)); + } + + register_cache_miss(state, &key); if let Some(sha) = state.config_dir.refresh(remote, Some(sha)) { let filter = match hogan::config::build_env_regex(env, Some(&state.environment_pattern)) { @@ -371,7 +423,7 @@ fn get_env( .find(|e| e.environment == env) { if let Err(e) = db::write_sql_env(&state.db_path, env, &sha, environment) { - warn!("Unable to write env {} to db {:?}", key, e); + warn!("Unable to write env {} {}::{} to db {:?}", key, sha, env, e); }; Some(insert_into_env_cache(state, &key, environment.clone())) } else { @@ -390,16 +442,9 @@ fn check_env_listing_cache(state: &ServerState, sha: &str) -> Option