From c8f214cfdc23452ee383bd32c9de49db1f2100bf Mon Sep 17 00:00:00 2001 From: Michal Moskal Date: Sat, 10 Feb 2024 01:06:05 +0000 Subject: [PATCH] start on gh controllers; fixes #55 --- Cargo.lock | 89 ++++++++++++++++++++++++++++++------------- aicirt/Cargo.toml | 2 + aicirt/src/main.rs | 95 +++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 158 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bbb184f7..c70cab30 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -244,7 +244,7 @@ dependencies = [ "lrlex", "lrpar", "lrtable", - "regex-automata 0.4.3", + "regex-automata 0.4.5", "rustc-hash", "serde", "serde_json", @@ -331,12 +331,14 @@ dependencies = [ "linux-futex", "log", "rayon", + "regex", "serde", "serde_json", "sha2", "thread-priority", "tokenizers", "ulock-sys", + "ureq", "uuid", "wasmtime", ] @@ -1625,6 +1627,29 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "hoot" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df22a4d90f1b0e65fe3e0d6ee6a4608cc4d81f4b2eb3e670f44bb6bde711e452" +dependencies = [ + "httparse", + "log", +] + +[[package]] +name = "hootbin" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "354e60868e49ea1a39c44b9562ad207c4259dc6eabf9863bf3b0f058c55cfdb2" +dependencies = [ + "fastrand", + "hoot", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "http" version = "0.2.11" @@ -2842,13 +2867,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.2" +version = "1.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.3", + "regex-automata 0.4.5", "regex-syntax 0.8.2", ] @@ -2871,9 +2896,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" dependencies = [ "aho-corasick", "memchr", @@ -3076,23 +3101,32 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.10" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" +checksum = "e87c9956bd9807afa1f77e0f7594af32566e830e088a5576d27c5b6f30f49d41" dependencies = [ "log", "ring", + "rustls-pki-types", "rustls-webpki", - "sct", + "subtle", + "zeroize", ] +[[package]] +name = "rustls-pki-types" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a716eb65e3158e90e17cd93d855216e27bde02745ab842f2cab4a39dba1bacf" + [[package]] name = "rustls-webpki" -version = "0.101.7" +version = "0.102.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" dependencies = [ "ring", + "rustls-pki-types", "untrusted", ] @@ -3419,16 +3453,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "security-framework" version = "2.9.2" @@ -3480,9 +3504,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.110" +version = "1.0.112" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fbd975230bada99c8bb618e0c365c2eefa219158d5c6c29610fd09ff1833257" +checksum = "4d1bd37ce2324cf3bf85e5a25f96eb4baf0d5aa6eba43e7ae8958870c4ec48ed" dependencies = [ "indexmap 2.1.0", "itoa", @@ -4169,16 +4193,18 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "2.9.1" +version = "2.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8cdd25c339e200129fe4de81451814e5228c9b771d57378817d6117cc2b3f97" +checksum = "0b52731d03d6bb2fd18289d4028aee361d6c28d44977846793b994b13cdcc64d" dependencies = [ "base64 0.21.5", "flate2", + "hootbin", "log", "native-tls", "once_cell", "rustls", + "rustls-pki-types", "rustls-webpki", "serde", "serde_json", @@ -4522,9 +4548,12 @@ checksum = "67761d8f8c0b3c13a5d34356274b10a40baba67fe9cfabbfc379a8b414e45de2" [[package]] name = "webpki-roots" -version = "0.25.3" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" +checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009" +dependencies = [ + "rustls-pki-types", +] [[package]] name = "which" @@ -4830,6 +4859,12 @@ dependencies = [ "syn 2.0.46", ] +[[package]] +name = "zeroize" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" + [[package]] name = "zip" version = "0.6.6" diff --git a/aicirt/Cargo.toml b/aicirt/Cargo.toml index 2f487e98..7f149aab 100644 --- a/aicirt/Cargo.toml +++ b/aicirt/Cargo.toml @@ -26,6 +26,8 @@ cap = "0.1.2" fxhash = "0.2.1" bincode = "1.3.3" uuid = "1.6.1" +regex = "1.10.3" +ureq = "2.9.5" [target.'cfg(target_os = "linux")'.dependencies] linux-futex = "0.2.0" diff --git a/aicirt/src/main.rs b/aicirt/src/main.rs index edc3a5fe..157ec22b 100644 --- a/aicirt/src/main.rs +++ b/aicirt/src/main.rs @@ -20,13 +20,15 @@ use base64::{self, Engine as _}; use clap::Parser; use hex; use hostimpl::GlobalInfo; +use regex::Regex; use serde_json::{json, Value}; use sha2::{Digest, Sha256}; use std::{ fs, + ops::Sub, path::PathBuf, sync::{Arc, Mutex}, - time::{Duration, Instant}, + time::{Duration, Instant, SystemTime}, }; use worker::{RtPostPreProcessArg, SeqWorkerHandle}; @@ -154,6 +156,12 @@ struct Stepper { pre_recv_timer: TimerRef, } +fn hex_hash_string(s: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(s); + hex::encode(hasher.finalize()) +} + impl ModuleRegistry { pub fn new(wasm_ctx: WasmContext, shm: Shm) -> Result { let forker = WorkerForker::new(wasm_ctx.clone(), shm); @@ -193,6 +201,11 @@ impl ModuleRegistry { self.cache_path.join(format!("{}.wasm", module_id)) } + fn url_path(&self, url: &str) -> PathBuf { + let hex = hex_hash_string(url); + self.cache_path.join(format!("url-{}.json", hex)) + } + fn elf_path(&self, module_id: &str) -> PathBuf { self.cache_path.join(format!("{}.elf", module_id)) } @@ -384,6 +397,86 @@ impl ModuleRegistry { Ok(json!(resp)) } + fn resolve_gh_module(&self, module_id: &str) -> Result { + if !module_id.starts_with("gh:") { + return Ok(module_id.to_string()); + } + ensure!( + Regex::new(r"^gh:[\./a-zA-Z0-9_-]+$") + .unwrap() + .is_match(module_id), + "invalid gh: module_id" + ); + let mut parts = module_id[3..] + .split('/') + .map(|s| s.to_string()) + .collect::>(); + ensure!( + 2 <= parts.len() && parts.len() <= 4, + "invalid gh: module_id" + ); + let mut ver = "latest".to_string(); + let last_part = parts.last().unwrap(); + let mut selector = "".to_string(); + if parts.len() > 2 + && (last_part == "latest" + || regex::Regex::new(r"^v\d+\.\d+") + .unwrap() + .is_match(last_part)) + { + ver = format!("tags/{}", parts.pop().unwrap()); + } + if parts.len() > 2 { + selector = parts.pop().unwrap(); + } + ensure!(parts.len() == 2, "invalid gh: module_id"); + + let url = format!( + "https://api.github.com/repos/{}/{}/releases/{}", + parts[0], parts[1], ver + ); + let cache_path = self.url_path(&url); + let meta = cache_path.metadata(); + if !(meta.is_ok() + && meta.unwrap().modified()? > SystemTime::now().sub(Duration::from_secs(120))) + { + log::info!("fetching {}", url); + let resp = ureq::get(&url) + .set("User-Agent", "AICI") + .set("Accept", "application/vnd.github+json") + .set("X-GitHub-Api-Version", "2022-11-28") + .call() + .map_err(|e| anyhow!("gh: fetch failed: {}", e))?; + std::fs::write(cache_path.clone(), resp.into_string()?)?; + } + let json: serde_json::Value = serde_json::from_slice(&std::fs::read(cache_path)?)?; + + let wasm_files = json["assets"] + .as_array() + .ok_or_else(|| anyhow!("no assets"))? + .iter() + .filter(|a| { + a["name"] + .as_str() + .map(|s| s.ends_with(".wasm") && s.contains(&selector)) + .unwrap_or(false) + }) + .collect::>(); + + ensure!(wasm_files.len() > 0, "no wasm files found"); + ensure!(wasm_files.len() == 1, "too many wasm files found"); + + let wasm_file = wasm_files[0]; + let _upd = wasm_file["updated_at"] + .as_str() + .ok_or_else(|| anyhow!("no updated_at"))?; + let _wasm_url = wasm_file["browser_download_url"] + .as_str() + .ok_or_else(|| anyhow!("no browser_download_url"))?; + + todo!() + } + fn instantiate(&mut self, mut req: InstantiateReq) -> Result { if valid_tagname(&req.module_id) { let taginfo = self.read_tag(&req.module_id)?;