From a9e841112f46eceef5aff2dbd568bce601d9ddf5 Mon Sep 17 00:00:00 2001 From: Leon Tan Date: Thu, 8 Aug 2024 14:30:42 +0200 Subject: [PATCH] feat(dfx-orbit): Version 0.2 (#308) This PR merges the new version of `dfx-orbit` into the orbit repo. Features: - `dfx-orbit asset upload`: automatically requests to commit the batch from orbit - `dfx-orbit asset check`: call to check the request against local sources. This is the call that the reviewers need - `dfx-orbit asset` now uses `sources` from `dfx.json` if the `canister` is an asset canister and no sources where provided manually Improvements and fixes: - Fixed issue with evidence computation - Adapted test framework to cover new functionality --------- Co-authored-by: Kepler Vital --- Cargo.lock | 85 +++++-- Cargo.toml | 11 +- tests/integration/Cargo.toml | 1 + tests/integration/src/dfx_orbit.rs | 51 +++- tests/integration/src/dfx_orbit/assets.rs | 123 ++++----- .../src/dfx_orbit/canister_call.rs | 8 +- tests/integration/src/dfx_orbit/me.rs | 6 +- tests/integration/src/dfx_orbit/review.rs | 7 +- tools/dfx-orbit/Cargo.toml | 5 +- tools/dfx-orbit/README.md | 91 ++++--- tools/dfx-orbit/src/args/asset.rs | 61 ++++- tools/dfx-orbit/src/cli.rs | 38 ++- tools/dfx-orbit/src/cli/asset.rs | 238 +++++++----------- tools/dfx-orbit/src/cli/asset/evidence.rs | 78 ++++++ tools/dfx-orbit/src/cli/asset/upload.rs | 86 +++++++ tools/dfx-orbit/src/cli/asset/util.rs | 95 +++++-- tools/dfx-orbit/src/dfx_extension_api.rs | 21 +- tools/dfx-orbit/src/lib.rs | 38 ++- tools/dfx-orbit/src/station_agent.rs | 19 +- 19 files changed, 697 insertions(+), 365 deletions(-) create mode 100644 tools/dfx-orbit/src/cli/asset/evidence.rs create mode 100644 tools/dfx-orbit/src/cli/asset/upload.rs diff --git a/Cargo.lock b/Cargo.lock index bbf69a21c..cb460d72e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -85,6 +85,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "allocator-api2" version = "0.2.18" @@ -443,6 +458,27 @@ dependencies = [ "subtle", ] +[[package]] +name = "brotli" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bs58" version = "0.4.0" @@ -1041,10 +1077,11 @@ checksum = "339544cc9e2c4dc3fc7149fd630c5f22263a4fdf18a98afd0075784968b5cf00" [[package]] name = "dfx-core" version = "0.0.1" -source = "git+https://github.com/dfinity/sdk.git?tag=0.20.2-beta.0#06eaa58f9c52d27a015cd8f754ecd3d0d0ba555b" +source = "git+https://github.com/dfinity/sdk.git?tag=0.22.0#d0c8be188d1a19c1908f148e3bc5331b62de780a" dependencies = [ "aes-gcm", "argon2", + "backoff", "bip32", "byte-unit", "bytes", @@ -1070,6 +1107,7 @@ dependencies = [ "semver", "serde", "serde_json", + "sha2 0.10.8", "slog", "tar", "tempfile", @@ -1082,10 +1120,11 @@ dependencies = [ [[package]] name = "dfx-core" version = "0.0.1" -source = "git+https://github.com/dfinity/sdk.git?rev=ae10a96b381cfce3d8ac5a6cb940d19224ea6d2e#ae10a96b381cfce3d8ac5a6cb940d19224ea6d2e" +source = "git+https://github.com/dfinity/sdk.git?rev=75c080ebae22a70578c06ddf1eda0b18ef091845#75c080ebae22a70578c06ddf1eda0b18ef091845" dependencies = [ "aes-gcm", "argon2", + "backoff", "bip32", "byte-unit", "bytes", @@ -1111,6 +1150,7 @@ dependencies = [ "semver", "serde", "serde_json", + "sha2 0.10.8", "slog", "tar", "tempfile", @@ -1122,14 +1162,15 @@ dependencies = [ [[package]] name = "dfx-orbit" -version = "0.1.0" +version = "0.2.0" dependencies = [ "anyhow", "candid", "candid_parser", "cap-std", "clap", - "dfx-core 0.0.1 (git+https://github.com/dfinity/sdk.git?tag=0.20.2-beta.0)", + "dfx-core 0.0.1 (git+https://github.com/dfinity/sdk.git?tag=0.22.0)", + "hex", "ic-agent", "ic-asset", "ic-certified-assets", @@ -1137,13 +1178,13 @@ dependencies = [ "serde", "serde_bytes", "serde_json", + "sha2 0.10.8", "slog", "slog-async", "slog-term", "station-api", "thiserror", "tokio", - "walkdir", ] [[package]] @@ -1934,8 +1975,8 @@ dependencies = [ [[package]] name = "ic-agent" -version = "0.35.0" -source = "git+https://github.com/dfinity/agent-rs.git?rev=8273d321e9a09fd8373bd4e38b0676ec6ad9c260#8273d321e9a09fd8373bd4e38b0676ec6ad9c260" +version = "0.36.0" +source = "git+https://github.com/dfinity/agent-rs.git?rev=be929fd7967249c879f48f2f494cbfc5805a7d98#be929fd7967249c879f48f2f494cbfc5805a7d98" dependencies = [ "async-lock 3.4.0", "backoff", @@ -1975,12 +2016,13 @@ dependencies = [ [[package]] name = "ic-asset" version = "0.20.0" -source = "git+https://github.com/dfinity/sdk.git?rev=ae10a96b381cfce3d8ac5a6cb940d19224ea6d2e#ae10a96b381cfce3d8ac5a6cb940d19224ea6d2e" +source = "git+https://github.com/dfinity/sdk.git?rev=75c080ebae22a70578c06ddf1eda0b18ef091845#75c080ebae22a70578c06ddf1eda0b18ef091845" dependencies = [ "backoff", + "brotli", "candid", "derivative", - "dfx-core 0.0.1 (git+https://github.com/dfinity/sdk.git?rev=ae10a96b381cfce3d8ac5a6cb940d19224ea6d2e)", + "dfx-core 0.0.1 (git+https://github.com/dfinity/sdk.git?rev=75c080ebae22a70578c06ddf1eda0b18ef091845)", "flate2", "futures", "futures-intrusive", @@ -2131,7 +2173,7 @@ dependencies = [ [[package]] name = "ic-certified-assets" version = "0.2.5" -source = "git+https://github.com/dfinity/sdk.git?rev=ae10a96b381cfce3d8ac5a6cb940d19224ea6d2e#ae10a96b381cfce3d8ac5a6cb940d19224ea6d2e" +source = "git+https://github.com/dfinity/sdk.git?rev=75c080ebae22a70578c06ddf1eda0b18ef091845#75c080ebae22a70578c06ddf1eda0b18ef091845" dependencies = [ "base64 0.13.1", "candid", @@ -2142,6 +2184,7 @@ dependencies = [ "ic-response-verification", "itertools 0.10.5", "num-traits", + "percent-encoding", "serde", "serde_bytes", "serde_cbor", @@ -2165,8 +2208,8 @@ dependencies = [ [[package]] name = "ic-identity-hsm" -version = "0.35.0" -source = "git+https://github.com/dfinity/agent-rs.git?rev=8273d321e9a09fd8373bd4e38b0676ec6ad9c260#8273d321e9a09fd8373bd4e38b0676ec6ad9c260" +version = "0.36.0" +source = "git+https://github.com/dfinity/agent-rs.git?rev=be929fd7967249c879f48f2f494cbfc5805a7d98#be929fd7967249c879f48f2f494cbfc5805a7d98" dependencies = [ "hex", "ic-agent", @@ -2236,8 +2279,8 @@ dependencies = [ [[package]] name = "ic-transport-types" -version = "0.35.0" -source = "git+https://github.com/dfinity/agent-rs.git?rev=8273d321e9a09fd8373bd4e38b0676ec6ad9c260#8273d321e9a09fd8373bd4e38b0676ec6ad9c260" +version = "0.36.0" +source = "git+https://github.com/dfinity/agent-rs.git?rev=be929fd7967249c879f48f2f494cbfc5805a7d98#be929fd7967249c879f48f2f494cbfc5805a7d98" dependencies = [ "candid", "hex", @@ -2252,8 +2295,8 @@ dependencies = [ [[package]] name = "ic-utils" -version = "0.35.0" -source = "git+https://github.com/dfinity/agent-rs.git?rev=8273d321e9a09fd8373bd4e38b0676ec6ad9c260#8273d321e9a09fd8373bd4e38b0676ec6ad9c260" +version = "0.36.0" +source = "git+https://github.com/dfinity/agent-rs.git?rev=be929fd7967249c879f48f2f494cbfc5805a7d98#be929fd7967249c879f48f2f494cbfc5805a7d98" dependencies = [ "async-trait", "candid", @@ -2358,6 +2401,7 @@ dependencies = [ "hex", "ic-cdk 0.13.2", "ic-ledger-types", + "itertools 0.13.0", "lazy_static", "num-bigint 0.4.5", "orbit-essentials", @@ -2443,6 +2487,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" diff --git a/Cargo.toml b/Cargo.toml index 3050eb0b6..c397e8818 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,21 +37,22 @@ candid = "0.10.3" candid_parser = "0.1.3" cap-std = "3.1.0" clap = { version = "4.5.7", features = ["derive"] } -dfx-core = { git = "https://github.com/dfinity/sdk.git", tag = "0.20.2-beta.0" } +dfx-core = { git = "https://github.com/dfinity/sdk.git", tag = "0.22.0" } convert_case = "0.6" futures = "0.3" getrandom = { version = "0.2", features = ["custom"] } hex = "0.4" # The ic-agent matches the one sed by bthe -ic-agent = { git = "https://github.com/dfinity/agent-rs.git", rev = "8273d321e9a09fd8373bd4e38b0676ec6ad9c260" } -ic-asset = { git = "https://github.com/dfinity/sdk.git", rev = "ae10a96b381cfce3d8ac5a6cb940d19224ea6d2e" } -ic-certified-assets = { git = "https://github.com/dfinity/sdk.git", rev = "ae10a96b381cfce3d8ac5a6cb940d19224ea6d2e" } +ic-agent = { git = "https://github.com/dfinity/agent-rs.git", rev = "be929fd7967249c879f48f2f494cbfc5805a7d98" } +ic-asset = { git = "https://github.com/dfinity/sdk.git", rev = "75c080ebae22a70578c06ddf1eda0b18ef091845" } +ic-certified-assets = { git = "https://github.com/dfinity/sdk.git", rev = "75c080ebae22a70578c06ddf1eda0b18ef091845" } ic-cdk = "0.13.2" ic-cdk-macros = "0.9" ic-cdk-timers = "0.7.0" ic-ledger-types = "0.10.0" ic-stable-structures = "0.6.4" -ic-utils = { git = "https://github.com/dfinity/agent-rs.git", rev = "8273d321e9a09fd8373bd4e38b0676ec6ad9c260" } +ic-utils = { git = "https://github.com/dfinity/agent-rs.git", rev = "be929fd7967249c879f48f2f494cbfc5805a7d98" } +itertools = "0.13.0" lazy_static = "1.4.0" mockall = "0.12.1" num-bigint = "0.4" diff --git a/tests/integration/Cargo.toml b/tests/integration/Cargo.toml index 743928c68..d5248d50f 100644 --- a/tests/integration/Cargo.toml +++ b/tests/integration/Cargo.toml @@ -12,6 +12,7 @@ hex = { workspace = true } orbit-essentials = { path = '../../libs/orbit-essentials', version = '0.0.2-alpha.3' } ic-cdk = { workspace = true } ic-ledger-types = { workspace = true } +itertools = { workspace = true } lazy_static = { workspace = true } num-bigint = { workspace = true } pocket-ic = { workspace = true } diff --git a/tests/integration/src/dfx_orbit.rs b/tests/integration/src/dfx_orbit.rs index 3e749aeb6..4df0d9948 100644 --- a/tests/integration/src/dfx_orbit.rs +++ b/tests/integration/src/dfx_orbit.rs @@ -5,12 +5,14 @@ use crate::{ }; use candid::Principal; use dfx_orbit::{dfx_extension_api::OrbitExtensionAgent, station_agent::StationConfig, DfxOrbit}; +use itertools::Itertools; use pocket_ic::PocketIc; use rand::Rng; use rand_chacha::{rand_core::SeedableRng, ChaCha8Rng}; use station_api::UserDTO; use std::{ cell::RefCell, + collections::BTreeMap, future::Future, hash::{DefaultHasher, Hash, Hasher}, path::Path, @@ -43,7 +45,17 @@ const IDENTITY_JSON: &str = " \"default\": \"default\" }"; -fn dfx_orbit_test(env: &mut PocketIc, test_func: F) -> F::Output +/// The test setup needs to be configurable +/// +/// This struct allows to gradually introduce configurations into the `dfx_orbit` tests +/// to allow testing more fine grained controls +#[derive(Debug, Clone, Default)] +struct DfxOrbitTestConfig { + /// Sets the asset canisters to be defined in the dfx.json, maps name tp list of paths + asset_canisters: BTreeMap>, +} + +fn dfx_orbit_test(env: &mut PocketIc, config: DfxOrbitTestConfig, test_func: F) -> F::Output where F: Future, { @@ -90,7 +102,7 @@ where *port.borrow() }); - setup_test_dfx_json(tmp_dir.path()); + setup_test_dfx_json(tmp_dir.path(), config); setup_identity(tmp_dir.path()); // Start the live environment @@ -123,15 +135,37 @@ fn setup_identity(dfx_root: &Path) { std::fs::write(default_id_path.join("identity.pem"), TEST_KEY).unwrap(); } -fn setup_test_dfx_json(dfx_root: &Path) { +/// Sets up a custom `dfx.json` from the provided `config` +fn setup_test_dfx_json(dfx_root: &Path, config: DfxOrbitTestConfig) { let port = PORT.with(|port| *port.borrow()); - let dfx_json = test_dfx_json_from_template(port); + let dfx_json = test_dfx_json_from_template(config, port); std::fs::write(dfx_root.join("dfx.json"), dfx_json).unwrap(); } -fn test_dfx_json_from_template(port: u16) -> String { +/// Generate a custom `dfx.json` from the provided `config` +fn test_dfx_json_from_template(config: DfxOrbitTestConfig, port: u16) -> String { + let asset_canisters = config + .asset_canisters + .iter() + .map(|(name, sources)| { + ( + name, + sources + .iter() + .map(|source| format!("\"{source}\"")) + .join(","), + ) + }) + .map(|(name, sources)| { + format!("\"{name}\": {{ \"source\": [{sources}], \"type\": \"assets\"}}") + }) + .join(","); + format!( "{{ + \"canisters\": {{ + {asset_canisters} + }}, \"networks\": {{ \"test\": {{ \"providers\": [ @@ -175,6 +209,7 @@ fn setup_dfx_user(env: &PocketIc, canister_ids: &CanisterIds) -> (Principal, Use (dfx_principal, dfx_user) } +/// Install the counter canister under given `canister_id` into the running IC fn setup_counter_canister(env: &mut PocketIc, canister_ids: &CanisterIds) -> Principal { // create and install the counter canister let canister_id = create_canister(env, canister_ids.station); @@ -192,6 +227,10 @@ fn setup_counter_canister(env: &mut PocketIc, canister_ids: &CanisterIds) -> Pri canister_id } +/// Fetches an asset from the local host and port +/// +/// This is a bit tricky, as the boundary node uses the `Referer` header to determine the +/// resource being fetched. async fn fetch_asset(canister_id: Principal, path: &str) -> Vec { let port = PORT.with(|port| *port.borrow()); let local_url = format!("http://localhost:{}{}", port, path); @@ -208,5 +247,3 @@ async fn fetch_asset(canister_id: Principal, path: &str) -> Vec { .unwrap() .into() } - -// TODO: Test canister update diff --git a/tests/integration/src/dfx_orbit/assets.rs b/tests/integration/src/dfx_orbit/assets.rs index 93647abb1..32fd2ff28 100644 --- a/tests/integration/src/dfx_orbit/assets.rs +++ b/tests/integration/src/dfx_orbit/assets.rs @@ -1,15 +1,4 @@ -use dfx_orbit::StationAgent; -use pocket_ic::PocketIc; -use rand::{thread_rng, Rng}; -use station_api::{ - AddRequestPolicyOperationInput, CallExternalCanisterOperationInput, - CallExternalCanisterResourceTargetDTO, CanisterMethodDTO, CreateRequestInput, - ExecutionMethodResourceTargetDTO, RequestOperationInput, RequestPolicyRuleDTO, - RequestSpecifierDTO, ValidationMethodResourceTargetDTO, -}; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use tempfile::tempdir; - +use super::DfxOrbitTestConfig; use crate::{ dfx_orbit::{ canister_call::permit_call_operation, dfx_orbit_test, fetch_asset, setup_dfx_orbit, @@ -19,16 +8,29 @@ use crate::{ utils::execute_request, CanisterIds, TestEnv, }; +use pocket_ic::PocketIc; +use rand::{thread_rng, Rng}; +use station_api::{ + AddRequestPolicyOperationInput, CallExternalCanisterResourceTargetDTO, + ExecutionMethodResourceTargetDTO, RequestOperationInput, RequestPolicyRuleDTO, + RequestSpecifierDTO, ValidationMethodResourceTargetDTO, +}; +use std::{ + collections::BTreeMap, + path::Path, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; +use tempfile::Builder; #[test] -fn assets_update() { +fn assets_upload() { let TestEnv { mut env, canister_ids, .. } = setup_new_env(); - let (dfx_principal, _dfx_user) = setup_dfx_user(&env, &canister_ids); + let (_dfx_principal, _dfx_user) = setup_dfx_user(&env, &canister_ids); // Install the assets canister under orbit control let asset_canister = create_canister(&mut env, canister_ids.station); @@ -40,25 +42,6 @@ fn assets_update() { Some(canister_ids.station), ); - // As admin: Setup the prepare permission in the asset canister for the dfx user - execute_request( - &env, - WALLET_ADMIN_USER, - canister_ids.station, - station_api::RequestOperationInput::CallExternalCanister( - CallExternalCanisterOperationInput { - validation_method: None, - execution_method: CanisterMethodDTO { - canister_id: asset_canister, - method_name: String::from("grant_permission"), - }, - arg: Some(StationAgent::request_prepare_permission_payload(dfx_principal).unwrap()), - execution_method_cycles: None, - }, - ), - ) - .unwrap(); - // As admin: Grant the user the call permission, set auto-approval for external calls permit_call_operation(&env, &canister_ids); set_auto_approve(&env, &canister_ids); @@ -66,7 +49,10 @@ fn assets_update() { // Setup a tmpdir, and store two assets in it // We generate the assets dyniamically, since we want to make sure we are not // fetching old assets - let asset_dir = tempdir().unwrap(); + // NOTE: Currently, the local asset computation skips hidden files while the + // remote version does not. This creates an issue if we just used tempdir(), as that + // uses `.` prefix. + let asset_dir = Builder::new().prefix("asset").tempdir().unwrap(); let asset_a = format!( "This is the current time: {}", SystemTime::now() @@ -84,40 +70,55 @@ fn assets_update() { ) .unwrap(); - dfx_orbit_test(&mut env, async { + let mut asset_canisters = BTreeMap::new(); + asset_canisters.insert( + String::from("test_asset_upload"), + vec![asset_dir.path().to_str().unwrap().to_string()], + ); + let config = DfxOrbitTestConfig { asset_canisters }; + + dfx_orbit_test(&mut env, config, async { // Setup the station agent - let mut dfx_orbit = setup_dfx_orbit(canister_ids.station).await; + let dfx_orbit = setup_dfx_orbit(canister_ids.station).await; + + // As dfx user: Request to have Prepare permission for asset_canister + let _response = dfx_orbit + .request_prepare_permission(asset_canister, None, None) + .await + .unwrap(); + tokio::time::sleep(Duration::from_secs(1)).await; + + // Test that we can retreive the sources from `dfx.json` + let sources = dfx_orbit.as_path_bufs("test_asset_upload", &[]).unwrap(); + let sources_path = sources + .iter() + .map(|pathbuf| pathbuf.as_path()) + .collect::>(); // As dfx user: Request to upload new files to the asset canister - let upload_request = dfx_orbit - .upload_assets( - asset_canister.to_string(), - vec![asset_dir.path().to_str().unwrap().to_string()], + let (batch_id, evidence) = dfx_orbit + .upload(asset_canister, &sources_path, false) + .await + .unwrap(); + let response = dfx_orbit + .request_commit_batch( + asset_canister, + batch_id.clone(), + evidence.clone(), + None, + None, ) .await .unwrap(); - // As dfx user: Request commitment of the batch - let _result = dfx_orbit - .station - .request(CreateRequestInput { - operation: station_api::RequestOperationInput::CallExternalCanister( - CallExternalCanisterOperationInput { - validation_method: None, - execution_method: CanisterMethodDTO { - canister_id: asset_canister, - method_name: String::from("commit_proposed_batch"), - }, - arg: Some( - StationAgent::commit_proposed_batch_payload(upload_request).unwrap(), - ), - execution_method_cycles: None, - }, - ), - title: None, - summary: None, - execution_plan: None, - }) + // Check whether the request passes the asset check + dfx_orbit + .check_evidence( + asset_canister, + response.request.id, + batch_id, + hex::encode(evidence), + ) .await .unwrap(); diff --git a/tests/integration/src/dfx_orbit/canister_call.rs b/tests/integration/src/dfx_orbit/canister_call.rs index 9365a856c..ff914315d 100644 --- a/tests/integration/src/dfx_orbit/canister_call.rs +++ b/tests/integration/src/dfx_orbit/canister_call.rs @@ -1,5 +1,7 @@ use crate::{ - dfx_orbit::{dfx_orbit_test, setup_counter_canister, setup_dfx_orbit, setup_dfx_user}, + dfx_orbit::{ + dfx_orbit_test, setup_counter_canister, setup_dfx_orbit, setup_dfx_user, DfxOrbitTestConfig, + }, setup::{setup_new_env, WALLET_ADMIN_USER}, utils::{ add_user, execute_request, submit_request_approval, update_raw, user_test_id, @@ -53,9 +55,9 @@ fn canister_call() { execution_plan: None, }; - let request = dfx_orbit_test(&mut env, async { + let request = dfx_orbit_test(&mut env, DfxOrbitTestConfig::default(), async { // Setup the station agent - let mut dfx_orbit = setup_dfx_orbit(canister_ids.station).await; + let dfx_orbit = setup_dfx_orbit(canister_ids.station).await; // Call the counter canister let request = dfx_orbit diff --git a/tests/integration/src/dfx_orbit/me.rs b/tests/integration/src/dfx_orbit/me.rs index a337a806e..ed3a127e3 100644 --- a/tests/integration/src/dfx_orbit/me.rs +++ b/tests/integration/src/dfx_orbit/me.rs @@ -1,5 +1,5 @@ use crate::{ - dfx_orbit::{dfx_orbit_test, setup_dfx_orbit, setup_dfx_user}, + dfx_orbit::{dfx_orbit_test, setup_dfx_orbit, setup_dfx_user, DfxOrbitTestConfig}, setup::setup_new_env, TestEnv, }; @@ -15,9 +15,9 @@ fn me() { let (_, dfx_user) = setup_dfx_user(&env, &canister_ids); - let response = dfx_orbit_test(&mut env, async { + let response = dfx_orbit_test(&mut env, DfxOrbitTestConfig::default(), async { // Setup the station agent - let mut dfx_orbit = setup_dfx_orbit(canister_ids.station).await; + let dfx_orbit = setup_dfx_orbit(canister_ids.station).await; // Call the counter canister dfx_orbit.station.me().await.unwrap() diff --git a/tests/integration/src/dfx_orbit/review.rs b/tests/integration/src/dfx_orbit/review.rs index 14b9e2853..a658aea0d 100644 --- a/tests/integration/src/dfx_orbit/review.rs +++ b/tests/integration/src/dfx_orbit/review.rs @@ -10,7 +10,8 @@ use station_api::{ use crate::{ dfx_orbit::{ canister_call::{permit_call_operation, set_four_eyes_on_call}, - dfx_orbit_test, setup_counter_canister, setup_dfx_orbit, TEST_PRINCIPAL, + dfx_orbit_test, setup_counter_canister, setup_dfx_orbit, DfxOrbitTestConfig, + TEST_PRINCIPAL, }, setup::{setup_new_env, WALLET_ADMIN_USER}, utils::{ @@ -71,9 +72,9 @@ fn review() { let ctr = update_raw(&env, canister_id, Principal::anonymous(), "read", vec![]).unwrap(); assert_eq!(ctr, 0_u32.to_le_bytes()); - dfx_orbit_test(&mut env, async { + dfx_orbit_test(&mut env, DfxOrbitTestConfig::default(), async { // Setup the station agent - let mut dfx_orbit = setup_dfx_orbit(canister_ids.station).await; + let dfx_orbit = setup_dfx_orbit(canister_ids.station).await; let list_request_response = dfx_orbit .station diff --git a/tools/dfx-orbit/Cargo.toml b/tools/dfx-orbit/Cargo.toml index 4fdf8d150..2c349461b 100644 --- a/tools/dfx-orbit/Cargo.toml +++ b/tools/dfx-orbit/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "dfx-orbit" -version = "0.1.0" +version = "0.2.0" description = "Command line tool for interacting with the Orbit digital asset manager on the ICP blockchain." authors.workspace = true edition.workspace = true @@ -20,10 +20,12 @@ serde_bytes.workspace = true serde_json.workspace = true cap-std.workspace = true dfx-core.workspace = true +hex.workspace = true ic-agent.workspace = true ic-asset.workspace = true ic-certified-assets.workspace = true ic-utils.workspace = true +sha2.workspace = true slog.workspace = true slog-term.workspace = true slog-async.workspace = true @@ -31,7 +33,6 @@ thiserror.workspace = true tokio = { workspace = true, features = ['rt'] } orbit-station-api = { path = "../../core/station/api", package = "station-api" } -walkdir = "2.5.0" [lib] doctest = false diff --git a/tools/dfx-orbit/README.md b/tools/dfx-orbit/README.md index 293ecd3e5..973a2914e 100644 --- a/tools/dfx-orbit/README.md +++ b/tools/dfx-orbit/README.md @@ -5,9 +5,15 @@ It is designed to work alongside `dfx` to allow a `dfx`-like workflow to manage ## Getting started -### Installation +### Prequisites + +This guide assumes, that the user has setup and is acquainted with the following tools: -Build the tool: +- A fairly recent rust toolchain. This tool is known to work on linux using rust `1.79.0`. +- A working `dfx` development setup. +- An internet identity and an Orbit account with the correct permissions. + +### Installation Currently, there are two ways of installing `dfx-orbit`: @@ -19,19 +25,22 @@ To get the most recent version of `dfx-orbit` without manually cloning the entir cargo install -f --git https://github.com/dfinity/orbit.git --bin dfx-orbit ``` -#### Install from the repository +#### Clone and install from the repository + +This version is potentially more useful, if you want to make patches or use a specific branch. ``` -$ cargo build -p dfx-orbit +git clone https://github.com/dfinity/orbit.git +cargo install -f --path tools/dfx-orbit/ ``` Verify that the tool works: ``` -$ ./target/debug/dfx-orbit --version +$ dfx-orbit --version dfx-orbit 0.1.0 -$ ./target/debug/dfx-orbit --help +$ dfx-orbit --help Command line tool for interacting with the Orbit digital asset manager on the ICP blockchain. Usage: dfx-orbit @@ -41,13 +50,8 @@ Commands: ... ``` -Add `dfx-orbit` to your `PATH`. - ### Connect to Orbit -> **NOTE**: This assumes that you already have a `dfx` setup working. -> If you need to set up a new identity, have a look at `dfx identity new`. - Connect your local dfx identity to your Orbit identity: - Log in to Orbit. @@ -84,8 +88,6 @@ Tell the command line tool where to find the orbit station: dfx-orbit me ``` -TODO: The Oisy canister ID is also called the wallet ID and the station ID. Consistent nomenclature that doesn't conflict with established terminology would be nice. - ### Grant permission to make requests You can check which permissions you have with: @@ -165,55 +167,34 @@ This will create an Orbit request. Once approved you will be able to propose can Suppose that you have built a new Wasm and put a copy at `./MY-CANISTER.wasm.gz`. To upgrade your canister to the new Wasm: ``` -dfx-orbit request canister install --mode upgrade --wasm ./MY-CANISTER.wasm.gz MY_CANISTER +dfx-orbit request canister install --mode upgrade MY_CANISTER --wasm ./MY-CANISTER.wasm.gz ``` ### Upload assets to a canister -We will assume that Orbit is a controller of the asset canister. If not, please adapt the following commands by using `dfx canister call` instead of `dfx-orbit request canister call`. +We will assume that Orbit is a controller of the asset canister. +If not, please transfer the control of the canister to the orbit station. #### Authorize the developer to upload assets Note: Uploaded assets are not published. They are only prepared for release. ``` -developer_principal="$(dfx identity get-principal)" -dfx-orbit request canister call frontend grant_permission " -( - record { - permission = variant { Prepare }; - to_principal = principal \"$developer_principal\"; - }, -) -" +dfx-orbit asset request-prepare-permission frontend ``` -When the request has been approved, check the list of principals permitted to prepare assets: +In case you want to verify, whether you have the `Prepare` permission on the asset canister, +run: ``` dfx canister call frontend list_permitted '(record { permission = variant { Prepare } })' ``` -#### Authorize the orbit station to commit assets - -Note: Committing uploaded assets causes them to be published on the asset canister web site. - -``` -station_principal="$(dfx-orbit station show | jq -r .station_id)" -dfx-orbit request canister call frontend grant_permission " -( - record { - permission = variant { Commit }; - to_principal = principal \"$station_principal\"; - }, -) -" -``` - -When the request has been approved, check the list of principals permitted to commit assets: +and check whether your principal is among the ones listed. +You can optain your own principal via: ``` -dfx canister call frontend list_permitted '(record { permission = variant { Commit } })' +dfx identity get-principal ``` #### Request an asset update @@ -224,14 +205,26 @@ A developer may upload one or more directories of HTTP assets with: dfx-orbit asset upload CANISTER_NAME SOME_DIR/ OTHER_DIR/ ``` -The developer may now request that the assets be published. The command for this is printed at the end of the upload command. Example: +This will upload the assets to the asset canister and then request the orbit station to publish +the assets. + +#### Verifying an asset update + +After the request has been made, the reviewers can locally verify the request: ``` -... -Jul 03 09:36:42.148 INFO Computing evidence. -Proposed batch_id: 5 -Assets have been uploaded. For the changes to take effect, run: -dfx-orbit request canister call frontend commit_proposed_batch '(record { batch_id = 5 : nat; evidence = blob "\e3\b0\c4\42\98\fc\1c\14\9a\fb\f4\c8\99\6f\b9\24\27\ae\41\e4\64\9b\93\4c\a4\95\99\1b\78\52\b8\55" })' +dfx-orbit asset check --then-approve CANISTER REQUEST_ID BATCH_ID SOME_DIR/ OTHER_DIR/ ``` +The exact command is printed in the output of `dfx-orbit asset upload` and must be distributed +from the proposer to the verifiers. + +> The verifiers needs to have the same set of data as was used in the request. +> How the verifier accomplishes this is outside the scope of this document. +> +> - The verifier might either download a tarball from the requester and manually verify the content +> - The verifier might check out a git revision and check that the content matches +> - If there are build scripts used while generating the assets, care must be taken to make +> the build step deterministic, such that verifiers can recreate the exact assets + Once the request has been approved, the changes will take effect. diff --git a/tools/dfx-orbit/src/args/asset.rs b/tools/dfx-orbit/src/args/asset.rs index a826140a7..42221c065 100644 --- a/tools/dfx-orbit/src/args/asset.rs +++ b/tools/dfx-orbit/src/args/asset.rs @@ -1,3 +1,4 @@ +use candid::Nat; use clap::{Parser, Subcommand}; /// Station management commands. @@ -10,8 +11,28 @@ pub struct AssetArgs { #[derive(Debug, Clone, Subcommand)] pub enum AssetArgsAction { + /// Request to grant this user Prepare permission for the asset canister + RequestPreparePermission(AssetReqeustPreparePermissionArgs), /// Upload assets to an asset canister Upload(AssetUploadArgs), + /// Compute local evidence + ComputeEvidence(AssetComputeEvidenceArgs), + /// Check an asset upload request + Check(AssetCheckArgs), +} + +#[derive(Debug, Clone, Parser)] +pub struct AssetReqeustPreparePermissionArgs { + /// The name of the asset canister targeted by this action + pub(crate) canister: String, + + /// The title of the request to grant Prepare permission + #[clap(long)] + pub(crate) title: Option, + + /// The summary of the request to grant Prepare permission + #[clap(long)] + pub(crate) summary: Option, } #[derive(Debug, Clone, Parser)] @@ -19,7 +40,45 @@ pub struct AssetUploadArgs { /// The name of the asset canister targeted by this action pub(crate) canister: String, + /// Do not abort the upload, if the evidence does not match between local and remote calculation + #[clap(long)] + pub(crate) ignore_evidence: bool, + + /// The title of the request to commit the batch + #[clap(long)] + pub(crate) title: Option, + + /// The summary of the request to commit the batch + #[clap(long)] + pub(crate) summary: Option, + /// The source directories to upload (multiple values possible) - #[clap(num_args = 1..)] pub(crate) files: Vec, } + +#[derive(Debug, Clone, Parser)] +pub struct AssetComputeEvidenceArgs { + /// The name of the asset canister targeted by this action + pub(crate) canister: String, + /// The source directories to compute evidence from (multiple values possible) + pub(crate) files: Vec, +} + +#[derive(Debug, Clone, Parser)] +pub struct AssetCheckArgs { + /// The name of the asset canister targeted by this action + pub(crate) canister: String, + + /// The ID of the request to commit the assets + pub(crate) request_id: String, + + /// The batch ID to commit to + pub(crate) batch_id: Nat, + + /// The source directories of the asset upload (multiple values possible) + pub(crate) files: Vec, + + /// Automatically approve the request, if the request's evidence matches the local evidence + #[clap(long)] + pub(crate) then_approve: bool, +} diff --git a/tools/dfx-orbit/src/cli.rs b/tools/dfx-orbit/src/cli.rs index e771b3987..8ac0a63f2 100644 --- a/tools/dfx-orbit/src/cli.rs +++ b/tools/dfx-orbit/src/cli.rs @@ -1,10 +1,11 @@ //! Implementation of the `dfx-orbit` commands. -mod asset; -mod station; +pub(crate) mod asset; +pub(crate) mod station; + +use orbit_station_api::{RequestApprovalStatusDTO, SubmitRequestApprovalInput}; -pub use crate::cli::asset::AssetUploadRequest; use crate::{ - args::{asset::AssetArgsAction, review::ReviewArgs, DfxOrbitArgs, DfxOrbitSubcommands}, + args::{review::ReviewArgs, DfxOrbitArgs, DfxOrbitSubcommands}, dfx_extension_api::OrbitExtensionAgent, DfxOrbit, }; @@ -28,15 +29,12 @@ pub async fn exec(args: DfxOrbitArgs) -> anyhow::Result<()> { Ok(()) } DfxOrbitSubcommands::Request(request_args) => { - let response = dfx_orbit + let request = dfx_orbit .station .request(request_args.into_create_request_input(&dfx_orbit)?) .await?; - let request_id = &response.request.id; - let request_url = dfx_orbit.station.request_url(request_id); - println!("Created request: {request_id}"); - println!("Request URL: {request_url}"); - println!("To view the request, run: dfx-orbit review id {request_id}"); + dfx_orbit.print_create_request_info(&request); + Ok(()) } DfxOrbitSubcommands::Review(review_args) => match review_args { @@ -68,8 +66,14 @@ pub async fn exec(args: DfxOrbitArgs) -> anyhow::Result<()> { )? ); - // TODO: Reaffirm user consent before progressing with submitting - if let Ok(submit) = args.try_into() { + if let Ok(submit) = SubmitRequestApprovalInput::try_from(args) { + let action = match submit.decision { + RequestApprovalStatusDTO::Approved => "approve", + RequestApprovalStatusDTO::Rejected => "reject", + }; + dfx_core::cli::ask_for_consent(&format!( + "Would you like to {action} this request?" + ))?; dfx_orbit.station.submit(submit).await?; }; @@ -77,16 +81,8 @@ pub async fn exec(args: DfxOrbitArgs) -> anyhow::Result<()> { } }, DfxOrbitSubcommands::Asset(asset_args) => { - match asset_args.action { - AssetArgsAction::Upload(upload_args) => { - dfx_orbit - .upload_assets(upload_args.canister, upload_args.files) - .await?; - } - } - + dfx_orbit.exec_asset(asset_args).await?; Ok(()) - // } _ => unreachable!(), } diff --git a/tools/dfx-orbit/src/cli/asset.rs b/tools/dfx-orbit/src/cli/asset.rs index 08a59ea39..5cd8731c3 100644 --- a/tools/dfx-orbit/src/cli/asset.rs +++ b/tools/dfx-orbit/src/cli/asset.rs @@ -1,163 +1,109 @@ //! Implements the `dfx-orbit canister upload-http-assets` CLI command. +mod evidence; +mod upload; mod util; -use crate::DfxOrbit; -use candid::{Nat, Principal}; -use ic_asset::canister_api::{ - methods::batch::compute_evidence, types::batch_upload::common::ComputeEvidenceArguments, +use crate::{ + args::asset::{AssetArgs, AssetArgsAction}, + DfxOrbit, }; -use ic_utils::canister::CanisterBuilder; -use serde::{Deserialize, Serialize}; -use serde_bytes::ByteBuf; -use slog::{info, warn}; -use std::{ - collections::HashMap, - path::{Path, PathBuf}, -}; -use walkdir::WalkDir; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AssetUploadRequest { - station_principal: Principal, - asset_canister_principal: Principal, - batch_id: Nat, - evidence: ByteBuf, +use candid::Principal; +use ic_utils::Canister; +use orbit_station_api::{RequestApprovalStatusDTO, SubmitRequestApprovalInput}; +use slog::Logger; + +pub struct AssetAgent<'agent> { + canister_agent: Canister<'agent>, + logger: Logger, } -// TODO: Use StationAgentResult instead of anyhow result impl DfxOrbit { - /// The main entry point for the `dfx orbit canister upload-http-assets` CLI. - pub async fn upload_assets( - &mut self, - canister: String, - files: Vec, - ) -> anyhow::Result { - // The path is needed in various forms. - let source_pathbufs: Vec = - files.iter().map(|source| PathBuf::from(&source)).collect(); - let source_paths: Vec<&Path> = source_pathbufs - .iter() - .map(|pathbuf| pathbuf.as_path()) - .collect(); - - let canister_id = self.canister_id(&canister)?; - let logger = self.dfx.logger().clone(); - - // Upload assets: - let canister_agent = CanisterBuilder::new() - .with_agent(self.interface.agent()) - .with_canister_id(canister_id) - .build()?; - let assets = assets_as_hash_map(&files); - let batch_id = ic_asset::upload_and_propose(&canister_agent, assets, &logger).await?; - println!("Proposed batch_id: {}", batch_id); - // Compute evidence locally: - let local_evidence = { - let local_evidence = - ic_asset::compute_evidence(&canister_agent, source_paths.as_ref(), &logger).await?; - escape_hex_string(&local_evidence) - }; - // Wait for the canister to compute evidence: - - // This part is stolen from ic_asset::sync::prepare_sync_for_proposal. Unfortunately the relevant functions are private. - // The docs explicitly include waiting for the evidence so this should really be made easier! See: https://github.com/dfinity/sdk/blob/2509e81e11e71dce4045c679686c952809525470/docs/design/asset-canister-interface.md?plain=1#L85 - let compute_evidence_arg = ComputeEvidenceArguments { - batch_id: batch_id.clone(), - max_iterations: Some(97), // 75% of max(130) = 97.5 - }; - info!(logger, "Computing evidence."); - let canister_evidence_bytes = loop { - if let Some(evidence) = compute_evidence(&canister_agent, &compute_evidence_arg).await? - { - break evidence; + pub async fn exec_asset(&mut self, args: AssetArgs) -> anyhow::Result<()> { + match args.action { + AssetArgsAction::RequestPreparePermission(args) => { + let canister_id = self.canister_id(&args.canister)?; + let request = self + .request_prepare_permission(canister_id, args.title, args.summary) + .await?; + self.print_create_request_info(&request); + + Ok(()) } - }; - let canister_evidence = blob_from_bytes(&canister_evidence_bytes); - - // TODO: Move this out of the agent into the tool - println!(r#"Proposed batch_id: {batch_id}"#); - if local_evidence == canister_evidence { - info!(logger, "Local evidence matches canister evidence."); - } else { - warn!(logger, "Local evidence does not match canister evidence:\n local: {local_evidence}\n canister:{canister_evidence}"); - } - println!(r#"Assets have been uploaded. For the changes to take effect, run:"#); - println!( - r#"dfx-orbit request canister call {canister} commit_proposed_batch '(record {{ batch_id = {batch_id} : nat; evidence = blob "{canister_evidence}" }})'"# - ); - - let upload_request = AssetUploadRequest { - station_principal: self.station.config.station_id, - asset_canister_principal: canister_id, - batch_id, - evidence: canister_evidence_bytes, - }; - - Ok(upload_request) - } -} + AssetArgsAction::Upload(args) => { + let pathbufs = self.as_path_bufs(&args.canister, &args.files)?; + let paths = Self::as_paths(&pathbufs); -// TODO: Implement request_upload_commit - -// TODO: Move all these funtions to util -/// Lists all the files at the given path. -/// -/// - Links are followed. -/// - Only files are returned. -/// - The files are sorted by name. -/// - Any files that cannot be read are ignored. -/// - The path includes the prefix. -fn list_assets(path: &str) -> Vec { - WalkDir::new(path) - .sort_by_file_name() - .follow_links(true) - .into_iter() - .filter_map(|e| e.ok()) - .filter(|entry| entry.file_type().is_file()) - .map(|entry| entry.into_path()) - .collect() -} + let canister_name = args.canister; + let canister_id = self.canister_id(&canister_name)?; + let (batch_id, evidence) = self + .upload(canister_id, &paths, args.ignore_evidence) + .await?; -/// A hash map of all assets. -/// -/// Note: Given that ordering in a HashMap is not deterministic, is this really the best API? -fn assets_as_hash_map(asset_dirs: &[String]) -> HashMap { - asset_dirs - .iter() - .flat_map(|asset_dir| { - list_assets(asset_dir).into_iter().map(move |asset_path| { - let relative_path = asset_path.strip_prefix(asset_dir).expect( - "Internal error: list_assets should have returned only files in the asset_dir", - ); - let http_path = format!( - "/{relative_path}", - relative_path = relative_path.to_string_lossy() - ); - (http_path, asset_path) - }) - }) - .collect() -} + let result = self + .request_commit_batch( + canister_id, + batch_id.clone(), + evidence, + args.title, + args.summary, + ) + .await?; + let request_id = result.request.id; -/// Converts a hex string into one escaped as in a candid blob. -fn escape_hex_string(s: &str) -> String { - let mut ans = String::with_capacity(s.len() + s.len() / 2); - for chunk in s.chars().collect::>()[..].chunks(2) { - ans.push('\\'); - for char in chunk { - ans.push(*char); + let files = args.files.join(" "); + println!("Created request to commit batches. To verify the batch against local files, run:"); + println!("dfx-orbit asset check {canister_name} {request_id} {batch_id} {files}"); + + Ok(()) + } + AssetArgsAction::ComputeEvidence(args) => { + let pathbufs = self.as_path_bufs(&args.canister, &args.files)?; + let paths = Self::as_paths(&pathbufs); + + let canister_id = self.canister_id(&args.canister)?; + let asset_agent = self.asset_agent(canister_id)?; + + let evidence = asset_agent.compute_evidence(&paths).await?; + println!("{evidence}"); + Ok(()) + } + AssetArgsAction::Check(args) => { + let pathbufs = self.as_path_bufs(&args.canister, &args.files)?; + let paths = Self::as_paths(&pathbufs); + + let canister_id = self.canister_id(&args.canister)?; + let asset_agent = self.asset_agent(canister_id)?; + + let evidence = asset_agent.compute_evidence(&paths).await?; + self.check_evidence( + canister_id, + args.request_id.clone(), + args.batch_id, + evidence, + ) + .await?; + + println!("Local evidence matches expected arguments"); + + if args.then_approve { + dfx_core::cli::ask_for_consent("Do you want to approve the request?")?; + let args = SubmitRequestApprovalInput { + decision: RequestApprovalStatusDTO::Approved, + request_id: args.request_id, + reason: None, + }; + self.station.submit(args).await?; + } + Ok(()) + } } } - ans -} -/// Converts a byte array into one escaped as a candid blob -fn blob_from_bytes(bytes: &[u8]) -> String { - let mut ans = String::with_capacity(bytes.len() + bytes.len() / 2); - for byte in bytes { - ans.push('\\'); - ans.push_str(&format!("{:02x}", byte)); + pub fn asset_agent(&self, canister_id: Principal) -> anyhow::Result { + Ok(AssetAgent { + canister_agent: self.canister_agent(canister_id)?, + logger: self.logger.clone(), + }) } - ans } diff --git a/tools/dfx-orbit/src/cli/asset/evidence.rs b/tools/dfx-orbit/src/cli/asset/evidence.rs new file mode 100644 index 000000000..669cd5ceb --- /dev/null +++ b/tools/dfx-orbit/src/cli/asset/evidence.rs @@ -0,0 +1,78 @@ +use super::AssetAgent; +use crate::DfxOrbit; +use anyhow::bail; +use candid::{Nat, Principal}; +use ic_certified_assets::types::CommitProposedBatchArguments; +use orbit_station_api::{CanisterMethodDTO, GetRequestInput, RequestOperationDTO}; +use serde_bytes::ByteBuf; +use sha2::{Digest, Sha256}; +use std::path::Path; + +impl DfxOrbit { + /// Check that the locally computed evidence will lead to the correcst sha256 checksum + /// of the args of the request + pub async fn check_evidence( + &self, + canister_id: Principal, + request_id: String, + batch_id: Nat, + evidence: String, + ) -> anyhow::Result<()> { + let request = self + .station + .review_id(GetRequestInput { + request_id: request_id.clone(), + }) + .await?; + + // Check: + // - Request is actually a CallExternalCanister + // - Target is the canister we are expecting + // - Method is `propose_commit_batch` + // - `arg_checksum` exists + let RequestOperationDTO::CallExternalCanister(request) = request.request.operation else { + bail!("{} is not an external canister request. Are you sure you have the correct request id?", {request_id}); + }; + let CanisterMethodDTO { + canister_id: request_canister_id, + method_name, + } = request.execution_method; + if request_canister_id != canister_id { + bail!( + "Canister id of the request {} does not match canister id of asset canister {}", + request_canister_id, + canister_id + ); + } + if &method_name != "commit_proposed_batch" { + bail!( + "Method name if the request is not \"commit_proposed_batch\", but instead \"{}\"", + method_name + ); + } + let Some(remote_checksum) = request.arg_checksum else { + bail!("The request has no arguments. This likely means that is is malformed."); + }; + + // Now we check that the argument that we construct locally matches the hash of the argument + let evidence = hex::decode(evidence)?; + let args = CommitProposedBatchArguments { + batch_id, + evidence: ByteBuf::from(evidence), + }; + let arg = candid::encode_one(args)?; + let local_checksum = hex::encode(Sha256::digest(arg)); + + if local_checksum != remote_checksum { + bail!("Local evidence does not match expected arguments"); + } + + Ok(()) + } +} + +impl AssetAgent<'_> { + pub async fn compute_evidence(&self, sources: &[&Path]) -> anyhow::Result { + Ok(ic_asset::compute_evidence(&self.canister_agent, sources, &self.logger).await?) + } +} diff --git a/tools/dfx-orbit/src/cli/asset/upload.rs b/tools/dfx-orbit/src/cli/asset/upload.rs new file mode 100644 index 000000000..e953cdb6a --- /dev/null +++ b/tools/dfx-orbit/src/cli/asset/upload.rs @@ -0,0 +1,86 @@ +use super::AssetAgent; +use crate::DfxOrbit; +use anyhow::bail; +use candid::{Nat, Principal}; +use ic_certified_assets::types::CommitProposedBatchArguments; +use orbit_station_api::{ + CallExternalCanisterOperationInput, CanisterMethodDTO, CreateRequestInput, + CreateRequestResponse, RequestOperationInput, +}; +use serde_bytes::ByteBuf; +use slog::{info, warn}; +use std::path::Path; + +impl DfxOrbit { + pub async fn upload( + &self, + canister_id: Principal, + sources: &[&Path], + ignore_evidence: bool, + ) -> anyhow::Result<(Nat, ByteBuf)> { + let asset_agent = self.asset_agent(canister_id)?; + let (batch_id, evidence) = asset_agent.upload_assets(sources).await?; + + let remote_evidence = hex::encode(&evidence); + let local_evidence = asset_agent.compute_evidence(sources).await?; + + if !ignore_evidence { + if local_evidence != remote_evidence { + warn!( + self.logger, + "Local evidence does not match remotely calculated evidence" + ); + warn!(self.logger, "Local: {local_evidence}"); + warn!(self.logger, "Remote: {remote_evidence}"); + bail!("Evidence did not match!"); + } else { + info!(self.logger, "Local and remote evidence match!"); + } + } + + Ok((batch_id, evidence)) + } + + pub async fn request_commit_batch( + &self, + canister_id: Principal, + batch_id: Nat, + evidence: ByteBuf, + title: Option, + summary: Option, + ) -> anyhow::Result { + let args = CommitProposedBatchArguments { batch_id, evidence }; + let arg = candid::encode_one(args)?; + + let response = self + .station + .request(CreateRequestInput { + operation: RequestOperationInput::CallExternalCanister( + CallExternalCanisterOperationInput { + validation_method: None, + execution_method: CanisterMethodDTO { + canister_id, + method_name: String::from("commit_proposed_batch"), + }, + arg: Some(arg), + execution_method_cycles: None, + }, + ), + title, + summary, + execution_plan: None, + }) + .await?; + + Ok(response) + } +} + +impl AssetAgent<'_> { + pub async fn upload_assets(&self, sources: &[&Path]) -> anyhow::Result<(Nat, ByteBuf)> { + Ok( + ic_asset::prepare_sync_for_proposal(&self.canister_agent, sources, &self.logger) + .await?, + ) + } +} diff --git a/tools/dfx-orbit/src/cli/asset/util.rs b/tools/dfx-orbit/src/cli/asset/util.rs index 749f6b2da..edfc8a099 100644 --- a/tools/dfx-orbit/src/cli/asset/util.rs +++ b/tools/dfx-orbit/src/cli/asset/util.rs @@ -1,30 +1,93 @@ +use std::path::{Path, PathBuf}; + +use crate::DfxOrbit; + +use super::AssetAgent; +use anyhow::{anyhow, bail}; use candid::Principal; -use ic_certified_assets::types::{ - CommitProposedBatchArguments, GrantPermissionArguments, Permission, +use dfx_core::config::model::dfinity::CanisterTypeProperties; +use ic_certified_assets::types::{GrantPermissionArguments, Permission}; +use orbit_station_api::{ + CallExternalCanisterOperationInput, CanisterMethodDTO, CreateRequestInput, + CreateRequestResponse, RequestOperationInput, }; -use crate::StationAgent; - -use super::AssetUploadRequest; +impl DfxOrbit { + /// Request from the station to grant the `Prepare` permission for the asset canister + pub async fn request_prepare_permission( + &self, + canister_id: Principal, + title: Option, + summary: Option, + ) -> anyhow::Result { + let me = self.own_principal()?; -impl StationAgent { - pub fn request_prepare_permission_payload( - canister: Principal, - ) -> Result, candid::Error> { let args = GrantPermissionArguments { - to_principal: canister, + to_principal: me, permission: Permission::Prepare, }; + let arg = candid::encode_one(args)?; - candid::encode_one(args) + let response = self + .station + .request(CreateRequestInput { + operation: RequestOperationInput::CallExternalCanister( + CallExternalCanisterOperationInput { + validation_method: None, + execution_method: CanisterMethodDTO { + canister_id, + method_name: String::from("grant_permission"), + }, + arg: Some(arg), + execution_method_cycles: None, + }, + ), + title, + summary, + execution_plan: None, + }) + .await?; + + Ok(response) } - pub fn commit_proposed_batch_payload( - upload_request: AssetUploadRequest, + pub fn as_path_bufs(&self, canister: &str, paths: &[String]) -> anyhow::Result> { + if paths.is_empty() { + let config = self.interface.config().ok_or_else(|| { + anyhow!("Could not read \"dfx.json\". Are you in the correct directory?") + })?; + + let canister_config = config + .get_config() + .canisters + .as_ref() + .ok_or_else(|| anyhow!("No canisters defined in this \"dfx.json\""))? + .get(canister) + .ok_or_else(|| anyhow!("Could not find {canister} in \"dfx.json\""))?; + + let CanisterTypeProperties::Assets { source, .. } = &canister_config.type_specific + else { + bail!("Canister {canister} is not an asset canister"); + }; + Ok(source.clone()) + } else { + Ok(paths.iter().map(|source| PathBuf::from(&source)).collect()) + } + } + + pub(crate) fn as_paths(paths: &[PathBuf]) -> Vec<&Path> { + paths.iter().map(|pathbuf| pathbuf.as_path()).collect() + } +} + +impl AssetAgent<'_> { + // TODO: Turn into a functionality + pub fn request_prepare_permission_payload( + canister: Principal, ) -> Result, candid::Error> { - let args = CommitProposedBatchArguments { - batch_id: upload_request.batch_id, - evidence: upload_request.evidence, + let args = GrantPermissionArguments { + to_principal: canister, + permission: Permission::Prepare, }; candid::encode_one(args) diff --git a/tools/dfx-orbit/src/dfx_extension_api.rs b/tools/dfx-orbit/src/dfx_extension_api.rs index 1f186bdca..6d3978d77 100644 --- a/tools/dfx-orbit/src/dfx_extension_api.rs +++ b/tools/dfx-orbit/src/dfx_extension_api.rs @@ -1,7 +1,7 @@ //! Placeholders for the proposed dfx extension API methods. use anyhow::Context; use dfx_core::interface::dfx::DfxInterface; -use slog::{o, Drain, Logger}; + use std::path::PathBuf; /// The name of the Orbit dfx extension. @@ -11,8 +11,6 @@ const ORBIT_EXTENSION_NAME: &str = "orbit"; pub struct OrbitExtensionAgent { /// The directory where all extension configuration files are stored, including those of other extensions. extensions_dir: cap_std::fs::Dir, - /// A logger; some public `sdk` repository methods require a specific type of logger so this is a compatible logger. - logger: Logger, } impl OrbitExtensionAgent { @@ -24,22 +22,7 @@ impl OrbitExtensionAgent { } fn new_from_dir(extensions_dir: cap_std::fs::Dir) -> Self { - let logger = { - let decorator = slog_term::TermDecorator::new().build(); - let drain = slog_term::FullFormat::new(decorator).build().fuse(); - let drain = slog_async::Async::new(drain).build().fuse(); - - slog::Logger::root(drain, o!()) - }; - Self { - extensions_dir, - logger, - } - } - - /// A logger; some public `sdk` repository methods require a specific type of logger so this is a compatible logger. - pub fn logger(&self) -> &Logger { - &self.logger + Self { extensions_dir } } /// Gets the extensions directory, typically at `~/.config/dfx/extensions` diff --git a/tools/dfx-orbit/src/lib.rs b/tools/dfx-orbit/src/lib.rs index 00ab7706f..694469923 100644 --- a/tools/dfx-orbit/src/lib.rs +++ b/tools/dfx-orbit/src/lib.rs @@ -13,8 +13,12 @@ pub mod station_agent; use candid::Principal; use dfx_core::{config::model::canister_id_store::CanisterIdStore, DfxInterface}; use dfx_extension_api::OrbitExtensionAgent; -pub use station_agent::StationAgent; +use ic_utils::{canister::CanisterBuilder, Canister}; +use orbit_station_api::CreateRequestResponse; +use slog::{o, Drain, Logger}; +pub use cli::asset::AssetAgent; +pub use station_agent::StationAgent; pub struct DfxOrbit { // The station agent that handles communication with the station pub station: StationAgent, @@ -22,6 +26,8 @@ pub struct DfxOrbit { pub dfx: OrbitExtensionAgent, // The dfx interface pub interface: DfxInterface, + /// A logger; some public `sdk` repository methods require a specific type of logger so this is a compatible logger. + logger: Logger, } impl DfxOrbit { @@ -32,17 +38,23 @@ impl DfxOrbit { .ok_or_else(|| anyhow::format_err!("No default station specified"))?; let interface = agent.dfx_interface().await?; + let decorator = slog_term::TermDecorator::new().build(); + let drain = slog_term::FullFormat::new(decorator).build().fuse(); + let drain = slog_async::Async::new(drain).build().fuse(); + let logger = slog::Logger::root(drain, o!()); + Ok(Self { station: StationAgent::new(interface.agent().clone(), config), dfx: agent, interface, + logger, }) } /// Gets the ID of a given canister name. If the name is already an ID, it is returned as is. pub fn canister_id(&self, canister_name: &str) -> anyhow::Result { let canister_id_store = CanisterIdStore::new( - self.dfx.logger(), + &self.logger, self.interface.network_descriptor(), self.interface.config(), )?; @@ -52,4 +64,26 @@ impl DfxOrbit { Ok(canister_id) } + + pub fn own_principal(&self) -> anyhow::Result { + self.interface + .identity() + .sender() + .map_err(anyhow::Error::msg) + } + + pub fn canister_agent(&self, canister_id: Principal) -> anyhow::Result { + Ok(CanisterBuilder::new() + .with_agent(self.interface.agent()) + .with_canister_id(canister_id) + .build()?) + } + + pub fn print_create_request_info(&self, response: &CreateRequestResponse) { + let request_id = &response.request.id; + let request_url = self.station.request_url(request_id); + println!("Created request: {request_id}"); + println!("Request URL: {request_url}"); + println!("To view the request, run: dfx-orbit review id {request_id}"); + } } diff --git a/tools/dfx-orbit/src/station_agent.rs b/tools/dfx-orbit/src/station_agent.rs index 4b13381ce..b3404e043 100644 --- a/tools/dfx-orbit/src/station_agent.rs +++ b/tools/dfx-orbit/src/station_agent.rs @@ -25,47 +25,44 @@ impl StationAgent { } pub async fn request( - &mut self, + &self, input: CreateRequestInput, ) -> StationAgentResult { self.update_orbit_typed("create_request", input).await } pub async fn submit( - &mut self, + &self, args: SubmitRequestApprovalInput, ) -> StationAgentResult { self.update_orbit_typed("submit_request_approval", args) .await } - pub async fn me(&mut self) -> StationAgentResult { + pub async fn me(&self) -> StationAgentResult { self.update_orbit_typed("me", ()).await } - pub async fn review_id( - &mut self, - args: GetRequestInput, - ) -> StationAgentResult { + pub async fn review_id(&self, args: GetRequestInput) -> StationAgentResult { self.update_orbit_typed("get_request", args).await } pub async fn review_list( - &mut self, + &self, args: ListRequestsInput, ) -> StationAgentResult { self.update_orbit_typed("list_requests", args).await } pub async fn review_next( - &mut self, + &self, args: GetNextApprovableRequestInput, ) -> StationAgentResult { self.update_orbit_typed("get_next_approvable_request", args) .await } - async fn update_orbit(&mut self, method_name: &str) -> UpdateBuilder { + async fn update_orbit(&self, method_name: &str) -> UpdateBuilder { self.agent.update(&self.config.station_id, method_name) } @@ -73,7 +70,7 @@ impl StationAgent { /// /// This version integrates candid encoding / decoding async fn update_orbit_typed( - &mut self, + &self, method_name: &str, request: Req, ) -> StationAgentResult