diff --git a/CHANGELOG.md b/CHANGELOG.md index 82d237cbae..0a48c6dbf0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ # UNRELEASED +### feat: facade pull ICP, ckBTC, ckETH ledger canisters + +The ledger canisters can be pulled even though they are not really "pullable". +The metadata like wasm_url and init_guide are hardcoded inside `dfx deps pull` logic. + +- ICP ledger: `ryjl3-tyaaa-aaaaa-aaaba-cai` +- ckBTC ledger: `mxzaz-hqaaa-aaaar-qaada-cai` +- ckETH ledger: `ss2fx-dyaaa-aaaar-qacoq-cai` + ### chore: update agent version in frontend templates, and include `resolve.dedupe` in Vite config ### chore: improve error message when trying to use the local replica when it is not running diff --git a/e2e/tests-dfx/deps.bash b/e2e/tests-dfx/deps.bash index 21d19fc55d..1cedfd5532 100644 --- a/e2e/tests-dfx/deps.bash +++ b/e2e/tests-dfx/deps.bash @@ -673,3 +673,172 @@ Installing canister: $CANISTER_ID_C (dep_c)" # this command will fail if the pulled.json is not correct assert_command dfx deps init } + +@test "dfx deps can facade pull ICP ledger" { + use_test_specific_cache_root # dfx deps pull will download files to cache + + dfx_new + jq '.canisters.e2e_project_backend.dependencies=["icp_ledger"]' dfx.json | sponge dfx.json + jq '.canisters.icp_ledger.type="pull"' dfx.json | sponge dfx.json + jq '.canisters.icp_ledger.id="ryjl3-tyaaa-aaaaa-aaaba-cai"' dfx.json | sponge dfx.json + + dfx_start + assert_command dfx deps pull --network local + assert_contains "Using facade dependencies for canister ryjl3-tyaaa-aaaaa-aaaba-cai." + + dfx identity new --storage-mode plaintext minter + assert_command_fail dfx deps init icp_ledger + assert_contains "1. Create a 'minter' identity: dfx identity new minter +2. Run the following multi-line command:" + + assert_command dfx deps init ryjl3-tyaaa-aaaaa-aaaba-cai --argument "(variant { + Init = record { + minting_account = \"$(dfx --identity minter ledger account-id)\"; + initial_values = vec {}; + send_whitelist = vec {}; + transfer_fee = opt record { e8s = 10_000 : nat64; }; + token_symbol = opt \"LICP\"; + token_name = opt \"Local ICP\"; + } +})" + + assert_command dfx deps deploy + + # Can mint tokens (transfer from minting_account) + assert_command dfx --identity minter canister call icp_ledger icrc1_transfer "( + record { + to = record { + owner = principal \"$(dfx --identity default identity get-principal)\"; + }; + amount = 1_000_000 : nat; + }, +)" + + assert_command dfx canister call icp_ledger icrc1_balance_of "( + record { + owner = principal \"$(dfx --identity default identity get-principal)\"; + }, +)" + assert_eq "(1_000_000 : nat)" +} + +@test "dfx deps can facade pull ckBTC ledger" { + [[ "$USE_POCKETIC" ]] && skip "skipped for pocketic which doesn't have ckBTC subnet" + + use_test_specific_cache_root # dfx deps pull will download files to cache + + dfx_new + jq '.canisters.e2e_project_backend.dependencies=["ckbtc_ledger"]' dfx.json | sponge dfx.json + jq '.canisters.ckbtc_ledger.type="pull"' dfx.json | sponge dfx.json + jq '.canisters.ckbtc_ledger.id="mxzaz-hqaaa-aaaar-qaada-cai"' dfx.json | sponge dfx.json + + dfx_start + assert_command dfx deps pull --network local + assert_contains "Using facade dependencies for canister mxzaz-hqaaa-aaaar-qaada-cai." + + dfx identity new --storage-mode plaintext minter + assert_command_fail dfx deps init ckbtc_ledger + assert_contains "1. Create a 'minter' identity: dfx identity new minter +2. Run the following multi-line command:" + + assert_command dfx deps init mxzaz-hqaaa-aaaar-qaada-cai --argument "(variant { + Init = record { + minting_account = record { owner = principal \"$(dfx --identity minter identity get-principal)\"; }; + transfer_fee = 10; + token_symbol = \"ckBTC\"; + token_name = \"ckBTC\"; + metadata = vec {}; + initial_balances = vec {}; + max_memo_length = opt 80; + archive_options = record { + num_blocks_to_archive = 1000; + trigger_threshold = 2000; + max_message_size_bytes = null; + cycles_for_archive_creation = opt 100_000_000_000_000; + node_max_memory_size_bytes = opt 3_221_225_472; + controller_id = principal \"2vxsx-fae\" + } + } +})" + + assert_command dfx deps deploy + + # Can mint tokens (transfer from minting_account) + assert_command dfx --identity minter canister call ckbtc_ledger icrc1_transfer "( + record { + to = record { + owner = principal \"$(dfx --identity default identity get-principal)\"; + }; + amount = 1_000_000 : nat; + }, +)" + + assert_command dfx canister call ckbtc_ledger icrc1_balance_of "( + record { + owner = principal \"$(dfx --identity default identity get-principal)\"; + }, +)" + assert_eq "(1_000_000 : nat)" +} + + +@test "dfx deps can facade pull ckETH ledger" { + [[ "$USE_POCKETIC" ]] && skip "skipped for pocketic which doesn't have ckETH subnet" + + use_test_specific_cache_root # dfx deps pull will download files to cache + + dfx_new + jq '.canisters.e2e_project_backend.dependencies=["cketh_ledger"]' dfx.json | sponge dfx.json + jq '.canisters.cketh_ledger.type="pull"' dfx.json | sponge dfx.json + jq '.canisters.cketh_ledger.id="ss2fx-dyaaa-aaaar-qacoq-cai"' dfx.json | sponge dfx.json + + dfx_start + assert_command dfx deps pull --network local + assert_contains "Using facade dependencies for canister ss2fx-dyaaa-aaaar-qacoq-cai." + + dfx identity new --storage-mode plaintext minter + assert_command_fail dfx deps init cketh_ledger + assert_contains "1. Create a 'minter' identity: dfx identity new minter +2. Run the following multi-line command:" + + assert_command dfx deps init ss2fx-dyaaa-aaaar-qacoq-cai --argument "(variant { + Init = record { + minting_account = record { owner = principal \"$(dfx --identity minter identity get-principal)\"; }; + decimals = opt 18; + max_memo_length = opt 80; + transfer_fee = 2_000_000_000_000; + token_symbol = \"ckETH\"; + token_name = \"ckETH\"; + feature_flags = opt record { icrc2 = true }; + metadata = vec {}; + initial_balances = vec {}; + archive_options = record { + num_blocks_to_archive = 1000; + trigger_threshold = 2000; + max_message_size_bytes = null; + cycles_for_archive_creation = opt 100_000_000_000_000; + node_max_memory_size_bytes = opt 3_221_225_472; + controller_id = principal \"2vxsx-fae\" + } + } +})" + + assert_command dfx deps deploy + + # Can mint tokens (transfer from minting_account) + assert_command dfx --identity minter canister call cketh_ledger icrc1_transfer "( + record { + to = record { + owner = principal \"$(dfx --identity default identity get-principal)\"; + }; + amount = 1_000_000 : nat; + }, +)" + + assert_command dfx canister call cketh_ledger icrc1_balance_of "( + record { + owner = principal \"$(dfx --identity default identity get-principal)\"; + }, +)" + assert_eq "(1_000_000 : nat)" +} diff --git a/src/dfx/src/commands/deps/pull.rs b/src/dfx/src/commands/deps/pull.rs index 3da9b7d9d8..cf42888104 100644 --- a/src/dfx/src/commands/deps/pull.rs +++ b/src/dfx/src/commands/deps/pull.rs @@ -1,31 +1,15 @@ use crate::lib::agent::create_anonymous_agent_environment; -use crate::lib::deps::{ - get_candid_path_in_project, get_pull_canisters_in_config, get_pulled_canister_dir, - get_pulled_service_candid_path, get_pulled_wasm_path, save_pulled_json, +use crate::lib::deps::pull::{ + copy_service_candid_to_project, download_all_and_generate_pulled_json, resolve_all_dependencies, }; -use crate::lib::deps::{PulledCanister, PulledJson}; +use crate::lib::deps::{get_pull_canisters_in_config, save_pulled_json}; use crate::lib::environment::Environment; use crate::lib::error::DfxResult; -use crate::lib::metadata::dfx::DfxMetadata; -use crate::lib::metadata::names::{CANDID_ARGS, CANDID_SERVICE, DFX}; use crate::lib::network::network_opt::NetworkOpt; use crate::lib::root_key::fetch_root_key_if_needed; -use crate::lib::state_tree::canister_info::read_state_tree_canister_module_hash; -use crate::lib::wasm::file::{decompress_bytes, read_wasm_module}; -use crate::util::download_file; -use anyhow::{anyhow, bail, Context}; -use candid::Principal; +use anyhow::anyhow; use clap::Parser; -use dfx_core::config::model::dfinity::Pullable; -use dfx_core::fs::composite::{ensure_dir_exists, ensure_parent_dir_exists}; -use fn_error_context::context; -use ic_agent::{Agent, AgentError}; -use ic_wasm::metadata::get_metadata; -use sha2::{Digest, Sha256}; -use slog::{error, info, trace, warn, Logger}; -use std::collections::{BTreeMap, BTreeSet, VecDeque}; -use std::io::Write; -use std::path::Path; +use slog::info; /// Pull canisters upon which the project depends. /// This command connects to the "ic" mainnet by default. @@ -74,289 +58,3 @@ pub async fn exec(env: &dyn Environment, opts: DepsPullOpts) -> DfxResult { save_pulled_json(&project_root, &pulled_json)?; Ok(()) } - -async fn resolve_all_dependencies( - agent: &Agent, - logger: &Logger, - pull_canisters_in_config: &BTreeMap, -) -> DfxResult> { - let mut canisters_to_resolve: VecDeque = - pull_canisters_in_config.values().cloned().collect(); - let mut checked = BTreeSet::new(); - while let Some(canister_id) = canisters_to_resolve.pop_front() { - if !checked.contains(&canister_id) { - checked.insert(canister_id); - let dependencies = get_dependencies(agent, logger, &canister_id).await?; - canisters_to_resolve.extend(dependencies.iter()); - } - } - let all_dependencies = checked.into_iter().collect::>(); - let mut message = String::new(); - message.push_str(&format!("Found {} dependencies:", all_dependencies.len())); - for id in &all_dependencies { - message.push('\n'); - message.push_str(&id.to_text()); - } - info!(logger, "{}", message); - Ok(all_dependencies) -} - -#[context("Failed to get dependencies of canister {canister_id}.")] -async fn get_dependencies( - agent: &Agent, - logger: &Logger, - canister_id: &Principal, -) -> DfxResult> { - info!(logger, "Fetching dependencies of canister {canister_id}..."); - let dfx_metadata = fetch_dfx_metadata(agent, canister_id).await?; - let dependencies = dfx_metadata.get_pullable()?.dependencies.clone(); - Ok(dependencies) -} - -async fn download_all_and_generate_pulled_json( - agent: &Agent, - logger: &Logger, - all_dependencies: &[Principal], -) -> DfxResult { - let mut any_download_fail = false; - let mut pulled_json = PulledJson::default(); - for canister_id in all_dependencies { - match download_and_generate_pulled_canister(agent, logger, *canister_id).await { - Ok(pulled_canister) => { - pulled_json.canisters.insert(*canister_id, pulled_canister); - } - Err(e) => { - error!(logger, "Failed to pull canister {canister_id}.\n{e}"); - any_download_fail = true; - } - } - } - - if any_download_fail { - bail!("Failed when pulling canisters."); - } - Ok(pulled_json) -} - -// Download canister wasm, then extract metadata from it to build a PulledCanister -async fn download_and_generate_pulled_canister( - agent: &Agent, - logger: &Logger, - canister_id: Principal, -) -> DfxResult { - info!(logger, "Pulling canister {canister_id}..."); - - let mut pulled_canister = PulledCanister::default(); - - let dfx_metadata = fetch_dfx_metadata(agent, &canister_id).await?; - let pullable = dfx_metadata.get_pullable()?; - - let hash_on_chain = get_hash_on_chain(agent, logger, canister_id, pullable).await?; - pulled_canister.wasm_hash = hex::encode(&hash_on_chain); - - // skip download if cache hit - let mut cache_hit = false; - - for gzip in [false, true] { - let path = get_pulled_wasm_path(&canister_id, gzip)?; - if path.exists() { - let bytes = dfx_core::fs::read(&path)?; - let hash_cache = Sha256::digest(bytes); - if hash_cache.as_slice() == hash_on_chain { - cache_hit = true; - pulled_canister.gzip = gzip; - pulled_canister.wasm_hash_download = hex::encode(hash_cache); - trace!(logger, "The canister wasm was found in the cache."); - } - break; - } - } - - if !cache_hit { - // delete files from previous pull - let pulled_canister_dir = get_pulled_canister_dir(&canister_id)?; - if pulled_canister_dir.exists() { - dfx_core::fs::remove_dir_all(&pulled_canister_dir)?; - } - dfx_core::fs::create_dir_all(&pulled_canister_dir)?; - - // lookup `wasm_url` in dfx metadata - let wasm_url = reqwest::Url::parse(&pullable.wasm_url)?; - - // download - let content = download_file(&wasm_url).await?; - - // hash check - let hash_download = Sha256::digest(&content); - pulled_canister.wasm_hash_download = hex::encode(hash_download); - - let gzip = decompress_bytes(&content).is_ok(); - pulled_canister.gzip = gzip; - let wasm_path = get_pulled_wasm_path(&canister_id, gzip)?; - - write_to_tempfile_then_rename(&content, &wasm_path)?; - } - - let wasm_path = get_pulled_wasm_path(&canister_id, pulled_canister.gzip)?; - - // extract `candid:service` and save as candid file in shared cache - let module = read_wasm_module(&wasm_path)?; - let candid_service = get_metadata_as_string(&module, CANDID_SERVICE, &wasm_path)?; - let service_candid_path = get_pulled_service_candid_path(&canister_id)?; - write_to_tempfile_then_rename(candid_service.as_bytes(), &service_candid_path)?; - - // extract `candid:args` - let candid_args = get_metadata_as_string(&module, CANDID_ARGS, &wasm_path)?; - pulled_canister.candid_args = candid_args; - - // extract `dfx` - let dfx_metadata_str = get_metadata_as_string(&module, DFX, &wasm_path)?; - let dfx_metadata: DfxMetadata = serde_json::from_str(&dfx_metadata_str)?; - let pullable = dfx_metadata.get_pullable()?; - pulled_canister.dependencies = pullable.dependencies.clone(); - pulled_canister.init_guide = pullable.init_guide.clone(); - pulled_canister.init_arg = pullable.init_arg.clone(); - - Ok(pulled_canister) -} - -async fn fetch_dfx_metadata(agent: &Agent, canister_id: &Principal) -> DfxResult { - match fetch_metadata(agent, canister_id, DFX).await? { - Some(dfx_metadata_raw) => { - let dfx_metadata_str = String::from_utf8(dfx_metadata_raw)?; - let dfx_metadata: DfxMetadata = serde_json::from_str(&dfx_metadata_str)?; - Ok(dfx_metadata) - } - None => { - bail!("`{DFX}` metadata not found in canister {canister_id}."); - } - } -} - -#[context("Failed to fetch metadata {metadata} of canister {canister_id}.")] -async fn fetch_metadata( - agent: &Agent, - canister_id: &Principal, - metadata: &str, -) -> DfxResult>> { - match agent - .read_state_canister_metadata(*canister_id, metadata) - .await - { - Ok(data) => Ok(Some(data)), - Err(agent_error) => match agent_error { - // replica returns such error - AgentError::HttpError(ref e) => { - let status = e.status; - let content = String::from_utf8(e.content.clone())?; - if status == 404 - && content.starts_with(&format!("Custom section {metadata} not found")) - { - Ok(None) - } else { - bail!(agent_error); - } - } - // ic-ref returns such error when the canister doesn't define the metadata - AgentError::LookupPathAbsent(_) => Ok(None), - _ => { - bail!(agent_error) - } - }, - } -} - -// Get expected hash of the canister wasm. -// If `wasm_hash` is specified in dfx metadata, use it. -// If `wasm_hash_url` is specified in dfx metadata, download the hash from the url. -// Otherwise, get the hash of the on chain canister. -async fn get_hash_on_chain( - agent: &Agent, - logger: &Logger, - canister_id: Principal, - pullable: &Pullable, -) -> DfxResult> { - if pullable.wasm_hash.is_some() && pullable.wasm_hash_url.is_some() { - warn!(logger, "Canister {canister_id} specified both `wasm_hash` and `wasm_hash_url`. `wasm_hash` will be used."); - }; - if let Some(wasm_hash_str) = &pullable.wasm_hash { - trace!( - logger, - "Canister {canister_id} specified a custom hash: {wasm_hash_str}" - ); - Ok(hex::decode(wasm_hash_str) - .with_context(|| format!("Failed to decode {wasm_hash_str} as sha256 hash."))?) - } else if let Some(wasm_hash_url) = &pullable.wasm_hash_url { - trace!( - logger, - "Canister {canister_id} specified a custom hash via url: {wasm_hash_url}" - ); - let wasm_hash_url = reqwest::Url::parse(wasm_hash_url) - .with_context(|| format!("{wasm_hash_url} is not a valid URL."))?; - let wasm_hash_content = download_file(&wasm_hash_url) - .await - .with_context(|| format!("Failed to download wasm_hash from {wasm_hash_url}."))?; - let wasm_hash_str = String::from_utf8(wasm_hash_content) - .with_context(|| format!("Content from {wasm_hash_url} is not valid text."))?; - // The content might contain the file name (usually from tools like shasum or sha256sum). - // We only need the hash part. - let wasm_hash_encoded = wasm_hash_str - .split_whitespace() - .next() - .with_context(|| format!("Content from {wasm_hash_url} is empty."))?; - Ok(hex::decode(wasm_hash_encoded) - .with_context(|| format!("Failed to decode {wasm_hash_encoded} as sha256 hash."))?) - } else { - match read_state_tree_canister_module_hash(agent, canister_id).await? { - Some(hash_on_chain) => Ok(hash_on_chain), - None => { - bail!( - "Canister {canister_id} doesn't have module hash. Perhaps it's not installed." - ); - } - } - } -} - -#[context("Failed to write to a tempfile then rename it to {}", path.display())] -fn write_to_tempfile_then_rename(content: &[u8], path: &Path) -> DfxResult { - assert!(path.is_absolute()); - let dir = dfx_core::fs::parent(path)?; - ensure_dir_exists(&dir)?; - let mut f = tempfile::NamedTempFile::new_in(&dir) - .with_context(|| format!("Failed to create a NamedTempFile in {dir:?}"))?; - f.write_all(content) - .with_context(|| format!("Failed to write the NamedTempFile at {:?}", f.path()))?; - dfx_core::fs::rename(f.path(), path)?; - Ok(()) -} - -#[context("Failed to copy candid path of pull dependency {name}")] -pub fn copy_service_candid_to_project( - project_root: &Path, - name: &str, - canister_id: &Principal, -) -> DfxResult { - let service_candid_path = get_pulled_service_candid_path(canister_id)?; - let path_in_project = get_candid_path_in_project(project_root, canister_id); - ensure_parent_dir_exists(&path_in_project)?; - dfx_core::fs::copy(&service_candid_path, &path_in_project)?; - dfx_core::fs::set_permissions_readwrite(&path_in_project)?; - Ok(()) -} - -fn get_metadata_as_string( - module: &walrus::Module, - section: &str, - wasm_path: &Path, -) -> DfxResult { - let metadata_bytes = get_metadata(module, section) - .with_context(|| format!("Failed to get {} metadata from {:?}", section, wasm_path))?; - let metadata = String::from_utf8(metadata_bytes.to_vec()).with_context(|| { - format!( - "Failed to read {} metadata from {:?} as UTF-8 text", - section, wasm_path - ) - })?; - Ok(metadata) -} diff --git a/src/dfx/src/lib/deps/mod.rs b/src/dfx/src/lib/deps/mod.rs index 456220ccda..90e7ba973d 100644 --- a/src/dfx/src/lib/deps/mod.rs +++ b/src/dfx/src/lib/deps/mod.rs @@ -15,13 +15,14 @@ use std::{ }; pub mod deploy; +pub mod pull; #[derive(Serialize, Deserialize, Default)] pub struct PulledJson { pub canisters: BTreeMap, } -#[derive(Serialize, Deserialize, Default)] +#[derive(Serialize, Deserialize, Default, Clone)] pub struct PulledCanister { /// Name of `type: pull` in dfx.json. Omitted if indirect dependency. #[serde(skip_serializing_if = "Option::is_none")] diff --git a/src/dfx/src/lib/deps/pull/facade.rs b/src/dfx/src/lib/deps/pull/facade.rs new file mode 100644 index 0000000000..aeae315eef --- /dev/null +++ b/src/dfx/src/lib/deps/pull/facade.rs @@ -0,0 +1,190 @@ +use candid::{pretty::candid::pp_args, Principal}; +use candid_parser::utils::{instantiate_candid, CandidSource}; +use dfx_core::config::cache::get_cache_root; +use dfx_core::fs::{composite::ensure_parent_dir_exists, read, read_to_string, write}; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; + +use super::{super::PulledCanister, write_to_tempfile_then_rename}; +use crate::lib::deps::{ + get_pulled_canister_dir, get_pulled_service_candid_path, get_pulled_wasm_path, +}; +use crate::lib::error::DfxResult; +use crate::util::{download_file, download_file_to_path}; + +static IC_REV: &str = "1eeb4d74deb00bd52739cbd6f37ce1dc72e0c76e"; + +#[derive(Debug)] +struct Facade { + wasm_url: String, + candid_url: String, + dependencies: Vec, + init_guide: String, +} + +lazy_static::lazy_static! { + static ref ICP_LEDGER: Principal=Principal::from_text("ryjl3-tyaaa-aaaaa-aaaba-cai").unwrap(); + static ref CKBTC_LEDGER: Principal=Principal::from_text("mxzaz-hqaaa-aaaar-qaada-cai").unwrap(); + static ref CKETH_LEDGER: Principal=Principal::from_text("ss2fx-dyaaa-aaaar-qacoq-cai").unwrap(); + static ref FACADE: HashMap = { + let mut m = HashMap::new(); + // https://internetcomputer.org/docs/current/developer-docs/defi/tokens/ledger/setup/icp_ledger_setup + m.insert( + *ICP_LEDGER, + Facade { + wasm_url: format!("https://download.dfinity.systems/ic/{IC_REV}/canisters/ledger-canister.wasm.gz"), + candid_url: format!("https://raw.githubusercontent.com/dfinity/ic/{IC_REV}/rs/ledger_suite/icp/ledger.did"), + dependencies:vec![], + init_guide: r#" +1. Create a 'minter' identity: dfx identity new minter +2. Run the following multi-line command: + +dfx deps init ryjl3-tyaaa-aaaaa-aaaba-cai --argument "(variant { + Init = record { + minting_account = \"$(dfx --identity minter ledger account-id)\"; + initial_values = vec {}; + send_whitelist = vec {}; + transfer_fee = opt record { e8s = 10_000 : nat64; }; + token_symbol = opt \"LICP\"; + token_name = opt \"Local ICP\"; + } +})" +"#.to_string(), + } + ); + // https://github.com/dfinity/ic/blob/master/rs/bitcoin/ckbtc/mainnet/README.md#installing-the-ledger-mxzaz-hqaaa-aaaar-qaada-cai + m.insert( + *CKBTC_LEDGER, + Facade { + wasm_url: format!("https://download.dfinity.systems/ic/{IC_REV}/canisters/ic-icrc1-ledger.wasm.gz"), + candid_url: format!("https://raw.githubusercontent.com/dfinity/ic/{IC_REV}/rs/ledger_suite/icrc1/ledger/ledger.did"), + dependencies:vec![], + init_guide: r#" +1. Create a 'minter' identity: dfx identity new minter +2. Run the following multi-line command: + +dfx deps init mxzaz-hqaaa-aaaar-qaada-cai --argument "(variant { + Init = record { + minting_account = record { owner = principal \"$(dfx --identity minter identity get-principal)\"; }; + transfer_fee = 10; + token_symbol = \"ckBTC\"; + token_name = \"ckBTC\"; + metadata = vec {}; + initial_balances = vec {}; + max_memo_length = opt 80; + archive_options = record { + num_blocks_to_archive = 1000; + trigger_threshold = 2000; + max_message_size_bytes = null; + cycles_for_archive_creation = opt 100_000_000_000_000; + node_max_memory_size_bytes = opt 3_221_225_472; + controller_id = principal \"2vxsx-fae\" + } + } +})" +"#.to_string(), + } + ); + // https://github.com/dfinity/ic/blob/master/rs/ethereum/cketh/mainnet/README.md#installing-the-ledger + m.insert( + *CKETH_LEDGER, + Facade { + wasm_url: format!("https://download.dfinity.systems/ic/{IC_REV}/canisters/ic-icrc1-ledger-u256.wasm.gz"), + candid_url: format!("https://raw.githubusercontent.com/dfinity/ic/{IC_REV}/rs/ledger_suite/icrc1/ledger/ledger.did"), + dependencies:vec![], + init_guide: r#" +1. Create a 'minter' identity: dfx identity new minter +2. Run the following multi-line command: + +dfx deps init ss2fx-dyaaa-aaaar-qacoq-cai --argument "(variant { + Init = record { + minting_account = record { owner = principal \"$(dfx --identity minter identity get-principal)\"; }; + decimals = opt 18; + max_memo_length = opt 80; + transfer_fee = 2_000_000_000_000; + token_symbol = \"ckETH\"; + token_name = \"ckETH\"; + feature_flags = opt record { icrc2 = true }; + metadata = vec {}; + initial_balances = vec {}; + archive_options = record { + num_blocks_to_archive = 1000; + trigger_threshold = 2000; + max_message_size_bytes = null; + cycles_for_archive_creation = opt 100_000_000_000_000; + node_max_memory_size_bytes = opt 3_221_225_472; + controller_id = principal \"2vxsx-fae\" + } + } +})" +"#.to_string(), + } + ); + m + }; +} + +pub(super) fn facade_dependencies(canister_id: &Principal) -> Option> { + FACADE + .get(canister_id) + .map(|facade| facade.dependencies.clone()) +} + +pub(super) async fn facade_download(canister_id: &Principal) -> DfxResult> { + if let Some(facade) = FACADE.get(canister_id) { + let mut pulled_canister = PulledCanister { + dependencies: facade.dependencies.clone(), + init_guide: facade.init_guide.clone(), + gzip: facade.wasm_url.ends_with(".gz"), + ..Default::default() + }; + let ic_rev_path = get_cache_root()? + .join("pulled") + .join(".facade") + .join(canister_id.to_text()); + let wasm_path = get_pulled_wasm_path(canister_id, pulled_canister.gzip)?; + let service_candid_path = get_pulled_service_candid_path(canister_id)?; + let mut cache_hit = false; + if ic_rev_path.exists() && wasm_path.exists() && service_candid_path.exists() { + let ic_rev = read_to_string(&ic_rev_path)?; + if ic_rev == IC_REV { + cache_hit = true; + } + } + if !cache_hit { + // delete files from previous pull + let pulled_canister_dir = get_pulled_canister_dir(canister_id)?; + if pulled_canister_dir.exists() { + dfx_core::fs::remove_dir_all(&pulled_canister_dir)?; + } + dfx_core::fs::create_dir_all(&pulled_canister_dir)?; + // download wasm and candid + let wasm_url = reqwest::Url::parse(&facade.wasm_url)?; + download_file_to_path(&wasm_url, &wasm_path).await?; + let candid_url = reqwest::Url::parse(&facade.candid_url)?; + let candid_bytes = download_file(&candid_url).await?; + let candid_service = String::from_utf8(candid_bytes)?; + write_to_tempfile_then_rename(candid_service.as_bytes(), &service_candid_path)?; + // write ic_rev for cache logic + ensure_parent_dir_exists(&ic_rev_path)?; + write(&ic_rev_path, IC_REV)?; + } + + // wasm_hash + let wasm_content = read(&wasm_path)?; + let wasm_hash = Sha256::digest(wasm_content).to_vec(); + pulled_canister.wasm_hash = hex::encode(&wasm_hash); + pulled_canister.wasm_hash_download = hex::encode(&wasm_hash); + + // candid_args + let candid_service = read_to_string(&service_candid_path)?; + let candid_source = CandidSource::Text(&candid_service); + let (args, _service) = instantiate_candid(candid_source)?; + let candid_args = pp_args(&args).pretty(80).to_string(); + pulled_canister.candid_args = candid_args; + + Ok(Some(pulled_canister)) + } else { + Ok(None) + } +} diff --git a/src/dfx/src/lib/deps/pull/mod.rs b/src/dfx/src/lib/deps/pull/mod.rs new file mode 100644 index 0000000000..5358eec4ee --- /dev/null +++ b/src/dfx/src/lib/deps/pull/mod.rs @@ -0,0 +1,322 @@ +use super::{ + get_candid_path_in_project, get_pulled_canister_dir, get_pulled_service_candid_path, + get_pulled_wasm_path, PulledCanister, PulledJson, +}; +use crate::lib::error::DfxResult; +use crate::lib::metadata::dfx::DfxMetadata; +use crate::lib::metadata::names::{CANDID_ARGS, CANDID_SERVICE, DFX}; +use crate::lib::state_tree::canister_info::read_state_tree_canister_module_hash; +use crate::lib::wasm::file::{decompress_bytes, read_wasm_module}; +use crate::util::download_file; +use anyhow::{bail, Context}; +use candid::Principal; +use dfx_core::config::model::dfinity::Pullable; +use dfx_core::fs::composite::{ensure_dir_exists, ensure_parent_dir_exists}; +use fn_error_context::context; +use ic_agent::{Agent, AgentError}; +use ic_wasm::metadata::get_metadata; +use sha2::{Digest, Sha256}; +use slog::{error, info, trace, warn, Logger}; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; +use std::io::Write; +use std::path::Path; + +mod facade; +use facade::{facade_dependencies, facade_download}; + +pub async fn resolve_all_dependencies( + agent: &Agent, + logger: &Logger, + pull_canisters_in_config: &BTreeMap, +) -> DfxResult> { + let mut canisters_to_resolve: VecDeque = + pull_canisters_in_config.values().cloned().collect(); + let mut checked = BTreeSet::new(); + while let Some(canister_id) = canisters_to_resolve.pop_front() { + if !checked.contains(&canister_id) { + checked.insert(canister_id); + let dependencies = if let Some(deps) = facade_dependencies(&canister_id) { + info!( + logger, + "Using facade dependencies for canister {canister_id}." + ); + deps + } else { + get_dependencies(agent, logger, &canister_id).await? + }; + canisters_to_resolve.extend(dependencies.iter()); + } + } + let all_dependencies = checked.into_iter().collect::>(); + let mut message = String::new(); + message.push_str(&format!("Found {} dependencies:", all_dependencies.len())); + for id in &all_dependencies { + message.push('\n'); + message.push_str(&id.to_text()); + } + info!(logger, "{}", message); + Ok(all_dependencies) +} + +#[context("Failed to get dependencies of canister {canister_id}.")] +async fn get_dependencies( + agent: &Agent, + logger: &Logger, + canister_id: &Principal, +) -> DfxResult> { + info!(logger, "Fetching dependencies of canister {canister_id}..."); + let dfx_metadata = fetch_dfx_metadata(agent, canister_id).await?; + let dependencies = dfx_metadata.get_pullable()?.dependencies.clone(); + Ok(dependencies) +} + +pub async fn download_all_and_generate_pulled_json( + agent: &Agent, + logger: &Logger, + all_dependencies: &[Principal], +) -> DfxResult { + let mut any_download_fail = false; + let mut pulled_json = PulledJson::default(); + for canister_id in all_dependencies { + match download_and_generate_pulled_canister(agent, logger, *canister_id).await { + Ok(pulled_canister) => { + pulled_json.canisters.insert(*canister_id, pulled_canister); + } + Err(e) => { + error!(logger, "Failed to pull canister {canister_id}.\n{e}"); + any_download_fail = true; + } + } + } + + if any_download_fail { + bail!("Failed when pulling canisters."); + } + Ok(pulled_json) +} + +// Download canister wasm, then extract metadata from it to build a PulledCanister +async fn download_and_generate_pulled_canister( + agent: &Agent, + logger: &Logger, + canister_id: Principal, +) -> DfxResult { + info!(logger, "Pulling canister {canister_id}..."); + if let Some(pulled_canister) = facade_download(&canister_id).await? { + return Ok(pulled_canister); + } + + let mut pulled_canister = PulledCanister::default(); + + let dfx_metadata = fetch_dfx_metadata(agent, &canister_id).await?; + let pullable = dfx_metadata.get_pullable()?; + + let hash_on_chain = get_hash_on_chain(agent, logger, canister_id, pullable).await?; + pulled_canister.wasm_hash = hex::encode(&hash_on_chain); + + // skip download if cache hit + let mut cache_hit = false; + + for gzip in [false, true] { + let path = get_pulled_wasm_path(&canister_id, gzip)?; + if path.exists() { + let bytes = dfx_core::fs::read(&path)?; + let hash_cache = Sha256::digest(bytes); + if hash_cache.as_slice() == hash_on_chain { + cache_hit = true; + pulled_canister.gzip = gzip; + pulled_canister.wasm_hash_download = hex::encode(hash_cache); + trace!(logger, "The canister wasm was found in the cache."); + } + break; + } + } + + if !cache_hit { + // delete files from previous pull + let pulled_canister_dir = get_pulled_canister_dir(&canister_id)?; + if pulled_canister_dir.exists() { + dfx_core::fs::remove_dir_all(&pulled_canister_dir)?; + } + dfx_core::fs::create_dir_all(&pulled_canister_dir)?; + + // lookup `wasm_url` in dfx metadata + let wasm_url = reqwest::Url::parse(&pullable.wasm_url)?; + + // download + let content = download_file(&wasm_url).await?; + + // hash check + let hash_download = Sha256::digest(&content); + pulled_canister.wasm_hash_download = hex::encode(hash_download); + + let gzip = decompress_bytes(&content).is_ok(); + pulled_canister.gzip = gzip; + let wasm_path = get_pulled_wasm_path(&canister_id, gzip)?; + + write_to_tempfile_then_rename(&content, &wasm_path)?; + } + + let wasm_path = get_pulled_wasm_path(&canister_id, pulled_canister.gzip)?; + + // extract `candid:service` and save as candid file in shared cache + let module = read_wasm_module(&wasm_path)?; + let candid_service = get_metadata_as_string(&module, CANDID_SERVICE, &wasm_path)?; + let service_candid_path = get_pulled_service_candid_path(&canister_id)?; + write_to_tempfile_then_rename(candid_service.as_bytes(), &service_candid_path)?; + + // extract `candid:args` + let candid_args = get_metadata_as_string(&module, CANDID_ARGS, &wasm_path)?; + pulled_canister.candid_args = candid_args; + + // extract `dfx` + let dfx_metadata_str = get_metadata_as_string(&module, DFX, &wasm_path)?; + let dfx_metadata: DfxMetadata = serde_json::from_str(&dfx_metadata_str)?; + let pullable = dfx_metadata.get_pullable()?; + pulled_canister.dependencies = pullable.dependencies.clone(); + pulled_canister.init_guide = pullable.init_guide.clone(); + pulled_canister.init_arg = pullable.init_arg.clone(); + + Ok(pulled_canister) +} + +async fn fetch_dfx_metadata(agent: &Agent, canister_id: &Principal) -> DfxResult { + match fetch_metadata(agent, canister_id, DFX).await? { + Some(dfx_metadata_raw) => { + let dfx_metadata_str = String::from_utf8(dfx_metadata_raw)?; + let dfx_metadata: DfxMetadata = serde_json::from_str(&dfx_metadata_str)?; + Ok(dfx_metadata) + } + None => { + bail!("`{DFX}` metadata not found in canister {canister_id}."); + } + } +} + +#[context("Failed to fetch metadata {metadata} of canister {canister_id}.")] +async fn fetch_metadata( + agent: &Agent, + canister_id: &Principal, + metadata: &str, +) -> DfxResult>> { + match agent + .read_state_canister_metadata(*canister_id, metadata) + .await + { + Ok(data) => Ok(Some(data)), + Err(agent_error) => match agent_error { + // replica returns such error + AgentError::HttpError(ref e) => { + let status = e.status; + let content = String::from_utf8(e.content.clone())?; + if status == 404 + && content.starts_with(&format!("Custom section {metadata} not found")) + { + Ok(None) + } else { + bail!(agent_error); + } + } + // ic-ref returns such error when the canister doesn't define the metadata + AgentError::LookupPathAbsent(_) => Ok(None), + _ => { + bail!(agent_error) + } + }, + } +} + +// Get expected hash of the canister wasm. +// If `wasm_hash` is specified in dfx metadata, use it. +// If `wasm_hash_url` is specified in dfx metadata, download the hash from the url. +// Otherwise, get the hash of the on chain canister. +async fn get_hash_on_chain( + agent: &Agent, + logger: &Logger, + canister_id: Principal, + pullable: &Pullable, +) -> DfxResult> { + if pullable.wasm_hash.is_some() && pullable.wasm_hash_url.is_some() { + warn!(logger, "Canister {canister_id} specified both `wasm_hash` and `wasm_hash_url`. `wasm_hash` will be used."); + }; + if let Some(wasm_hash_str) = &pullable.wasm_hash { + trace!( + logger, + "Canister {canister_id} specified a custom hash: {wasm_hash_str}" + ); + Ok(hex::decode(wasm_hash_str) + .with_context(|| format!("Failed to decode {wasm_hash_str} as sha256 hash."))?) + } else if let Some(wasm_hash_url) = &pullable.wasm_hash_url { + trace!( + logger, + "Canister {canister_id} specified a custom hash via url: {wasm_hash_url}" + ); + let wasm_hash_url = reqwest::Url::parse(wasm_hash_url) + .with_context(|| format!("{wasm_hash_url} is not a valid URL."))?; + let wasm_hash_content = download_file(&wasm_hash_url) + .await + .with_context(|| format!("Failed to download wasm_hash from {wasm_hash_url}."))?; + let wasm_hash_str = String::from_utf8(wasm_hash_content) + .with_context(|| format!("Content from {wasm_hash_url} is not valid text."))?; + // The content might contain the file name (usually from tools like shasum or sha256sum). + // We only need the hash part. + let wasm_hash_encoded = wasm_hash_str + .split_whitespace() + .next() + .with_context(|| format!("Content from {wasm_hash_url} is empty."))?; + Ok(hex::decode(wasm_hash_encoded) + .with_context(|| format!("Failed to decode {wasm_hash_encoded} as sha256 hash."))?) + } else { + match read_state_tree_canister_module_hash(agent, canister_id).await? { + Some(hash_on_chain) => Ok(hash_on_chain), + None => { + bail!( + "Canister {canister_id} doesn't have module hash. Perhaps it's not installed." + ); + } + } + } +} + +#[context("Failed to write to a tempfile then rename it to {}", path.display())] +fn write_to_tempfile_then_rename(content: &[u8], path: &Path) -> DfxResult { + assert!(path.is_absolute()); + let dir = dfx_core::fs::parent(path)?; + ensure_dir_exists(&dir)?; + let mut f = tempfile::NamedTempFile::new_in(&dir) + .with_context(|| format!("Failed to create a NamedTempFile in {dir:?}"))?; + f.write_all(content) + .with_context(|| format!("Failed to write the NamedTempFile at {:?}", f.path()))?; + dfx_core::fs::rename(f.path(), path)?; + Ok(()) +} + +#[context("Failed to copy candid path of pull dependency {name}")] +pub fn copy_service_candid_to_project( + project_root: &Path, + name: &str, + canister_id: &Principal, +) -> DfxResult { + let service_candid_path = get_pulled_service_candid_path(canister_id)?; + let path_in_project = get_candid_path_in_project(project_root, canister_id); + ensure_parent_dir_exists(&path_in_project)?; + dfx_core::fs::copy(&service_candid_path, &path_in_project)?; + dfx_core::fs::set_permissions_readwrite(&path_in_project)?; + Ok(()) +} + +fn get_metadata_as_string( + module: &walrus::Module, + section: &str, + wasm_path: &Path, +) -> DfxResult { + let metadata_bytes = get_metadata(module, section) + .with_context(|| format!("Failed to get {} metadata from {:?}", section, wasm_path))?; + let metadata = String::from_utf8(metadata_bytes.to_vec()).with_context(|| { + format!( + "Failed to read {} metadata from {:?} as UTF-8 text", + section, wasm_path + ) + })?; + Ok(metadata) +}