Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: ✨ Add fullnet script for demo #298

Merged
merged 10 commits into from
Dec 26, 2024
Merged

feat: ✨ Add fullnet script for demo #298

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
46e4236
feat: :technologist: Improve return type of `loadFileInStorage` RPC m…
ffarall Dec 19, 2024
e11f2c1
feat: :loud_sound: Make error log acutally `error!` out and not `debug!`
ffarall Dec 19, 2024
5106213
fix: :bug: Fix passing down `--extrinsic-time-out` config parameter i…
ffarall Dec 19, 2024
aae98fa
feat: :sparkles: Add demo script for `fullnet` initialised
ffarall Dec 19, 2024
c4f7712
fix: :bug: Use new data structure response from `loadFileInStorage` RPC
ffarall Dec 19, 2024
875218f
Merge branch 'main' into demo/mbf-demo
ffarall Dec 26, 2024
4ab8567
fix: :white_check_mark: Use new return type of `loadFileInStorage`
ffarall Dec 26, 2024
836533e
fix: :white_check_mark: Fix access to file metadata after RPC change
ffarall Dec 26, 2024
6833b05
fix: :white_check_mark: Fix destructuring RPC result properly
ffarall Dec 26, 2024
c64b936
fix: :adhesive_bandage: Remove dangling `only: true`
ffarall Dec 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion api-augment/dist/types/interfaces/augment-api-rpc.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ import type {
CheckpointChallenge,
FileMetadata,
GetFileFromFileStorageResult,
LoadFileInStorageResult,
SaveFileToDisk
} from "@storagehub/api-augment/interfaces/storagehubclient";
export type __AugmentedRpc = AugmentedRpc<() => unknown>;
Expand Down Expand Up @@ -1146,7 +1147,7 @@ declare module "@polkadot/rpc-core/types/jsonrpc" {
location: Text | string,
owner: AccountId32 | string | Uint8Array,
bucket_id: H256 | string | Uint8Array
) => Observable<FileMetadata>
) => Observable<LoadFileInStorageResult>
>;
/**
* Remove keys of BCSV type for the Blockchain Service.
Expand Down
2 changes: 2 additions & 0 deletions api-augment/dist/types/interfaces/augment-types.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1270,6 +1270,7 @@ import type {
GetUsersWithDebtOverThresholdError,
IncompleteFileStatus,
IsStorageRequestOpenToVolunteersError,
LoadFileInStorageResult,
MainStorageProviderId,
MerklePatriciaRoot,
Multiaddresses,
Expand Down Expand Up @@ -1883,6 +1884,7 @@ declare module "@polkadot/types/types/registry" {
LegacyTransaction: LegacyTransaction;
Limits: Limits;
LimitsTo264: LimitsTo264;
LoadFileInStorageResult: LoadFileInStorageResult;
LocalValidationData: LocalValidationData;
LockIdentifier: LockIdentifier;
LookupSource: LookupSource;
Expand Down
5 changes: 5 additions & 0 deletions api-augment/dist/types/interfaces/storagehubclient/types.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,11 @@ export interface IsStorageRequestOpenToVolunteersError extends Enum {
}
/** @name Key */
export interface Key extends H256 {}
/** @name LoadFileInStorageResult */
export interface LoadFileInStorageResult extends Struct {
readonly file_key: H256;
readonly file_metadata: FileMetadata;
}
/** @name MainStorageProviderId */
export interface MainStorageProviderId extends H256 {}
/** @name MerklePatriciaRoot */
Expand Down
3 changes: 2 additions & 1 deletion api-augment/src/interfaces/augment-api-rpc.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ import type {
CheckpointChallenge,
FileMetadata,
GetFileFromFileStorageResult,
LoadFileInStorageResult,
SaveFileToDisk
} from "@storagehub/api-augment/interfaces/storagehubclient";

Expand Down Expand Up @@ -1101,7 +1102,7 @@ declare module "@polkadot/rpc-core/types/jsonrpc" {
location: Text | string,
owner: AccountId32 | string | Uint8Array,
bucket_id: H256 | string | Uint8Array
) => Observable<FileMetadata>
) => Observable<LoadFileInStorageResult>
>;
/**
* Remove keys of BCSV type for the Blockchain Service.
Expand Down
2 changes: 2 additions & 0 deletions api-augment/src/interfaces/augment-types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1276,6 +1276,7 @@ import type {
GetUsersWithDebtOverThresholdError,
IncompleteFileStatus,
IsStorageRequestOpenToVolunteersError,
LoadFileInStorageResult,
MainStorageProviderId,
MerklePatriciaRoot,
Multiaddresses,
Expand Down Expand Up @@ -1890,6 +1891,7 @@ declare module "@polkadot/types/types/registry" {
LegacyTransaction: LegacyTransaction;
Limits: Limits;
LimitsTo264: LimitsTo264;
LoadFileInStorageResult: LoadFileInStorageResult;
LocalValidationData: LocalValidationData;
LockIdentifier: LockIdentifier;
LookupSource: LookupSource;
Expand Down
6 changes: 6 additions & 0 deletions api-augment/src/interfaces/storagehubclient/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,12 @@ export interface IsStorageRequestOpenToVolunteersError extends Enum {
/** @name Key */
export interface Key extends H256 {}

/** @name LoadFileInStorageResult */
export interface LoadFileInStorageResult extends Struct {
readonly file_key: H256;
readonly file_metadata: FileMetadata;
}

/** @name MainStorageProviderId */
export interface MainStorageProviderId extends H256 {}

Expand Down
21 changes: 17 additions & 4 deletions client/rpc/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use jsonrpsee::{
proc_macros::rpc,
types::error::{ErrorObjectOwned as JsonRpseeError, INTERNAL_ERROR_CODE, INTERNAL_ERROR_MSG},
};
use log::{debug, error};
use log::{debug, error, info};
use sp_api::ProvideRuntimeApi;
use sp_blockchain::HeaderBackend;
use tokio::{fs, fs::create_dir_all, sync::RwLock};
Expand Down Expand Up @@ -33,6 +33,12 @@ pub struct CheckpointChallenge {
pub should_remove_file: bool,
}

#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub struct LoadFileInStorageResult {
pub file_key: H256,
pub file_metadata: FileMetadata,
}

pub struct StorageHubClientRpcConfig<FL, FSH> {
pub file_storage: Arc<RwLock<FL>>,
pub forest_storage_handler: FSH,
Expand Down Expand Up @@ -102,7 +108,7 @@ pub trait StorageHubClientApi {
location: String,
owner: AccountId32,
bucket_id: H256,
) -> RpcResult<FileMetadata>;
) -> RpcResult<LoadFileInStorageResult>;

#[method(name = "saveFileToDisk")]
async fn save_file_to_disk(
Expand Down Expand Up @@ -215,7 +221,7 @@ where
location: String,
owner: AccountId32,
bucket_id: H256,
) -> RpcResult<FileMetadata> {
) -> RpcResult<LoadFileInStorageResult> {
// Open file in the local file system.
let mut file = File::open(PathBuf::from(file_path.clone())).map_err(into_rpc_error)?;

Expand Down Expand Up @@ -282,7 +288,14 @@ where
.insert_file_with_data(file_key, file_metadata.clone(), file_data_trie)
.map_err(into_rpc_error)?;

Ok(file_metadata)
let result = LoadFileInStorageResult {
file_key,
file_metadata,
};

info!(target: LOG_TARGET, "File loaded successfully: {:?}", result);

Ok(result)
}

async fn save_file_to_disk(
Expand Down
2 changes: 1 addition & 1 deletion node/src/tasks/bsp_upload_file.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -792,7 +792,7 @@ where
.await;

if let Err(e) = result {
debug!(
error!(
target: LOG_TARGET,
"Failed to volunteer for file {:?}: {:?}",
file_key,
Expand Down
171 changes: 86 additions & 85 deletions test/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,87 +1,88 @@
{
"name": "@storagehub/test",
"version": "0.1.0",
"description": "Test package for storagehub",
"main": "index.js",
"type": "module",
"engines": {
"node": "23.x.x"
},
"scripts": {
"bundle-types": "cd ../types-bundle && pnpm i && pnpm build && pnpm fmt:fix && cd ../test",
"typegen": "pnpm bundle-types && cd ../api-augment && pnpm scrape && pnpm generate:all && pnpm build && pnpm fmt:fix; cd ../test",
"fmt": "biome format .",
"fmt:fix": "biome format . --write",
"lint": "biome lint .",
"typecheck": "tsc --noEmit",
"crossbuild:mac": "DOCKER_DEFAULT_PLATFORM=linux/amd64 pnpm tsx scripts/crossBuildMac.ts",
"docker:build": "DOCKER_DEFAULT_PLATFORM=linux/amd64 pnpm tsx scripts/buildLocalDocker.ts",
"docker:start": "docker compose -f ../docker/local-node-compose.yml -p sh_dev_node up -d",
"docker:start:latest": "docker compose -f ../docker/latest-node-compose.yml -p sh_dev_node up -d",
"docker:stop": "docker compose -f ../docker/local-node-compose.yml -p sh_dev_node down",
"docker:stop:all": "docker rm -vf $(docker ps -a --filter 'ancestor=storage-hub:local' -q) $(docker ps -a --filter 'name=sh-postgres' -q) $(docker ps -a --filter 'name=toxiproxy' -q)",
"docker:stop:latest": "docker compose -f ../docker/latest-node-compose.yml -p sh_dev_node down",
"docker:start:bspnet": "pnpm tsx scripts/bspNetBootstrap.ts",
"docker:start:fullnet": "INDEXER=1 pnpm tsx scripts/fullNetBootstrap.ts",
"docker:start:generateBenchmarkProofs": "pnpm tsx scripts/generateBenchmarkProofs.ts",
"docker:stop:fullnet": "docker compose -f ../docker/fullnet-base-template.yml down --remove-orphans && docker volume prune -f",
"docker:stop:bspnet": "docker compose -f ../docker/bspnet-base-template.yml down --remove-orphans && docker volume prune -f",
"docker:stop:generateBenchmarkProofs": "docker compose -f ../docker/local-dev-bsp-compose.yml down --remove-orphans && docker volume prune -f",
"zombie:run:latest": "SH_IMAGE=docker.io/moonsonglabs/storage-hub:latest pnpm zombienet spawn configs/simple.toml",
"zombie:run:local": "DOCKER_BUILDKIT=0 SH_IMAGE=storage-hub:local pnpm zombienet spawn configs/simple.toml",
"zombie:run:native": "pnpm tsx scripts/downloadPolkadot.ts 1.9.0 && pnpm zombienet spawn configs/simpleNative.toml",
"zombie:run:full": "SH_IMAGE=docker.io/moonsonglabs/storage-hub:latest pnpm zombienet spawn configs/fullNetwork.toml",
"zombie:run:full:native": "pnpm tsx scripts/downloadPolkadot.ts 1.9.0 && pnpm zombienet spawn configs/fullNetworkNative.toml",
"zombie:setup:native": "pnpm tsx scripts/fullNetworkSetup.ts",
"zombie:test:native": "pnpm tsx scripts/downloadPolkadot.ts 1.9.0 && pnpm zombienet test --provider native configs/simpleNative.zndsl",
"zombie:test:local": "SH_IMAGE=storage-hub:local pnpm zombienet test configs/simple.zndsl",
"zombie:test:latest": "SH_IMAGE=docker.io/moonsonglabs/storage-hub:latest pnpm zombienet test configs/simple.zndsl",
"test:full": "node --no-deprecation --import tsx --test ./suites/zombie/**.spec.ts",
"test:fullnet": "NODE_OPTIONS='--no-deprecation' pnpm tsx scripts/checkRunning.ts && node --no-deprecation --test-concurrency 1 --import tsx --test ./suites/integration/msp/**.test.ts",
"test:fullnet:only": "NODE_OPTIONS='--no-deprecation' pnpm tsx scripts/checkRunning.ts && node --no-deprecation --test-concurrency 1 --import tsx --test --test-only ./suites/integration/msp/**.test.ts",
"test:bspnet": "NODE_OPTIONS='--no-deprecation' pnpm tsx scripts/checkRunning.ts && node --no-deprecation --test-concurrency 1 --import tsx --test ./suites/integration/bsp/**.test.ts",
"test:bspnet:only": "NODE_OPTIONS='--no-deprecation' pnpm tsx scripts/checkRunning.ts && node --no-deprecation --import tsx --test-concurrency 1 --test --test-only ./suites/integration/bsp/**.test.ts",
"test:user": "NODE_OPTIONS='--no-deprecation' pnpm tsx scripts/checkRunning.ts && node --no-deprecation --test-concurrency 1 --import tsx --test ./suites/integration/user/**.test.ts",
"test:user:only": "NODE_OPTIONS='--no-deprecation' pnpm tsx scripts/checkRunning.ts && node --no-deprecation --test-concurrency 1 --import tsx --test --test-only ./suites/integration/user/**.test.ts",
"test:node": "node --no-deprecation --import tsx --test ./suites/solo-node/**/**.test.ts",
"test:node:only": "node --no-deprecation --import tsx --test --test-only ./suites/solo-node/**/**.test.ts",
"test:node:single": "node --no-deprecation --import tsx --test --test-name-pattern=$FILTER ./suites/solo-node/**/**.test.ts"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"@polkadot/api": "12.4.2",
"@polkadot/api-augment": "12.4.2",
"@polkadot/keyring": "13.0.2",
"@polkadot/types": "12.4.2",
"@polkadot/util": "13.0.2",
"@polkadot/util-crypto": "13.0.2",
"@polkadot/wasm-crypto": "7.3.2",
"@reporters/github": "1.7.0",
"@storagehub/api-augment": "workspace:*",
"@storagehub/types-bundle": "workspace:*",
"@types/tmp": "^0.2.6",
"@zombienet/cli": "1.3.109",
"@zombienet/utils": "0.0.25",
"docker-compose": "1.1.0",
"dockerode": "4.0.2",
"dotenv": "16.4.5",
"inquirer": "10.2.0",
"json-bigint": "^1.0.0",
"postgres": "^3.4.5",
"strip-ansi": "^7.1.0",
"testcontainers": "10.13.0",
"tmp": "0.2.3",
"tsx": "4.19.0",
"yaml": "2.5.1"
},
"devDependencies": {
"@biomejs/biome": "1.8.3",
"@types/dockerode": "3.3.31",
"@types/inquirer": "9.0.7",
"@types/json-bigint": "1.0.4",
"@types/node": "22.5.2",
"typescript": "5.5.4"
}
"name": "@storagehub/test",
"version": "0.1.0",
"description": "Test package for storagehub",
"main": "index.js",
"type": "module",
"engines": {
"node": "23.x.x"
},
"scripts": {
"bundle-types": "cd ../types-bundle && pnpm i && pnpm build && pnpm fmt:fix && cd ../test",
"typegen": "pnpm bundle-types && cd ../api-augment && pnpm scrape && pnpm generate:all && pnpm build && pnpm fmt:fix; cd ../test",
"fmt": "biome format .",
"fmt:fix": "biome format . --write",
"lint": "biome lint .",
"typecheck": "tsc --noEmit",
"crossbuild:mac": "DOCKER_DEFAULT_PLATFORM=linux/amd64 pnpm tsx scripts/crossBuildMac.ts",
"docker:build": "DOCKER_DEFAULT_PLATFORM=linux/amd64 pnpm tsx scripts/buildLocalDocker.ts",
"docker:start": "docker compose -f ../docker/local-node-compose.yml -p sh_dev_node up -d",
"docker:start:latest": "docker compose -f ../docker/latest-node-compose.yml -p sh_dev_node up -d",
"docker:stop": "docker compose -f ../docker/local-node-compose.yml -p sh_dev_node down",
"docker:stop:all": "docker rm -vf $(docker ps -a --filter 'ancestor=storage-hub:local' -q) $(docker ps -a --filter 'name=sh-postgres' -q) $(docker ps -a --filter 'name=toxiproxy' -q)",
"docker:stop:latest": "docker compose -f ../docker/latest-node-compose.yml -p sh_dev_node down",
"docker:start:bspnet": "pnpm tsx scripts/bspNetBootstrap.ts",
"docker:start:fullnet": "INDEXER=1 pnpm tsx scripts/fullNetBootstrap.ts",
"docker:start:fullnet:initialised": "INDEXER=1 pnpm tsx scripts/fullNetBootstrapInitialised.ts",
"docker:start:generateBenchmarkProofs": "pnpm tsx scripts/generateBenchmarkProofs.ts",
"docker:stop:fullnet": "docker compose -f ../docker/fullnet-base-template.yml down --remove-orphans && docker volume prune -f",
"docker:stop:bspnet": "docker compose -f ../docker/bspnet-base-template.yml down --remove-orphans && docker volume prune -f",
"docker:stop:generateBenchmarkProofs": "docker compose -f ../docker/local-dev-bsp-compose.yml down --remove-orphans && docker volume prune -f",
"zombie:run:latest": "SH_IMAGE=docker.io/moonsonglabs/storage-hub:latest pnpm zombienet spawn configs/simple.toml",
"zombie:run:local": "DOCKER_BUILDKIT=0 SH_IMAGE=storage-hub:local pnpm zombienet spawn configs/simple.toml",
"zombie:run:native": "pnpm tsx scripts/downloadPolkadot.ts 1.9.0 && pnpm zombienet spawn configs/simpleNative.toml",
"zombie:run:full": "SH_IMAGE=docker.io/moonsonglabs/storage-hub:latest pnpm zombienet spawn configs/fullNetwork.toml",
"zombie:run:full:native": "pnpm tsx scripts/downloadPolkadot.ts 1.9.0 && pnpm zombienet spawn configs/fullNetworkNative.toml",
"zombie:setup:native": "pnpm tsx scripts/fullNetworkSetup.ts",
"zombie:test:native": "pnpm tsx scripts/downloadPolkadot.ts 1.9.0 && pnpm zombienet test --provider native configs/simpleNative.zndsl",
"zombie:test:local": "SH_IMAGE=storage-hub:local pnpm zombienet test configs/simple.zndsl",
"zombie:test:latest": "SH_IMAGE=docker.io/moonsonglabs/storage-hub:latest pnpm zombienet test configs/simple.zndsl",
"test:full": "node --no-deprecation --import tsx --test ./suites/zombie/**.spec.ts",
"test:fullnet": "NODE_OPTIONS='--no-deprecation' pnpm tsx scripts/checkRunning.ts && node --no-deprecation --test-concurrency 1 --import tsx --test ./suites/integration/msp/**.test.ts",
"test:fullnet:only": "NODE_OPTIONS='--no-deprecation' pnpm tsx scripts/checkRunning.ts && node --no-deprecation --test-concurrency 1 --import tsx --test --test-only ./suites/integration/msp/**.test.ts",
"test:bspnet": "NODE_OPTIONS='--no-deprecation' pnpm tsx scripts/checkRunning.ts && node --no-deprecation --test-concurrency 1 --import tsx --test ./suites/integration/bsp/**.test.ts",
"test:bspnet:only": "NODE_OPTIONS='--no-deprecation' pnpm tsx scripts/checkRunning.ts && node --no-deprecation --import tsx --test-concurrency 1 --test --test-only ./suites/integration/bsp/**.test.ts",
"test:user": "NODE_OPTIONS='--no-deprecation' pnpm tsx scripts/checkRunning.ts && node --no-deprecation --test-concurrency 1 --import tsx --test ./suites/integration/user/**.test.ts",
"test:user:only": "NODE_OPTIONS='--no-deprecation' pnpm tsx scripts/checkRunning.ts && node --no-deprecation --test-concurrency 1 --import tsx --test --test-only ./suites/integration/user/**.test.ts",
"test:node": "node --no-deprecation --import tsx --test ./suites/solo-node/**/**.test.ts",
"test:node:only": "node --no-deprecation --import tsx --test --test-only ./suites/solo-node/**/**.test.ts",
"test:node:single": "node --no-deprecation --import tsx --test --test-name-pattern=$FILTER ./suites/solo-node/**/**.test.ts"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"@polkadot/api": "12.4.2",
"@polkadot/api-augment": "12.4.2",
"@polkadot/keyring": "13.0.2",
"@polkadot/types": "12.4.2",
"@polkadot/util": "13.0.2",
"@polkadot/util-crypto": "13.0.2",
"@polkadot/wasm-crypto": "7.3.2",
"@reporters/github": "1.7.0",
"@storagehub/api-augment": "workspace:*",
"@storagehub/types-bundle": "workspace:*",
"@types/tmp": "^0.2.6",
"@zombienet/cli": "1.3.109",
"@zombienet/utils": "0.0.25",
"docker-compose": "1.1.0",
"dockerode": "4.0.2",
"dotenv": "16.4.5",
"inquirer": "10.2.0",
"json-bigint": "^1.0.0",
"postgres": "^3.4.5",
"strip-ansi": "^7.1.0",
"testcontainers": "10.13.0",
"tmp": "0.2.3",
"tsx": "4.19.0",
"yaml": "2.5.1"
},
"devDependencies": {
"@biomejs/biome": "1.8.3",
"@types/dockerode": "3.3.31",
"@types/inquirer": "9.0.7",
"@types/json-bigint": "1.0.4",
"@types/node": "22.5.2",
"typescript": "5.5.4"
}
}
Loading
Loading