From 16042ae28bf9f4dc42e940c9610f7c9bc1ec9b41 Mon Sep 17 00:00:00 2001 From: Abhishek Harde <47945971+abhiyana@users.noreply.github.com> Date: Mon, 4 Dec 2023 16:34:01 +0530 Subject: [PATCH] docs: add read.md file and docstring inside starlark package (#144) * fix: wait long for validation from relaychain node * fix: mangata local, testnet, mainnet node issue * docs: add read.md documentation file * docs: add doc strings inside code * fix: fix ident in main.star and parachain.star * chore: change file name from READ.md to README.md --------- Co-authored-by: Shreyas S Bhat <35568964+shreyasbhat0@users.noreply.github.com> --- README.md | 180 ++++++++++++++++++++++++++++- local.json | 7 +- main.star | 17 ++- package_io/build-spec.star | 39 +++++-- package_io/constant.star | 2 +- package_io/polkadot_js_app.star | 4 +- parachain/parachain.star | 43 +++++++ parachain/static_files/images.star | 2 +- relaychain/relay-chain.star | 65 ++++++++++- 9 files changed, 340 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 2033abc..17bb736 100644 --- a/README.md +++ b/README.md @@ -1 +1,179 @@ -# Polkadot Kurtosis Package +# Polkadot-kurtosis-package + +Polkadot-kurtosis-package is a tool built leveraging the power of Kurtosis, a developer platform for packaging and launching environments. This tool simplifies the process of setting up various local, testnet, and mainnet network configurations and scenarios for Polkadot parachains. + +## About + +The primary goal of this project is to streamline the setup of Polkadot parachain environments using the Kurtosis platform. With just a few one-liners, developers can package and launch environments tailored to their needs. + +## Setup and Requirements + +Ensure the following prerequisites are met before using the Polkadot-kurtosis-package: +- Docker installed on your machine[https://www.docker.com/] +- Kurtosis installed on your machine[https://www.kurtosis.com/] + +# Integrated Parachains + +List of integrated parachains within the Polkadot-kurtosis-package. + +- acala +- ajuna +- bifrost +- centrifuge +- clover +- frequency +- integritee +- interlay +- kilt +- kylin +- litentry +- manta +- moonbeam +- moonsama +- nodle +- parallel +- pendulum +- phala +- polkadex +- subsocial +- zeitgeist +- encointer +- altair +- bajun +- calamari +- karura +- khala +- kintsugi-btc +- litmus +- mangata +- moonriver +- robonomics +- subzero +- turing + + + +## Chopstick Compatibility + +The package also supports Chopsticks, a tool offering a user-friendly approach to locally branching existing Substrate-based chains. It enables block replay, multi-block forking, and more. + +### Parachains Compatible with Chopsticks + +- astar +- basilisk +- acala +- centrifuge +- composable-polkadot +- hydradx +- imbu +- interlay +- karura +- mandala +- mangata +- moonbase +- moonbeam +- moonriver +- nodle-eden +- picasso-kusama +- picasso-rococo +- polkadex +- shibuya +- shiden +- statemine +- statemint +-ß tinkernet + +Check more info on Chopsticks, refer to the [Chopsticks]. + +## Zombienet + +Zombienet is a CLI tool that facilitates the creation and testing of ephemeral Polkadot/Substrate networks. It offers a straightforward interface for developers to spawn and examine these temporary networks, enabling them to thoroughly evaluate their applications and protocols within a simulated real-world environment. + + +## Usage + +To use the package, run the following command inside the root directory of your project: + +```bash +kurtosis run . --enclave 'enclavename' --args-file=path/to/config/file +``` + +For detailed instructions on writing the configuration file, refer to the [Configuration File Guidelines] + +Certainly! Let's create a section in your README file to explain how to write the configuration file using the provided example. I'll include a breakdown of each field and provide explanations: + +# Configuration File Guidelines + +To use the Polkadot-kurtosis-package, you need to create a configuration file specifying the desired network setup. Below is an example configuration file along with explanations for each field: + +```json +{ + "chain-type": "testnet", + "relaychain": { + "name": "rococo", + "nodes": [ + { + "name": "alice", + "node-type": "validator", + "port": 9944, + "prometheus": false + }, + { + "name": "bob", + "node-type": "full", + "port": 9945, + "prometheus": false + } + ] + }, + "para": [ + { + "name":"kilt", + "nodes": [ + { + "name": "alice", + "node-type": "validator", + "prometheus": false + }, + { + "name": "bob", + "node-type": "full", + "prometheus": false + } + ] + } + ], + "chopstick": { + "xcm": false, + "relaychain":"", + "parachains": ["acala"] + } +} +``` + +## Configuration Fields: + +- **chain-type:** Specifies the type of the network (e.g., "localnet","testnet", "mainnet"). +- **relaychain:** Configuration for the relay chain. (When chain-type is "testnet" or "mainenet", the "relaychain" can be empty dictonary) + - **name:** Name of the relay chain (e.g., "rococo-local", "rococo", "polkadot" or "kusama"). + - **nodes:** List of nodes on the relay chain, each with: + - **name:** Node name (e.g., "alice"). + - **node-type:** Node type, can be "validator" or "full". + - **port:** Port number for the node (e.g., 9944). + - **prometheus:** Whether Prometheus monitoring is enabled (true/false). +- **para:** List of parachains, each with: + - **name:** Parachain name (e.g., "kilt"). + - **nodes:** List of nodes on the parachain, similar to relay chain nodes. + - **name:** Node name (e.g., "alice"). + - **node-type:** Node type, can be "callator" or "full". + - **prometheus:** Whether Prometheus monitoring is enabled (true/false). +- **chopstick:** Configuration for Chopstick integration. + - **xcm:** Whether XCM (Cross-Chain Messaging) is enabled (true/false). + - **relaychain:** Name of the relay chain for Chopstick integration. + - **parachains:** List of parachains compatible with Chopsticks. + +Feel free to modify this example configuration file based on your specific network requirements. + +## Contributing + +We welcome contributions to enhance and expand the functionality of the Polkadot-kurtosis-package. Feel free to fork the repository, make your changes, and submit a pull request. diff --git a/local.json b/local.json index 8f3b3b1..9621519 100644 --- a/local.json +++ b/local.json @@ -1,5 +1,5 @@ { - "chain-type": "local", + "chain-type": "testnet", "relaychain": { "name": "rococo", "nodes": [ @@ -21,7 +21,7 @@ "para": [ { - "name":"acala", + "name":"mangata", "nodes": [ { "name": "alice", @@ -36,5 +36,6 @@ } ] } - ] + ], + "explorer": false } \ No newline at end of file diff --git a/main.star b/main.star index 34b12de..851bc47 100644 --- a/main.star +++ b/main.star @@ -3,8 +3,19 @@ relay_chain = import_module("./relaychain/relay-chain.star") package = import_module("./package_io/build-spec.star") promethues = import_module("./package_io/promethues.star") grafana = import_module("./package_io/grafana.star") +explorer = import_module("./package_io/polkadot_js_app.star") def run(plan, args): + """ + Main function to run the Polkadot relay and parachain setup. + + Args: + plan (object): The Kurtosis plan object for orchestrating the test. + args (dict): Dictionary containing arguments for configuring the setup. + + Returns: + dict: Service details containing information about relay chains, parachains, and Prometheus. + """ plan.upload_files(src = "./parachain/static_files/configs", name = "configs") prometheus_template = read_file("./package_io/static_files/prometheus.yml.tmpl") @@ -33,4 +44,8 @@ def run(plan, args): service_details["prometheus"] = prometheus_address grafana.launch_grafana(plan, grafana) - return service_details + #run the polkadot js App explorer + if args["explorer"] == True: + service_details["explorer"] = explorer.run_pokadot_js_app(plan, "wss://127.0.0.1:9944") + + return service_details \ No newline at end of file diff --git a/package_io/build-spec.star b/package_io/build-spec.star index 286661f..d3c077a 100644 --- a/package_io/build-spec.star +++ b/package_io/build-spec.star @@ -1,4 +1,16 @@ + def create_service_for_build_spec(plan, service_name, image, build_file): + """Create a service based on a build specification. + + Args: + plan (object): The execution plan to add the service to. + service_name (str): Name of the service. + image (str): Docker image for the service. + build_file (str): Path to the build file. + + Returns: + object: The created service. + """ files = { "/app": "configs", } @@ -6,23 +18,36 @@ def create_service_for_build_spec(plan, service_name, image, build_file): files["/build"] = build_file service = plan.add_service( - name = service_name, - config = ServiceConfig( - image = image, - files = files, - entrypoint = ["/bin/sh"], + name=service_name, + config=ServiceConfig( + image=image, + files=files, + entrypoint=["/bin/sh"], ), ) return service def create_edit_and_build_spec(plan, service_name, image, chain_name, command, build_file): + """Create, edit, and build a service based on a build specification. + + Args: + plan (object): The execution plan to add the service to. + service_name (str): Name of the service. + image (str): Docker image for the service. + chain_name (str): Name of the chain. + command (list): Command to execute inside the Docker container. + build_file (str): Path to the build file. + + Returns: + object: The created and built service. + """ service = create_service_for_build_spec(plan, service_name, image, build_file) - result = plan.exec(service_name = service_name, recipe = command) + result = plan.exec(service_name=service_name, recipe=command) plan.verify(result["code"], "==", 0) - plan.store_service_files(service_name = service_name, src = "/tmp/{0}.json".format(chain_name), name = service_name) + plan.store_service_files(service_name=service_name, src="/tmp/{0}.json".format(chain_name), name=service_name) plan.remove_service(service_name) diff --git a/package_io/constant.star b/package_io/constant.star index f3c7d34..99a7068 100644 --- a/package_io/constant.star +++ b/package_io/constant.star @@ -5,7 +5,7 @@ CURL_JQ_IMAGE = "badouralix/curl-jq" NODE_IMAGE = "hugobyte/parachain-node-modules" PARA_SLOT_REGISTER_SERVICE_NAME = "para-slot-registration" BINARY_COMMAND_CHAINS = ["manta", "khala", "phala", "clover", "calamari", "subzero", "robonomics"] -NO_WS_PORT = ["acala", "frequency", "moonbeam", "karura", "ajuna", "bajun", "centrifuge", "moonsama", "encointer", "moonriver", "altair"] +NO_WS_PORT = ["acala", "frequency", "moonbeam", "karura", "ajuna", "bajun", "centrifuge", "moonsama", "encointer", "moonriver", "altair", "mangata"] DIFFERENT_IMAGES_FOR_MAINNET = { "centrifuge": "centrifugeio/centrifuge-chain:main-latest", diff --git a/package_io/polkadot_js_app.star b/package_io/polkadot_js_app.star index 265c26f..40b0022 100644 --- a/package_io/polkadot_js_app.star +++ b/package_io/polkadot_js_app.star @@ -21,5 +21,5 @@ def run_pokadot_js_app(plan, ws_url): "WS_URL": ws_url, } ) - plan.add_service(name="polkadot-js", config=service_config) - \ No newline at end of file + service_details = plan.add_service(name="polkadot-js", config=service_config) + return service_details \ No newline at end of file diff --git a/parachain/parachain.star b/parachain/parachain.star index 6c3db7c..41eb6f3 100644 --- a/parachain/parachain.star +++ b/parachain/parachain.star @@ -5,6 +5,18 @@ parachain_list = import_module("./static_files/images.star") node_setup = import_module("./node_setup.star") def spawn_parachain(plan, chain_name, image, command, build_file): + """Spawn a parachain node with specified configuration. + + Args: + plan (object): The Kurtosis plan. + chain_name (str): Name of the parachain. + image (str): Docker image for the parachain node. + command (list): Command to execute inside service. + build_file (str): Path to the build spec file. + + Returns: + dict: The service details of spawned parachain node. + """ files = { "/app": "configs", } @@ -27,6 +39,17 @@ def spawn_parachain(plan, chain_name, image, command, build_file): return parachain_node def start_local_parachain_node(plan, args, parachain_config, para_id): + """Start local parachain nodes based on configuration. + + Args: + plan (object): The Kurtosis plan. + args (dict): arguments for configuration. + parachain_config (dict): Configuration for the parachain. + para_id (int): Parachain ID. + + Returns: + list: List of dictionaries containing service details of parachain nodes. + """ parachain = parachain_config["name"].lower() parachain_details = parachain_list.parachain_images[parachain] image = parachain_details["image"] @@ -58,6 +81,16 @@ def start_local_parachain_node(plan, args, parachain_config, para_id): return parachain_final def start_nodes(plan, args, relay_chain_ip): + """Start multiple parachain nodes. + + Args: + plan (object): The kurtosis plan. + args (dict): arguments for configuration. + relay_chain_ip (str): IP address of the relay chain. + + Returns: + list: List of dictionaries containing service details of each parachain. + """ parachains = args["para"] final_parachain_details = [] for parachain in parachains: @@ -71,6 +104,16 @@ def start_nodes(plan, args, relay_chain_ip): return final_parachain_details def run_testnet_mainnet(plan, parachain, args): + """Run a testnet or mainnet based on configuration. + + Args: + plan (object): The kurtosis plan. + parachain (dict): Configuration for the parachain. + args (dict): arguments for configuration. + + Returns: + list: List of dictionaries containing details of each parachain node. + """ if args["chain-type"] == "testnet": main_chain = "rococo" if parachain["name"] == "ajuna": diff --git a/parachain/static_files/images.star b/parachain/static_files/images.star index 3bfe5d7..d68f03c 100644 --- a/parachain/static_files/images.star +++ b/parachain/static_files/images.star @@ -148,7 +148,7 @@ parachain_images = { "mangata": { "image": "mangatasolutions/mangata-node:feature-post-3rdparty-rewards-fast", "entrypoint": "/mangata/node", - "base": ["rococo-local", "mangata-rococo", "mangata-kusama"], + "base": ["mangata-rococo-local", "mangata-rococo", "mangata-kusama"], }, "moonriver": { "image": "moonbeamfoundation/moonbeam:sha-519bd694", diff --git a/relaychain/relay-chain.star b/relaychain/relay-chain.star index e63d491..4c00584 100644 --- a/relaychain/relay-chain.star +++ b/relaychain/relay-chain.star @@ -1,4 +1,14 @@ def start_relay_chain(plan, args): + """ + Starts relay chain nodes based on the provided arguments. + + Args: + plan (object): The Kurtosis plan object for orchestrating the test. + args (dict): Dictionary containing arguments for configuring the relay chain setup. + + Returns: + list: List of dictionaries containing service details of started relay chain nodes. + """ name = args["chain-type"] chain = args["relaychain"]["name"] final_details=[] @@ -30,29 +40,65 @@ def start_relay_chain(plan, args): return final_details + def start_test_main_net_relay_nodes(plan, args): + """ + Starts testnet/mainnet relay nodes based on the provided arguments. + + Args: + plan (object): The Kurtosis plan object for orchestrating the test. + args (dict): Dictionary containing arguments for configuring the relay node setup. + + Returns: + list: List of dictionaries containing service details of started relay nodes. + """ name = args["chain-type"] chain = args["relaychain"]["name"] + if name == "testnet": if chain != "rococo" and chain != "westend": fail("Please provide rococo or westend as relaychain for testnet") elif name == "mainnet": if chain != "polkadot" and chain != "kusama": fail("Please provide polkadot or kusama as relaychain for mainnet") + relay_node_details = start_relay_chain(plan, args) return relay_node_details + def spawn_multiple_relay(plan, count): - list = ["alice", "bob", "dave", "charlie"] + """ + Spawns multiple local relay chain nodes. + + Args: + plan (object): The Kurtosis plan object for orchestrating the test. + count (int): Number of relay nodes to spawn. + """ + node_list = ["alice", "bob", "dave", "charlie"] port = 9944 for i in range(0, count): port = port + count - start_relay_chain_local(plan, list[i], port) + start_relay_chain_local(plan, node_list[i], port) + def start_relay_chains_local(plan, args): + """ + Starts local relay chain nodes based on the provided arguments. + + Args: + plan (object): The Kurtosis plan object for orchestrating the test. + args (dict): Dictionary containing arguments for configuring the relay chain setup. + + Returns: + list: List of dictionaries containing sevice details of started relay chain nodes. + """ relay_nodes = args["relaychain"]["nodes"] - final_details=[] + + if len(relay_nodes) < 2: + fail("relay nodes must contain at least two nodes") + + final_details = [] prometheus_port = 9615 for node in relay_nodes: relay_detail = {} @@ -63,7 +109,20 @@ def start_relay_chains_local(plan, args): prometheus_port = prometheus_port + 1 return final_details + def start_relay_chain_local(plan, name, port, prometheus_port): + """ + Starts a local relay chain node based on the provided arguments. + + Args: + plan (object): The Kurtosis plan + name (str): Name of the relay chain node. + port (int): Port number for the relay chain node. + prometheus_port (int): Port number for Prometheus. + + Returns: + object: Service details of the started relay chain node. + """ exec_command = ["bin/sh", "-c", "polkadot --base-path=/data --chain=/app/raw-polkadot.json --validator --rpc-external --rpc-cors=all --name=alice --{0} --rpc-methods=unsafe --execution=wasm --prometheus-external".format(name)] service_details = plan.add_service( name = "polkadot-{0}".format(name),