From 63d6772597c3626d71015391a4871c081b463d0a Mon Sep 17 00:00:00 2001 From: riyasng12 Date: Tue, 9 Jan 2024 17:36:10 +0530 Subject: [PATCH 1/4] refactor: refactor public ports for relay and para chains --- package_io/constant.star | 43 ++++++++++- package_io/promethues.star | 2 +- package_io/utils.star | 17 ++++- parachain/node_setup.star | 109 ++++++++++++++++++++++++---- parachain/parachain.star | 108 ++++++++++++++-------------- relaychain/relay-chain.star | 140 +++++++++++++++++++++++------------- 6 files changed, 302 insertions(+), 117 deletions(-) diff --git a/package_io/constant.star b/package_io/constant.star index 8c481b0..2cb4b70 100644 --- a/package_io/constant.star +++ b/package_io/constant.star @@ -5,7 +5,7 @@ CURL_JQ_IMAGE = "badouralix/curl-jq" NODE_IMAGE = "hugobyte/parachain-node-modules" PARA_SLOT_REGISTER_SERVICE_NAME = "para-slot-registration" BINARY_COMMAND_CHAINS = ["manta", "khala", "phala", "clover", "calamari", "subzero", "robonomics"] -NO_WS_PORT = ["acala", "frequency", "moonbeam", "karura", "ajuna", "bajun", "centrifuge", "moonsama", "encointer", "moonriver", "altair", "mangata", "khala", "phala", "turing", "bifrost", "khala", "phala"] +NO_WS_PORT = ["acala", "frequency", "moonbeam", "karura", "ajuna", "bajun", "centrifuge", "moonsama", "encointer", "moonriver", "altair", "mangata", "khala", "phala", "turing", "bifrost", "khala", "phala", "nodle"] DIFFERENT_IMAGES_FOR_MAINNET = { "centrifuge": "centrifugeio/centrifuge-chain:main-latest", @@ -24,3 +24,44 @@ DIFFERENT_IMAGES_FOR_TESTNET = { } CHAIN_COMMAND = ["manta", "moonsama", "interlay", "kintsugi-btc", "polkadex", "centrifuge", "altair", "robonomics", "kilt"] + +KUSAMA_PARACHAINS = [ + "altair", + "bajun", + "bifrost", + "calamari", + "encointer", + "khala", + "kintsugi-btc", + "litmus", + "mangata", + "moonriver", + "robonomics", + "subzero", + "turing" +] + +POLKADOT_PARACHAINS = [ + "acala", + "ajuna", + "bifrost", + "centrifuge", + "clover", + "frequency", + "integritee", + "interlay", + "karura", + "kilt", + "kylin", + "litentry", + "manta", + "moonbeam", + "moonsama", + "nodle", + "parallel", + "pendulum", + "phala", + "polkadex", + "subsocial", + "zeitgeist" +] \ No newline at end of file diff --git a/package_io/promethues.star b/package_io/promethues.star index c49df73..b83081b 100644 --- a/package_io/promethues.star +++ b/package_io/promethues.star @@ -83,7 +83,7 @@ def get_config(config_files_artifact_name): def new_config_template_data(plan, args, service_details): metrics_jobs = [] for service in service_details: - if service_details[service]["prometheus"] == True: + if "prometheus" in service_details[service] and service_details[service]["prometheus"] == True: ip = service_details[service]["ip_address"] port_number = service_details[service]["prometheus_port"] endpoint = "{0}:{1}".format(ip, port_number) diff --git a/package_io/utils.star b/package_io/utils.star index c3d5bf6..5528197 100644 --- a/package_io/utils.star +++ b/package_io/utils.star @@ -1,3 +1,5 @@ +constant = import_module("./constant.star") + NOT_PROVIDED_APPLICATION_PROTOCOL = "" NOT_PROVIDED_WAIT = "not-provided-wait" @@ -70,7 +72,20 @@ def check_config_validity(plan, args): plan.print("config for parachain is valid") else: return fail("parachain node-type can be only validator/full/collator") - + + if args["relaychain"] != {}: + chain = args["relaychain"]["name"] + if chain == "polkadot": + if len(args["para"]) != 0: + for para in args["para"]: + if para["name"] not in constant.POLKADOT_PARACHAINS: + return fail("Invalid parachain for POLKADOT") + if chain == "kusama": + if len(args["para"]) != 0: + for para in args["para"]: + if para["name"] not in constant.KUSAMA_PARACHAINS: + return fail("Invalid parachain for KUSAMA") + def upload_files(plan): plan.upload_files(src = "../parachain/static_files/configs", name = "configs") plan.upload_files(src = "../parachain/static_files/javascript", name = "javascript") diff --git a/parachain/node_setup.star b/parachain/node_setup.star index 6b3b9fb..1788219 100644 --- a/parachain/node_setup.star +++ b/parachain/node_setup.star @@ -1,11 +1,28 @@ -def run_testnet_node_with_entrypoint(plan, image, chain_name, execute_command): +def run_testnet_node_with_entrypoint(plan, prometheus, image, chain_name, execute_command, rpc_port = None, prometheus_port = None, lib2lib_port = None): + + ports = { + "ws": PortSpec(9947, transport_protocol = "TCP"), + "lib2lib": PortSpec(30333, transport_protocol = "TCP") + } + + public_ports = {} + + if rpc_port != None : + public_ports["ws"] = PortSpec(rpc_port, transport_protocol = "TCP") + + if lib2lib_port != None: + public_ports["lib2lib"] = PortSpec(lib2lib_port, transport_protocol = "TCP") + + if prometheus: + ports["metrics"] = PortSpec(9615, transport_protocol = "TCP", application_protocol = "http") + + if prometheus_port != None: + public_ports["metrics"] = PortSpec(prometheus_port, transport_protocol = "TCP", application_protocol = "http") + service_config = ServiceConfig( image = image, - ports = { - "ws": PortSpec(9947, transport_protocol = "TCP"), - "metrics": PortSpec(9615, transport_protocol = "TCP", application_protocol = "http"), - "lib": PortSpec(30333), - }, + ports = ports, + public_ports = public_ports, files = { "/app": "configs", }, @@ -15,14 +32,31 @@ def run_testnet_node_with_entrypoint(plan, image, chain_name, execute_command): return parachain -def run_testnet_node_with_command(plan, image, chain_name, execute_command): +def run_testnet_node_with_command(plan, prometheus, image, chain_name, execute_command, rpc_port = None, prometheus_port = None, lib2lib_port = None): + + ports = { + "ws": PortSpec(9947, transport_protocol = "TCP"), + "lib": PortSpec(30333) + } + + public_ports = {} + + if rpc_port != None : + public_ports["ws"] = PortSpec(rpc_port, transport_protocol = "TCP") + + if lib2lib_port != None: + public_ports["lib"] = PortSpec(lib2lib_port, transport_protocol = "TCP") + + if prometheus: + ports["metrics"] = PortSpec(9615, transport_protocol = "TCP", application_protocol = "http") + + if prometheus_port != None: + public_ports["metrics"] = PortSpec(prometheus_port, transport_protocol = "TCP", application_protocol = "http") + service_config = ServiceConfig( image = image, - ports = { - "ws": PortSpec(9947, transport_protocol = "TCP"), - "metrics": PortSpec(9615, transport_protocol = "TCP", application_protocol = "http"), - "lib": PortSpec(30333), - }, + ports = ports, + public_ports = public_ports, files = { "/app": "configs", }, @@ -31,3 +65,54 @@ def run_testnet_node_with_command(plan, image, chain_name, execute_command): parachain = plan.add_service(name = "{0}".format(chain_name), config = service_config) return parachain + +def spawn_parachain(plan, prometheus, chain_name, image, command, build_file, rpc_port = None, prometheus_port = None, lib2lib_port = None): + """Spawn a parachain node with specified configuration. + + Args: + plan (object): The Kurtosis plan. + chain_name (str): Name of the parachain. + image (str): Docker image for the parachain node. + command (list): Command to execute inside service. + build_file (str): Path to the build spec file. + + Returns: + dict: The service details of spawned parachain node. + """ + files = { + "/app": "configs", + } + if build_file != None: + files["/build"] = build_file + + ports = { + "ws": PortSpec(9946, transport_protocol = "TCP", application_protocol = "http"), + "lib": PortSpec(30333, transport_protocol = "TCP", application_protocol = "http"), + } + + public_ports = {} + + if rpc_port != None : + public_ports["ws"] = PortSpec(rpc_port, transport_protocol = "TCP", application_protocol = "http") + + if lib2lib_port != None: + public_ports["lib2lib"] = PortSpec(lib2lib_port, transport_protocol = "TCP", application_protocol = "http") + + if prometheus: + ports["metrics"] = PortSpec(9615, transport_protocol = "TCP", application_protocol = "http") + + if prometheus_port != None: + public_ports["metrics"] = PortSpec(prometheus_port, transport_protocol = "TCP", application_protocol = "http") + + parachain_node = plan.add_service( + name = "{}".format(chain_name), + config = ServiceConfig( + image = image, + files = files, + ports = ports, + public_ports = public_ports, + entrypoint = command, + ), + ) + + return parachain_node diff --git a/parachain/parachain.star b/parachain/parachain.star index 8bec0e9..32d88a1 100644 --- a/parachain/parachain.star +++ b/parachain/parachain.star @@ -5,42 +5,7 @@ parachain_list = import_module("./static_files/images.star") node_setup = import_module("./node_setup.star") utils = import_module("../package_io/utils.star") -def spawn_parachain(plan, chain_name, image, command, build_file): - """Spawn a parachain node with specified configuration. - - Args: - plan (object): The Kurtosis plan. - chain_name (str): Name of the parachain. - image (str): Docker image for the parachain node. - command (list): Command to execute inside service. - build_file (str): Path to the build spec file. - - Returns: - dict: The service details of spawned parachain node. - """ - files = { - "/app": "configs", - } - if build_file != None: - files["/build"] = build_file - - parachain_node = plan.add_service( - name = "{}".format(chain_name), - config = ServiceConfig( - image = image, - files = files, - ports = { - "ws": PortSpec(9946, transport_protocol = "TCP", application_protocol = "http"), - "metrics": PortSpec(9615, transport_protocol = "TCP", application_protocol = "http"), - "lib": PortSpec(30333, transport_protocol = "TCP", application_protocol = "http"), - }, - entrypoint = command, - ), - ) - - return parachain_node - -def start_local_parachain_node(plan, args, parachain_config, para_id): +def start_local_parachain_node(plan, args, parachain_config, para_id, rpc_port = [], prometheus_port = [], lib2lib_port = []): """Start local parachain nodes based on configuration. Args: @@ -61,8 +26,12 @@ def start_local_parachain_node(plan, args, parachain_config, para_id): raw_service = build_spec.create_parachain_build_spec_with_para_id(plan, image, binary, chain_name, chain_base, para_id) parachain_final = {} - for node in parachain_config["nodes"]: + for idx, node in enumerate(parachain_config["nodes"]): parachain_detail = {} + rpc_port_value = rpc_port[idx] if rpc_port else None + prometheus_port_value = prometheus_port[idx] if prometheus_port else None + lib2lib_port_value = lib2lib_port[idx] if lib2lib_port else None + if parachain in constant.NO_WS_PORT: exec_comexec_commandmand = [ "/bin/bash", @@ -75,19 +44,27 @@ def start_local_parachain_node(plan, args, parachain_config, para_id): "-c", "{0} --base-path=/tmp/{1} --chain=/build/{1}-raw.json --ws-port=9946 --port=30333 --rpc-port=9933 --ws-external --rpc-external --prometheus-external --rpc-cors=all --{2} --collator --rpc-methods=unsafe --force-authoring --execution=wasm -- --chain=/app/raw-polkadot.json --execution=wasm".format(binary, chain_name, node["name"]), ] - parachain_spawn_detail = spawn_parachain(plan, "{0}-{1}-{2}".format(parachain, node["name"].lower(), args["chain-type"]), image, exec_comexec_commandmand, build_file = raw_service.name) + + build_file = raw_service.name + parachain_spawn_detail = node_setup.spawn_parachain(plan, node["prometheus"], "{0}-{1}-{2}".format(parachain, node["name"].lower(), args["chain-type"]), image, exec_comexec_commandmand, build_file, rpc_port_value, prometheus_port_value, lib2lib_port_value) parachain_detail["service_name"] = parachain_spawn_detail.name parachain_detail["endpoint"] = utils.get_service_url("ws", parachain_spawn_detail.ip_address, parachain_spawn_detail.ports["ws"].number) - parachain_detail["endpoint_prometheus"] = utils.get_service_url("tcp", parachain_spawn_detail.ip_address, parachain_spawn_detail.ports["metrics"].number) - parachain_detail["service_name"] = parachain_spawn_detail.name - parachain_detail["prometheus_port"] = parachain_spawn_detail.ports["metrics"].number parachain_detail["ip_address"] = parachain_spawn_detail.ip_address parachain_detail["prometheus"] = node["prometheus"] parachain_detail["node-type"] = node["node-type"] + if node["prometheus"] == True: + parachain_detail["prometheus_port"] = parachain_spawn_detail.ports["metrics"].number + if prometheus_port_value != None or prometheus_port_value != 0: + parachain_detail["prometheus_public_port"] = prometheus_port_value + parachain_detail["endpoint_prometheus"] = utils.get_service_url("tcp", "127.0.0.1", prometheus_port_value) + if rpc_port_value != None: + parachain_detail["endpoint_public"] = utils.get_service_url("ws", "127.0.0.1", rpc_port_value) + parachain_final[parachain_spawn_detail.name] = parachain_detail + return parachain_final -def start_nodes(plan, args, relay_chain_ip): +def start_nodes(plan, args, relay_chain_ip, rpc_port = [[]], prometheus_port = [[]], lib2lib_port = [[]]): """Start multiple parachain nodes. Args: @@ -100,14 +77,20 @@ def start_nodes(plan, args, relay_chain_ip): """ parachains = args["para"] final_parachain_details = {} - for parachain in parachains: + for idx, parachain in enumerate(parachains): para_id = register_para_slot.register_para_id(plan, relay_chain_ip) - parachain_details = start_local_parachain_node(plan, args, parachain, para_id) + + rpc_port_idx = rpc_port[idx] if idx < len(rpc_port) else [] + prometheus_port_idx = prometheus_port[idx] if idx < len(prometheus_port) else [] + lib2lib_port_idx = lib2lib_port[idx] if idx < len(lib2lib_port) else [] + + parachain_details = start_local_parachain_node(plan, args, parachain, para_id, rpc_port_idx, prometheus_port_idx, lib2lib_port_idx) register_para_slot.onboard_genesis_state_and_wasm(plan, para_id, parachain["name"], relay_chain_ip) final_parachain_details.update(parachain_details) + return final_parachain_details -def run_testnet_mainnet(plan, parachain, args): +def run_testnet_mainnet(plan, parachain, args, rpc_port = [], prometheus_port = [], lib2lib_port = []): """Run a testnet or mainnet based on configuration. Args: @@ -176,7 +159,14 @@ def run_testnet_mainnet(plan, parachain, args): common_command = [x for x in common_command if x != "--port=30333"] final_parachain_info = {} - for node in parachain["nodes"]: + for idx, node in enumerate(parachain["nodes"]): + rpc_port_value = rpc_port[idx] if rpc_port else None + prometheus_port_value = prometheus_port[idx] if prometheus_port else None + lib2lib_port_value = lib2lib_port[idx] if lib2lib_port else None + + if prometheus_port_value == 0: + prometheus_port_value = None + command = common_command command = command + ["--name={0}".format(node["name"])] if node["node-type"] == "collator": @@ -195,27 +185,37 @@ def run_testnet_mainnet(plan, parachain, args): binary = parachain_details["entrypoint"] command = [binary] + command node_info = {} - node_details = node_setup.run_testnet_node_with_entrypoint(plan, image, "{0}-{1}-{2}".format(parachain["name"], node["name"], args["chain-type"]), command) + node_details = node_setup.run_testnet_node_with_entrypoint(plan, node["prometheus"], image, "{0}-{1}-{2}".format(parachain["name"], node["name"], args["chain-type"]), command, rpc_port_value, prometheus_port_value, lib2lib_port_value) node_info["service_name"] = node_details.name node_info["endpoint"] = utils.get_service_url("ws", node_details.ip_address, node_details.ports["ws"].number) - node_info["endpoint_prometheus"] = utils.get_service_url("tcp", node_details.ip_address, node_details.ports["metrics"].number) - node_info["service_name"] = node_details.name node_info["ip_address"] = node_details.ip_address - node_info["prometheus_port"] = node_details.ports["metrics"].number node_info["prometheus"] = node["prometheus"] node_info["node-type"] = node["node-type"] + if node["prometheus"] == True: + node_info["prometheus_port"] = node_details.ports["metrics"].number + if prometheus_port_value != None: + node_info["prometheus_public_port"] = prometheus_port_value + node_info["endpoint_prometheus"] = utils.get_service_url("tcp", "127.0.0.1", prometheus_port_value) + if rpc_port_value != None: + node_info["endpoint_public"] = utils.get_service_url("ws", "127.0.0.1", rpc_port_value) + final_parachain_info[node_details.name] = node_info else: node_info = {} - node_details = node_setup.run_testnet_node_with_command(plan, image, "{0}-{1}-{2}".format(parachain["name"], node["name"], args["chain-type"]), command) + node_details = node_setup.run_testnet_node_with_command(plan, node["prometheus"], image, "{0}-{1}-{2}".format(parachain["name"], node["name"], args["chain-type"]), command, rpc_port_value, prometheus_port_value, lib2lib_port_value) node_info["service_name"] = node_details.name node_info["endpoint"] = utils.get_service_url("ws", node_details.ip_address, node_details.ports["ws"].number) - node_info["endpoint_prometheus"] = utils.get_service_url("tcp", node_details.ip_address, node_details.ports["metrics"].number) - node_info["service_name"] = node_details.name node_info["ip_address"] = node_details.ip_address - node_info["prometheus_port"] = node_details.ports["metrics"].number node_info["prometheus"] = node["prometheus"] node_info["node-type"] = node["node-type"] + if node["prometheus"] == True: + node_info["prometheus_port"] = node_details.ports["metrics"].number + if prometheus_port_value != None: + node_info["prometheus_public_port"] = prometheus_port_value + node_info["endpoint_prometheus"] = utils.get_service_url("tcp", "127.0.0.1", prometheus_port_value) + if rpc_port_value != None: + node_info["endpoint_public"] = utils.get_service_url("ws", "127.0.0.1", rpc_port_value) + final_parachain_info[node_details.name] = node_info return final_parachain_info diff --git a/relaychain/relay-chain.star b/relaychain/relay-chain.star index 09d6102..455ef3b 100644 --- a/relaychain/relay-chain.star +++ b/relaychain/relay-chain.star @@ -1,7 +1,7 @@ utils = import_module("../package_io/utils.star") -def start_relay_chain(plan, args): +def start_relay_chain(plan, args, rpc_port = [], prometheus_port = [], lib2lib_port = []): """ Starts relay chain nodes based on the provided arguments. @@ -16,45 +16,66 @@ def start_relay_chain(plan, args): chain = args["relaychain"]["name"] final_details = {} - for relay_node in args["relaychain"]["nodes"]: - - - command = "polkadot --rpc-external --rpc-cors=all --rpc-methods=unsafe --chain {0} --name={1} --execution=wasm --prometheus-external".format(chain, relay_node["name"]) - - if relay_node["node-type"] == "validator": - command = command + " --validator --insecure-validator-i-know-what-i-do" - elif relay_node["node-type"] == "archive": - command = command + " --pruning=archive" - - plan.print(command) - - exec_command = ["bin/sh", "-c", command] + ports = { + "ws": PortSpec(9944, transport_protocol = "TCP"), + "lib2lib": PortSpec(30333, transport_protocol = "TCP") + } + + relay_nodes = args["relaychain"]["nodes"] + for idx, relay_node in enumerate(relay_nodes): + public_ports = {} + + rpc_port_value = rpc_port[idx] if rpc_port else None + prometheus_port_value = prometheus_port[idx] if prometheus_port else None + lib2lib_port_value = lib2lib_port[idx] if lib2lib_port else None + + + if rpc_port_value != None : + public_ports["ws"] = PortSpec(rpc_port_value, transport_protocol = "TCP") + + if lib2lib_port_value != None: + public_ports["lib2lib"] = PortSpec(lib2lib_port_value, transport_protocol = "TCP") + + if relay_node["prometheus"]: + ports["metrics"] = PortSpec(9615, transport_protocol="TCP") + + if prometheus_port_value == 0: + prometheus_port_value = None + + if prometheus_port_value != None: + public_ports["metrics"] = PortSpec(prometheus_port_value, transport_protocol = "TCP") + + exec_command = ["bin/sh", "-c", "polkadot --rpc-external --rpc-cors=all --rpc-methods=unsafe --chain {0} --name={1} --execution=wasm --prometheus-external".format(chain, relay_node["name"])] service_details = plan.add_service( name = "{0}-{1}-{2}".format(name, chain, relay_node["name"]), config = ServiceConfig( image = "parity/polkadot:latest", - ports = { - "ws": PortSpec(9944, transport_protocol = "TCP"), - "metrics": PortSpec(9615, transport_protocol = "TCP"), - }, + ports = ports, + public_ports = public_ports, entrypoint = exec_command, ), ) relay_node_details = {} relay_node_details["endpoint"] = utils.get_service_url("ws", service_details.ip_address, service_details.ports["ws"].number) - relay_node_details["endpoint_prometheus"] = utils.get_service_url("tcp" , service_details.ip_address, service_details.ports["metrics"].number) relay_node_details["service_name"] = service_details.name - relay_node_details["prometheus_port"] = service_details.ports["metrics"].number relay_node_details["prometheus"] = relay_node["prometheus"] relay_node_details["ip_address"] = service_details.ip_address relay_node_details["node-type"] = relay_node["node-type"] + if relay_node["prometheus"] == True: + relay_node_details["prometheus_port"] = service_details.ports["metrics"].number + if prometheus_port_value != None: + relay_node_details["prometheus_public_port"] = prometheus_port_value + relay_node_details["endpoint_prometheus"] = utils.get_service_url("tcp", "127.0.0.1", prometheus_port_value) + if rpc_port_value != None: + relay_node_details["endpoint_public"] = utils.get_service_url("ws", "127.0.0.1", rpc_port_value) + final_details[service_details.name] = relay_node_details return final_details -def start_test_main_net_relay_nodes(plan, args): +def start_test_main_net_relay_nodes(plan, args, rpc_port = [], prometheus_port = [], lib2lib_port = []): """ Starts testnet/mainnet relay nodes based on the provided arguments. @@ -75,25 +96,11 @@ def start_test_main_net_relay_nodes(plan, args): if chain != "polkadot" and chain != "kusama": fail("Please provide polkadot or kusama as relaychain for mainnet") - relay_node_details = start_relay_chain(plan, args) + relay_node_details = start_relay_chain(plan, args, rpc_port, prometheus_port, lib2lib_port) return relay_node_details -def spawn_multiple_relay(plan, count): - """ - Spawns multiple local relay chain nodes. - - Args: - plan (object): The Kurtosis plan object for orchestrating the test. - count (int): Number of relay nodes to spawn. - """ - node_list = ["alice", "bob", "dave", "charlie"] - port = 9944 - for i in range(0, count): - port = port + count - start_relay_chain_local(plan, node_list[i]) - -def start_relay_chains_local(plan, args): +def start_relay_chains_local(plan, args, rpc_port = [], prometheus_port = [], lib2lib_port = []): """ Starts local relay chain nodes based on the provided arguments. @@ -104,27 +111,48 @@ def start_relay_chains_local(plan, args): Returns: list: List of dictionaries containing sevice details of started relay chain nodes. """ + name = args["chain-type"] + chain = args["relaychain"]["name"] + + if name == "local": + if chain != "rococo-local": + fail("Please provide rococo-local as relaychain for localnet") + relay_nodes = args["relaychain"]["nodes"] if len(relay_nodes) < 2: fail("relay nodes must contain at least two nodes") final_details = {} - for node in relay_nodes: + for idx, node in enumerate(relay_nodes): relay_detail = {} - service_details = start_relay_chain_local(plan, args, node["name"]) + rpc_port_value = rpc_port[idx] if rpc_port else None + prometheus_port_value = prometheus_port[idx] if prometheus_port else None + lib2lib_port_value = lib2lib_port[idx] if lib2lib_port else None + + if prometheus_port_value == 0: + prometheus_port_value = None + + service_details = start_relay_chain_local(plan, args, node["name"], node["prometheus"], rpc_port_value, prometheus_port_value, lib2lib_port_value) + relay_detail["endpoint"] = utils.get_service_url("ws", service_details.ip_address, service_details.ports["ws"].number) - relay_detail["endpoint_prometheus"] = utils.get_service_url("tcp" , service_details.ip_address, service_details.ports["metrics"].number) relay_detail["service_name"] = service_details.name - relay_detail["prometheus_port"] = service_details.ports["metrics"].number relay_detail["prometheus"] = node["prometheus"] relay_detail["ip_address"] = service_details.ip_address relay_detail["node-type"] = node["node-type"] + if node["prometheus"] == True: + relay_detail["prometheus_port"] = service_details.ports["metrics"].number + if prometheus_port_value != None: + relay_detail["prometheus_public_port"] = prometheus_port_value + relay_detail["endpoint_prometheus"] = utils.get_service_url("tcp", "127.0.0.1", prometheus_port_value) + if rpc_port_value != None: + relay_detail["endpoint_public"] = utils.get_service_url("ws", "127.0.0.1", rpc_port_value) + final_details[service_details.name] = relay_detail return final_details -def start_relay_chain_local(plan, args, name): +def start_relay_chain_local(plan, args, name, prometheus, rpc_port = None, prometheus_port = None, lib2lib_port = None): """ Starts a local relay chain node based on the provided arguments. @@ -135,6 +163,25 @@ def start_relay_chain_local(plan, args, name): Returns: object: Service details of the started relay chain node. """ + ports = { + "ws": PortSpec(9944, transport_protocol = "TCP"), + "lib2lib": PortSpec(30333, transport_protocol = "TCP"), + } + + public_ports = {} + + if rpc_port != None : + public_ports["ws"] = PortSpec(rpc_port, transport_protocol = "TCP") + + if lib2lib_port != None: + public_ports["lib2lib"] = PortSpec(lib2lib_port, transport_protocol = "TCP") + + if prometheus: + ports["metrics"] = PortSpec(9615, transport_protocol="TCP") + + if prometheus_port != None: + public_ports["metrics"] = PortSpec(prometheus_port, transport_protocol = "TCP") + exec_command = ["bin/sh", "-c", "polkadot --base-path=/data --chain=/app/raw-polkadot.json --validator --rpc-external --port=30333 --rpc-cors=all --name=alice --{0} --rpc-methods=unsafe --execution=wasm --prometheus-external --insecure-validator-i-know-what-i-do".format(name)] service_details = plan.add_service( name = "{0}-{1}".format(args["relaychain"]["name"], name), @@ -143,12 +190,9 @@ def start_relay_chain_local(plan, args, name): files = { "/app": "configs", }, - ports = { - "ws": PortSpec(9944, transport_protocol = "TCP"), - "metrics": PortSpec(9615, transport_protocol = "TCP"), - "lib2lib": PortSpec(30333, transport_protocol = "TCP"), - }, - entrypoint = exec_command, + ports = ports, + public_ports = public_ports, + entrypoint = exec_command, ), ) return service_details From 99eb0e2d64adf7d60c15ece5b07560cfd5bc3de1 Mon Sep 17 00:00:00 2001 From: riyasng12 Date: Wed, 10 Jan 2024 15:31:17 +0530 Subject: [PATCH 2/4] refactor: refactor args and ports allocation --- local.json | 6 +- main.star | 43 ++++++++------ package_io/constant.star | 42 +------------ package_io/grafana.star | 2 +- package_io/promethues.star | 4 +- package_io/utils.star | 69 ++++++++++++++++------ parachain/node_setup.star | 4 +- parachain/parachain.star | 110 +++++++++++++++++----------------- relaychain/relay-chain.star | 114 +++++++++++++++--------------------- 9 files changed, 187 insertions(+), 207 deletions(-) diff --git a/local.json b/local.json index 1ac491b..4cae165 100644 --- a/local.json +++ b/local.json @@ -1,7 +1,7 @@ { - "chain-type": "local", + "chain_type": "local", "relaychain": { - "name": "rococo", + "name": "rococo-local", "nodes": [ { "name": "alice", @@ -17,7 +17,7 @@ ] }, - "para": [ + "parachains": [ { "name":"acala", "nodes": [ diff --git a/main.star b/main.star index da11646..41ea136 100644 --- a/main.star +++ b/main.star @@ -3,10 +3,10 @@ relay_chain = import_module("./relaychain/relay-chain.star") package = import_module("./package_io/build-spec.star") promethues = import_module("./package_io/promethues.star") grafana = import_module("./package_io/grafana.star") -explorer = import_module("./package_io/polkadot_js_app.star") +explorer_js = import_module("./package_io/polkadot_js_app.star") utils = import_module("./package_io/utils.star") -def run(plan, args): +def run(plan, chain_type = "local", relaychain = None, parachains = None, explorer = False): """ Main function to run the Polkadot relay and parachain setup. @@ -17,11 +17,11 @@ def run(plan, args): Returns: dict: Service details containing information about relay chains, parachains, and Prometheus. """ - service_details = run_polkadot_setup(plan, args) + service_details = run_polkadot_setup(plan, chain_type, relaychain, parachains, explorer) return service_details -def run_polkadot_setup(plan, args): +def run_polkadot_setup(plan, chain_type, relaychain, parachains, explorer): """ Main function to run the Polkadot relay and parachain setup. @@ -33,42 +33,51 @@ def run_polkadot_setup(plan, args): dict: Service details containing information about relay chains, parachains, and Prometheus. """ - utils.check_config_validity(plan, args) - + chain_type, relaychain, parachains = utils.convert_to_lowercase(chain_type, relaychain, parachains) + utils.check_config_validity(plan, chain_type, relaychain, parachains) utils.upload_files(plan) service_details = {} - if args["chain-type"] == "local": - relay_chain_details = relay_chain.start_relay_chains_local(plan, args) + if chain_type == "local": + relay_chain_details = relay_chain.start_relay_chains_local(plan, chain_type, relaychain["name"], relaychain["nodes"]) polkadot_service_name = None for key in relay_chain_details: polkadot_service_name = key break service_details.update(relay_chain_details) - parachain_details = parachain.start_nodes(plan, args, relay_chain_details[polkadot_service_name]["ip_address"]) + parachain_details = parachain.start_nodes(plan, chain_type, parachains, relay_chain_details[polkadot_service_name]["ip_address"]) service_details.update(parachain_details) else: - if len(args["relaychain"]) != 0: - relay_node_details = relay_chain.start_test_main_net_relay_nodes(plan, args) + if len(relaychain) != 0: + relay_node_details = relay_chain.start_test_main_net_relay_nodes(plan, chain_type, relaychain["name"], relaychain["nodes"]) service_details.update(relay_node_details) - for paras in args["para"]: - parachain_info = parachain.run_testnet_mainnet(plan, paras, args) + for paras in parachains: + parachain_info = parachain.run_testnet_mainnet(plan, chain_type, relaychain["name"], paras) service_details.update(parachain_info) #run prometheus , if it returs some endpoint then grafana will up - prometheus_service_details = promethues.launch_prometheus(plan, args, service_details) + prometheus_service_details = promethues.launch_prometheus(plan, service_details) if len(prometheus_service_details) != 0: service_details.update(prometheus_service_details) if prometheus_service_details["prometheus"]["endpoint"].startswith("http://"): - grafana_service_details = grafana.launch_grafana(plan, grafana) + grafana_service_details = grafana.launch_grafana(plan) service_details.update(grafana_service_details) #run the polkadot js App explorer - if args["explorer"] == True: - explorer_service_details = explorer.run_pokadot_js_app(plan, "ws://127.0.0.1:9944") + if explorer == True: + explorer_service_details = explorer_js.run_pokadot_js_app(plan, "ws://127.0.0.1:9944") service_details.update(explorer_service_details) return service_details +def convert_to_lowercase(data): + if type(data) == dict: + return {key: convert_to_lowercase(value) for key, value in data.items()} + elif type(data) == list: + return [convert_to_lowercase(item) for item in data] + elif type(data) == str: + return data.lower() + else: + return data \ No newline at end of file diff --git a/package_io/constant.star b/package_io/constant.star index 2cb4b70..1e3205d 100644 --- a/package_io/constant.star +++ b/package_io/constant.star @@ -25,43 +25,5 @@ DIFFERENT_IMAGES_FOR_TESTNET = { CHAIN_COMMAND = ["manta", "moonsama", "interlay", "kintsugi-btc", "polkadex", "centrifuge", "altair", "robonomics", "kilt"] -KUSAMA_PARACHAINS = [ - "altair", - "bajun", - "bifrost", - "calamari", - "encointer", - "khala", - "kintsugi-btc", - "litmus", - "mangata", - "moonriver", - "robonomics", - "subzero", - "turing" -] - -POLKADOT_PARACHAINS = [ - "acala", - "ajuna", - "bifrost", - "centrifuge", - "clover", - "frequency", - "integritee", - "interlay", - "karura", - "kilt", - "kylin", - "litentry", - "manta", - "moonbeam", - "moonsama", - "nodle", - "parallel", - "pendulum", - "phala", - "polkadex", - "subsocial", - "zeitgeist" -] \ No newline at end of file +KUSAMA_PARACHAINS = ["altair", "bajun", "bifrost", "calamari", "encointer", "khala", "kintsugi-btc", "litmus", "mangata", "moonriver", "robonomics", "subzero", "turing"] +POLKADOT_PARACHAINS = ["acala", "ajuna", "bifrost", "centrifuge", "clover", "frequency", "integritee", "interlay", "karura", "kilt", "kylin", "litentry", "manta", "moonbeam", "moonsama", "nodle", "parallel", "pendulum", "phala", "polkadex", "subsocial", "zeitgeist"] diff --git a/package_io/grafana.star b/package_io/grafana.star index 96de448..2e3fd5c 100644 --- a/package_io/grafana.star +++ b/package_io/grafana.star @@ -24,7 +24,7 @@ def prometheus_grafana_service(plan, service_name, image, port, command, build_f return service -def launch_grafana(plan, args): +def launch_grafana(plan): command = ["/run.sh"] service = prometheus_grafana_service(plan, "grafana", "grafana/grafana-dev:10.3.0-147071", 3000, command, None) diff --git a/package_io/promethues.star b/package_io/promethues.star index b83081b..af1340b 100644 --- a/package_io/promethues.star +++ b/package_io/promethues.star @@ -20,12 +20,10 @@ USED_PORTS = { def launch_prometheus( plan, - args, service_details ): template_data = new_config_template_data( plan, - args, service_details, ) @@ -80,7 +78,7 @@ def get_config(config_files_artifact_name): ], ) -def new_config_template_data(plan, args, service_details): +def new_config_template_data(plan, service_details): metrics_jobs = [] for service in service_details: if "prometheus" in service_details[service] and service_details[service]["prometheus"] == True: diff --git a/package_io/utils.star b/package_io/utils.star index 5528197..f34035f 100644 --- a/package_io/utils.star +++ b/package_io/utils.star @@ -55,38 +55,69 @@ def get_service_url(protocol ,ip_address, ports): return url -def check_config_validity(plan, args): - if len(args["relaychain"]) != 0: - for node in args["relaychain"]["nodes"]: +def check_config_validity(plan, chain_type, relaychain, parachains): + if relaychain != {}: + chain_name = relaychain["name"] + relay_nodes = relaychain["nodes"] + + if chain_type == "testnet": + if chain_name != "rococo" and chain_name != "westend": + return fail("Please provide rococo or westend as relaychain for testnet") + elif chain_type == "mainnet": + if chain_name != "polkadot" and chain_name != "kusama": + return fail("Please provide polkadot or kusama as relaychain for mainnet") + elif chain_type == "local": + if chain_name != "rococo-local": + return fail("Please provide rococo-local as relaychain for localnet") + elif len(relay_nodes) < 2: + return fail("relay nodes must contain at least two nodes for localnet") + + if chain_name == "polkadot": + if len(parachains) != 0: + for para in parachains: + if para["name"] not in constant.POLKADOT_PARACHAINS: + return fail("Invalid parachain for POLKADOT") + + if chain_name == "kusama": + if len(parachains) != 0: + for para in parachains: + if para["name"] not in constant.KUSAMA_PARACHAINS: + return fail("Invalid parachain for KUSAMA") + + if len(relaychain) != 0: + for node in relay_nodes: if len(node) != 0: if node["node-type"] in ["validator", "full", "archive"]: plan.print("config for relaynodes is valid") else: return fail("relaychain node-type can be only validator/full") - if len(args["para"]) != 0: - for para in args["para"]: + if len(parachains) != 0: + for para in parachains: if len(para["nodes"]) != 0: for node in para["nodes"]: if node["node-type"] in ["validator", "full", "collator"]: plan.print("config for parachain is valid") else: return fail("parachain node-type can be only validator/full/collator") - - if args["relaychain"] != {}: - chain = args["relaychain"]["name"] - if chain == "polkadot": - if len(args["para"]) != 0: - for para in args["para"]: - if para["name"] not in constant.POLKADOT_PARACHAINS: - return fail("Invalid parachain for POLKADOT") - if chain == "kusama": - if len(args["para"]) != 0: - for para in args["para"]: - if para["name"] not in constant.KUSAMA_PARACHAINS: - return fail("Invalid parachain for KUSAMA") - + def upload_files(plan): plan.upload_files(src = "../parachain/static_files/configs", name = "configs") plan.upload_files(src = "../parachain/static_files/javascript", name = "javascript") + +def convert_to_lowercase(chain_type, relaychain, parachains): + chain_type = chain_type.lower() + relaychain["name"] = relaychain["name"].lower() + for node in relaychain["nodes"]: + node["name"] = node["name"].lower() + node["node-type"] = node["node-type"].lower() + + for para in parachains: + para["name"] = para["name"].lower() + for node in para["nodes"]: + node["name"] = node["name"].lower() + node["node-type"] = node["node-type"].lower() + + return chain_type, relaychain, parachains + \ No newline at end of file diff --git a/parachain/node_setup.star b/parachain/node_setup.star index 1788219..b91511b 100644 --- a/parachain/node_setup.star +++ b/parachain/node_setup.star @@ -66,7 +66,7 @@ def run_testnet_node_with_command(plan, prometheus, image, chain_name, execute_c return parachain -def spawn_parachain(plan, prometheus, chain_name, image, command, build_file, rpc_port = None, prometheus_port = None, lib2lib_port = None): +def spawn_parachain(plan, prometheus, image, chain_name, execute_command, build_file, rpc_port = None, prometheus_port = None, lib2lib_port = None): """Spawn a parachain node with specified configuration. Args: @@ -111,7 +111,7 @@ def spawn_parachain(plan, prometheus, chain_name, image, command, build_file, rp files = files, ports = ports, public_ports = public_ports, - entrypoint = command, + entrypoint = execute_command, ), ) diff --git a/parachain/parachain.star b/parachain/parachain.star index 32d88a1..441826a 100644 --- a/parachain/parachain.star +++ b/parachain/parachain.star @@ -5,7 +5,7 @@ parachain_list = import_module("./static_files/images.star") node_setup = import_module("./node_setup.star") utils = import_module("../package_io/utils.star") -def start_local_parachain_node(plan, args, parachain_config, para_id, rpc_port = [], prometheus_port = [], lib2lib_port = []): +def start_local_parachain_node(plan, chain_type, parachain, para_id): """Start local parachain nodes based on configuration. Args: @@ -17,22 +17,28 @@ def start_local_parachain_node(plan, args, parachain_config, para_id, rpc_port = Returns: list: List of dictionaries containing service details of parachain nodes. """ - parachain = parachain_config["name"].lower() - parachain_details = parachain_list.parachain_images[parachain] + chain_name = parachain["name"] + parachain_details = parachain_list.parachain_images[chain_name] image = parachain_details["image"] binary = parachain_details["entrypoint"] chain_base = parachain_details["base"][0] - chain_name = parachain raw_service = build_spec.create_parachain_build_spec_with_para_id(plan, image, binary, chain_name, chain_base, para_id) parachain_final = {} - for idx, node in enumerate(parachain_config["nodes"]): + + for node in parachain["nodes"]: parachain_detail = {} - rpc_port_value = rpc_port[idx] if rpc_port else None - prometheus_port_value = prometheus_port[idx] if prometheus_port else None - lib2lib_port_value = lib2lib_port[idx] if lib2lib_port else None + + if "ports" in node: + rpc_port = node["ports"]["rpc_port"] + lib2lib_port = node["ports"]["lib2lib_port"] + prometheus_port = node["ports"]["prometheus_port"] if node["prometheus"] else None + else: + rpc_port = None + lib2lib_port = None + prometheus_port = None - if parachain in constant.NO_WS_PORT: + if chain_name in constant.NO_WS_PORT: exec_comexec_commandmand = [ "/bin/bash", "-c", @@ -46,7 +52,7 @@ def start_local_parachain_node(plan, args, parachain_config, para_id, rpc_port = ] build_file = raw_service.name - parachain_spawn_detail = node_setup.spawn_parachain(plan, node["prometheus"], "{0}-{1}-{2}".format(parachain, node["name"].lower(), args["chain-type"]), image, exec_comexec_commandmand, build_file, rpc_port_value, prometheus_port_value, lib2lib_port_value) + parachain_spawn_detail = node_setup.spawn_parachain(plan, node["prometheus"], image, "{0}-{1}-{2}".format(chain_name, node["name"], chain_type), exec_comexec_commandmand, build_file, rpc_port, prometheus_port, lib2lib_port) parachain_detail["service_name"] = parachain_spawn_detail.name parachain_detail["endpoint"] = utils.get_service_url("ws", parachain_spawn_detail.ip_address, parachain_spawn_detail.ports["ws"].number) parachain_detail["ip_address"] = parachain_spawn_detail.ip_address @@ -54,17 +60,17 @@ def start_local_parachain_node(plan, args, parachain_config, para_id, rpc_port = parachain_detail["node-type"] = node["node-type"] if node["prometheus"] == True: parachain_detail["prometheus_port"] = parachain_spawn_detail.ports["metrics"].number - if prometheus_port_value != None or prometheus_port_value != 0: - parachain_detail["prometheus_public_port"] = prometheus_port_value - parachain_detail["endpoint_prometheus"] = utils.get_service_url("tcp", "127.0.0.1", prometheus_port_value) - if rpc_port_value != None: - parachain_detail["endpoint_public"] = utils.get_service_url("ws", "127.0.0.1", rpc_port_value) + if prometheus_port != None: + parachain_detail["prometheus_public_port"] = prometheus_port + parachain_detail["endpoint_prometheus"] = utils.get_service_url("tcp", "127.0.0.1", prometheus_port) + if rpc_port != None: + parachain_detail["endpoint_public"] = utils.get_service_url("ws", "127.0.0.1", rpc_port) parachain_final[parachain_spawn_detail.name] = parachain_detail return parachain_final -def start_nodes(plan, args, relay_chain_ip, rpc_port = [[]], prometheus_port = [[]], lib2lib_port = [[]]): +def start_nodes(plan, chain_type, parachains, relay_chain_ip): """Start multiple parachain nodes. Args: @@ -75,22 +81,17 @@ def start_nodes(plan, args, relay_chain_ip, rpc_port = [[]], prometheus_port = [ Returns: list: List of dictionaries containing service details of each parachain. """ - parachains = args["para"] final_parachain_details = {} - for idx, parachain in enumerate(parachains): - para_id = register_para_slot.register_para_id(plan, relay_chain_ip) - - rpc_port_idx = rpc_port[idx] if idx < len(rpc_port) else [] - prometheus_port_idx = prometheus_port[idx] if idx < len(prometheus_port) else [] - lib2lib_port_idx = lib2lib_port[idx] if idx < len(lib2lib_port) else [] - - parachain_details = start_local_parachain_node(plan, args, parachain, para_id, rpc_port_idx, prometheus_port_idx, lib2lib_port_idx) + + for parachain in parachains: + para_id = register_para_slot.register_para_id(plan, relay_chain_ip) + parachain_details = start_local_parachain_node(plan, chain_type, parachain, para_id) register_para_slot.onboard_genesis_state_and_wasm(plan, para_id, parachain["name"], relay_chain_ip) final_parachain_details.update(parachain_details) return final_parachain_details -def run_testnet_mainnet(plan, parachain, args, rpc_port = [], prometheus_port = [], lib2lib_port = []): +def run_testnet_mainnet(plan, chain_type, relaychain_name, parachain): """Run a testnet or mainnet based on configuration. Args: @@ -101,8 +102,7 @@ def run_testnet_mainnet(plan, parachain, args, rpc_port = [], prometheus_port = Returns: list: List of dictionaries containing details of each parachain node. """ - if args["chain-type"] == "testnet": - main_chain = "rococo" + if chain_type == "testnet": if parachain["name"] == "ajuna": parachain["name"] = "bajun" parachain_details = parachain_list.parachain_images[parachain["name"]] @@ -110,16 +110,15 @@ def run_testnet_mainnet(plan, parachain, args, rpc_port = [], prometheus_port = base = parachain_details["base"][1] if parachain["name"] in constant.DIFFERENT_IMAGES_FOR_TESTNET: - image = constant.DIFFERENT_IMAGES_FOR_TESTNET[parachain["name"].lower()] + image = constant.DIFFERENT_IMAGES_FOR_TESTNET[parachain["name"]] else: - main_chain = "polkadot" - parachain_details = parachain_list.parachain_images[parachain["name"].lower()] + parachain_details = parachain_list.parachain_images[parachain["name"]] image = parachain_details["image"] base = parachain_details["base"][2] if parachain["name"] in constant.DIFFERENT_IMAGES_FOR_MAINNET: - image = constant.DIFFERENT_IMAGES_FOR_MAINNET[parachain["name"].lower()] + image = constant.DIFFERENT_IMAGES_FOR_MAINNET[parachain["name"]] if base == None: fail("Tesnet is not there for {}".format(parachain["name"])) @@ -154,18 +153,21 @@ def run_testnet_mainnet(plan, parachain, args, rpc_port = [], prometheus_port = if parachain["name"] == "altair" or parachain["name"] == "centrifuge": common_command = common_command + ["--database=auto"] - if parachain["name"] == "subzero" and args["chain-type"] == "mainnet": + if parachain["name"] == "subzero" and chain_type == "mainnet": common_command = [x for x in common_command if x != "--chain="] common_command = [x for x in common_command if x != "--port=30333"] final_parachain_info = {} - for idx, node in enumerate(parachain["nodes"]): - rpc_port_value = rpc_port[idx] if rpc_port else None - prometheus_port_value = prometheus_port[idx] if prometheus_port else None - lib2lib_port_value = lib2lib_port[idx] if lib2lib_port else None - - if prometheus_port_value == 0: - prometheus_port_value = None + for node in parachain["nodes"]: + + if "ports" in node: + rpc_port = node["ports"]["rpc_port"] + lib2lib_port = node["ports"]["lib2lib_port"] + prometheus_port = node["ports"]["prometheus_port"] if node["prometheus"] else None + else: + rpc_port = None + lib2lib_port = None + prometheus_port = None command = common_command command = command + ["--name={0}".format(node["name"])] @@ -176,16 +178,16 @@ def run_testnet_mainnet(plan, parachain, args, rpc_port = [], prometheus_port = command = command + ["--validator"] if parachain["name"] in constant.CHAIN_COMMAND: - command = command + ["--", "--chain={0}".format(main_chain)] + command = command + ["--", "--chain={0}".format(relaychain_name)] - if parachain["name"] == "kilt-spiritnet" and args["chain-type"] == "testnet": + if parachain["name"] == "kilt-spiritnet" and chain_type == "testnet": command = command + ["--", "--chain=/node/dev-specs/kilt-parachain/peregrine-relay.json"] if parachain["name"] in constant.BINARY_COMMAND_CHAINS: binary = parachain_details["entrypoint"] command = [binary] + command node_info = {} - node_details = node_setup.run_testnet_node_with_entrypoint(plan, node["prometheus"], image, "{0}-{1}-{2}".format(parachain["name"], node["name"], args["chain-type"]), command, rpc_port_value, prometheus_port_value, lib2lib_port_value) + node_details = node_setup.run_testnet_node_with_entrypoint(plan, node["prometheus"], image, "{0}-{1}-{2}".format(parachain["name"], node["name"], chain_type), command, rpc_port, prometheus_port, lib2lib_port) node_info["service_name"] = node_details.name node_info["endpoint"] = utils.get_service_url("ws", node_details.ip_address, node_details.ports["ws"].number) node_info["ip_address"] = node_details.ip_address @@ -193,17 +195,17 @@ def run_testnet_mainnet(plan, parachain, args, rpc_port = [], prometheus_port = node_info["node-type"] = node["node-type"] if node["prometheus"] == True: node_info["prometheus_port"] = node_details.ports["metrics"].number - if prometheus_port_value != None: - node_info["prometheus_public_port"] = prometheus_port_value - node_info["endpoint_prometheus"] = utils.get_service_url("tcp", "127.0.0.1", prometheus_port_value) - if rpc_port_value != None: - node_info["endpoint_public"] = utils.get_service_url("ws", "127.0.0.1", rpc_port_value) + if prometheus_port != None: + node_info["prometheus_public_port"] = prometheus_port + node_info["endpoint_prometheus"] = utils.get_service_url("tcp", "127.0.0.1", prometheus_port) + if rpc_port != None: + node_info["endpoint_public"] = utils.get_service_url("ws", "127.0.0.1", rpc_port) final_parachain_info[node_details.name] = node_info else: node_info = {} - node_details = node_setup.run_testnet_node_with_command(plan, node["prometheus"], image, "{0}-{1}-{2}".format(parachain["name"], node["name"], args["chain-type"]), command, rpc_port_value, prometheus_port_value, lib2lib_port_value) + node_details = node_setup.run_testnet_node_with_command(plan, node["prometheus"], image, "{0}-{1}-{2}".format(parachain["name"], node["name"], chain_type), command, rpc_port, prometheus_port, lib2lib_port) node_info["service_name"] = node_details.name node_info["endpoint"] = utils.get_service_url("ws", node_details.ip_address, node_details.ports["ws"].number) node_info["ip_address"] = node_details.ip_address @@ -211,11 +213,11 @@ def run_testnet_mainnet(plan, parachain, args, rpc_port = [], prometheus_port = node_info["node-type"] = node["node-type"] if node["prometheus"] == True: node_info["prometheus_port"] = node_details.ports["metrics"].number - if prometheus_port_value != None: - node_info["prometheus_public_port"] = prometheus_port_value - node_info["endpoint_prometheus"] = utils.get_service_url("tcp", "127.0.0.1", prometheus_port_value) - if rpc_port_value != None: - node_info["endpoint_public"] = utils.get_service_url("ws", "127.0.0.1", rpc_port_value) + if prometheus_port != None: + node_info["prometheus_public_port"] = prometheus_port + node_info["endpoint_prometheus"] = utils.get_service_url("tcp", "127.0.0.1", prometheus_port) + if rpc_port != None: + node_info["endpoint_public"] = utils.get_service_url("ws", "127.0.0.1", rpc_port) final_parachain_info[node_details.name] = node_info return final_parachain_info diff --git a/relaychain/relay-chain.star b/relaychain/relay-chain.star index 455ef3b..d617e4e 100644 --- a/relaychain/relay-chain.star +++ b/relaychain/relay-chain.star @@ -1,7 +1,7 @@ utils = import_module("../package_io/utils.star") -def start_relay_chain(plan, args, rpc_port = [], prometheus_port = [], lib2lib_port = []): +def start_relay_chain(plan, chain_type, chain_name, relay_nodes): """ Starts relay chain nodes based on the provided arguments. @@ -12,8 +12,6 @@ def start_relay_chain(plan, args, rpc_port = [], prometheus_port = [], lib2lib_p Returns: list: List of dictionaries containing service details of started relay chain nodes. """ - name = args["chain-type"] - chain = args["relaychain"]["name"] final_details = {} ports = { @@ -21,34 +19,33 @@ def start_relay_chain(plan, args, rpc_port = [], prometheus_port = [], lib2lib_p "lib2lib": PortSpec(30333, transport_protocol = "TCP") } - relay_nodes = args["relaychain"]["nodes"] - - for idx, relay_node in enumerate(relay_nodes): + for relay_node in relay_nodes: public_ports = {} - - rpc_port_value = rpc_port[idx] if rpc_port else None - prometheus_port_value = prometheus_port[idx] if prometheus_port else None - lib2lib_port_value = lib2lib_port[idx] if lib2lib_port else None - - if rpc_port_value != None : - public_ports["ws"] = PortSpec(rpc_port_value, transport_protocol = "TCP") + if "ports" in relay_node: + rpc_port = relay_node["ports"]["rpc_port"] + lib2lib_port = relay_node["ports"]["lib2lib_port"] + prometheus_port = relay_node["ports"]["prometheus_port"] if relay_node["prometheus"] else None + else: + rpc_port = None + lib2lib_port = None + prometheus_port = None + + if rpc_port != None : + public_ports["ws"] = PortSpec(rpc_port, transport_protocol = "TCP") - if lib2lib_port_value != None: - public_ports["lib2lib"] = PortSpec(lib2lib_port_value, transport_protocol = "TCP") + if lib2lib_port != None: + public_ports["lib2lib"] = PortSpec(lib2lib_port, transport_protocol = "TCP") if relay_node["prometheus"]: ports["metrics"] = PortSpec(9615, transport_protocol="TCP") - if prometheus_port_value == 0: - prometheus_port_value = None - - if prometheus_port_value != None: - public_ports["metrics"] = PortSpec(prometheus_port_value, transport_protocol = "TCP") + if prometheus_port != None: + public_ports["metrics"] = PortSpec(prometheus_port, transport_protocol = "TCP") - exec_command = ["bin/sh", "-c", "polkadot --rpc-external --rpc-cors=all --rpc-methods=unsafe --chain {0} --name={1} --execution=wasm --prometheus-external".format(chain, relay_node["name"])] + exec_command = ["bin/sh", "-c", "polkadot --rpc-external --rpc-cors=all --rpc-methods=unsafe --chain {0} --name={1} --execution=wasm --prometheus-external".format(chain_name, relay_node["name"])] service_details = plan.add_service( - name = "{0}-{1}-{2}".format(name, chain, relay_node["name"]), + name = "{0}-{1}-{2}".format(chain_type, chain_name, relay_node["name"]), config = ServiceConfig( image = "parity/polkadot:latest", ports = ports, @@ -65,17 +62,17 @@ def start_relay_chain(plan, args, rpc_port = [], prometheus_port = [], lib2lib_p relay_node_details["node-type"] = relay_node["node-type"] if relay_node["prometheus"] == True: relay_node_details["prometheus_port"] = service_details.ports["metrics"].number - if prometheus_port_value != None: - relay_node_details["prometheus_public_port"] = prometheus_port_value - relay_node_details["endpoint_prometheus"] = utils.get_service_url("tcp", "127.0.0.1", prometheus_port_value) - if rpc_port_value != None: - relay_node_details["endpoint_public"] = utils.get_service_url("ws", "127.0.0.1", rpc_port_value) + if prometheus_port != None: + relay_node_details["prometheus_public_port"] = prometheus_port + relay_node_details["endpoint_prometheus"] = utils.get_service_url("tcp", "127.0.0.1", prometheus_port) + if rpc_port != None: + relay_node_details["endpoint_public"] = utils.get_service_url("ws", "127.0.0.1", rpc_port) final_details[service_details.name] = relay_node_details return final_details -def start_test_main_net_relay_nodes(plan, args, rpc_port = [], prometheus_port = [], lib2lib_port = []): +def start_test_main_net_relay_nodes(plan, chain_type, chain_name, relay_nodes): """ Starts testnet/mainnet relay nodes based on the provided arguments. @@ -86,21 +83,12 @@ def start_test_main_net_relay_nodes(plan, args, rpc_port = [], prometheus_port = Returns: list: List of dictionaries containing service details of started relay nodes. """ - name = args["chain-type"] - chain = args["relaychain"]["name"] - - if name == "testnet": - if chain != "rococo" and chain != "westend": - fail("Please provide rococo or westend as relaychain for testnet") - elif name == "mainnet": - if chain != "polkadot" and chain != "kusama": - fail("Please provide polkadot or kusama as relaychain for mainnet") - relay_node_details = start_relay_chain(plan, args, rpc_port, prometheus_port, lib2lib_port) + relay_node_details = start_relay_chain(plan, chain_type, chain_name, relay_nodes) return relay_node_details -def start_relay_chains_local(plan, args, rpc_port = [], prometheus_port = [], lib2lib_port = []): +def start_relay_chains_local(plan, chain_type, chain_name, relay_nodes): """ Starts local relay chain nodes based on the provided arguments. @@ -111,48 +99,38 @@ def start_relay_chains_local(plan, args, rpc_port = [], prometheus_port = [], li Returns: list: List of dictionaries containing sevice details of started relay chain nodes. """ - name = args["chain-type"] - chain = args["relaychain"]["name"] - - if name == "local": - if chain != "rococo-local": - fail("Please provide rococo-local as relaychain for localnet") - - relay_nodes = args["relaychain"]["nodes"] - - if len(relay_nodes) < 2: - fail("relay nodes must contain at least two nodes") - final_details = {} - for idx, node in enumerate(relay_nodes): + for node in relay_nodes: relay_detail = {} - rpc_port_value = rpc_port[idx] if rpc_port else None - prometheus_port_value = prometheus_port[idx] if prometheus_port else None - lib2lib_port_value = lib2lib_port[idx] if lib2lib_port else None - - if prometheus_port_value == 0: - prometheus_port_value = None + if "ports" in node: + rpc_port = node["ports"]["rpc_port"] + lib2lib_port = node["ports"]["lib2lib_port"] + prometheus_port = node["ports"]["prometheus_port"] if node["prometheus"] else None + else: + rpc_port = None + lib2lib_port = None + prometheus_port = None - service_details = start_relay_chain_local(plan, args, node["name"], node["prometheus"], rpc_port_value, prometheus_port_value, lib2lib_port_value) + service_details = start_relay_chain_local(plan, chain_name, node["name"], node["prometheus"], rpc_port, prometheus_port, lib2lib_port) relay_detail["endpoint"] = utils.get_service_url("ws", service_details.ip_address, service_details.ports["ws"].number) relay_detail["service_name"] = service_details.name relay_detail["prometheus"] = node["prometheus"] relay_detail["ip_address"] = service_details.ip_address relay_detail["node-type"] = node["node-type"] - if node["prometheus"] == True: + if node["prometheus"]: relay_detail["prometheus_port"] = service_details.ports["metrics"].number - if prometheus_port_value != None: - relay_detail["prometheus_public_port"] = prometheus_port_value - relay_detail["endpoint_prometheus"] = utils.get_service_url("tcp", "127.0.0.1", prometheus_port_value) - if rpc_port_value != None: - relay_detail["endpoint_public"] = utils.get_service_url("ws", "127.0.0.1", rpc_port_value) + if prometheus_port != None: + relay_detail["prometheus_public_port"] = prometheus_port + relay_detail["endpoint_prometheus"] = utils.get_service_url("tcp", "127.0.0.1", prometheus_port) + if rpc_port != None: + relay_detail["endpoint_public"] = utils.get_service_url("ws", "127.0.0.1", rpc_port) final_details[service_details.name] = relay_detail return final_details -def start_relay_chain_local(plan, args, name, prometheus, rpc_port = None, prometheus_port = None, lib2lib_port = None): +def start_relay_chain_local(plan, chain_name, node_name, prometheus, rpc_port = None, prometheus_port = None, lib2lib_port = None): """ Starts a local relay chain node based on the provided arguments. @@ -182,9 +160,9 @@ def start_relay_chain_local(plan, args, name, prometheus, rpc_port = None, prome if prometheus_port != None: public_ports["metrics"] = PortSpec(prometheus_port, transport_protocol = "TCP") - exec_command = ["bin/sh", "-c", "polkadot --base-path=/data --chain=/app/raw-polkadot.json --validator --rpc-external --port=30333 --rpc-cors=all --name=alice --{0} --rpc-methods=unsafe --execution=wasm --prometheus-external --insecure-validator-i-know-what-i-do".format(name)] + exec_command = ["bin/sh", "-c", "polkadot --base-path=/data --chain=/app/raw-polkadot.json --validator --rpc-external --port=30333 --rpc-cors=all --name=alice --{0} --rpc-methods=unsafe --execution=wasm --prometheus-external --insecure-validator-i-know-what-i-do".format(node_name)] service_details = plan.add_service( - name = "{0}-{1}".format(args["relaychain"]["name"], name), + name = "{0}-{1}".format(chain_name, node_name), config = ServiceConfig( image = "parity/polkadot:latest", files = { From 3b5830a3dda2b0ea95404b65a66115c34fd91386 Mon Sep 17 00:00:00 2001 From: riyasng12 Date: Wed, 10 Jan 2024 16:07:57 +0530 Subject: [PATCH 3/4] chore: removed unused function --- main.star | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/main.star b/main.star index 41ea136..8eeb6f1 100644 --- a/main.star +++ b/main.star @@ -70,14 +70,4 @@ def run_polkadot_setup(plan, chain_type, relaychain, parachains, explorer): explorer_service_details = explorer_js.run_pokadot_js_app(plan, "ws://127.0.0.1:9944") service_details.update(explorer_service_details) - return service_details - -def convert_to_lowercase(data): - if type(data) == dict: - return {key: convert_to_lowercase(value) for key, value in data.items()} - elif type(data) == list: - return [convert_to_lowercase(item) for item in data] - elif type(data) == str: - return data.lower() - else: - return data \ No newline at end of file + return service_details \ No newline at end of file From 90ae6ce3cbe5ab2dbeb0b29841fd32322182321c Mon Sep 17 00:00:00 2001 From: riyasng12 Date: Wed, 10 Jan 2024 17:25:06 +0530 Subject: [PATCH 4/4] chore: changed node-type to node_type --- README.md | 12 ++++++------ local.json | 8 ++++---- package_io/utils.star | 12 ++++++------ parachain/parachain.star | 10 +++++----- relaychain/relay-chain.star | 4 ++-- testdata/updated_config.json | 8 ++++---- 6 files changed, 27 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 17bb736..942a9e9 100644 --- a/README.md +++ b/README.md @@ -114,13 +114,13 @@ To use the Polkadot-kurtosis-package, you need to create a configuration file sp "nodes": [ { "name": "alice", - "node-type": "validator", + "node_type": "validator", "port": 9944, "prometheus": false }, { "name": "bob", - "node-type": "full", + "node_type": "full", "port": 9945, "prometheus": false } @@ -132,12 +132,12 @@ To use the Polkadot-kurtosis-package, you need to create a configuration file sp "nodes": [ { "name": "alice", - "node-type": "validator", + "node_type": "validator", "prometheus": false }, { "name": "bob", - "node-type": "full", + "node_type": "full", "prometheus": false } ] @@ -158,14 +158,14 @@ To use the Polkadot-kurtosis-package, you need to create a configuration file sp - **name:** Name of the relay chain (e.g., "rococo-local", "rococo", "polkadot" or "kusama"). - **nodes:** List of nodes on the relay chain, each with: - **name:** Node name (e.g., "alice"). - - **node-type:** Node type, can be "validator" or "full". + - **node_type:** Node type, can be "validator" or "full". - **port:** Port number for the node (e.g., 9944). - **prometheus:** Whether Prometheus monitoring is enabled (true/false). - **para:** List of parachains, each with: - **name:** Parachain name (e.g., "kilt"). - **nodes:** List of nodes on the parachain, similar to relay chain nodes. - **name:** Node name (e.g., "alice"). - - **node-type:** Node type, can be "callator" or "full". + - **node_type:** Node type, can be "callator" or "full". - **prometheus:** Whether Prometheus monitoring is enabled (true/false). - **chopstick:** Configuration for Chopstick integration. - **xcm:** Whether XCM (Cross-Chain Messaging) is enabled (true/false). diff --git a/local.json b/local.json index 4cae165..f8d0499 100644 --- a/local.json +++ b/local.json @@ -5,12 +5,12 @@ "nodes": [ { "name": "alice", - "node-type": "validator", + "node_type": "validator", "prometheus": false }, { "name": "bob", - "node-type": "full", + "node_type": "full", "prometheus": true } @@ -23,13 +23,13 @@ "nodes": [ { "name": "alice", - "node-type": "validator", + "node_type": "validator", "prometheus": false }, { "name": "bob", - "node-type": "full", + "node_type": "full", "prometheus": true } ] diff --git a/package_io/utils.star b/package_io/utils.star index f34035f..421e6e3 100644 --- a/package_io/utils.star +++ b/package_io/utils.star @@ -87,19 +87,19 @@ def check_config_validity(plan, chain_type, relaychain, parachains): if len(relaychain) != 0: for node in relay_nodes: if len(node) != 0: - if node["node-type"] in ["validator", "full", "archive"]: + if node["node_type"] in ["validator", "full", "archive"]: plan.print("config for relaynodes is valid") else: - return fail("relaychain node-type can be only validator/full") + return fail("relaychain node_type can be only validator/full") if len(parachains) != 0: for para in parachains: if len(para["nodes"]) != 0: for node in para["nodes"]: - if node["node-type"] in ["validator", "full", "collator"]: + if node["node_type"] in ["validator", "full", "collator"]: plan.print("config for parachain is valid") else: - return fail("parachain node-type can be only validator/full/collator") + return fail("parachain node_type can be only validator/full/collator") def upload_files(plan): plan.upload_files(src = "../parachain/static_files/configs", name = "configs") @@ -111,13 +111,13 @@ def convert_to_lowercase(chain_type, relaychain, parachains): relaychain["name"] = relaychain["name"].lower() for node in relaychain["nodes"]: node["name"] = node["name"].lower() - node["node-type"] = node["node-type"].lower() + node["node_type"] = node["node_type"].lower() for para in parachains: para["name"] = para["name"].lower() for node in para["nodes"]: node["name"] = node["name"].lower() - node["node-type"] = node["node-type"].lower() + node["node_type"] = node["node_type"].lower() return chain_type, relaychain, parachains \ No newline at end of file diff --git a/parachain/parachain.star b/parachain/parachain.star index 441826a..50c5702 100644 --- a/parachain/parachain.star +++ b/parachain/parachain.star @@ -57,7 +57,7 @@ def start_local_parachain_node(plan, chain_type, parachain, para_id): parachain_detail["endpoint"] = utils.get_service_url("ws", parachain_spawn_detail.ip_address, parachain_spawn_detail.ports["ws"].number) parachain_detail["ip_address"] = parachain_spawn_detail.ip_address parachain_detail["prometheus"] = node["prometheus"] - parachain_detail["node-type"] = node["node-type"] + parachain_detail["node_type"] = node["node_type"] if node["prometheus"] == True: parachain_detail["prometheus_port"] = parachain_spawn_detail.ports["metrics"].number if prometheus_port != None: @@ -171,10 +171,10 @@ def run_testnet_mainnet(plan, chain_type, relaychain_name, parachain): command = common_command command = command + ["--name={0}".format(node["name"])] - if node["node-type"] == "collator": + if node["node_type"] == "collator": command = command + ["--collator"] - if node["node-type"] == "validator": + if node["node_type"] == "validator": command = command + ["--validator"] if parachain["name"] in constant.CHAIN_COMMAND: @@ -192,7 +192,7 @@ def run_testnet_mainnet(plan, chain_type, relaychain_name, parachain): node_info["endpoint"] = utils.get_service_url("ws", node_details.ip_address, node_details.ports["ws"].number) node_info["ip_address"] = node_details.ip_address node_info["prometheus"] = node["prometheus"] - node_info["node-type"] = node["node-type"] + node_info["node_type"] = node["node_type"] if node["prometheus"] == True: node_info["prometheus_port"] = node_details.ports["metrics"].number if prometheus_port != None: @@ -210,7 +210,7 @@ def run_testnet_mainnet(plan, chain_type, relaychain_name, parachain): node_info["endpoint"] = utils.get_service_url("ws", node_details.ip_address, node_details.ports["ws"].number) node_info["ip_address"] = node_details.ip_address node_info["prometheus"] = node["prometheus"] - node_info["node-type"] = node["node-type"] + node_info["node_type"] = node["node_type"] if node["prometheus"] == True: node_info["prometheus_port"] = node_details.ports["metrics"].number if prometheus_port != None: diff --git a/relaychain/relay-chain.star b/relaychain/relay-chain.star index d617e4e..0d711ad 100644 --- a/relaychain/relay-chain.star +++ b/relaychain/relay-chain.star @@ -59,7 +59,7 @@ def start_relay_chain(plan, chain_type, chain_name, relay_nodes): relay_node_details["service_name"] = service_details.name relay_node_details["prometheus"] = relay_node["prometheus"] relay_node_details["ip_address"] = service_details.ip_address - relay_node_details["node-type"] = relay_node["node-type"] + relay_node_details["node_type"] = relay_node["node_type"] if relay_node["prometheus"] == True: relay_node_details["prometheus_port"] = service_details.ports["metrics"].number if prometheus_port != None: @@ -117,7 +117,7 @@ def start_relay_chains_local(plan, chain_type, chain_name, relay_nodes): relay_detail["service_name"] = service_details.name relay_detail["prometheus"] = node["prometheus"] relay_detail["ip_address"] = service_details.ip_address - relay_detail["node-type"] = node["node-type"] + relay_detail["node_type"] = node["node_type"] if node["prometheus"]: relay_detail["prometheus_port"] = service_details.ports["metrics"].number if prometheus_port != None: diff --git a/testdata/updated_config.json b/testdata/updated_config.json index 86e4f4e..14db386 100644 --- a/testdata/updated_config.json +++ b/testdata/updated_config.json @@ -5,12 +5,12 @@ "nodes": [ { "name": "alice", - "node-type": "validator", + "node_type": "validator", "port": 9944 }, { "name": "bob", - "node-type": "full", + "node_type": "full", "port": 9945 } ] @@ -20,11 +20,11 @@ "nodes": [ { "name": "alice", - "node-type": "collator" + "node_type": "collator" }, { "name": "bob", - "node-type": "full" + "node_type": "full" } ] }