From 11e02519ce31b50f5f8f69ba343886609fa28969 Mon Sep 17 00:00:00 2001 From: abhiyana Date: Wed, 13 Dec 2023 18:49:28 +0530 Subject: [PATCH 1/2] refactor: refactor service config return by polkadot service --- local.json | 6 ++-- main.star | 38 +++++++++++---------- package_io/grafana.star | 14 +++++++- package_io/polkadot_js_app.star | 13 ++++++-- package_io/promethues.star | 59 +++++++++++---------------------- package_io/utils.star | 6 ++++ parachain/parachain.star | 52 ++++++++++++++++++----------- relaychain/relay-chain.star | 28 ++++++++++++---- 8 files changed, 128 insertions(+), 88 deletions(-) diff --git a/local.json b/local.json index 9621519..3481c25 100644 --- a/local.json +++ b/local.json @@ -13,7 +13,7 @@ "name": "bob", "node-type": "full", "port": 9945, - "prometheus": false + "prometheus": true } ] @@ -32,10 +32,10 @@ { "name": "bob", "node-type": "full", - "prometheus": false + "prometheus": true } ] } ], - "explorer": false + "explorer": true } \ No newline at end of file diff --git a/main.star b/main.star index 8e2df70..310f22e 100644 --- a/main.star +++ b/main.star @@ -20,35 +20,37 @@ def run(plan, args): plan.upload_files(src = "./parachain/static_files/javascript", name = "javascript") prometheus_template = read_file("./package_io/static_files/prometheus.yml.tmpl") - service_details = {"relaychains": {}, "parachains": {}, "prometheus": ""} + service_details = {} if args["chain-type"] == "local": relay_chain_details = relay_chain.start_relay_chains_local(plan, args) - service_details["relaychains"] = relay_chain_details - parachain_details = parachain.start_nodes(plan, args, relay_chain_details[0]["service_details"].ip_address) - service_details["parachains"] = parachain_details + polkadot_service_name = None + for key in relay_chain_details: + polkadot_service_name = key + break + plan.print(polkadot_service_name) + service_details.update(relay_chain_details) + parachain_details = parachain.start_nodes(plan, args, relay_chain_details[polkadot_service_name]["ip_address"]) + service_details.update(parachain_details) else: if len(args["relaychain"]) != 0: - relay_node_detals = relay_chain.start_test_main_net_relay_nodes(plan, args) - service_details["relaychains"] = relay_node_detals - final_parachain_detail = [] + relay_node_details = relay_chain.start_test_main_net_relay_nodes(plan, args) + service_details.update(relay_node_details) for paras in args["para"]: - parachain_details = {} - parachain_details["service_name"] = "parachain_service_" + paras["name"] - parachain_details["parachain_name"] = paras["name"] parachain_info = parachain.run_testnet_mainnet(plan, paras, args) - parachain_details["nodes"] = parachain_info - final_parachain_detail.append(parachain_details) - service_details["parachains"] = final_parachain_detail + service_details.update(parachain_info) #run prometheus , if it returs some endpoint then grafana will up - prometheus_address = promethues.launch_prometheus(plan, args, service_details, prometheus_template) - if prometheus_address.startswith("http://"): - service_details["prometheus"] = prometheus_address - grafana.launch_grafana(plan, grafana) + prometheus_service_details = promethues.launch_prometheus(plan, args, service_details, prometheus_template) + + service_details.update(prometheus_service_details) + if prometheus_service_details["prometheus"]["endpoint"].startswith("http://"): + grafana_service_details = grafana.launch_grafana(plan, grafana) + service_details.update(grafana_service_details) #run the polkadot js App explorer if args["explorer"] == True: - service_details["explorer"] = explorer.run_pokadot_js_app(plan, "ws://127.0.0.1:9944") + explorer_service_details = explorer.run_pokadot_js_app(plan, "ws://127.0.0.1:9944") + service_details.update(explorer_service_details) return service_details diff --git a/package_io/grafana.star b/package_io/grafana.star index d462c70..96de448 100644 --- a/package_io/grafana.star +++ b/package_io/grafana.star @@ -1,3 +1,5 @@ +utils = import_module("./utils.star") + def prometheus_grafana_service(plan, service_name, image, port, command, build_file = None): files = { "/app": "configs", @@ -24,4 +26,14 @@ def prometheus_grafana_service(plan, service_name, image, port, command, build_f def launch_grafana(plan, args): command = ["/run.sh"] - prometheus_grafana_service(plan, "grafana", "grafana/grafana-dev:10.3.0-147071", 3000, command, None) + service = prometheus_grafana_service(plan, "grafana", "grafana/grafana-dev:10.3.0-147071", 3000, command, None) + + grafana_service_details = {} + all_grafana_service_details = {} + + grafana_service_details["service_name"] = "grafana" + grafana_service_details["endpoint"] = utils.get_service_url("http", service.ip_address, 3000) + + all_grafana_service_details[service.name] = grafana_service_details + + return all_grafana_service_details diff --git a/package_io/polkadot_js_app.star b/package_io/polkadot_js_app.star index 689c219..ae55d6e 100644 --- a/package_io/polkadot_js_app.star +++ b/package_io/polkadot_js_app.star @@ -1,6 +1,8 @@ """ This file contain code for running the polkadot-js App """ + +utils = import_module("./utils.star") def run_pokadot_js_app(plan, ws_url): """ This function will run the service for polkadot Js App @@ -21,5 +23,12 @@ def run_pokadot_js_app(plan, ws_url): "WS_URL": ws_url, } ) - service_details = plan.add_service(name="explorer", config=service_config) - return service_details \ No newline at end of file + service_details = plan.add_service(name="polkadot-js-explorer", config=service_config) + + all_explorer_service_details = {} + explorer_service_details = {} + explorer_service_details["service_name"] = service_details.name + explorer_service_details["endpoint"] = utils.get_service_url("http", service_details.ip_address, 80) + explorer_service_details["endpoint_public"] = utils.get_service_url("http", "127.0.0.1", 80) + all_explorer_service_details[service_details.name] = explorer_service_details + return all_explorer_service_details \ No newline at end of file diff --git a/package_io/promethues.star b/package_io/promethues.star index f09e0e6..4dce446 100644 --- a/package_io/promethues.star +++ b/package_io/promethues.star @@ -43,13 +43,19 @@ def launch_prometheus( "prometheus-config", ) + prometheus_service_details = {} + all_prometheus_service_details = {} config = get_config(config_files_artifact_name) prometheus_service = plan.add_service(SERVICE_NAME, config) private_ip_address = prometheus_service.ip_address prometheus_service_http_port = prometheus_service.ports[HTTP_PORT_ID].number + prometheus_service_details["service_name"] = SERVICE_NAME + prometheus_service_details["endpoint"] = "http://{0}:{1}".format(private_ip_address, prometheus_service_http_port) - return "http://{0}:{1}".format(private_ip_address, prometheus_service_http_port) + all_prometheus_service_details[prometheus_service.name] = prometheus_service_details + + return all_prometheus_service_details def get_config(config_files_artifact_name): config_file_path = shared_utils.path_join( @@ -75,48 +81,23 @@ def get_config(config_files_artifact_name): def new_config_template_data(plan, args, service_details): metrics_jobs = [] - - if len(args["relaychain"]) != 0: - relay_nodes = args["relaychain"]["nodes"] - for node in relay_nodes: - if node["prometheus"] == True: - for relay_chain in service_details["relaychains"]: - node_name = relay_chain["service_details"].name - if node_name.endswith(node["name"]): - ip = relay_chain["service_details"].ip_address - port_number = relay_chain["service_details"].ports["metrics"].number - endpoint = "{0}:{1}".format(ip, port_number) - metrics_jobs.append( - new_metrics_job( - job_name = "relay_service_{}".format(node["name"]), - endpoint = endpoint, - scrape_interval = "5s", - ), - ) - - for parachain in args["para"]: - for node in parachain["nodes"]: - if node["prometheus"] == True: - for para_chain in service_details["parachains"]: - for para_chain_node in para_chain["nodes"]: - service_name = para_chain_node["node_details"].name - string = "{}-{}-{}".format(parachain["name"], node["name"], args["chain-type"]) - if string == service_name: - ip = para_chain_node["node_details"].ip_address - port_number = para_chain_node["node_details"].ports["metrics"].number - endpoint = "{0}:{1}".format(ip, port_number) - metrics_jobs.append( - new_metrics_job( - job_name = "parachain_{}_service_{}".format(parachain["name"], node["name"]), - endpoint = endpoint, - scrape_interval = "5s", - ), - ) - + for service in service_details: + if service_details[service]["prometheus"] == True: + ip = service_details[service]["ip_address"] + port_number = service_details[service]["prometheus_port"] + endpoint = "{0}:{1}".format(ip, port_number) + metrics_jobs.append( + new_metrics_job( + job_name = service_details[service]["service_name"], + endpoint = endpoint, + scrape_interval = "5s", + ), + ) return { "MetricsJobs": metrics_jobs, } + def new_metrics_job( job_name, endpoint, diff --git a/package_io/utils.star b/package_io/utils.star index 1c38812..b2c96cd 100644 --- a/package_io/utils.star +++ b/package_io/utils.star @@ -46,3 +46,9 @@ def read_file_from_service(plan, service_name, filename): ), ) return output["output"] + + +def get_service_url(protocol ,ip_address, ports): + url = "{0}://{1}:{2}".format(protocol, ip_address, ports) + return url + diff --git a/parachain/parachain.star b/parachain/parachain.star index f5ec8bd..6e3129b 100644 --- a/parachain/parachain.star +++ b/parachain/parachain.star @@ -3,6 +3,7 @@ register_para_slot = import_module("./register-para-id.star") constant = import_module("../package_io/constant.star") parachain_list = import_module("./static_files/images.star") node_setup = import_module("./node_setup.star") +utils = import_module("../package_io/utils.star") def spawn_parachain(plan, chain_name, image, command, build_file): """Spawn a parachain node with specified configuration. @@ -70,14 +71,19 @@ def start_local_parachain_node(plan, args, parachain_config, para_id): "-c", "{0} --chain=/build/{1}-raw.json --ws-port=9944 --rpc-port=9933 --ws-external --rpc-external --prometheus-external --rpc-cors=all --name={1} --collator --rpc-methods=unsafe --force-authoring --execution=wasm -- --chain=/app/raw-polkadot.json --execution=wasm".format(binary, chain_name), ] - parachain_final = [] - parachain_detail = {} + parachain_final = {} for node in parachain_config["nodes"]: parachain_detail = {} parachain_spawn_detail = spawn_parachain(plan, "{0}-{1}-{2}".format(parachain, node["name"], args["chain-type"]), image, exec_comexec_commandmand, build_file = raw_service.name) - parachain_detail["node_details"] = parachain_spawn_detail - parachain_detail["nodename"] = node["name"] - parachain_final.append(parachain_detail) + parachain_detail["service_name"] = parachain_spawn_detail.name + parachain_detail["endpoint"] = utils.get_service_url("ws", parachain_spawn_detail.ip_address, parachain_spawn_detail.ports["ws"].number) + parachain_detail["endpoint_prometheus"] = utils.get_service_url("tcp" , parachain_spawn_detail.ip_address, parachain_spawn_detail.ports["metrics"].number) + parachain_detail["service_name"] = parachain_spawn_detail.name + parachain_detail["prometheus_port"] = parachain_spawn_detail.ports["metrics"].number + parachain_detail["ip_address"] = parachain_spawn_detail.ip_address + parachain_detail["prometheus"] = node["prometheus"] + parachain_detail["node"] = node["node-type"] + parachain_final[parachain_spawn_detail.name] = parachain_detail return parachain_final def start_nodes(plan, args, relay_chain_ip): @@ -92,15 +98,12 @@ def start_nodes(plan, args, relay_chain_ip): list: List of dictionaries containing service details of each parachain. """ parachains = args["para"] - final_parachain_details = [] + final_parachain_details = {} for parachain in parachains: - parachain_details = {} para_id = register_para_slot.register_para_id(plan, relay_chain_ip) - parachain_details["nodes"] = start_local_parachain_node(plan, args, parachain, para_id) - parachain_details["service_name"] = "parachain_service_" + parachain["name"] - parachain_details["parachain_name"] = parachain["name"] + parachain_details = start_local_parachain_node(plan, args, parachain, para_id) register_para_slot.onboard_genesis_state_and_wasm(plan, para_id, parachain["name"], relay_chain_ip) - final_parachain_details.append(parachain_details) + final_parachain_details.update(parachain_details) return final_parachain_details def run_testnet_mainnet(plan, parachain, args): @@ -171,7 +174,7 @@ def run_testnet_mainnet(plan, parachain, args): common_command = [x for x in common_command if x != "--chain="] common_command = [x for x in common_command if x != "--port=30333"] - final_parachain_info = [] + final_parachain_info = {} for node in parachain["nodes"]: command = common_command command = command + ["--name={0}".format(node["name"])] @@ -192,15 +195,26 @@ def run_testnet_mainnet(plan, parachain, args): command = [binary] + command node_info = {} node_details = node_setup.run_testnet_node_with_entrypoint(plan, image, "{0}-{1}-{2}".format(parachain["name"], node["name"], args["chain-type"]), command) - node_info["nodename"] = node["name"] - node_info["node_details"] = node_details - final_parachain_info.append(node_info) + node_info["service_name"] = node_details.name + node_info["endpoint"] = utils.get_service_url("ws", node_details.ip_address, node_details.ports["ws"].number) + node_info["endpoint_prometheus"] = utils.get_service_url("tcp" , node_details.ip_address, node_details.ports["metrics"].number) + node_info["service_name"] = node_details.name + node_info["ip_address"] = node_details.ip_address + node_info["prometheus_port"] = node_details.ports["metrics"].number + node_info["prometheus"] = node["prometheus"] + node_info["node"] = node["node-type"] + final_parachain_info[node_details.name] = node_info else: node_info = {} node_details = node_setup.run_testnet_node_with_command(plan, image, "{0}-{1}-{2}".format(parachain["name"], node["name"], args["chain-type"]), command) - node_info["nodename"] = node["name"] - node_info["node_details"] = node_details - final_parachain_info.append(node_info) - + node_info["service_name"] = node_details.name + node_info["endpoint"] = utils.get_service_url("ws", node_details.ip_address, node_details.ports["ws"].number) + node_info["endpoint_prometheus"] = utils.get_service_url("tcp" , node_details.ip_address, node_details.ports["metrics"].number) + node_info["service_name"] = node_details.name + node_info["ip_address"] = node_details.ip_address + node_info["prometheus_port"] = node_details.ports["metrics"].number + node_info["prometheus"] = node["prometheus"] + node_info["node"] = node["node-type"] + final_parachain_info[node_details.name] = node_info return final_parachain_info diff --git a/relaychain/relay-chain.star b/relaychain/relay-chain.star index 77793fa..0bb4891 100644 --- a/relaychain/relay-chain.star +++ b/relaychain/relay-chain.star @@ -1,3 +1,6 @@ +utils = import_module("../package_io/utils.star") + + def start_relay_chain(plan, args): """ Starts relay chain nodes based on the provided arguments. @@ -11,7 +14,7 @@ def start_relay_chain(plan, args): """ name = args["chain-type"] chain = args["relaychain"]["name"] - final_details = [] + final_details = {} prometheus = 9615 for relay_node in args["relaychain"]["nodes"]: @@ -33,10 +36,16 @@ def start_relay_chain(plan, args): ), ) prometheus += 1 + relay_node_details = {} - relay_node_details["service_details"] = service_details + relay_node_details["endpoint_public"] = utils.get_service_url("ws", "127.0.0.1", service_details.ports["ws"].number) + relay_node_details["endpoint"] = utils.get_service_url("ws", service_details.ip_address, service_details.ports["ws"].number) + relay_node_details["endpoint_prometheus"] = utils.get_service_url("tcp" , service_details.ip_address, service_details.ports["metrics"].number) relay_node_details["service_name"] = service_details.name - final_details.append(relay_node_details) + relay_node_details["prometheus_port"] = service_details.ports["metrics"].number + relay_node_details["prometheus"] = relay_node["prometheus"] + relay_node_details["ip_address"] = service_details.ip_address + final_details[service_details.name] = relay_node_details return final_details @@ -95,15 +104,22 @@ def start_relay_chains_local(plan, args): if len(relay_nodes) < 2: fail("relay nodes must contain at least two nodes") - final_details = [] + final_details = {} prometheus_port = 9615 for node in relay_nodes: relay_detail = {} service_details = start_relay_chain_local(plan, args, node["name"], node["port"], prometheus_port) - relay_detail["service_details"] = service_details + relay_detail["endpoint_public"] = utils.get_service_url("ws", "127.0.0.1", service_details.ports["ws"].number) + relay_detail["endpoint"] = utils.get_service_url("ws", service_details.ip_address, service_details.ports["ws"].number) + relay_detail["endpoint_prometheus"] = utils.get_service_url("tcp" , service_details.ip_address, service_details.ports["metrics"].number) relay_detail["service_name"] = service_details.name - final_details.append(relay_detail) + relay_detail["prometheus_port"] = service_details.ports["metrics"].number + relay_detail["prometheus"] = node["prometheus"] + relay_detail["ip_address"] = service_details.ip_address + final_details[service_details.name] = relay_detail prometheus_port = prometheus_port + 1 + + return final_details def start_relay_chain_local(plan, args, name, port, prometheus_port): From e202f936b8bc1e92e5daa3366d50f56976cc18cd Mon Sep 17 00:00:00 2001 From: abhiyana Date: Thu, 14 Dec 2023 11:00:27 +0530 Subject: [PATCH 2/2] fix: fix issue of prometheus service details return as string --- main.star | 26 +++++++++++++++++++++----- package_io/promethues.star | 2 +- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/main.star b/main.star index 310f22e..8d6059c 100644 --- a/main.star +++ b/main.star @@ -9,6 +9,21 @@ def run(plan, args): """ Main function to run the Polkadot relay and parachain setup. + Args: + plan (object): The Kurtosis plan object for orchestrating the test. + args (dict): Dictionary containing arguments for configuring the setup. + + Returns: + dict: Service details containing information about relay chains, parachains, and Prometheus. + """ + service_details = run_polkadot_setup(plan, args) + return service_details + + +def run_polkadot_setup(plan, args): + """ + Main function to run the Polkadot relay and parachain setup. + Args: plan (object): The Kurtosis plan object for orchestrating the test. args (dict): Dictionary containing arguments for configuring the setup. @@ -28,7 +43,6 @@ def run(plan, args): for key in relay_chain_details: polkadot_service_name = key break - plan.print(polkadot_service_name) service_details.update(relay_chain_details) parachain_details = parachain.start_nodes(plan, args, relay_chain_details[polkadot_service_name]["ip_address"]) service_details.update(parachain_details) @@ -43,10 +57,11 @@ def run(plan, args): #run prometheus , if it returs some endpoint then grafana will up prometheus_service_details = promethues.launch_prometheus(plan, args, service_details, prometheus_template) - service_details.update(prometheus_service_details) - if prometheus_service_details["prometheus"]["endpoint"].startswith("http://"): - grafana_service_details = grafana.launch_grafana(plan, grafana) - service_details.update(grafana_service_details) + if len(prometheus_service_details) != 0: + service_details.update(prometheus_service_details) + if prometheus_service_details["prometheus"]["endpoint"].startswith("http://"): + grafana_service_details = grafana.launch_grafana(plan, grafana) + service_details.update(grafana_service_details) #run the polkadot js App explorer if args["explorer"] == True: @@ -54,3 +69,4 @@ def run(plan, args): service_details.update(explorer_service_details) return service_details + diff --git a/package_io/promethues.star b/package_io/promethues.star index 4dce446..413725a 100644 --- a/package_io/promethues.star +++ b/package_io/promethues.star @@ -29,7 +29,7 @@ def launch_prometheus( ) if len(template_data["MetricsJobs"]) == 0: - return "No Prometheus is available" + return {} template_and_data = shared_utils.new_template_and_data( config_template,