diff --git a/config/provider_examples/lava_example_archive_methodroute.yml b/config/provider_examples/lava_example_archive_method_route.yml similarity index 84% rename from config/provider_examples/lava_example_archive_methodroute.yml rename to config/provider_examples/lava_example_archive_method_route.yml index e8cbf3bad9..5e4f76b9dc 100644 --- a/config/provider_examples/lava_example_archive_methodroute.yml +++ b/config/provider_examples/lava_example_archive_method_route.yml @@ -2,14 +2,14 @@ endpoints: - api-interface: tendermintrpc chain-id: LAV1 network-address: - address: "127.0.0.1:2220" + address: "127.0.0.1:2224" node-urls: - url: ws://127.0.0.1:26657/websocket - url: http://127.0.0.1:26657 - url: http://127.0.0.1:26657 addons: - archive - - url: https://trustless-api.com + - url: http://127.0.0.1:4444 methods: - block - block_by_hash @@ -18,7 +18,7 @@ endpoints: - api-interface: grpc chain-id: LAV1 network-address: - address: "127.0.0.1:2220" + address: "127.0.0.1:2224" node-urls: - url: 127.0.0.1:9090 - url: 127.0.0.1:9090 @@ -27,7 +27,7 @@ endpoints: - api-interface: rest chain-id: LAV1 network-address: - address: "127.0.0.1:2220" + address: "127.0.0.1:2224" node-urls: - url: http://127.0.0.1:1317 - url: http://127.0.0.1:1317 diff --git a/cookbook/projects/policy_all_chains_with_extension.yml b/cookbook/projects/policy_all_chains_with_extension.yml index 491e9bd047..59c25bcb00 100644 --- a/cookbook/projects/policy_all_chains_with_extension.yml +++ b/cookbook/projects/policy_all_chains_with_extension.yml @@ -110,6 +110,26 @@ Policy: extensions: - "archive" mixed: true + - chain_id: OSMOSIS + requirements: + - collection: + api_interface: "rest" + type: "GET" + extensions: + - "archive" + mixed: true + - collection: + api_interface: "grpc" + type: "" + extensions: + - "archive" + mixed: true + - collection: + api_interface: "tendermintrpc" + type: "" + extensions: + - "archive" + mixed: true - chain_id: COSMOSHUB requirements: - collection: diff --git a/cookbook/specs/solana.json b/cookbook/specs/solana.json index 5089de1331..07d746c62f 100755 --- a/cookbook/specs/solana.json +++ b/cookbook/specs/solana.json @@ -964,7 +964,91 @@ "stateful": 0 }, "extra_compute_units": 0 + } + ], + "headers": [], + "inheritance_apis": [], + "parse_directives": [ + { + "function_template": "{\"jsonrpc\":\"2.0\",\"method\":\"getLatestBlockhash\",\"params\":[{\"commitment\":\"finalized\"}],\"id\":1}", + "function_tag": "GET_BLOCKNUM", + "result_parsing": { + "parser_arg": [ + "0", + "context", + "slot" + ], + "parser_func": "PARSE_CANONICAL" + }, + "api_name": "getLatestBlockhash" }, + { + "function_tag": "GET_BLOCK_BY_NUM", + "function_template": "{\"jsonrpc\":\"2.0\",\"method\":\"getBlock\",\"params\":[%d,{\"transactionDetails\":\"none\",\"rewards\":false}],\"id\":1}", + "result_parsing": { + "parser_arg": [ + "0", + "blockhash" + ], + "parser_func": "PARSE_CANONICAL", + "encoding": "base64" + }, + "api_name": "getBlock" + } + ], + "verifications": [ + { + "name": "version", + "parse_directive": { + "function_template": "{\"jsonrpc\":\"2.0\",\"method\":\"getVersion\",\"params\":[],\"id\":1}", + "function_tag": "VERIFICATION", + "result_parsing": { + "parser_arg": [ + "0", + "solana-core" + ], + "parser_func": "PARSE_CANONICAL" + }, + "api_name": "getVersion" + }, + "values": [ + { + "expected_value": "*" + } + ] + }, + { + "name": "tokens-owner-indexed", + "parse_directive": { + "function_template": "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"getTokenAccountsByOwner\",\"params\":[\"4Qkev8aNZcqFNSRhQzwyLMFSsi94jHqE8WNVTJzTP99F\",{\"programId\":\"TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA\"},{\"encoding\":\"jsonParsed\"}]}", + "function_tag": "VERIFICATION", + "result_parsing": { + "parser_arg": [ + "0", + "value" + ], + "parser_func": "PARSE_CANONICAL" + }, + "api_name": "getTokenAccountsByOwner" + }, + "values": [ + { + "expected_value": "*", + "severity": "Warning" + } + ] + } + ] + }, + { + "enabled": true, + "collection_data": { + "api_interface": "jsonrpc", + "internal_path": "/ws", + "type": "POST", + "add_on": "" + }, + "apis": [ { "name": "accountSubscribe", "block_parsing": { @@ -1294,30 +1378,17 @@ "inheritance_apis": [], "parse_directives": [ { - "function_template": "{\"jsonrpc\":\"2.0\",\"method\":\"getLatestBlockhash\",\"params\":[{\"commitment\":\"finalized\"}],\"id\":1}", "function_tag": "GET_BLOCKNUM", "result_parsing": { - "parser_arg": [ - "0", - "context", - "slot" - ], - "parser_func": "PARSE_CANONICAL" - }, - "api_name": "getLatestBlockhash" + "parser_func": "DEFAULT" + } }, { + "function_template": "%d", "function_tag": "GET_BLOCK_BY_NUM", - "function_template": "{\"jsonrpc\":\"2.0\",\"method\":\"getBlock\",\"params\":[%d,{\"transactionDetails\":\"none\",\"rewards\":false}],\"id\":1}", "result_parsing": { - "parser_arg": [ - "0", - "blockhash" - ], - "parser_func": "PARSE_CANONICAL", - "encoding": "base64" - }, - "api_name": "getBlock" + "parser_func": "DEFAULT" + } }, { "function_tag": "SUBSCRIBE", @@ -1401,49 +1472,7 @@ "api_name": "voteUnsubscribe" } ], - "verifications": [ - { - "name": "version", - "parse_directive": { - "function_template": "{\"jsonrpc\":\"2.0\",\"method\":\"getVersion\",\"params\":[],\"id\":1}", - "function_tag": "VERIFICATION", - "result_parsing": { - "parser_arg": [ - "0", - "solana-core" - ], - "parser_func": "PARSE_CANONICAL" - }, - "api_name": "getVersion" - }, - "values": [ - { - "expected_value": "*" - } - ] - }, - { - "name": "tokens-owner-indexed", - "parse_directive": { - "function_template": "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"getTokenAccountsByOwner\",\"params\":[\"4Qkev8aNZcqFNSRhQzwyLMFSsi94jHqE8WNVTJzTP99F\",{\"programId\":\"TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA\"},{\"encoding\":\"jsonParsed\"}]}", - "function_tag": "VERIFICATION", - "result_parsing": { - "parser_arg": [ - "0", - "value" - ], - "parser_func": "PARSE_CANONICAL" - }, - "api_name": "getTokenAccountsByOwner" - }, - "values": [ - { - "expected_value": "*", - "severity": "Warning" - } - ] - } - ] + "verifications": [] } ] }, diff --git a/protocol/chainlib/chainproxy/rpcclient/handler.go b/protocol/chainlib/chainproxy/rpcclient/handler.go index bb1aa31199..acedfd97d9 100755 --- a/protocol/chainlib/chainproxy/rpcclient/handler.go +++ b/protocol/chainlib/chainproxy/rpcclient/handler.go @@ -237,13 +237,16 @@ func (h *handler) handleImmediate(msg *JsonrpcMessage) bool { h.handleSubscriptionResultTendermint(msg) return true case msg.isEthereumNotification(): - if strings.HasSuffix(msg.Method, notificationMethodSuffix) { + if strings.HasSuffix(msg.Method, ethereumNotificationMethodSuffix) { h.handleSubscriptionResultEthereum(msg) return true + } else if strings.HasSuffix(msg.Method, solanaNotificationMethodSuffix) { + h.handleSubscriptionResultSolana(msg) + return true } return false case msg.isStarkNetPathfinderNotification(): - if strings.HasSuffix(msg.Method, notificationMethodSuffix) { + if strings.HasSuffix(msg.Method, ethereumNotificationMethodSuffix) { h.handleSubscriptionResultStarkNetPathfinder(msg) return true } @@ -258,7 +261,7 @@ func (h *handler) handleImmediate(msg *JsonrpcMessage) bool { } func (h *handler) handleSubscriptionResultStarkNetPathfinder(msg *JsonrpcMessage) { - var result starkNetPathfinderSubscriptionResult + var result integerIdSubscriptionResult if err := json.Unmarshal(msg.Result, &result); err != nil { utils.LavaFormatTrace("Dropping invalid starknet pathfinder subscription message", utils.LogAttr("err", err), @@ -290,6 +293,21 @@ func (h *handler) handleSubscriptionResultEthereum(msg *JsonrpcMessage) { } } +func (h *handler) handleSubscriptionResultSolana(msg *JsonrpcMessage) { + var result integerIdSubscriptionResult + if err := json.Unmarshal(msg.Params, &result); err != nil { + utils.LavaFormatTrace("Dropping invalid solana subscription message", + utils.LogAttr("err", err), + utils.LogAttr("params", string(msg.Params)), + ) + h.log.Debug("Dropping invalid subscription message") + return + } + if h.clientSubs[strconv.Itoa(result.ID)] != nil { + h.clientSubs[strconv.Itoa(result.ID)].deliver(msg) + } +} + func (h *handler) handleSubscriptionResultTendermint(msg *JsonrpcMessage) { var result tendermintSubscriptionResult if err := json.Unmarshal(msg.Result, &result); err != nil { diff --git a/protocol/chainlib/chainproxy/rpcclient/json.go b/protocol/chainlib/chainproxy/rpcclient/json.go index 794ad3ebe4..84ab2e0a6e 100755 --- a/protocol/chainlib/chainproxy/rpcclient/json.go +++ b/protocol/chainlib/chainproxy/rpcclient/json.go @@ -33,11 +33,12 @@ import ( ) const ( - Vsn = "2.0" - serviceMethodSeparator = "_" - subscribeMethodSuffix = "_subscribe" - unsubscribeMethodSuffix = "_unsubscribe" - notificationMethodSuffix = "_subscription" + Vsn = "2.0" + serviceMethodSeparator = "_" + subscribeMethodSuffix = "_subscribe" + unsubscribeMethodSuffix = "_unsubscribe" + ethereumNotificationMethodSuffix = "_subscription" + solanaNotificationMethodSuffix = "Notification" defaultWriteTimeout = 10 * time.Second // used if context has no deadline ) @@ -49,7 +50,7 @@ type ethereumSubscriptionResult struct { Result json.RawMessage `json:"result,omitempty"` } -type starkNetPathfinderSubscriptionResult struct { +type integerIdSubscriptionResult struct { ID int `json:"subscription"` Result json.RawMessage `json:"result,omitempty"` } diff --git a/protocol/chainlib/chainproxy/rpcclient/subscription.go b/protocol/chainlib/chainproxy/rpcclient/subscription.go index 803ecc171c..cc882364ac 100755 --- a/protocol/chainlib/chainproxy/rpcclient/subscription.go +++ b/protocol/chainlib/chainproxy/rpcclient/subscription.go @@ -181,7 +181,7 @@ func (n *Notifier) send(sub *Subscription, data json.RawMessage) error { ctx := context.Background() return n.h.conn.writeJSON(ctx, &JsonrpcMessage{ Version: Vsn, - Method: n.namespace + notificationMethodSuffix, + Method: n.namespace + ethereumNotificationMethodSuffix, Params: params, }) } diff --git a/protocol/integration/protocol_test.go b/protocol/integration/protocol_test.go index a0ed5126d9..a5b2b86f84 100644 --- a/protocol/integration/protocol_test.go +++ b/protocol/integration/protocol_test.go @@ -2154,7 +2154,7 @@ func TestArchiveProvidersRetryOnParsedHash(t *testing.T) { ChainId: specId, SeenBlock: 1005, BlocksHashesToHeights: []*pairingtypes.BlockHashToHeight{{Hash: blockHash, Height: spectypes.NOT_APPLICABLE}}, - }) // caching in the portal doesn't care about hashes, and we don't have data on finalization yet + }) // caching in the consumer doesn't care about hashes, and we don't have data on finalization yet cancel() if err != nil { continue diff --git a/protocol/rpcconsumer/rpcconsumer_server.go b/protocol/rpcconsumer/rpcconsumer_server.go index a1c6e55823..a774f0e1da 100644 --- a/protocol/rpcconsumer/rpcconsumer_server.go +++ b/protocol/rpcconsumer/rpcconsumer_server.go @@ -632,7 +632,7 @@ func (rpccs *RPCConsumerServer) sendRelayToProvider( SharedStateId: sharedStateId, SeenBlock: protocolMessage.RelayPrivateData().SeenBlock, BlocksHashesToHeights: rpccs.newBlocksHashesToHeightsSliceFromRequestedBlockHashes(protocolMessage.GetRequestedBlocksHashes()), - }) // caching in the portal doesn't care about hashes, and we don't have data on finalization yet + }) // caching in the consumer doesn't care about hashes, and we don't have data on finalization yet cancel() reply := cacheReply.GetReply() diff --git a/scripts/init_chain_commands.sh b/scripts/init_chain_commands.sh index 295f201531..75faa7345d 100755 --- a/scripts/init_chain_commands.sh +++ b/scripts/init_chain_commands.sh @@ -52,6 +52,7 @@ PROVIDERSTAKE="500000000000ulava" PROVIDER1_LISTENER="127.0.0.1:2221" PROVIDER2_LISTENER="127.0.0.1:2222" PROVIDER3_LISTENER="127.0.0.1:2223" +# PROVIDER4_LISTENER="127.0.0.1:2224" sleep 4 @@ -67,7 +68,7 @@ lavad tx gov vote $(latest_vote) yes -y --from alice --gas-adjustment "1.5" --ga echo; echo "#### Buy DefaultPlan subscription for user1 ####" lavad tx subscription buy DefaultPlan $(lavad keys show user1 -a) --enable-auto-renewal -y --from user1 --gas-adjustment "1.5" --gas "auto" --gas-prices $GASPRICE # wait_count_blocks 2 -# lavad tx project set-policy $(lavad keys show user1 -a)-admin ./cookbook/projects/policy_all_chains_with_addon.yml -y --from user1 --gas-adjustment "1.5" --gas "auto" --gas-prices $GASPRICE +# lavad tx project set-policy $(lavad keys show user1 -a)-admin ./cookbook/projects/policy_all_chains_with_extension.yml -y --from user1 --gas-adjustment "1.5" --gas "auto" --gas-prices $GASPRICE # MANTLE CHAINS="ETH1,SEP1,HOL1,OSMOSIS,FTM250,CELO,LAV1,OSMOSIST,ALFAJORES,ARB1,ARBN,APT1,STRK,JUN1,COSMOSHUB,POLYGON1,EVMOS,OPTM,BASES,CANTO,SUIT,SOLANA,BSC,AXELAR,AVAX,FVM,NEAR,SQDSUBGRAPH,AGR,AGRT,KOIIT,AVAXT,CELESTIATM" @@ -82,6 +83,9 @@ lavad tx pairing bulk-stake-provider $BASE_CHAINS $PROVIDERSTAKE "$PROVIDER2_LIS echo; echo "#### Staking provider 3 ####" lavad tx pairing bulk-stake-provider $BASE_CHAINS $PROVIDERSTAKE "$PROVIDER3_LISTENER,1" 1 $(operator_address) -y --delegate-commission 50 --from servicer3 --provider-moniker "servicer3" --gas-adjustment "1.5" --gas "auto" --gas-prices $GASPRICE +# echo; echo "#### Staking provider 4 ####" +# lavad tx pairing bulk-stake-provider $BASE_CHAINS $PROVIDERSTAKE "$PROVIDER4_LISTENER,1" 1 $(operator_address) -y --delegate-commission 50 --from servicer4 --provider-moniker "servicer4" --gas-adjustment "1.5" --gas "auto" --gas-prices $GASPRICE + echo; echo "#### Waiting 1 block ####" wait_count_blocks 1 diff --git a/scripts/pre_setups/init_eth_archive_mix.sh b/scripts/pre_setups/init_eth_archive_mix.sh index 2d9291a578..134dfcb21e 100755 --- a/scripts/pre_setups/init_eth_archive_mix.sh +++ b/scripts/pre_setups/init_eth_archive_mix.sh @@ -67,7 +67,7 @@ screen -d -m -S provider$i bash -c "source ~/.bashrc; lavap rpcprovider \ $EXTRA_PROVIDER_FLAGS --geolocation 1 --log_level debug --from servicer$i --chain-id lava 2>&1 | tee $LOGS_DIR/PROVIDER$i.log" && sleep 0.25 screen -d -m -S portals bash -c "source ~/.bashrc; lavap rpcconsumer consumer_examples/ethereum_example.yml\ -$EXTRA_PORTAL_FLAGS --cache-be "127.0.0.1:7778" --geolocation 1 --debug-relays --log_level debug --from user1 --chain-id lava --allow-insecure-provider-dialing 2>&1 | tee $LOGS_DIR/PORTAL.log" && sleep 0.25 +$EXTRA_PORTAL_FLAGS --cache-be "127.0.0.1:7778" --geolocation 1 --debug-relays --log_level debug --from user1 --chain-id lava --allow-insecure-provider-dialing 2>&1 | tee $LOGS_DIR/CONSUMER.log" && sleep 0.25 echo "--- setting up screens done ---" screen -ls diff --git a/scripts/setup_providers.sh b/scripts/setup_providers.sh index a3b5fa9b0d..127640da41 100755 --- a/scripts/setup_providers.sh +++ b/scripts/setup_providers.sh @@ -100,9 +100,9 @@ $EXTRA_PROVIDER_FLAGS --geolocation "$GEOLOCATION" --log_level debug --from serv # $PROVIDER3_LISTENER MANTLE jsonrpc '$MANTLE_JRPC' \ echo; echo "#### Starting consumer ####" -# Setup Portal +# Setup Consumer screen -d -m -S portals bash -c "source ~/.bashrc; lavap rpcconsumer consumer_examples/full_consumer_example.yml\ -$EXTRA_PORTAL_FLAGS --cache-be "127.0.0.1:7778" --geolocation "$GEOLOCATION" --debug-relays --log_level debug --from user1 --chain-id lava --allow-insecure-provider-dialing --strategy distributed 2>&1 | tee $LOGS_DIR/PORTAL.log" && sleep 0.25 +$EXTRA_PORTAL_FLAGS --cache-be "127.0.0.1:7778" --geolocation "$GEOLOCATION" --debug-relays --log_level debug --from user1 --chain-id lava --allow-insecure-provider-dialing --strategy distributed 2>&1 | tee $LOGS_DIR/CONSUMER.log" && sleep 0.25 # 127.0.0.1:3385 MANTLE jsonrpc \ echo "--- setting up screens done ---" diff --git a/scripts/test/httpServer.py b/scripts/test/httpServer.py index 94358ac9d6..ddbe4e77a4 100644 --- a/scripts/test/httpServer.py +++ b/scripts/test/httpServer.py @@ -1,6 +1,8 @@ from http.server import BaseHTTPRequestHandler, HTTPServer import sys +payload_ret = "OK" + class RequestHandler(BaseHTTPRequestHandler): def do_GET(self): self.print_request() @@ -26,10 +28,11 @@ def print_request(self): print(f"Body:\n{body.decode('utf-8')}") # Send a response back to the client + response = payload_ret.encode('utf-8') self.send_response(200) - self.send_header("Content-type", "text/html") + self.send_header("Content-type", "application/json") self.end_headers() - self.wfile.write(b"OK") + self.wfile.write(response) def run_server(port=8000): server_address = ('', port) @@ -40,6 +43,8 @@ def run_server(port=8000): if __name__ == '__main__': if len(sys.argv) > 1: port = int(sys.argv[1]) + if len(sys.argv) > 2: + payload_ret = sys.argv[2] run_server(port) else: run_server() \ No newline at end of file diff --git a/scripts/test/jail_provider_test.sh b/scripts/test/jail_provider_test.sh index 878754b542..f190e70376 100755 --- a/scripts/test/jail_provider_test.sh +++ b/scripts/test/jail_provider_test.sh @@ -74,11 +74,11 @@ $PROVIDER4_LISTENER LAV1 tendermintrpc '$LAVA_RPC,$LAVA_RPC' \ $PROVIDER4_LISTENER LAV1 grpc '$LAVA_GRPC' \ $EXTRA_PROVIDER_FLAGS --chain-id=lava --metrics-listen-address ":7780" --geolocation 1 --log_level debug --from servicer4 2>&1 | tee $LOGS_DIR/PROVIDER4.log" -# Setup Portal +# Setup Consumer screen -d -m -S portals bash -c "source ~/.bashrc; lava-protocol rpcconsumer \ 127.0.0.1:3333 ETH1 jsonrpc \ 127.0.0.1:3360 LAV1 rest 127.0.0.1:3361 LAV1 tendermintrpc 127.0.0.1:3362 LAV1 grpc \ -$EXTRA_PORTAL_FLAGS --metrics-listen-address ":7779" --geolocation 1 --log_level debug --from user1 --chain-id lava --allow-insecure-provider-dialing 2>&1 | tee $LOGS_DIR/PORTAL.log" +$EXTRA_PORTAL_FLAGS --metrics-listen-address ":7779" --geolocation 1 --log_level debug --from user1 --chain-id lava --allow-insecure-provider-dialing 2>&1 | tee $LOGS_DIR/CONSUMER.log" # need to wait 8 epochs for the provider to be jail eligible diff --git a/scripts/test_spec_full.sh b/scripts/test_spec_full.sh index 9971f9959c..cd504f47ec 100755 --- a/scripts/test_spec_full.sh +++ b/scripts/test_spec_full.sh @@ -206,7 +206,7 @@ done echo "[+]generated consumer config: $output_consumer_yaml" cat $output_consumer_yaml if [ "$dry" = false ]; then - screen -d -m -S consumers bash -c "source ~/.bashrc; lavap rpcconsumer testutil/debugging/logs/consumer.yml $EXTRA_PORTAL_FLAGS --geolocation 1 --debug-relays --log_level debug --from user1 --chain-id lava --allow-insecure-provider-dialing --metrics-listen-address ":7779" 2>&1 | tee $LOGS_DIR/PORTAL.log" + screen -d -m -S consumers bash -c "source ~/.bashrc; lavap rpcconsumer testutil/debugging/logs/consumer.yml $EXTRA_PORTAL_FLAGS --geolocation 1 --debug-relays --log_level debug --from user1 --chain-id lava --allow-insecure-provider-dialing --metrics-listen-address ":7779" 2>&1 | tee $LOGS_DIR/CONSUMER.log" echo "[+] letting providers start and running health check then running command with flags: $test_consumer_command_args" sleep 10