From c6b16a29fae7c91ad29df8a48aed496c6c667550 Mon Sep 17 00:00:00 2001 From: Ardian Date: Tue, 12 Dec 2023 15:20:10 +0100 Subject: [PATCH] feat: add support for wrapped tools --- packages/packages.json | 10 ++-- packages/valory/agents/mech/aea-config.yaml | 6 +-- packages/valory/services/mech/service.yaml | 2 +- packages/valory/skills/mech_abci/skill.yaml | 2 +- .../skills/task_execution/behaviours.py | 12 +++-- .../valory/skills/task_execution/skill.yaml | 8 ++-- .../skills/task_execution/utils/ipfs.py | 46 +++++++++++++++++++ .../skills/task_execution/utils/task.py | 7 +-- tools/stability_ai_request/component.yaml | 4 +- 9 files changed, 75 insertions(+), 22 deletions(-) diff --git a/packages/packages.json b/packages/packages.json index 3119b9b5..63eb8b89 100644 --- a/packages/packages.json +++ b/packages/packages.json @@ -2,13 +2,13 @@ "dev": { "connection/valory/websocket_client/0.1.0": "bafybeiflmystocxaqblhpzqlcop2vkhsknpzjx2jomohomaxamwskeokzm", "skill/valory/contract_subscription/0.1.0": "bafybeicyugrkx5glat4p4ezwf6i7oduh26eycfie6ftd4uxrknztzl3ik4", - "agent/valory/mech/0.1.0": "bafybeiezyvels4ylygglro3isx7cvjaku2e5cfhb2vjrw6b3gxjioa4h4a", - "skill/valory/mech_abci/0.1.0": "bafybeifktlrkncnb5hjtwc7xqjag2pzmtciogp6nsm4t5umrqhmkmzyl5i", + "agent/valory/mech/0.1.0": "bafybeibrn5iap4c2ciagiemoe2sfo6myk3gdyu5n4rpjttkehr5oq4263m", + "skill/valory/mech_abci/0.1.0": "bafybeid3uhvakfymmly2uqezynq7a5xdqg7u4pmtas3cnxaxyhtfnbuff4", "contract/valory/agent_mech/0.1.0": "bafybeicgt3bwogpwv7t57xuxzcq7vxmoq4glcjoprabb6d4ajtgjwzsrzi", - "service/valory/mech/0.1.0": "bafybeicme3nvcsgvid2q4k2xuhg4y4h6hzfzws4a2ikuowsuquoehjy6re", + "service/valory/mech/0.1.0": "bafybeiaxqz5m6774k4suzmth5v7qnxc4wbsz6ywv6elrrgbxypjzygsngm", "protocol/valory/acn_data_share/0.1.0": "bafybeih5ydonnvrwvy2ygfqgfabkr47s4yw3uqxztmwyfprulwfsoe7ipq", "skill/valory/task_submission_abci/0.1.0": "bafybeia6b7rzrbrhijkctxpwzeywqvsdqm2j5u5ee6fbmrp6t3x2afdase", - "skill/valory/task_execution/0.1.0": "bafybeidmgp7dwelaxbul532vsebllmb6tyqbnvrifosmxwkzwgvyekyrve", + "skill/valory/task_execution/0.1.0": "bafybeiffc4nqpk3ri7kq5tuljbgnxqmhfj3brpznr6ujxqaif24tj64kcq", "contract/valory/agent_registry/0.1.0": "bafybeiargayav6yiztdnwzejoejstcx4idssch2h4f5arlgtzj3tgsgfmu", "protocol/valory/websocket_client/0.1.0": "bafybeih43mnztdv3v2hetr2k3gezg7d3yj4ur7cxdvcyaqhg65e52s5sf4", "skill/valory/websocket_client/0.1.0": "bafybeidwntmkk4b2ixq5454ycbkknclqx7a6vpn7aqpm2nw3duszqrxvta", @@ -39,6 +39,6 @@ "contract/valory/gnosis_safe_proxy_factory/0.1.0": "bafybeidvcgnbbmjtha6tl7st2ysna2l3iiqfhnzjfasyvml5unecozeeka", "contract/valory/gnosis_safe/0.1.0": "bafybeidll7frtdsq2ckiluazkwk4zpkp7natt4kdejjehwvykqwzyuf6ei", "contract/valory/multisend/0.1.0": "bafybeig5byt5urg2d2bsecufxe5ql7f4mezg3mekfleeh32nmuusx66p4y", - "connection/valory/http_server/0.22.0": "bafybeiesafy2af2wakqvsal4hz6ldycb73j5w62szhwy25iyhlf2wgyg44" + "connection/valory/http_server/0.22.0": "bafybeib6axuzanyktdidtygeibsm426fw4a2dcpc5ppdwsjblaboeepmry" } } \ No newline at end of file diff --git a/packages/valory/agents/mech/aea-config.yaml b/packages/valory/agents/mech/aea-config.yaml index ef297e6c..ffd37357 100644 --- a/packages/valory/agents/mech/aea-config.yaml +++ b/packages/valory/agents/mech/aea-config.yaml @@ -12,7 +12,7 @@ connections: - valory/ipfs:0.1.0:bafybeihx7wb5hngjobw2salzqqryrhxvmxfuw7o2npjyqd2talmh2flqeq - valory/ledger:0.19.0:bafybeia47rr37ianvwsh77tjjpv3nwif5sywhhy2fbdshnz4a2icwln76a - valory/p2p_libp2p_client:0.1.0:bafybeihge56dn3xep2dzomu7rtvbgo4uc2qqh7ljl3fubqdi2lq44gs5lq -- valory/http_server:0.22.0:bafybeiesafy2af2wakqvsal4hz6ldycb73j5w62szhwy25iyhlf2wgyg44 +- valory/http_server:0.22.0:bafybeib6axuzanyktdidtygeibsm426fw4a2dcpc5ppdwsjblaboeepmry - valory/websocket_client:0.1.0:bafybeiflmystocxaqblhpzqlcop2vkhsknpzjx2jomohomaxamwskeokzm contracts: - valory/agent_mech:0.1.0:bafybeicgt3bwogpwv7t57xuxzcq7vxmoq4glcjoprabb6d4ajtgjwzsrzi @@ -37,8 +37,8 @@ skills: - valory/abstract_abci:0.1.0:bafybeiflcfufixmsrhobf56bn5745m2iipcfqyulwk2qegtnagb3kvaaxi - valory/abstract_round_abci:0.1.0:bafybeiaqcl7h2famylusiffigwem7tevkcsyocdu5xd42jkmgq6kvowzgq - valory/contract_subscription:0.1.0:bafybeicyugrkx5glat4p4ezwf6i7oduh26eycfie6ftd4uxrknztzl3ik4 -- valory/mech_abci:0.1.0:bafybeifktlrkncnb5hjtwc7xqjag2pzmtciogp6nsm4t5umrqhmkmzyl5i -- valory/task_execution:0.1.0:bafybeidmgp7dwelaxbul532vsebllmb6tyqbnvrifosmxwkzwgvyekyrve +- valory/mech_abci:0.1.0:bafybeid3uhvakfymmly2uqezynq7a5xdqg7u4pmtas3cnxaxyhtfnbuff4 +- valory/task_execution:0.1.0:bafybeiffc4nqpk3ri7kq5tuljbgnxqmhfj3brpznr6ujxqaif24tj64kcq - valory/registration_abci:0.1.0:bafybeic2ynseiak7jpta7jfwuqwyp453b4p7lolr4wihxmpn633uekv5am - valory/reset_pause_abci:0.1.0:bafybeidzajbe3erygeh2xbd6lrjv7nsptznjuzrt24ykgvhgotdeyhfnba - valory/task_submission_abci:0.1.0:bafybeia6b7rzrbrhijkctxpwzeywqvsdqm2j5u5ee6fbmrp6t3x2afdase diff --git a/packages/valory/services/mech/service.yaml b/packages/valory/services/mech/service.yaml index 22a5f6be..388c4174 100644 --- a/packages/valory/services/mech/service.yaml +++ b/packages/valory/services/mech/service.yaml @@ -7,7 +7,7 @@ license: Apache-2.0 fingerprint: README.md: bafybeif7ia4jdlazy6745ke2k2x5yoqlwsgwr6sbztbgqtwvs3ndm2p7ba fingerprint_ignore_patterns: [] -agent: valory/mech:0.1.0:bafybeiezyvels4ylygglro3isx7cvjaku2e5cfhb2vjrw6b3gxjioa4h4a +agent: valory/mech:0.1.0:bafybeibrn5iap4c2ciagiemoe2sfo6myk3gdyu5n4rpjttkehr5oq4263m number_of_agents: 4 deployment: agent: diff --git a/packages/valory/skills/mech_abci/skill.yaml b/packages/valory/skills/mech_abci/skill.yaml index 76a75636..9069dc2a 100644 --- a/packages/valory/skills/mech_abci/skill.yaml +++ b/packages/valory/skills/mech_abci/skill.yaml @@ -15,7 +15,7 @@ fingerprint: models.py: bafybeic3pjxw7py6jpiaaxjtcufzcjmyldj2fdhpkik5qnj4hpruuxcu4q fingerprint_ignore_patterns: [] connections: -- valory/http_server:0.22.0:bafybeiesafy2af2wakqvsal4hz6ldycb73j5w62szhwy25iyhlf2wgyg44 +- valory/http_server:0.22.0:bafybeib6axuzanyktdidtygeibsm426fw4a2dcpc5ppdwsjblaboeepmry contracts: [] protocols: - valory/http:1.0.0:bafybeiejoqgv7finfxo3rcvvovrlj5ccrbgxodjq43uo26ylpowsa3llfe diff --git a/packages/valory/skills/task_execution/behaviours.py b/packages/valory/skills/task_execution/behaviours.py index be421477..fb7f083a 100644 --- a/packages/valory/skills/task_execution/behaviours.py +++ b/packages/valory/skills/task_execution/behaviours.py @@ -49,6 +49,7 @@ from packages.valory.protocols.ledger_api import LedgerApiMessage from packages.valory.skills.task_execution.models import Params from packages.valory.skills.task_execution.utils.ipfs import ( + ComponentPackageLoader, get_ipfs_file_hash, to_multihash, ) @@ -73,7 +74,7 @@ def __init__(self, **kwargs: Any): self._executor = ProcessPoolExecutor(max_workers=1) self._executing_task: Optional[Dict[str, Any]] = None self._tools_to_file_hash: Dict[str, str] = {} - self._all_tools: Dict[str, str] = {} + self._all_tools: Dict[str, Tuple[str, str]] = {} self._inflight_tool_req: Optional[str] = None self._done_task: Optional[Dict[str, Any]] = None self._last_polling: Optional[float] = None @@ -183,9 +184,11 @@ def _download_tools(self) -> None: def _handle_get_tool(self, message: IpfsMessage, dialogue: Dialogue) -> None: """Handle get tool response""" - tool_py = list(message.files.values())[0] + _component_yaml, tool_py, callable_method = ComponentPackageLoader.load( + message.files + ) tool_req = cast(str, self._inflight_tool_req) - self._all_tools[tool_req] = tool_py + self._all_tools[tool_req] = tool_py, callable_method self._inflight_tool_req = None def _populate_from_block(self) -> None: @@ -361,8 +364,9 @@ def _submit_task(self, fn: Any, *args: Any, **kwargs: Any) -> Future: def _prepare_task(self, task_data: Dict[str, Any]) -> None: """Prepare the task.""" tool_task = AnyToolAsTask() - tool_py = self._all_tools[task_data["tool"]] + tool_py, callable_method = self._all_tools[task_data["tool"]] task_data["tool_py"] = tool_py + task_data["callable_method"] = callable_method task_data["api_keys"] = self.params.api_keys future = self._submit_task(tool_task.execute, **task_data) executing_task = cast(Dict[str, Any], self._executing_task) diff --git a/packages/valory/skills/task_execution/skill.yaml b/packages/valory/skills/task_execution/skill.yaml index 1cbc34ae..d948e59c 100644 --- a/packages/valory/skills/task_execution/skill.yaml +++ b/packages/valory/skills/task_execution/skill.yaml @@ -7,13 +7,13 @@ license: Apache-2.0 aea_version: '>=1.0.0, <2.0.0' fingerprint: __init__.py: bafybeidqhvvlnthkbnmrdkdeyjyx2f2ab6z4xdgmagh7welqnh2v6wczx4 - behaviours.py: bafybeiahzfpgcjpvbhpfkbifrkzb6qaj2gesnkswdi2lj7yszhe27dvjtm + behaviours.py: bafybeia4ohotkawmpee6k7jr3jin2cnuoikxnm2r7vd3mgapewbt6ea6vy dialogues.py: bafybeid4zxalqdlo5mw4yfbuf34hx4jp5ay5z6chm4zviwu4cj7fudtwca handlers.py: bafybeidbt5ezj74cgfogk3w4uw4si2grlnk5g54veyumw7g5yh6gdscywu models.py: bafybeihc2kmymmh5oousjddbc7xujqbk5niermuqak2dhtgryukzq5wxeq utils/__init__.py: bafybeiccdijaigu6e5p2iruwo5mkk224o7ywedc7nr6xeu5fpmhjqgk24e - utils/ipfs.py: bafybeicuaj23qrcdv6ly4j7yo6il2r5plozhd6mwvcp5acwqbjxb2t3u2i - utils/task.py: bafybeiakokty64m5cqp72drrpvfckhruldlwcge5hcc2bsy2ujk6nnrazq + utils/ipfs.py: bafybeidinbdqkidix44ibz5hug7inkcbijooag53gr5mtbaa72tk335uqq + utils/task.py: bafybeieuziu7owtk543z3umgmayhjh67klftk7vrhz24l6rlaii5lvkqh4 fingerprint_ignore_patterns: [] connections: - valory/ledger:0.19.0:bafybeia47rr37ianvwsh77tjjpv3nwif5sywhhy2fbdshnz4a2icwln76a @@ -104,4 +104,6 @@ dependencies: version: ==1.0.3 py-multicodec: version: ==0.2.1 + pyyaml: + version: <=6.0.1,>=3.10 is_abstract: false diff --git a/packages/valory/skills/task_execution/utils/ipfs.py b/packages/valory/skills/task_execution/utils/ipfs.py index 3882a049..364a3a28 100644 --- a/packages/valory/skills/task_execution/utils/ipfs.py +++ b/packages/valory/skills/task_execution/utils/ipfs.py @@ -17,6 +17,9 @@ # # ------------------------------------------------------------------------------ """This module contains helpers for IPFS interaction.""" +from typing import Any, Dict, Tuple + +import yaml from aea.helpers.cid import CID from multibase import multibase from multicodec import multicodec @@ -46,3 +49,46 @@ def to_multihash(hash_string: str) -> bytes: # Convert the multihash bytes to a hexadecimal string hex_multihash = multihash_bytes.hex() return hex_multihash[6:] + + +class ComponentPackageLoader: + """Component package loader.""" + + @staticmethod + def load(serialized_objects: Dict[str, str]) -> Tuple[Dict[str, Any], str, str]: + """ + Load a custom component package. + + :param serialized_objects: the serialized objects. + :return: the component.yaml, entry_point.py and callable as tuple. + """ + # the package MUST contain a component.yaml file + if "component.yaml" not in serialized_objects: + raise ValueError( + "Invalid component package. " + "The package MUST contain a component.yaml." + ) + + # load the component.yaml file + component_yaml = yaml.safe_load(serialized_objects["component.yaml"]) + if "entry_point" not in component_yaml or "callable" not in component_yaml: + raise ValueError( + "Invalid component package. " + "The component.yaml file MUST contain the 'entry_point' and 'callable' keys." + ) + + # the name of the script that needs to be executed + entry_point_name = component_yaml["entry_point"] + + # load the script + if entry_point_name not in serialized_objects: + raise ValueError( + f"Invalid component package. " + f"{entry_point_name} is not present in the component package." + ) + entry_point = serialized_objects[entry_point_name] + + # the method that needs to be called + callable_method = component_yaml["callable"] + + return component_yaml, entry_point, callable_method diff --git a/packages/valory/skills/task_execution/utils/task.py b/packages/valory/skills/task_execution/utils/task.py index af5fefca..2d4c9f42 100644 --- a/packages/valory/skills/task_execution/utils/task.py +++ b/packages/valory/skills/task_execution/utils/task.py @@ -28,8 +28,9 @@ class AnyToolAsTask: def execute(self, *args: Any, **kwargs: Any) -> Any: """Execute the task.""" tool_py = kwargs.pop("tool_py") - if "run" in globals(): - del globals()["run"] + callable_method = kwargs.pop("callable_method") + if callable_method in globals(): + del globals()[callable_method] exec(tool_py, globals()) # pylint: disable=W0122 # nosec - method = globals()["run"] + method = globals()[callable_method] return method(*args, **kwargs) diff --git a/tools/stability_ai_request/component.yaml b/tools/stability_ai_request/component.yaml index 2303fed7..b3bb9eee 100644 --- a/tools/stability_ai_request/component.yaml +++ b/tools/stability_ai_request/component.yaml @@ -7,8 +7,8 @@ license: Apache-2.0 aea_version: '>=1.0.0, <2.0.0' fingerprint: __init__.py: bafybeidey4syohls5hxmso6qsp5p4uhtzle5txv2mlbym6ktjzknich6oa - stability_ai_request.py: bafybeibqwl52cnz64cysjd2jnjijuakdvyrffapxq65cdzx6g65gu42deq + stabilityai_request.py: bafybeibqwl52cnz64cysjd2jnjijuakdvyrffapxq65cdzx6g65gu42deq fingerprint_ignore_patterns: [] -entry_point: stability_ai_request.py +entry_point: stabilityai_request.py callable: run dependencies: {} \ No newline at end of file