Merge branch 'main' into feature/multi-endpoints

integrations/blip-image-captioning-large/README.md

@@ -0,0 +1,52 @@
+# BLIP Integrations Examples
+
+BLIP integrations examples offer an easy-to-follow guide to setting up and using BLIP models. Ensure you have the following software installed:
+
+- Python (v3.10+ recommended)
+- Poetry (A Python packaging and dependency management tool)
+
+## Setup
+
+1. For the demo to work, you need to get HuggingFaceAPI Token:
+
+    1. Visit [HuggingFace](https://huggingface.co/).
+    2. Sign up or log in.
+    3. Navigate to `Profile -> Settings -> Access Tokens`.
+    4. Copy an existing token or create a new one.
+
+2. In the `integrations/blip-image-captioning-large/src` directory, create a `.env` file and set your HuggingFaceAPI Token:
+
+    ```
+    export HUGGING_FACE_ACCESS_TOKEN="{Your HuggingFaceAPI Token}"
+    ```
+
+3. To load the environment variables from `.env` and install the dependencies:
+
+    ```bash
+    cd src
+    source .env
+    poetry install
+    ```
+
+## Running The Main Script
+
+To run the project, use the command:
+
+```
+poetry run python main.py
+```
+
+
+After running the command, a request is sent to the agent every 10 minutes. The results can be seen in the console. Look for the following output in the logs:
+
+```
+Adding agent to Bureau: {agent_address}
+```
+
+Copy the {agent_address} value and replace AI_MODEL_AGENT_ADDRESS with this value in blip_user.py.
+
+In the src/blip_user.py file, there exists a variable named IMAGE_FILE. Modify this file path to point to the specific image you wish to caption.
+
+The default image is located at "images/man_umbrella.jpg".
+
+Feel free to utilize this feature to creatively generate captions for your images with BLIP, leveraging the model from "https://huggingface.co/Salesforce/blip-image-captioning-large". Enjoy experimenting with it!

integrations/blip-image-captioning-large/pyproject.toml

@@ -0,0 +1,10 @@
+[tool.poetry]
+name = "blip-image-captioning-large"
+version = "0.1.0"
+description = "Integrating BLIP image captioning into UAgent Ecosystem"
+authors = ["Gaurav Kumar"]
+
+[tool.poetry.dependencies]
+python = ">=3.10,<3.12"
+uagents = "*"
+requests = "^2.31.0"

integrations/blip-image-captioning-large/src/agents/blip_agent.py

@@ -0,0 +1,79 @@
+# Import all necessary modules and resources from various libraries
+from uagents import Agent, Context, Protocol
+from messages.basic import CaptionRequest, CaptionResponse, Error
+from uagents.setup import fund_agent_if_low
+import os
+import requests
+import base64
+
+# Get the HUGGING_FACE_ACCESS_TOKEN from environment variable or default to a placeholder string if not found.
+HUGGING_FACE_ACCESS_TOKEN = os.getenv(
+    "HUGGING_FACE_ACCESS_TOKEN", "HUGGING FACE secret phrase :)")
+
+BLIP_URL = "https://api-inference.huggingface.co/models/Salesforce/blip-image-captioning-large"
+
+# Define headers for HTTP request, including content type and authorization details
+HEADERS = {
+    "Authorization": f"Bearer {HUGGING_FACE_ACCESS_TOKEN}"
+}
+
+# Create an agent with predefined properties
+agent = Agent(
+    name="blip_agent",
+    seed=HUGGING_FACE_ACCESS_TOKEN,
+    port=8001,
+    endpoint=["http://127.0.0.1:8001/submit"],
+)
+
+# Ensure the agent has enough funds
+fund_agent_if_low(agent.wallet.address())
+
+
+# decodes the base64 string into an image, sends a POST request to the BLIP_URL containing the image data,
+# and returns the response from the model as a JSON object. If an error occurs during the process,
+# the function will inform the user about the specific error that occurred.
+async def get_image_caption(ctx: Context, sender: str, imagedata: str):
+    try:
+        # Encoding the image data from ASCII to bytes
+        imagedata = imagedata.encode("ascii")
+        # Converting the image data from base64 to bytes
+        imageBytes = base64.b64decode(imagedata)
+
+        # Sending POST request to BLIP_URL with image bytes
+        response = requests.post(BLIP_URL, headers=HEADERS, data=imageBytes)
+
+        # If the request response is not successful (non-200 code), send error message
+        if response.status_code != 200:
+            await ctx.send(sender, Error(error=f"Error: {response.json().get('error')}"))
+            return
+
+        # Parse the first message (image caption) from the response
+        message = response.json()[0]
+        # Send the parsed response back to the sender/user
+        await ctx.send(sender, CaptionResponse.parse_obj(message))
+        return
+
+    # If an unknown exception occurs, send a generic error message to the sender/user
+    except Exception as ex:
+        await ctx.send(sender, Error(error=f"Sorry, I wasn't able to complete your request this time. Error detail: {ex}"))
+        return
+
+# Create an instance of Protocol with a label "BlipImageCaptioning"
+blip_agent = Protocol(name="BlipImageCaptioning", version="0.1.0")
+
+
+@blip_agent.on_message(model=CaptionRequest, replies={CaptionResponse, Error})
+async def handle_request(ctx: Context, sender: str, request: CaptionRequest):
+    # Log the request details
+    ctx.logger.info(f"Got request from  {sender}")
+
+    # Get the image caption from blip model
+    await get_image_caption(ctx, sender, request.image_data)
+
+
+# Include the protocol with the agent, publish_manifest will make the protocol details available on Agentverse.
+agent.include(blip_agent, publish_manifest=True)
+
+# Define the main entry point of the application
+if __name__ == "__main__":
+    blip_agent.run()

integrations/blip-image-captioning-large/src/agents/blip_user.py

@@ -0,0 +1,70 @@
+from uagents import Agent, Context, Protocol  # Import necessary modules
+# Import Basic messages
+from messages.basic import CaptionRequest, CaptionResponse, Error
+
+# Import the fund_agent_if_low function
+from uagents.setup import fund_agent_if_low
+
+import base64
+
+IMAGE_FILE = "sample-images/gandhi.jpg"
+
+# AI model agent address
+AI_MODEL_AGENT_ADDRESS = "agent1q2gdm20yg3krr0k5nt8kf794ftm46x2cs7xyrm2gqc9ga5lulz9xsp4l4kk"
+
+# Define user agent with specified parameters
+user = Agent(
+    name="blip_user",
+    port=8000,
+    endpoint=["http://127.0.0.1:8000/submit"],
+)
+
+# Check and top up the agent's fund if low
+fund_agent_if_low(user.wallet.address())
+
+
+@user.on_event("startup")
+async def initialize_storage(ctx: Context):
+    ctx.storage.set("captionCreated", False)
+
+
+# Create an instance of Protocol with a label "BlipImageCaptioningUser"
+blip_user = Protocol(name="BlipImageCaptioningUser", version="0.1.0")
+
+
+# This is an asynchronous function that is set to run at intervals of 30 sec.
+# It opens the specified IMAGE_FILE, reads it and encodes the image in base64 format.
+# Afterwards, it sends a request with the encoded data to the AI uagent's address.
+@blip_user.on_interval(period=30, messages=CaptionRequest)
+async def image_caption(ctx: Context):
+    captionCreated = ctx.storage.get("captionCreated")
+
+    if not captionCreated:
+        # Opening the file in read binary mode
+        with open(IMAGE_FILE, "rb") as f:
+            # Encoding the image data to base64
+            data = base64.b64encode(f.read()).decode('ascii')
+        # Using the context to send the request to the desired address with the image data
+        await ctx.send(AI_MODEL_AGENT_ADDRESS, CaptionRequest(image_data=data))
+
+
+# Define a function to handle and
+# log data received from the AI model agent
+@blip_user.on_message(model=CaptionResponse)
+async def handle_data(ctx: Context, sender: str, caption: CaptionResponse):
+    ctx.logger.info(f"image caption => {caption.generated_text}")
+    ctx.storage.set("captionCreated", True)
+
+# Define a function to handle and log errors occurred during process
+
+
+@blip_user.on_message(model=Error)
+async def handle_error(ctx: Context, sender: str, error: Error):
+    ctx.logger.info(f"Got error from uagent: {error}")
+
+# Include the protocol with the agent, publish_manifest will make the protocol details available on Agentverse.
+user.include(blip_user, publish_manifest=True)
+
+# Initiate the image captioning task
+if __name__ == "__main__":
+    blip_user.run()

integrations/blip-image-captioning-large/src/main.py

@@ -0,0 +1,13 @@
+from uagents import Bureau
+
+from agents.blip_agent import agent
+from agents.blip_user import user
+
+
+if __name__ == "__main__":
+    bureau = Bureau(endpoint="http://127.0.0.1:8000/submit", port=8000)
+    print(f"Adding agent to Bureau: {agent.address}")
+    bureau.add(agent)
+    print(f"Adding user agent to Bureau: {user.address}")
+    bureau.add(user)
+    bureau.run()

integrations/blip-image-captioning-large/src/messages/basic.py

@@ -0,0 +1,13 @@
+from uagents import Model
+
+
+class CaptionRequest(Model):
+    image_data: str
+
+
+class Error(Model):
+    error: str
+
+
+class CaptionResponse(Model):
+    generated_text: str