Add bioengine app docs

bioimageio/engine/__init__.py

@@ -65,4 +65,6 @@ async def launch_all_bioengine_apps(server):
     apps = load_apps()
     logger.info("Starting all bioengine apps...")
     await launch_all_apps(server, apps, continue_on_error=True)
-    logger.info("All bioengine apps are started.")
+    logger.info("All bioengine apps are started.")
+
+

bioimageio/engine/__main__.py

@@ -40,9 +40,14 @@ def connect_server(server_url, login_required=False):
     loop.create_task(connect_server(server_url))
     loop.run_forever()
 
+def serve_ray_apps():
+    from ray import serve
+    from bioimageio.engine.ray_app_loader import app
+    serve.run(app)
 
 if __name__ == '__main__':
     fire.Fire({
         "start_server": start_server,
-        "connect_server": connect_server
+        "connect_server": connect_server,
+        "serve_ray_apps": serve_ray_apps,
     })

bioimageio/engine/apps/imagej/run_imagej.py

@@ -8,7 +8,7 @@
 import xarray as xr
 from jpype import JOverride, JImplements
 import argparse
-from imjoy_rpc.hypha import connect_to_server
+from hypha_rpc import connect_to_server
 
 
 logger = logging.getLogger(__name__)

bioimageio/engine/ray_app_loader.py

@@ -101,6 +101,7 @@ async def __call__(self, request: Request):
         app_deployment = serve.deployment(name=app_info.name)(app_info.app_class).bind()
         ray_apps[app_info.name] = app_deployment
 
+print("Loaded apps:", ray_apps.keys())
 # Getting config from environment
 server_url = os.environ.get("HYPHA_SERVER_URL")
 workspace = os.environ.get("HYPHA_WORKSPACE")
@@ -112,6 +113,6 @@ async def __call__(self, request: Request):
 
 if __name__ == "__main__":
     serve.start()
-    serve.run(app, name="hypha-apps")
+    serve.run(app, name="bioengine-apps")
     import asyncio
     asyncio.get_event_loop().run_forever()

bioimageio/engine/ray_apps/cellpose.py

@@ -0,0 +1,15 @@
+from hypha_rpc import api
+
+class CellposeModel:
+    def __init__(self):
+        # Load model
+        pass
+
+    def predict(self, image: str) -> str:
+        pass
+
+    def train(self, data: str, config: str) -> str:
+        pass
+
+
+api.export(CellposeModel, {"name": "cellpose", "type": "ray"})

docs/api.md

@@ -33,7 +33,7 @@ For different Python runtimes (e.g. native Python vs Pyodide), we provide two ve
 ```python
 import asyncio
 import numpy as np
-from imjoy_rpc.hypha import connect_to_server
+from hypha_rpc import connect_to_server
 
 async def main():
     server = await connect_to_server(
@@ -66,7 +66,7 @@ if __name__ == "__main__":
 
 ```python
 import numpy as np
-from imjoy_rpc.hypha.sync import connect_to_server
+from hypha_rpc.sync import connect_to_server
 
 def main():
     server = connect_to_server(
@@ -98,7 +98,7 @@ if __name__ == "__main__":
 
 > [!NOTE]
 > In Python, the recommended way to interact with the server to use asynchronous functions with `asyncio`. However, if you need to use synchronous functions,
-> you can use `from imjoy_rpc.hypha.sync import login, connect_to_server` (available since `hypha-rpc>=0.5.25.post0`) instead.
+> you can use `from hypha_rpc.sync import login, connect_to_server` (available since `hypha-rpc>=0.5.25.post0`) instead.
 > They have the exact same arguments as the asynchronous versions. For more information, see [Synchronous Wrapper](https://github.com/imjoy-team/hypha-rpc/blob/master/hypha-rpc-v2.md#synchronous-wrapper)
 
 > <strong>💡 Tip </strong><br>

docs/bioengine-apps.md

@@ -0,0 +1,234 @@
+# BioEngine Apps
+
+## 1. Introduction
+
+**BioEngine** is an advanced web platform designed to democratize AI in life sciences by offering cloud-powered tools for bioimage analysis. It enables users to create interactive annotation tools, host AI models for inference and training, and integrate these capabilities seamlessly into existing workflows. BioEngine's architecture consists of several key components, including **Hypha**, **NVIDIA Triton Inference Server**, **Ray**, and **ImJoy plugins**. These components work together to support two main types of applications:
+
+- **UI Apps**: Web-based applications that interact with BioEngine's Compute Apps through the Hypha server.
+- **Compute Apps**: Applications that provide computational services, such as AI model inference and training, which can be accessed by UI Apps.
+
+This documentation will guide you through the steps to create and deploy both UI and Compute Apps using the BioEngine platform.
+
+---
+
+## 2. Architecture Overview
+
+The architecture of BioEngine is designed to be modular, scalable, and easy to integrate with various bioimaging software environments. Below is an overview of the core components:
+
+- **Hypha**: The central communication hub of BioEngine, Hypha is an RPC-based framework that facilitates communication between different components of the platform. It orchestrates the interactions between UI Apps and Compute Apps, enabling real-time data exchange and service discovery.
+
+- **NVIDIA Triton Inference Server**: This server handles AI model inference, allowing BioEngine to efficiently manage GPU resources. Triton supports multiple AI frameworks, including TensorFlow, PyTorch, and ONNX, making it versatile for various bioimage analysis tasks.
+
+- **Ray**: Ray is a distributed computing framework used in BioEngine for scalable AI model training and inference. It manages distributed tasks across a cluster, ensuring efficient use of computational resources.
+
+- **ImJoy Plugins**: ImJoy is a plugin-based architecture that allows users to create interactive, web-based tools that can communicate with BioEngine's Compute Apps. These plugins can be customized to perform specific tasks, such as image processing or AI model inference.
+
+![BioEngine Architecture](path_to_architecture_image)
+
+The diagram above illustrates how these components interact within the BioEngine platform. UI Apps communicate with the Hypha server, which routes requests to the appropriate Compute App for processing.
+
+---
+
+## 3. Getting Started
+
+### Prerequisites
+
+To create BioEngine Apps, you need to ensure that the following prerequisites are met:
+
+- **Hypha Server**: The Hypha server should be set up and running. This server will act as the central communication point for your BioEngine Apps.
+- **Docker**: Docker should be installed on your system to manage containerized applications. For larger deployments, Kubernetes should be configured.
+- **Programming Knowledge**: Basic knowledge of JavaScript (for UI Apps) and Python (for Compute Apps) is required.
+
+### Installation
+
+1. **Clone the BioEngine repository**:
+
+   ```bash
+   git clone https://github.com/your-org/bioengine.git
+   cd bioengine
+   ```
+
+2. **Set up the environment**:
+
+   Ensure that Docker is installed on your system. For Kubernetes deployments, set up your cluster using the provided Helm charts.
+
+3. **Deploy the BioEngine stack**:
+
+   Use Docker Compose for local deployment or Kubernetes for scalable deployment. 
+
+   For Docker Compose:
+
+   ```bash
+   docker-compose up -d
+   ```
+
+   For Kubernetes:
+
+   ```bash
+   kubectl apply -f k8s-deployment.yaml
+   ```
+
+4. **Access the BioEngine Dashboard**:
+
+   Once the deployment is complete, access the BioEngine dashboard by navigating to `http://localhost:PORT` in your browser. The dashboard provides access to the API documentation and management interfaces for your BioEngine Apps.
+
+---
+
+## 4. Creating a BioEngine UI App
+
+### Example UI App
+
+The following example demonstrates how to create a simple UI App that connects to the BioEngine platform and registers a service for AI model inference.
+
+```html
+<script type="module">
+  import { connectToServer } from "https://cdn.jsdelivr.net/npm/[email protected]/dist/hypha-rpc-websocket.mjs";
+
+  const server = await connectToServer({
+    server_url: "https://ai.aicell.io",
+    workspace: "default",
+  });
+
+  server.export({
+    id: "my-ui-service",
+    getImage() {
+      // Logic to retrieve and return an image for processing
+      return "data:image/png;base64,...";
+    },
+  });
+</script>
+```
+
+**Explanation**:
+- **Connecting to the Hypha server**: The `connectToServer` function initiates a WebSocket connection to the Hypha server, using the provided server URL and workspace ID.
+- **Registering the service**: The `server.export` function registers a service named `my-ui-service` that can retrieve an image from the UI for processing by a Compute App.
+
+### Detailed Steps:
+
+1. **Develop the UI Logic**:
+   - Implement the logic for your UI App. This could involve creating an interface for users to upload images, adjust settings, or visualize results.
+
+2. **Connect to Hypha Server**:
+   - Use the provided API to establish a connection to the Hypha server. This connection allows your UI App to discover and interact with Compute Apps registered on the same server.
+
+3. **Register UI Services**:
+   - Define and register services within your UI App that will interact with Compute Apps. For example, a `getImage` service might allow a Compute App to request an image for AI model inference.
+
+4. **Deploy and Test the UI App**:
+   - Once the UI App is developed, deploy it within your BioEngine environment. Load the app in a browser and test its functionality by interacting with the registered Compute Apps.
+
+---
+
+## 5. Creating a BioEngine Compute App
+
+### Example Compute App
+
+Here’s an example of a Compute App that provides an interface for a Cellpose AI model used for cell segmentation. This app is built using Ray and is designed to be accessible via the Hypha server.
+
+```python
+from hypha_rpc import api
+
+class CellposeModel:
+    def __init__(self):
+        # Load the pre-trained Cellpose model
+        self.model = load_cellpose_model()
+
+    def predict(self, image: str) -> str:
+        # Perform prediction using the loaded model
+        results = self.model.predict(image)
+        return results
+
+    def train(self, data: str, config: str) -> str:
+        # Train the model with provided data and configuration
+        training_results = self.model.train(data, config)
+        return training_results
+
+api.export(CellposeModel, {"name": "cellpose"})
+```
+
+**Explanation**:
+- **Loading the Model**: The `CellposeModel` class initializes by loading the pre-trained Cellpose model, which is used for cell segmentation.
+- **Predict and Train Methods**: The `predict` method takes an image as input and returns segmentation results. The `train` method allows users to train the model with new data and configuration settings.
+- **Exporting the Service**: The `api.export` function registers the `CellposeModel` as a service on the Hypha server, making it accessible to UI Apps.
+
+### Detailed Steps:
+
+1. **Define the Compute App Logic**:
+   - Implement the core logic for your Compute App, which might involve loading AI models, processing data, or handling training tasks.
+
+2. **Connect to Hypha Server**:
+   - Use the Hypha RPC API to export your Compute App as a service. This makes your app discoverable and accessible to other components within the BioEngine platform.
+
+3. **Deploy the Compute App**:
+   - Containerize your Compute App using Docker, and deploy it within your BioEngine environment. Ensure it is registered with the Hypha server and is ready to handle requests from UI Apps.
+
+4. **Test and Optimize**:
+   - Test the Compute App by sending requests from a UI App. Monitor performance and optimize as needed, particularly if working with large datasets or complex models.
+
+---
+
+## 6. Technical Explanation
+
+**Communication between UI Apps and Compute Apps**:
+- The **Hypha server** acts as a middleware, facilitating communication between UI Apps and Compute Apps. It handles service registration, discovery, and message routing.
+- **WebSocket connections** are used for real-time data exchange, allowing UI Apps to interact with Compute Apps without significant delays.
+- **Service Registration**: Each service within a UI or Compute App is registered with the Hypha server, enabling seamless discovery and interaction between different components.
+
+**Resource Management**:
+- **Ray** efficiently manages distributed computing tasks across a cluster, ensuring that available resources are used effectively. This is particularly important for training large AI models or processing high-throughput data.
+- **NVIDIA Triton** optimizes AI model inference by dynamically scheduling tasks based on system load and user demand, ensuring that GPU resources are allocated where needed.
+
+---
+
+## 7. Example Workflow
+
+**Step-by-Step Workflow**:
+
+1. **Develop and Deploy the Compute App**:
+   - Write the model logic in Python, implement necessary methods for prediction and training, and containerize the app.
+   - Deploy the Compute App in a Docker or Kubernetes environment, ensuring it connects to the Hypha server.
+
+2. **Develop the UI App**:
+   - Create a user-friendly web interface using JavaScript/TypeScript. Implement services that will interact with the Compute App for tasks like image retrieval and result visualization.
+
+3. **Test the Integration**:
+   - Load the UI App in a browser, interact with the Compute App by sending requests, and visualize the results. Test various scenarios to ensure robustness and reliability.
+
+4. **Optimize and Deploy**:
+   - Optimize both the UI and Compute Apps based on performance testing. Once ready, deploy
+
+ them in the production environment for end-users.
+
+---
+
+## 8. Results
+
+**Expected Outcomes**:
+
+- **Real-time AI Inference**: Users can perform real-time AI model inference directly from the browser, with results processed and returned by the Compute App.
+- **Scalable AI Model Training**: The Compute App, powered by Ray, can handle large-scale model training, distributing tasks across multiple nodes to accelerate the process.
+- **Interactive Visualization**: The UI App provides immediate feedback and visualization of results, allowing users to interact with and adjust their analysis in real-time.
+
+**Case Study**:
+- **Cell Segmentation with Cellpose**: A UI App was developed to upload cell images, which were then processed by a Compute App running the Cellpose model. The results were visualized directly within the UI, enabling researchers to refine their analysis parameters on the fly.
+
+---
+
+## 9. Conclusion
+
+Creating BioEngine Apps allows you to harness the power of cloud-based AI tools for bioimage analysis. With its modular architecture and support for both UI and Compute Apps, BioEngine offers flexibility and scalability, making advanced bioimage analysis accessible to a broad audience. Whether developing a simple annotation tool or a complex AI model training pipeline, BioEngine provides the necessary infrastructure to bring your projects to life.
+
+**Future Directions**:
+- Continued development will focus on enhancing scalability, optimizing resource management, and expanding support for additional AI frameworks and bioimaging tools.
+
+---
+
+## 10. Acknowledgements
+
+- **EU Horizon Europe**: This project is funded by the EU’s Horizon Europe program, grant no. 101057970, which supports research and innovation in advanced technologies.
+- **NVIDIA**: Thanks to NVIDIA for providing the Triton Inference Server and computing credits, which are integral to the BioEngine platform's performance.
+- **Bioimaging Community**: Special thanks to the testers and participants in the AI4Life Stockholm Hackathon 2023 for their valuable feedback, which has helped refine BioEngine's features.
+
+**Contact and Support**:
+- **Community Feedback**: We encourage you to try BioEngine and share your experiences on our GitHub issues page or the image.sc forum. Your input is crucial for the continual refinement of this platform.
+- **Support**: For any questions or support, please contact us via the details provided on the BioEngine dashboard.

notebooks/1-bioengine-engine-tutorial.ipynb

@@ -28,7 +28,7 @@
     },
     {
       "cell_type": "code",
-      "source": "try:\n    # For pyodide in the browser\n    import micropip\n    await micropip.install(['hypha-rpc', 'kaibu-utils', 'pyodide-http', 'requests'])\n    \n    # 2. Patch requests\n    import pyodide_http\n    pyodide_http.patch_all()  # Patch all libraries\nexcept ImportError:\n    # For native python with pip\n    import subprocess\n    subprocess.call(['pip', 'install', 'hypha-rpc', 'kaibu-utils'])\n\nimport io\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport numpy as np\n#from pyotritonclient import execute\nfrom imjoy_rpc.hypha import connect_to_server\nfrom kaibu_utils import fetch_image\n\n\n",
+      "source": "try:\n    # For pyodide in the browser\n    import micropip\n    await micropip.install(['hypha-rpc', 'kaibu-utils', 'pyodide-http', 'requests'])\n    \n    # 2. Patch requests\n    import pyodide_http\n    pyodide_http.patch_all()  # Patch all libraries\nexcept ImportError:\n    # For native python with pip\n    import subprocess\n    subprocess.call(['pip', 'install', 'hypha-rpc', 'kaibu-utils'])\n\nimport io\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport numpy as np\n#from pyotritonclient import execute\nfrom hypha_rpc import connect_to_server\nfrom kaibu_utils import fetch_image\n\n\n",
       "metadata": {
         "trusted": true
       },

notebooks/3-kaibu-geojson.ipynb

@@ -32,7 +32,7 @@
     },
     {
       "cell_type": "code",
-      "source": "from js import fetch\nimport io\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom skimage.transform import resize\nfrom kaibu_utils import fetch_image, features_to_mask, mask_to_features\n\nfrom imjoy_rpc.hypha import connect_to_server\nserver = await connect_to_server(\n    {\"name\": \"test client\", \"server_url\": \"https://ai.imjoy.io/\", \"method_timeout\": 3000}\n)\ntriton = await server.get_service(\"triton-client\")\n\n\ndef display_image(image, mask):\n    # display the output\n    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(8, 4))\n    ax1.imshow(image)\n    ax1.set_title('input image')\n    ax2.imshow(mask)\n    ax2.set_title('predicted mask')\n    plt.show()\n\nasync def cellpose_segment(image, diameter=100):\n    results = await triton.execute(inputs=[image.transpose(2, 0, 1), {'diameter': diameter}],\n                model_name='cellpose-python',\n                decode_bytes=True)\n    mask = results['mask'][0].copy()\n    return mask\n\nimage = await fetch_image(\"https://images.proteinatlas.org/61448/1319_C10_2_blue_red_green.jpg\")\nimage = image.astype('float32')\nimage = resize(image, (image.shape[0] // 4, image.shape[1] // 4), anti_aliasing=True).astype('float32')\nprint(image.shape)\nplt.imshow(image.astype('uint8'))\nplt.show()",
+      "source": "from js import fetch\nimport io\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom skimage.transform import resize\nfrom kaibu_utils import fetch_image, features_to_mask, mask_to_features\n\nfrom hypha_rpc import connect_to_server\nserver = await connect_to_server(\n    {\"name\": \"test client\", \"server_url\": \"https://ai.imjoy.io/\", \"method_timeout\": 3000}\n)\ntriton = await server.get_service(\"triton-client\")\n\n\ndef display_image(image, mask):\n    # display the output\n    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(8, 4))\n    ax1.imshow(image)\n    ax1.set_title('input image')\n    ax2.imshow(mask)\n    ax2.set_title('predicted mask')\n    plt.show()\n\nasync def cellpose_segment(image, diameter=100):\n    results = await triton.execute(inputs=[image.transpose(2, 0, 1), {'diameter': diameter}],\n                model_name='cellpose-python',\n                decode_bytes=True)\n    mask = results['mask'][0].copy()\n    return mask\n\nimage = await fetch_image(\"https://images.proteinatlas.org/61448/1319_C10_2_blue_red_green.jpg\")\nimage = image.astype('float32')\nimage = resize(image, (image.shape[0] // 4, image.shape[1] // 4), anti_aliasing=True).astype('float32')\nprint(image.shape)\nplt.imshow(image.astype('uint8'))\nplt.show()",
       "metadata": {
         "trusted": true
       },