feat: allow loading multiple resources from loader

src/llmling/config/runtime.py

@@ -317,7 +317,10 @@ async def load_resource_by_uri(self, uri: str) -> LoadedResource:
             resolved_uri, resource = await self.resolve_resource_uri(uri)
             loader = self._loader_registry.get_loader(resource)
             loader = loader.create(resource, loader.get_name_from_uri(resolved_uri))
-            return await loader.load(processor_registry=self._processor_registry)
+            async for res in loader.load(processor_registry=self._processor_registry):
+                return res  # Return first resource
+            msg = "No resources loaded"
+            raise exceptions.ResourceError(msg)  # noqa: TRY301
         except Exception as exc:
             msg = f"Failed to load resource from URI {uri}"
             raise exceptions.ResourceError(msg) from exc

src/llmling/resources/base.py

@@ -10,7 +10,6 @@
 from typing import TYPE_CHECKING, Any, ClassVar, TypeVar, cast, overload
 import urllib.parse
 
-import logfire
 import upath
 
 from llmling.completions.protocols import CompletionProvider
@@ -24,6 +23,8 @@
 
 
 if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+
     from llmling.processors.registry import ProcessorRegistry
 
 
@@ -208,31 +209,31 @@ def resource_type(self) -> str:
         return fields["resource_type"].default  # type: ignore
 
     @overload
-    async def load(
+    def load(
         self,
         context: LoaderContext[TResource],
         processor_registry: ProcessorRegistry | None = None,
-    ) -> LoadedResource: ...
+    ) -> AsyncIterator[LoadedResource]: ...
 
     @overload
-    async def load(
+    def load(
         self,
         context: TResource,
         processor_registry: ProcessorRegistry | None = None,
-    ) -> LoadedResource: ...
+    ) -> AsyncIterator[LoadedResource]: ...
 
     @overload
-    async def load(
+    def load(
         self,
         context: None = None,
         processor_registry: ProcessorRegistry | None = None,
-    ) -> LoadedResource: ...
+    ) -> AsyncIterator[LoadedResource]: ...
 
     async def load(
         self,
-        context: LoaderContext[TResource] | TResource | None = None,
+        context: LoaderContext[TResource] | None = None,
         processor_registry: ProcessorRegistry | None = None,
-    ) -> LoadedResource:
+    ) -> AsyncIterator[LoadedResource]:
         """Load and process content.
 
         Args:
@@ -262,18 +263,18 @@ async def load(
             case _:
                 msg = f"Invalid context type: {type(context)}"
                 raise exceptions.LoaderError(msg)
-        with logfire.span(
-            "Loading resource",
-            resource_type=self.resource_type,
-            name=name,
-        ):
-            return await self._load_impl(resource, name, processor_registry)
+
+        generator = self._load_impl(resource, name, processor_registry)
+        # Then yield from the generator
+        async for result in generator:
+            yield result
 
     @abstractmethod
     async def _load_impl(
         self,
         resource: TResource,
         name: str,
         processor_registry: ProcessorRegistry | None,
-    ) -> LoadedResource:
+    ) -> AsyncIterator[LoadedResource]:
         """Implementation of actual loading logic."""
+        yield NotImplemented  # type: ignore

src/llmling/resources/loaders/callable.py

@@ -10,6 +10,8 @@
 
 
 if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+
     from llmling.processors.registry import ProcessorRegistry
     from llmling.resources.models import LoadedResource
 
@@ -29,7 +31,7 @@ async def _load_impl(
         resource: CallableResource,
         name: str,
         processor_registry: ProcessorRegistry | None,
-    ) -> LoadedResource:
+    ) -> AsyncIterator[LoadedResource]:
         """Execute callable and load result."""
         try:
             kwargs = resource.keyword_args
@@ -39,7 +41,7 @@ async def _load_impl(
                 processed = await processor_registry.process(content, procs)
                 content = processed.content
             meta = {"import_path": resource.import_path, "args": resource.keyword_args}
-            return create_loaded_resource(
+            yield create_loaded_resource(
                 content=content,
                 source_type="callable",
                 uri=self.create_uri(name=name),

src/llmling/resources/loaders/cli.py

@@ -12,6 +12,8 @@
 
 
 if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+
     from llmling.processors.registry import ProcessorRegistry
     from llmling.resources.models import LoadedResource
 
@@ -31,7 +33,7 @@ async def _load_impl(
         resource: CLIResource,
         name: str,
         processor_registry: ProcessorRegistry | None,
-    ) -> LoadedResource:
+    ) -> AsyncIterator[LoadedResource]:
         """Execute command and load output."""
         command = cmd if isinstance((cmd := resource.command), str) else " ".join(cmd)
         try:
@@ -63,7 +65,7 @@ async def _load_impl(
                 processed = await processor_registry.process(content, procs)
                 content = processed.content
             meta = {"command": command, "exit_code": proc.returncode}
-            return create_loaded_resource(
+            yield create_loaded_resource(
                 content=content,
                 source_type="cli",
                 uri=self.create_uri(name=name),

src/llmling/resources/loaders/image.py

@@ -15,6 +15,7 @@
 
 
 if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
     import os
 
     from llmling.processors.registry import ProcessorRegistry
@@ -63,7 +64,7 @@ async def _load_impl(
         resource: ImageResource,
         name: str,
         processor_registry: ProcessorRegistry | None,
-    ) -> LoadedResource:
+    ) -> AsyncIterator[LoadedResource]:
         """Load and process image content."""
         try:
             path_obj = upath.UPath(resource.path)
@@ -87,7 +88,7 @@ async def _load_impl(
             if resource.alt_text:
                 placeholder_text = f"{placeholder_text} - {resource.alt_text}"
 
-            return create_loaded_resource(
+            yield create_loaded_resource(
                 content=placeholder_text,
                 source_type="image",
                 uri=self.create_uri(name=name),

src/llmling/resources/loaders/path.py

@@ -15,6 +15,8 @@
 logger = get_logger(__name__)
 
 if TYPE_CHECKING:
+    from collections.abc import AsyncGenerator
+
     from llmling.processors.registry import ProcessorRegistry
     from llmling.resources.models import LoadedResource
 
@@ -82,25 +84,56 @@ async def _load_impl(
         resource: PathResource,
         name: str,
         processor_registry: ProcessorRegistry | None,
-    ) -> LoadedResource:
-        """Load content from a file or URL."""
+    ) -> AsyncGenerator[LoadedResource, None]:
+        """Load content from file(s)."""
         try:
             path = UPath(resource.path)
-            content = path.read_text("utf-8")
-
-            if processor_registry and (procs := resource.processors):
-                processed = await processor_registry.process(content, procs)
-                content = processed.content
-            meta = {"type": "path", "path": str(path), "scheme": path.protocol}
-            return create_loaded_resource(
-                content=content,
-                source_type="path",
-                uri=self.create_uri(name=name),
-                mime_type=self.supported_mime_types[0],
-                name=resource.description or path.name,
-                description=resource.description,
-                additional_metadata=meta,
-            )
+
+            if path.is_dir():
+                # Handle directory recursively
+                for file_path in path.rglob("*"):
+                    if file_path.is_file():
+                        content = file_path.read_text("utf-8")
+                        if processor_registry and (procs := resource.processors):
+                            processed = await processor_registry.process(content, procs)
+                            content = processed.content
+
+                        yield create_loaded_resource(
+                            content=content,
+                            source_type="path",
+                            uri=self.create_uri(
+                                name=file_path.name
+                            ),  # Use filename for URI
+                            mime_type=self.supported_mime_types[0],
+                            name=resource.description or file_path.name,
+                            description=resource.description,
+                            additional_metadata={
+                                "type": "path",
+                                "path": str(file_path),
+                                "scheme": file_path.protocol,
+                                "relative_to": str(path),  # Add original directory
+                            },
+                        )
+            else:
+                # Handle single file
+                content = path.read_text("utf-8")
+                if processor_registry and (procs := resource.processors):
+                    processed = await processor_registry.process(content, procs)
+                    content = processed.content
+
+                yield create_loaded_resource(
+                    content=content,
+                    source_type="path",
+                    uri=self.create_uri(name=name),
+                    mime_type=self.supported_mime_types[0],
+                    name=resource.description or path.name,
+                    description=resource.description,
+                    additional_metadata={
+                        "type": "path",
+                        "path": str(path),
+                        "scheme": path.protocol,
+                    },
+                )
         except Exception as exc:
             msg = f"Failed to load content from {resource.path}"
             raise exceptions.LoaderError(msg) from exc

src/llmling/resources/loaders/source.py

@@ -11,6 +11,8 @@
 
 
 if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+
     from llmling.processors.registry import ProcessorRegistry
     from llmling.resources.models import LoadedResource
 
@@ -30,7 +32,7 @@ async def _load_impl(
         resource: SourceResource,
         name: str,
         processor_registry: ProcessorRegistry | None,
-    ) -> LoadedResource:
+    ) -> AsyncIterator[LoadedResource]:
         """Load Python source content."""
         try:
             content = importing.get_module_source(
@@ -43,7 +45,7 @@ async def _load_impl(
                 processed = await processor_registry.process(content, procs)
                 content = processed.content
 
-            return create_loaded_resource(
+            yield create_loaded_resource(
                 content=content,
                 source_type="source",
                 uri=self.create_uri(name=name),

src/llmling/resources/loaders/text.py

@@ -9,6 +9,8 @@
 
 
 if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+
     from llmling.processors.registry import ProcessorRegistry
     from llmling.resources.models import LoadedResource
 
@@ -28,15 +30,15 @@ async def _load_impl(
         resource: TextResource,
         name: str,
         processor_registry: ProcessorRegistry | None,
-    ) -> LoadedResource:
-        """Implement actual loading logic."""
+    ) -> AsyncIterator[LoadedResource]:
+        """Load text content."""
         try:
             content = resource.content
             if processor_registry and (procs := resource.processors):
                 processed = await processor_registry.process(content, procs)
                 content = processed.content
 
-            return create_loaded_resource(
+            yield create_loaded_resource(
                 content=content,
                 source_type="text",
                 uri=self.create_uri(name=name),
@@ -46,6 +48,5 @@ async def _load_impl(
                 additional_metadata={"type": "text"},
             )
         except Exception as exc:
-            logger.exception("Failed to load text content")
-            msg = "Failed to load text content"
+            msg = f"Failed to load text content: {exc}"
             raise exceptions.LoaderError(msg) from exc