Update todos

README.md

@@ -116,6 +116,29 @@ parallel_map_operation:
       word_count: integer
 ```
 
+```yaml
+extract_info:
+  type: map
+  model: gpt-4o-mini
+  output:
+    schema:
+      officer_name: string
+      suspect_name: string
+  prompts:
+    - name: officer
+      prompt: |
+      Infer the officer's name from this police interrogation transcript:
+        { { input.transcript } }
+      output_keys:
+        - officer_name
+    - name: suspect
+      prompt: |
+      Infer the suspect's name from this police interrogation transcript:
+        { { input.transcript } }
+      output_keys:
+        - suspect_name
+```
+
 ### Filter
 
 The Filter operation selects items from the input data based on a condition.

motion/operations/base.py

@@ -29,8 +29,8 @@ def __init__(
         self.default_model = default_model
         self.max_threads = max_threads
         self.console = console or Console()
-        self.num_retries_on_validation_failure = self.config.get(
-            "num_retries_on_validation_failure", 0
+        self.num_retries_on_validate_failure = self.config.get(
+            "num_retries_on_validate_failure", 0
         )
         self.syntax_check()
 

motion/operations/filter.py

@@ -137,7 +137,7 @@ def validation_fn(response: Dict[str, Any]):
                 ),
                 validation_fn=validation_fn,
                 val_rule=self.config.get("validate", []),
-                num_retries=self.num_retries_on_validation_failure,
+                num_retries=self.num_retries_on_validate_failure,
                 console=self.console,
             )
 

motion/operations/map.py

@@ -105,7 +105,7 @@ def validation_fn(response: Dict[str, Any]):
                     ),
                     validation_fn=validation_fn,
                     val_rule=self.config.get("validate", []),
-                    num_retries=self.num_retries_on_validation_failure,
+                    num_retries=self.num_retries_on_validate_failure,
                     console=self.console,
                 )
             else:
@@ -119,7 +119,7 @@ def validation_fn(response: Dict[str, Any]):
                     ),
                     validation_fn=validation_fn,
                     val_rule=self.config.get("validate", []),
-                    num_retries=self.num_retries_on_validation_failure,
+                    num_retries=self.num_retries_on_validate_failure,
                     console=self.console,
                 )
 
@@ -287,7 +287,7 @@ def validation_fn(response: Dict[str, Any]):
                 llm_call_fn,
                 validation_fn,
                 prompt_config.get("validate", []),
-                self.num_retries_on_validation_failure,
+                self.num_retries_on_validate_failure,
                 self.console,
             )
 

todos.md

@@ -53,10 +53,12 @@ TODO:
   - [x] In map optimizer: optimize the reduce operator for each chunk size plan
   - [x] In reduce optimizer: synthesize resolver if need be
   - [x] In resolve optimizer, support list-type reduce keys
-- [ ] Operator reordering (Aug 21 & 22)
+- [ ] Operator reordering (Aug 23)
   - [ ] support equivalence: map -> unnest -> reduce might be same as split -> gather -> map -> unnest -> reduce (no need to have a reduce right after map)
 - [x] Run tests in CI
 - [x] Support retry on validation failure
+- [ ] Break down split into split + gather (Aug 21 & 22)
+- [ ] Support prompts exceeding context windows; figure out how to throw out data / prioritize elements
 - [ ] Support retries in the optimizers
 - [ ] Write tests for optimizers
 - [ ] Filter optimizer
@@ -84,3 +86,4 @@ Things to think about
 - Try various combine prompts in the reduce optimizer
 - Filter optimizer: we should recursively optimize reduces if the reduce isn't good on its own
 - Support retry on val failure for operations beyond map/filter
+- If reduce input is too big to fit in the prompt, prompt for a map operation

workloads/medical/map.yaml

@@ -21,7 +21,7 @@ operations:
       If a piece of information is not found, return an empty string.
     validate:
       - "output['medication_info'].startswith('Medication Info: ')"
-    num_retries_on_validation_failure: 1
+    num_retries_on_validate_failure: 1
 
     # gleaning:
     #   num_rounds: 1