some update to prevent errors when transforming data

run-llama · Aug 28, 2023 · f983b86 · f983b86
1 parent 7172088
commit f983b86
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/llama_hub/wordlift/base.py b/llama_hub/wordlift/base.py
@@ -108,6 +108,10 @@ def transform_data(self, data: dict) -> List[Document]:
             metadata_fields = self.configure_options.get('metadata_fields', [])
 
             for item in data:
+                if not all(key in item for key in text_fields):
+                    logging.warning(
+                        f"Skipping document due to missing text fields: {item}")
+                    continue
                 row = {}
                 for key, value in item.items():
                     if key in text_fields or key in metadata_fields: