* avoid unaligned columns when converting and processing the dataset …

…due to missing column or unaligned data type
modelscope · Nov 6, 2023 · 473b088 · 473b088
1 parent 5fc1f62
commit 473b088
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 2 deletions.
diff --git a/data_juicer/ops/filter/image_aspect_ratio_filter.py b/data_juicer/ops/filter/image_aspect_ratio_filter.py
@@ -50,6 +50,8 @@ def compute_stats(self, sample, context=False):
 
         # there is no image in this sample
         if self.image_key not in sample or not sample[self.image_key]:
+            sample[Fields.stats][StatsKeys.aspect_ratios] = np.array(
+                [], dtype=np.float64)
             return sample
 
         # load images

diff --git a/tools/multimodal/ds2dj/llava2dj.py b/tools/multimodal/ds2dj/llava2dj.py
@@ -265,9 +265,8 @@ def main(
             new_sample = {
                 'id': id,
                 text_key: text,
+                image_key: images,
             }
-            if images:
-                new_sample[image_key] = images
             writer.write(new_sample)
     logger.info(f'Store the target dataset into [{target_ds_path}].')