From af8a39dc7b39847700b08b32bfb922efba5c6adc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= Date: Wed, 27 Mar 2024 03:10:38 +0100 Subject: [PATCH] Fix tests in MacOS/Windows --- tests/data/test_tinystories.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/data/test_tinystories.py b/tests/data/test_tinystories.py index c67cc8e081..d0318995bf 100644 --- a/tests/data/test_tinystories.py +++ b/tests/data/test_tinystories.py @@ -7,12 +7,13 @@ from torch.utils._pytree import tree_map -def fake_chunk(path, data): - def fn(_): - for story in data: - yield torch.tensor(story) +def tokenize(data): + for story in data: + yield torch.tensor(story) + - optimize(fn=fn, inputs=[None] * len(data), output_dir=str(path), num_workers=1, chunk_bytes="200MB") +def fake_chunk(path, data): + optimize(fn=tokenize, inputs=[data] * len(data), output_dir=str(path), num_workers=1, chunk_bytes="200MB") @pytest.mark.xfail(raises=IndexError, strict=True) # requires https://github.com/Lightning-AI/litdata/pull/77