From 85eaee9c7954cb52adbfd0b8c4649adecd899e67 Mon Sep 17 00:00:00 2001 From: Sagi Polaczek Date: Wed, 18 Dec 2024 20:34:58 +0200 Subject: [PATCH] done after hooks --- fuse/data/tokenizers/modular_tokenizer/op.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fuse/data/tokenizers/modular_tokenizer/op.py b/fuse/data/tokenizers/modular_tokenizer/op.py index fb622a07..42489cd0 100644 --- a/fuse/data/tokenizers/modular_tokenizer/op.py +++ b/fuse/data/tokenizers/modular_tokenizer/op.py @@ -8,11 +8,12 @@ from huggingface_hub import HfApi, snapshot_download from huggingface_hub.utils import SoftTemporaryDirectory, validate_hf_hub_args -from fuse.data import OpBase, get_sample_id +from fuse.data import OpBase from fuse.data.tokenizers.modular_tokenizer.inject_utils import ( InjectorToModularTokenizerLib, ) from fuse.data.tokenizers.modular_tokenizer.modular_tokenizer import ModularTokenizer +from fuse.data.utils.sample import get_sample_id from fuse.utils import NDict