From f253196b85b91f508a42904e444152d2afa7c670 Mon Sep 17 00:00:00 2001 From: siyunzhao Date: Thu, 25 Apr 2024 08:04:52 +0000 Subject: [PATCH 1/3] fix tokenizer bug --- llmlingua/prompt_compressor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/llmlingua/prompt_compressor.py b/llmlingua/prompt_compressor.py index 91d9e4f..30a2239 100644 --- a/llmlingua/prompt_compressor.py +++ b/llmlingua/prompt_compressor.py @@ -992,7 +992,6 @@ def get_prefix_length(self, prefix: str, text: str): cur_prefix = self.tokenizer.decode(full_input_ids[:i]) if cur_prefix == prefix: break - assert self.tokenizer.decode(full_input_ids[i:]) == text[:100] return i def get_condition_ppl( From a53f46bc4da7c436dd8d9976f847a7e0b9cb3653 Mon Sep 17 00:00:00 2001 From: siyunzhao Date: Thu, 25 Apr 2024 08:36:55 +0000 Subject: [PATCH 2/3] exclude macros-14 unittest for environment conflict --- .github/workflows/unittest.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index a10ada6..f4923ee 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -19,6 +19,9 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, windows-2019] python-version: ["3.9", "3.10", "3.11"] + exclude: + - os: macos-14 + python-version: '3.9' steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} From 991be67b9bedc8abac4d88dba02f9b1e48fde211 Mon Sep 17 00:00:00 2001 From: siyunzhao Date: Thu, 25 Apr 2024 08:41:55 +0000 Subject: [PATCH 3/3] exclude macros-latest on python3.9 unittest for environment conflict --- .github/workflows/unittest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index f4923ee..7c1197b 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -20,7 +20,7 @@ jobs: os: [ubuntu-latest, macos-latest, windows-2019] python-version: ["3.9", "3.10", "3.11"] exclude: - - os: macos-14 + - os: macos-latest python-version: '3.9' steps: - uses: actions/checkout@v4