diff --git a/README.md b/README.md index b988c0f..81a1c04 100644 --- a/README.md +++ b/README.md @@ -49,11 +49,11 @@ For details on the evaluation, please see Section 4.2 of the paper. | :-- | --: | --: | | [LLaVA-1.6-Mistral-7B](https://llava-vl.github.io/blog/2024-01-30-llava-next/) | 14.32 | 41.10 | | [Japanese Stable VLM](https://huggingface.co/stabilityai/japanese-stable-vlm) | -*1 | 40.50 | -| [Heron BLIP Japanese StableLM Base 7B llava-620k](https://huggingface.co/turing-motors/heron-chat-blip-ja-stablelm-base-7b-v1-llava-620k) | 8.73*2 | 27.37*2 | +| [Heron BLIP Japanese StableLM Base 7B llava-620k](https://huggingface.co/turing-motors/heron-chat-blip-ja-stablelm-base-7b-v1-llava-620k) | 14.51 | 33.26 | | [EvoVLM-JP-v1-7B](https://huggingface.co/SakanaAI/EvoVLM-JP-v1-7B) | **19.70** | **51.25** | * \*1: Japanese Stable VLM cannot be evaluated using the JA-VG-VQA-500 dataset because this model has used this dataset for training. -* \*2: We are checking with the authors to see if this current results are valid. + diff --git a/configs/vlm/heron-v1.yaml b/configs/vlm/heron-v1.yaml index 9ab4f5f..3311cf8 100644 --- a/configs/vlm/heron-v1.yaml +++ b/configs/vlm/heron-v1.yaml @@ -1,8 +1,8 @@ model: target: evomerge.HeronV1 params: - # model_path: turing-motors/heron-chat-blip-ja-stablelm-base-7b-v1-llava-620k - model_path: turing-motors/heron-chat-blip-ja-stablelm-base-7b-v1 + model_path: turing-motors/heron-chat-blip-ja-stablelm-base-7b-v1-llava-620k + # model_path: turing-motors/heron-chat-blip-ja-stablelm-base-7b-v1 template: ja-heron-v1 model_kwargs: torch_dtype: torch.float16 @@ -18,10 +18,10 @@ eval: - target: evomerge.eval.JaVGVQA params: loader_kwargs: - batch_size: 4 + batch_size: 1 num_workers: 2 - target: evomerge.eval.JaVLMBenchIntheWild params: loader_kwargs: - batch_size: 4 + batch_size: 1 num_workers: 2 \ No newline at end of file