run-audio-alignment-mistral.sh

WANDB_PROJECT=audio-alignment-mistral \
torchrun --nproc_per_node 8 \
-m audio_alignment \
--model_name_or_path mesolitica/malaysian-mistral-7b-32k-instructions-v4 \
--audio_encoder_name_or_path mesolitica/malaysian-whisper-small \
--train_file mosaic-audio \
--per_device_train_batch_size 8 \
--gradient_accumulation_steps 1 \
--num_train_epochs 3 \
--save_strategy "steps" \
--save_steps 200 \
--save_total_limit 2 \
--learning_rate 1e-4 \
--weight_decay 0. \
--warmup_ratio 0.03 \
--logging_steps 1 \
--do_train \
--bf16 True \
--gradient_checkpointing True \
--output_dir "audio-alignment-mistral" \
--use_flash_attention2 True \
--block_size 2048