From 8d0fa4c6c976da5d7754745312d15e65624afac1 Mon Sep 17 00:00:00 2001 From: dm4 Date: Thu, 22 Feb 2024 16:47:34 +0800 Subject: [PATCH] [CI] llama: add gemma and llava jobs Signed-off-by: dm4 --- .github/workflows/llama.yml | 61 ++++++++++++++++++++++++++++++--- wasmedge-ggml/llava/src/main.rs | 16 +++++++++ 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/.github/workflows/llama.yml b/.github/workflows/llama.yml index 19593a0..895448c 100644 --- a/.github/workflows/llama.yml +++ b/.github/workflows/llama.yml @@ -51,7 +51,33 @@ jobs: --nn-preload default:GGML:AUTO:tinyllama-1.1b-chat-v0.3.Q5_K_M.gguf \ target/wasm32-wasi/release/wasmedge-ggml-llama.wasm \ default \ - '<|im_start|>system\nYou are an AI assistant, always answer as short as possible<|im_end|>\n<|im_start|>user\nWhere is the capital of Japan?<|im_end|>\n<|im_start|>assistant' + $'<|im_start|>system\nYou are an AI assistant, always answer as short as possible<|im_end|>\n<|im_start|>user\nWhere is the capital of Japan?<|im_end|>\n<|im_start|>assistant' + + - name: Gemma 2B + run: | + cd wasmedge-ggml/gemma + curl -LO https://huggingface.co/second-state/Gemma-2b-it-GGUF/resolve/main/gemma-2b-it-Q5_K_M.gguf + cargo build --target wasm32-wasi --release + wasmedge --dir .:. \ + --nn-preload default:GGML:AUTO:gemma-2b-it-Q5_K_M.gguf \ + target/wasm32-wasi/release/wasmedge-ggml-gemma.wasm \ + default \ + 'user Where is the capital of Japan? model' + + - name: Llava v1.5 7B + run: | + cd wasmedge-ggml/llava + curl -LO https://huggingface.co/mys/ggml_llava-v1.5-7b/resolve/main/ggml-model-q5_k.gguf + curl -LO https://huggingface.co/mys/ggml_llava-v1.5-7b/resolve/main/mmproj-model-f16.gguf + curl -LO https://llava-vl.github.io/static/images/monalisa.jpg + cargo build --target wasm32-wasi --release + wasmedge --dir .:. \ + --env mmproj=mmproj-model-f16.gguf \ + --env image=monalisa.jpg \ + --nn-preload default:GGML:AUTO:ggml-model-q5_k.gguf \ + target/wasm32-wasi/release/wasmedge-ggml-llava.wasm \ + default \ + $'You are a helpful, respectful and honest assistant. Always answer as short as possible, while being safe.\nUSER:\nDo you know who drew this painting?\nASSISTANT:' - name: llama2 7b run: | @@ -63,7 +89,7 @@ jobs: --nn-preload default:GGML:AUTO:llama-2-7b-chat.Q5_K_M.gguf \ target/wasm32-wasi/release/wasmedge-ggml-llama.wasm \ default \ - '[INST] <>\nYou are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you do not know the answer to a question, please do not share false information.\n<>\nWhat is the capital of Japan?[/INST]' + $'[INST] <>\nYou are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you do not know the answer to a question, please do not share false information.\n<>\nWhat is the capital of Japan?[/INST]' - name: Build llama-stream run: | @@ -117,7 +143,34 @@ jobs: --nn-preload default:GGML:AUTO:tinyllama-1.1b-chat-v0.3.Q5_K_M.gguf \ target/wasm32-wasi/release/wasmedge-ggml-llama.wasm \ default \ - '<|im_start|>system\nYou are an AI assistant<|im_end|>\n<|im_start|>user\nWhere is the capital of Japan?<|im_end|>\n<|im_start|>assistant' + $'<|im_start|>system\nYou are an AI assistant<|im_end|>\n<|im_start|>user\nWhere is the capital of Japan?<|im_end|>\n<|im_start|>assistant' + + - name: Gemma 2B + run: | + cd wasmedge-ggml/gemma + curl -LO https://huggingface.co/second-state/Gemma-2b-it-GGUF/resolve/main/gemma-2b-it-Q5_K_M.gguf + cargo build --target wasm32-wasi --release + wasmedge --dir .:. \ + --env n_gpu_layers=0 \ + --nn-preload default:GGML:AUTO:gemma-2b-it-Q5_K_M.gguf \ + target/wasm32-wasi/release/wasmedge-ggml-gemma.wasm \ + default \ + 'user Where is the capital of Japan? model' + + - name: Llava v1.5 7B + run: | + cd wasmedge-ggml/llava + curl -LO https://huggingface.co/mys/ggml_llava-v1.5-7b/resolve/main/ggml-model-q5_k.gguf + curl -LO https://huggingface.co/mys/ggml_llava-v1.5-7b/resolve/main/mmproj-model-f16.gguf + curl -LO https://llava-vl.github.io/static/images/monalisa.jpg + cargo build --target wasm32-wasi --release + wasmedge --dir .:. \ + --env mmproj=mmproj-model-f16.gguf \ + --env image=monalisa.jpg \ + --nn-preload default:GGML:AUTO:ggml-model-q5_k.gguf \ + target/wasm32-wasi/release/wasmedge-ggml-llava.wasm \ + default \ + $'You are a helpful, respectful and honest assistant. Always answer as short as possible, while being safe.\nUSER:\nDo you know who drew this painting?\nASSISTANT:' - name: llama2 7b run: | @@ -129,7 +182,7 @@ jobs: --nn-preload default:GGML:AUTO:llama-2-7b-chat.Q5_K_M.gguf \ target/wasm32-wasi/release/wasmedge-ggml-llama.wasm \ default \ - '[INST] <>\nYou are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you do not know the answer to a question, please do not share false information.\n<>\nWhat is the capital of Japan?[/INST]' + $'[INST] <>\nYou are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you do not know the answer to a question, please do not share false information.\n<>\nWhat is the capital of Japan?[/INST]' - name: Build llama-stream run: | diff --git a/wasmedge-ggml/llava/src/main.rs b/wasmedge-ggml/llava/src/main.rs index 69c3933..919fe7c 100644 --- a/wasmedge-ggml/llava/src/main.rs +++ b/wasmedge-ggml/llava/src/main.rs @@ -72,6 +72,22 @@ fn main() { .init_execution_context() .expect("Failed to init context"); + // If there is a third argument, use it as the prompt and enter non-interactive mode. + // This is mainly for the CI workflow. + if args.len() >= 3 { + let prompt = &args[2]; + println!("Prompt:\n{}", prompt); + let tensor_data = prompt.as_bytes().to_vec(); + context + .set_input(0, wasi_nn::TensorType::U8, &[1], &tensor_data) + .expect("Failed to set input"); + println!("Response:"); + context.compute().expect("Failed to compute"); + let output = get_output_from_context(&context); + println!("{}", output.trim()); + std::process::exit(0); + } + let mut saved_prompt = String::new(); let system_prompt = String::from("You are a helpful, respectful and honest assistant. Always answer as short as possible, while being safe." ); let image_placeholder = "";