[Example] ggml: add basic example with CI (#110)

* [Example] ggml: add basic example

Signed-off-by: dm4 <[email protected]>

* [CI] llama: add job for StarCoder 2 model

Signed-off-by: dm4 <[email protected]>

---------

Signed-off-by: dm4 <[email protected]>

.github/workflows/llama.yml

@@ -100,6 +100,20 @@ jobs:
                 default \
                 $'[INST] <<SYS>>\nYou are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe.  Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you do not know the answer to a question, please do not share false information.\n<</SYS>>\nWhat is the capital of Japan?[/INST]'
 
+          - name: StarCoder 2 7B
+            run: |
+              test -f ~/.wasmedge/env && source ~/.wasmedge/env
+              cd wasmedge-ggml/basic
+              curl -LO https://huggingface.co/second-state/StarCoder2-7B-GGUF/resolve/main/starcoder2-7b-Q5_K_M.gguf
+              cargo build --target wasm32-wasi --release
+              time wasmedge --dir .:. \
+                --env n_gpu_layers="$NGL" \
+                --env n_predict=100 \
+                --nn-preload default:GGML:AUTO:starcoder2-7b-Q5_K_M.gguf \
+                target/wasm32-wasi/release/wasmedge-ggml-basic.wasm \
+                default \
+                'def print_hello_world():'
+
           - name: Build llama-stream
             run: |
               cd wasmedge-ggml/llama-stream

wasmedge-ggml/basic/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "wasmedge-ggml-basic"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+serde_json = "1.0"
+wasmedge-wasi-nn = "0.7.0"

wasmedge-ggml/basic/README.md

@@ -0,0 +1,54 @@
+
+# Basic Example For WASI-NN with GGML Backend
+
+> [!NOTE]
+> Please refer to the [wasmedge-ggml/README.md](../README.md) for the general introduction and the setup of the WASI-NN plugin with GGML backend. This document will focus on the specific example for the models without prompt templates.
+
+## Get the Model
+
+This example is for the models without prompt template. For example, the `StarCoder2` model.
+
+Download the model:
+
+```bash
+curl -LO https://huggingface.co/second-state/StarCoder2-7B-GGUF/resolve/main/starcoder2-7b-Q5_K_M.gguf
+```
+
+## Parameters
+
+> [!NOTE]
+> Please check the parameters section of [wasmedge-ggml/README.md](https://github.com/second-state/WasmEdge-WASINN-examples/tree/master/wasmedge-ggml#parameters) first.
+
+- For GPU offloading, please adjust the `n-gpu-layers` options to the number of layers that you want to offload to the GPU.
+- When using the `StarCoder2` model, the `n-predict` option can be used to adjust the number of predictions. Since the inference may not stop as expected, it is recommended to set a limit for the number of predictions.
+
+## Execute
+
+Execute the WASM with the `wasmedge` using the named model feature to preload a large model:
+
+```console
+$ wasmedge --dir .:. \
+  --env n_predict=100 \
+  --nn-preload default:GGML:AUTO:/disk/starcoder2-7b-Q5_K_M.gguf \
+  wasmedge-ggml-basic.wasm default
+
+USER:
+def print_hello_world():
+ASSISTANT:
+
+    print("Hello World!")
+
+def main():
+    print_hello_world()
+
+
+if __name__ == "__main__":
+    main()/README.md
+# python-learning
+
+This repository is for learning Python.
+
+## References
+
+* [Python 3.8.0 Documentation](https://docs.python.org/3/)
+```

wasmedge-ggml/basic/src/main.rs

@@ -0,0 +1,152 @@
+use serde_json::json;
+use serde_json::Value;
+use std::env;
+use std::io;
+use wasmedge_wasi_nn::{
+    self, BackendError, Error, ExecutionTarget, GraphBuilder, GraphEncoding, GraphExecutionContext,
+    TensorType,
+};
+
+fn read_input() -> String {
+    loop {
+        let mut answer = String::new();
+        io::stdin()
+            .read_line(&mut answer)
+            .expect("Failed to read line");
+        if !answer.is_empty() && answer != "\n" && answer != "\r\n" {
+            return answer.trim().to_string();
+        }
+    }
+}
+
+fn get_options_from_env() -> Value {
+    let mut options = json!({});
+    if let Ok(val) = env::var("enable_log") {
+        options["enable-log"] =
+            serde_json::from_str(val.as_str()).expect("invalid enable-log value (true/false)")
+    } else {
+        options["enable-log"] = serde_json::from_str("false").unwrap()
+    }
+    if let Ok(val) = env::var("ctx_size") {
+        options["ctx-size"] =
+            serde_json::from_str(val.as_str()).expect("invalid ctx-size value (unsigned integer)")
+    } else {
+        options["ctx-size"] = serde_json::from_str("512").unwrap()
+    }
+    if let Ok(val) = env::var("n_gpu_layers") {
+        options["n-gpu-layers"] =
+            serde_json::from_str(val.as_str()).expect("invalid ngl (unsigned integer)")
+    } else {
+        options["n-gpu-layers"] = serde_json::from_str("100").unwrap()
+    }
+    if let Ok(val) = env::var("n_predict") {
+        options["n-predict"] =
+            serde_json::from_str(val.as_str()).expect("invalid n-predict (unsigned integer)")
+    } else {
+        options["n-predict"] = serde_json::from_str("512").unwrap()
+    }
+
+    options
+}
+
+fn set_data_to_context(context: &mut GraphExecutionContext, data: Vec<u8>) -> Result<(), Error> {
+    context.set_input(0, TensorType::U8, &[1], &data)
+}
+
+#[allow(dead_code)]
+fn set_metadata_to_context(
+    context: &mut GraphExecutionContext,
+    data: Vec<u8>,
+) -> Result<(), Error> {
+    context.set_input(1, TensorType::U8, &[1], &data)
+}
+
+fn get_data_from_context(context: &GraphExecutionContext, index: usize) -> String {
+    // Preserve for 4096 tokens with average token length 6
+    const MAX_OUTPUT_BUFFER_SIZE: usize = 4096 * 6;
+    let mut output_buffer = vec![0u8; MAX_OUTPUT_BUFFER_SIZE];
+    let mut output_size = context
+        .get_output(index, &mut output_buffer)
+        .expect("Failed to get output");
+    output_size = std::cmp::min(MAX_OUTPUT_BUFFER_SIZE, output_size);
+
+    return String::from_utf8_lossy(&output_buffer[..output_size]).to_string();
+}
+
+fn get_output_from_context(context: &GraphExecutionContext) -> String {
+    get_data_from_context(context, 0)
+}
+
+#[allow(dead_code)]
+fn get_metadata_from_context(context: &GraphExecutionContext) -> Value {
+    serde_json::from_str(&get_data_from_context(context, 1)).expect("Failed to get metadata")
+}
+
+fn main() {
+    let args: Vec<String> = env::args().collect();
+    let model_name: &str = &args[1];
+
+    // Set options for the graph. Check our README for more details:
+    // https://github.com/second-state/WasmEdge-WASINN-examples/tree/master/wasmedge-ggml#parameters
+    let mut options = get_options_from_env();
+    // Set the stream-stdout option to true to make the response more interactive.
+    options["stream-stdout"] = serde_json::from_str("true").unwrap();
+    // We set the temperature to 0.2 in this example to make the response more consistent.
+    options["temp"] = Value::from(0.1);
+
+    // Create graph and initialize context.
+    let graph = GraphBuilder::new(GraphEncoding::Ggml, ExecutionTarget::AUTO)
+        .config(serde_json::to_string(&options).expect("Failed to serialize options"))
+        .build_from_cache(model_name)
+        .expect("Failed to build graph");
+    let mut context = graph
+        .init_execution_context()
+        .expect("Failed to init context");
+
+    // If there is a third argument, use it as the prompt and enter non-interactive mode.
+    // This is mainly for the CI workflow.
+    if args.len() >= 3 {
+        let prompt = &args[2];
+        println!("Prompt:\n{}", prompt);
+        let tensor_data = prompt.as_bytes().to_vec();
+        context
+            .set_input(0, TensorType::U8, &[1], &tensor_data)
+            .expect("Failed to set input");
+        println!("Response:");
+        context.compute().expect("Failed to compute");
+        let output = get_output_from_context(&context);
+        println!("{}", output.trim());
+        std::process::exit(0);
+    }
+
+    loop {
+        println!("USER:");
+        let input = read_input();
+
+        // Set prompt to the input tensor.
+        set_data_to_context(&mut context, input.as_bytes().to_vec()).expect("Failed to set input");
+
+        // Execute the inference.
+        println!("ASSISTANT:");
+        match context.compute() {
+            Ok(_) => (),
+            Err(Error::BackendError(BackendError::ContextFull)) => {
+                println!("\n[INFO] Context full, we'll reset the context and continue.");
+            }
+            Err(Error::BackendError(BackendError::PromptTooLong)) => {
+                println!("\n[INFO] Prompt too long, we'll reset the context and continue.");
+            }
+            Err(err) => {
+                println!("\n[ERROR] {}", err);
+            }
+        }
+
+        // Retrieve the output.
+        let output = get_output_from_context(&context);
+        if let Some(true) = options["stream-stdout"].as_bool() {
+            println!();
+        } else {
+            println!("{}", output.trim());
+        }
+    }
+}