Merge pull request #9 from Multi-V-VM/bench

Bench

.github/workflows/build-macos.yml

@@ -21,7 +21,7 @@ jobs:
     - uses: actions/checkout@v2
 
     - name: Install Dependency
-      run: brew install cxxopts fmt llvm@14
+      run: brew install cxxopts fmt llvm@14 google-benchmark
 
     - name: Checkout
       run: git submodule update --init

.github/workflows/build-ubuntu.yml

@@ -21,7 +21,7 @@ jobs:
     - uses: actions/checkout@v2
 
     - name: Install Dependency
-      run: sudo apt install libssl-dev gcc-13 g++-13 libprotobuf-c-dev libprotobuf-dev libgrpc-dev libfmt-dev llvm-14-dev libedit-dev libcxxopts-dev libpfm4-dev liblldb-dev
+      run: sudo apt install libssl-dev gcc-13 g++-13 libprotobuf-c-dev libprotobuf-dev libgrpc-dev libfmt-dev llvm-14-dev libedit-dev libcxxopts-dev libpfm4-dev liblldb-dev libbenchmark-dev
 
     - name: Checkout
       run: git submodule update --init
@@ -36,7 +36,7 @@ jobs:
     - name: Install Instrumented WASI SDK
       run: |
         cd /opt
-        sudo wget https://asplos.dev/wasi-sdk-20.0-linux.tar.gz
+        sudo wget http://108.181.27.85/wasi-sdk-20.0-linux.tar.gz
         sudo tar -xzf wasi-sdk-*.tar.gz
 
     # - name: Make WASM LD no check features

.github/workflows/build-windows.yml

@@ -34,7 +34,7 @@ jobs:
         git clone https://github.com/microsoft/vcpkg
         cd vcpkg
         ./bootstrap-vcpkg.bat
-        ./vcpkg install fmt:x64-windows cxxopts:x64-windows
+        ./vcpkg install fmt:x64-windows cxxopts:x64-windows benchmark:x64-windows
         echo "C:/tools/msys64/usr/bin/" >> "$GITHUB_PATH"
         C:/tools/msys64/usr/bin/pacman.exe --noconfirm -S gcc
 
@@ -53,7 +53,7 @@ jobs:
     - name: Install LLVM SDK
       run: |
         cd "C:/Program Files/"
-        wget https://asplos.dev/LLVM.zip -O LLVM.zip
+        wget http://108.181.27.85/LLVM.zip -O LLVM.zip
         unzip LLVM.zip
         cp "C:/Program Files/LLVM/tools/llvm/wasm-ld.exe" D:/wasi-sdk/bin/wasm-ld.exe
         

.gitignore

@@ -21,4 +21,5 @@ build
 .idea
 cmake-*
 .cache
-.DS_Store
+.DS_Store
+foo

.gitmodules

@@ -1,15 +1,16 @@
 [submodule "lib/wasm-micro-runtime"]
 	path = lib/wasm-micro-runtime
-	url = https://github.com/Multi-V-VM/wasm-micro-runtime/
+	url = https://github.com/Multi-V-VM/wasm-micro-runtime.git
 [submodule "lib/yalantinglibs"]
 	path = lib/yalantinglibs
-	url = https://github.com/alibaba/yalantinglibs
+	url = https://github.com/alibaba/yalantinglibs.git
 [submodule "bench/gapbs"]
 	path = bench/gapbs
 	url = https://github.com/victoryang00/gapbs.git
 [submodule "bench/clickhouse"]
 	path = bench/clickhouse
 	url = https://github.com/victoryang00/ClickHouse.git
+	branch = Alucardik/WASM_for_clickhouse-local
 [submodule "bench/llama"]
 	path = bench/llama
-	url = https://github.com/second-state/WasmEdge-WASINN-examples
+	url = https://github.com/victoryang00/llamma-wamr.git

CMakeLists.txt

@@ -1,9 +1,10 @@
 cmake_minimum_required(VERSION 3.22)
 project(MVVM LANGUAGES C CXX)
-set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD 23)
 
-find_package(FMT REQUIRED)
+find_package(fmt REQUIRED)
 # find_package(LLVM REQUIRED)
+find_package(benchmark REQUIRED)
 find_package(cxxopts REQUIRED)
 include(CTest)
 if(WIN32)
@@ -19,7 +20,6 @@ if(APPLE)
     else()
         set(LLVM_LLDB_LIB /usr/local/Cellar/llvm@14/14.0.6/lib/liblldb.dylib)
     endif()
-    set(WAMR_DISABLE_HW_BOUND_CHECK 1)
 elseif(LINUX)
     set(WAMR_BUILD_PLATFORM "linux")
     set(LLVM_LLDB_LIB /usr/lib/liblldb.so)
@@ -28,7 +28,7 @@ else()
     set(WAMR_BUILD_LIBC_UVWASI 1)
     set(CMAKE_MSVC_RUNTIME_LIBRARY "")
     set(CMAKE_GENERATOR_PLATFORM "")
-    set(CMAKE_CXX_FLAGS "/MD")
+    set(CMAKE_CXX_FLAGS "/MDd")
     set(LLVM_LLDB_LIB uv -lws2_32 "C:/Program Files/LLVM/lib/liblldb.lib")
 endif()
 
@@ -78,6 +78,7 @@ set(WAMR_BUILD_REF_TYPES 0)
 set(WAMR_BUILD_SIMD 1)
 set(WAMR_BUILD_CUSTOM_NAME_SECTION 1)
 set(WAMR_BUILD_CHECKPOINT_RESTORE 1)
+set(WAMR_DISABLE_HW_BOUND_CHECK 1)
 
 set(CMAKE_SHARED_LINKER_FLAGS "-Wl,-U,_get_ext_lib_export_apis")
 set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}")
@@ -93,15 +94,15 @@ include(${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake)
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/lib/yalantinglibs/include ${LLVM_INCLUDE_DIRS})
 
 file(GLOB_RECURSE SOURCE_FILES src/logging.cpp src/wamr*.cpp)
+add_library(MVVM_export ${SOURCE_FILES} ${UNCOMMON_SHARED_SOURCE})
+add_executable(MVVM_restore src/restore.cpp ${UNCOMMON_SHARED_SOURCE})
+add_executable(MVVM_checkpoint src/checkpoint.cpp ${UNCOMMON_SHARED_SOURCE})
 
-add_executable(MVVM_restore ${SOURCE_FILES} src/restore.cpp  ${UNCOMMON_SHARED_SOURCE})
-add_executable(MVVM_checkpoint ${SOURCE_FILES} src/checkpoint.cpp  ${UNCOMMON_SHARED_SOURCE})
-
-target_link_libraries(MVVM_restore fmt::fmt cxxopts::cxxopts vmlib -lm -ldl -lpthread ${LLVM_LLDB_LIB} ${LLVM_AVAILABLE_LIBS})
-target_link_libraries(MVVM_checkpoint fmt::fmt cxxopts::cxxopts vmlib -lm -ldl -lpthread ${LLVM_LLDB_LIB} ${LLVM_AVAILABLE_LIBS})
+target_link_libraries(MVVM_export fmt::fmt vmlib -lm -ldl -lpthread ${LLVM_LLDB_LIB} ${LLVM_AVAILABLE_LIBS})
+target_link_libraries(MVVM_restore fmt::fmt cxxopts::cxxopts MVVM_export vmlib -lm -ldl -lpthread ${LLVM_LLDB_LIB} ${LLVM_AVAILABLE_LIBS})
+target_link_libraries(MVVM_checkpoint fmt::fmt cxxopts::cxxopts MVVM_export vmlib -lm -ldl -lpthread ${LLVM_LLDB_LIB} ${LLVM_AVAILABLE_LIBS})
 add_definitions(-DMVVM_INTERP=1)
 add_definitions(-DMVVM_DEBUG=1)
-add_definitions(-DMVVM_WASI=1)
 add_definitions(-DCXXOPTS_NO_RTTI=1)
 add_subdirectory(test)
 add_subdirectory(bench)

bench/CMakeLists.txt

@@ -1,36 +1,58 @@
-find_package(benchmark REQUIRED)
-add_executable(mvvm_benchmark bench.cpp)
+add_executable(MVVM_benchmark bench.cpp)
+target_link_libraries(MVVM_benchmark benchmark::benchmark MVVM_export)
+
+if(WIN32)
+    set(WASI_SDK "D:/wasi-sdk")
+    set(EMSDK_DIR "D:/emsdk/upstream")
+    set(WASI_EXE_POSTFIX ".exe")
+else()
+    set(WASI_SDK "/opt/wasi-sdk")
+    set(EMSDK_DIR "/opt/emsdk/upstream")
+    set(WASI_EXE_POSTFIX "")
+endif()
 
-# Link benchmark library to your test target
-target_link_libraries(mvvm_benchmark benchmark)
 include(ExternalProject)
 
-# add_subdirectory(gapbs)
-# add_subdirectory(clickhouse)
-# ExternalProject_Add(gapbs
-# SOURCE_DIR        ${CMAKE_CURRENT_SOURCE_DIR}/gapbs
-# UPDATE_COMMAND    ""
-# PATCH_COMMAND     ""
-# CONFIGURE_COMMAND  ${CMAKE_COMMAND}
-#                     -DCMAKE_C_COMPILER_WORKS=1 -DCMAKE_CXX_COMPILER_WORKS=1
-#                     -GNinja
-#                     -DWASI_SDK_DIR=${WASI_SDK_DIR}
-#                     -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK_DIR}/share/cmake/wasi-sdk.cmake
-#                     -DCMAKE_SYSROOT=${WASI_SDK_DIR}/share/wasi-sysroot
-#                     ${CMAKE_CURRENT_SOURCE_DIR}/gapbs
-# BUILD_COMMAND     ${CMAKE_COMMAND} --build .
-# )
+ExternalProject_Add(gapbs
+        SOURCE_DIR        ${CMAKE_CURRENT_SOURCE_DIR}/gapbs
+        UPDATE_COMMAND    ""
+        PATCH_COMMAND     ""
+        CONFIGURE_COMMAND  ${CMAKE_COMMAND}
+        -GNinja -DWASI_SDK_DIR=${WASI_SDK} -DWASI_SDK_PREFIX=${WASI_SDK}
+        -DCMAKE_C_COMPILER=${WASI_SDK}/bin/clang${WASI_EXE_POSTFIX}
+        -DCMAKE_CXX_COMPILER=${WASI_SDK}/bin/clang++${WASI_EXE_POSTFIX}
+        -DCMAKE_CXX_FLAGS="-fno-exceptions"
+        -DCMAKE_SYSROOT=${WASI_SDK}/share/wasi-sysroot
+        ${CMAKE_CURRENT_SOURCE_DIR}/gapbs
+        BUILD_COMMAND     ${CMAKE_COMMAND} --build .
+        INSTALL_COMMAND   ""
+)
+if(NOT WIN32)
+    ExternalProject_Add(clickhouse
+            SOURCE_DIR        ${CMAKE_CURRENT_SOURCE_DIR}/clickhouse
+            UPDATE_COMMAND    ""
+            PATCH_COMMAND     ""
+            CONFIGURE_COMMAND  ${CMAKE_COMMAND}
+            -GNinja
+            -DCMAKE_C_COMPILER=${EMSDK_DIR}/bin/clang${WASI_EXE_POSTFIX}
+            -DCMAKE_CXX_COMPILER=${EMSDK_DIR}/bin/clang++${WASI_EXE_POSTFIX}
+            -DCMAKE_SYSROOT=${EMSDK_DIR}/emscripten/cache/sysroot -DEMSCRIPTEN=1
+            ${CMAKE_CURRENT_SOURCE_DIR}/clickhouse
+            BUILD_COMMAND     ${CMAKE_COMMAND} --build .
+            INSTALL_COMMAND   ""
+    )
 
-# ExternalProject_Add(clickhouse
-# SOURCE_DIR        ${CMAKE_CURRENT_SOURCE_DIR}/clickhouse
-# UPDATE_COMMAND    ""
-# PATCH_COMMAND     ""
-# CONFIGURE_COMMAND  ${CMAKE_COMMAND}
-#                     -DCMAKE_C_COMPILER_WORKS=1 -DCMAKE_CXX_COMPILER_WORKS=1
-#                     -GNinja
-#                     -DWASI_SDK_DIR=${WASI_SDK_DIR}
-#                     -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK_DIR}/share/cmake/wasi-sdk.cmake
-#                     -DCMAKE_SYSROOT=${WASI_SDK_DIR}/share/wasi-sysroot
-#                     ${CMAKE_CURRENT_SOURCE_DIR}/clickhouse
-# BUILD_COMMAND     ${CMAKE_COMMAND} --build .
-# )
+    ExternalProject_Add(llama
+            SOURCE_DIR        ${CMAKE_CURRENT_SOURCE_DIR}/llama
+            UPDATE_COMMAND    ""
+            PATCH_COMMAND     ""
+            CONFIGURE_COMMAND  ${CMAKE_COMMAND}
+            -GNinja -DWASI_SDK_DIR=${WASI_SDK} -DWASI_SDK_PREFIX=${WASI_SDK}
+            -DCMAKE_C_COMPILER=${WASI_SDK}/bin/clang${WASI_EXE_POSTFIX}
+            -DCMAKE_CXX_COMPILER=${WASI_SDK}/bin/clang++${WASI_EXE_POSTFIX}
+            -DCMAKE_SYSROOT=${WASI_SDK}/share/wasi-sysroot
+            ${CMAKE_CURRENT_SOURCE_DIR}/llama
+            BUILD_COMMAND     ${CMAKE_COMMAND} --build .
+            INSTALL_COMMAND   ""
+    )
+endif()

bench/bench.cpp

@@ -1,13 +1,69 @@
+#define _ALLOW_ITERATOR_DEBUG_LEVEL_MISMATCH 1
 #include <benchmark/benchmark.h>
 #include <wamr.h>
 
+WAMRInstance * wamr;
+
+void serialize_to_file(WASMExecEnv *instance) {
+    /** Sounds like AoT/JIT is in this?*/
+    // Note: insert fd
+    // a fast restore interface for benchmarking
+    std::ifstream stdoutput;
+}
+//./bfs -g10 -n0 > test/out/generate-g10.out
+// \033[92mPASS\033[0m Generates g10
+//./bfs -u10 -n0 > test/out/generate-u10.out
+// \033[91mFAIL\033[0m Generates u10
+//./bfs -f test/graphs/4.gr -n0 > test/out/load-4.gr.out
+// \033[92mPASS\033[0m Load 4.gr
+//./bfs -f test/graphs/4.el -n0 > test/out/load-4.el.out
+// \033[92mPASS\033[0m Load 4.el
+//./bfs -f test/graphs/4.wel -n0 > test/out/load-4.wel.out
+// \033[92mPASS\033[0m Load 4.wel
+//./bfs -f test/graphs/4.graph -n0 > test/out/load-4.graph.out
+// \033[92mPASS\033[0m Load 4.graph
+//./bfs -f test/graphs/4w.graph -n0 > test/out/load-4w.graph.out
+// \033[92mPASS\033[0m Load 4w.graph
+//./bfs -f test/graphs/4.mtx -n0 > test/out/load-4.mtx.out
+// \033[92mPASS\033[0m Load 4.mtx
+//./bfs -f test/graphs/4w.mtx -n0 > test/out/load-4w.mtx.out
+// \033[92mPASS\033[0m Load 4w.mtx
+//./bc -g10 -vn1 > test/out/verify-bc-g10.out
+// \033[92mPASS\033[0m Verify bc
+//./bfs -g10 -vn1 > test/out/verify-bfs-g10.out
+// \033[92mPASS\033[0m Verify bfs
+//./cc -g10 -vn1 > test/out/verify-cc-g10.out
+// \033[92mPASS\033[0m Verify cc
+//./cc_sv -g10 -vn1 > test/out/verify-cc_sv-g10.out
+// \033[92mPASS\033[0m Verify cc_sv
+//./pr -g10 -vn1 > test/out/verify-pr-g10.out
+// \033[92mPASS\033[0m Verify pr
+//./pr_spmv -g10 -vn1 > test/out/verify-pr_spmv-g10.out
+// \033[92mPASS\033[0m Verify pr_spmv
+//./sssp -g10 -vn1 > test/out/verify-sssp-g10.out
+// \033[92mPASS\033[0m Verify sssp
+//./tc -g10 -vn1 > test/out/verify-tc-g10.out
 static void BM_gapbs(benchmark::State& state) {
+    auto dir = {std::string("./")};
+    auto map_dir = {std::string("./")};
+    std::vector<string> env = {};
+    std::vector<std::string> cmd = {"bfs.wasm","bfs.wasm", "bfs.wasm", "bfs.wasm", "bfs.wasm", "bfs.wasm", "bfs.wasm", "bfs.wasm", "bfs.wasm", "bc.wasm", "bfs.wasm", "cc.wasm", "cc_sv.wasm", "pr.wasm", "pr_spmv.wasm", "sssp.wasm", "tc.wasm"};
+    std::vector<std::vector<string>> arg = {{"-g10","-n0"},{"-u10","-n0"},{"-f","test/graphs/4.gr","-n0"},{"-f","test/graphs/4.el","-n0"},{"-f","test/graphs/4.wel","-n0"},{"-f","test/graphs/4.graph","-n0"},{"-f","test/graphs/4w.graph","-n0"},{"-f","test/graphs/4.mtx","-n0"},{"-f","test/graphs/4w.mtx","-n0"},{"-g10","-vn1"},{"-g10","-vn1"},{"-g10","-vn1"},{"-g10","-vn1"},{"-g10","-vn1"},{"-g10","-vn1"},{"-g10","-vn1"},{"-g10","-vn1"}};
+    std::vector<string> addr = {};
+    std::vector<string> ns_pool = {};
+    auto is_jit = false;
     for (auto _ : state) {
-        // Your code to benchmark goes here
+        for (auto const &[k, v] : cmd | enumerate) {
+            wamr = new WAMRInstance(v.c_str(), is_jit);
+            wamr->set_wasi_args(dir, map_dir, env, arg[k], addr, ns_pool);
+            wamr->instantiate();
+            wamr->invoke_main();
+        }
     }
 }
 BENCHMARK(BM_gapbs);
 
+#if !defined(__WIN32__)
 static void BM_clickhouse(benchmark::State& state) {
     for (auto _ : state) {
         // Your code to benchmark goes here
@@ -16,10 +72,38 @@ static void BM_clickhouse(benchmark::State& state) {
 BENCHMARK(BM_clickhouse);
 
 static void BM_llama(benchmark::State& state) {
+    auto dir = {std::string("./")};
+    auto map_dir = {std::string("./")};
+    std::vector<string> env = {};
+    std::vector<string> arg = {};
+    std::vector<string> addr = {};
+    std::vector<string> ns_pool = {};
+    auto is_jit = false;
     for (auto _ : state) {
-        // Your code to benchmark goes here
+        wamr = new WAMRInstance("./llama.wasm", is_jit);
+        wamr->set_wasi_args(dir, map_dir, env, arg, addr, ns_pool);
+        wamr->instantiate();
+        wamr->invoke_main();
     }
 }
 BENCHMARK(BM_llama);
 
+static void BM_llama_aot(benchmark::State& state) {
+    auto dir = {std::string("./")};
+    auto map_dir = {std::string("./")};
+    std::vector<string> env = {};
+    std::vector<string> arg = {};
+    std::vector<string> addr = {};
+    std::vector<string> ns_pool = {};
+    auto is_jit = false;
+    for (auto _ : state) {
+        wamr = new WAMRInstance("./llama.aot", is_jit);
+        wamr->set_wasi_args(dir, map_dir, env, arg, addr, ns_pool);
+        wamr->instantiate();
+        wamr->invoke_main();
+    }
+}
+BENCHMARK(BM_llama_aot);
+#endif
+
 BENCHMARK_MAIN();

include/logging.h

@@ -18,9 +18,18 @@
 #include <iostream>
 #include <list>
 #include <sstream>
-#include <cstdarg>
+#include <ranges>
 #include <string>
 
+/** Barry's work*/
+struct Enumerate : std::ranges::range_adaptor_closure<Enumerate> {
+    template <std::ranges::viewable_range R> constexpr auto operator()(R &&r) const {
+        return std::views::zip(std::views::iota(0), (R &&)r);
+    }
+};
+
+inline constexpr Enumerate enumerate;
+
 using std::list;
 using std::string;
 struct LocationInfo {