From d1671502178ab5eb360a9bb9bd346ac971ec7e2c Mon Sep 17 00:00:00 2001 From: victoryang00 Date: Fri, 25 Aug 2023 06:34:20 +0000 Subject: [PATCH 01/15] init fd restore --- include/wamr.h | 8 ++-- include/wamr_wasi_context.h | 1 + src/checkpoint.cpp | 4 +- src/wamr.cpp | 81 +++++++++++++++++++++++++++---------- src/wamr_wasi_context.cpp | 18 +++++++-- 5 files changed, 82 insertions(+), 30 deletions(-) diff --git a/include/wamr.h b/include/wamr.h index 62a06e4..b14694e 100644 --- a/include/wamr.h +++ b/include/wamr.h @@ -8,9 +8,9 @@ #include "bh_read_file.h" #include "logging.h" #include "wamr_exec_env.h" +#include "wamr_export.h" #include "wamr_read_write.h" #include "wasm_runtime.h" -#include "wamr_export.h" class WAMRInstance { WASMExecEnv *exec_env{}; @@ -26,7 +26,7 @@ class WAMRInstance { std::vector arg_; std::vector addr_; std::vector ns_pool_; - std::map> fd_map_; + std::map> fd_map_; // add offset to pair->tuple, 3rd param 'int' bool is_jit; char *buffer{}; @@ -57,9 +57,9 @@ class WAMRInstance { const std::vector &addr_list, const std::vector &ns_lookup_pool); int invoke_main(); - int invoke_fopen(uint32 fd,const std::string& path, uint32 option); + int invoke_fopen(std::string &path, uint32 option); int invoke_fseek(uint32 fd, uint32 offset); - int invoke_preopen(uint32 fd,const std::string& path); + int invoke_preopen(uint32 fd, const std::string &path); ~WAMRInstance(); }; #endif // MVVM_WAMR_H diff --git a/include/wamr_wasi_context.h b/include/wamr_wasi_context.h index d7c2278..be9bd44 100644 --- a/include/wamr_wasi_context.h +++ b/include/wamr_wasi_context.h @@ -29,6 +29,7 @@ struct WAMRAddrPool { uint8 mask; }; struct WAMRWASIContext { + std::map> fd_map; std::vector dir; std::vector map_dir; WAMRArgvEnvironValues argv_environ; diff --git a/src/checkpoint.cpp b/src/checkpoint.cpp index 144833d..b84955f 100644 --- a/src/checkpoint.cpp +++ b/src/checkpoint.cpp @@ -16,8 +16,8 @@ WAMRInstance *wamr = nullptr; std::ostringstream re{}; auto writer = FwriteStream("test.bin"); -std::vector> as; -std::mutex as_mtx; +constinit std::vector> as; +constinit std::mutex as_mtx; /**fopen, fseek*/ void insert_fd(int fd, const char *path, int flags, int offset) { printf("\n #insert_fd(fd,filename,flags, offset) %d %s %d %d \n\n",fd, path,flags, offset); diff --git a/src/wamr.cpp b/src/wamr.cpp index a729e32..12b90cd 100644 --- a/src/wamr.cpp +++ b/src/wamr.cpp @@ -73,18 +73,38 @@ int WAMRInstance::invoke_main() { return wasm_runtime_call_wasm(exec_env, func, 0, nullptr); } -int WAMRInstance::invoke_fopen(uint32 fd, const std::string &path, uint32 option) { - if (!(func = wasm_runtime_lookup_function(module_inst, "open", "($i)i"))) { - LOGV(ERROR) << "The wasi open function is not found."; - return -1; +int WAMRInstance::invoke_fopen(std::string &path, uint32 option) { + auto name = "__wasilibc_open_nomode"; + if (!(func = wasm_runtime_lookup_function(module_inst, name, nullptr))) { + LOGV(ERROR) << "The wasi fopen function is not found."; + auto target_module = get_module_instance()->e; + for (int i = 0; i < target_module->function_count; i++) { + auto cur_func = &target_module->functions[i]; + if (cur_func->is_import_func) { + LOGV(DEBUG) << cur_func->u.func_import->field_name; + if (!strcmp(cur_func->u.func_import->field_name, name)) { + + func = ((WASMFunctionInstanceCommon *)cur_func); + break; + } + + } else { + LOGV(DEBUG) << cur_func->u.func->field_name; + + if (!strcmp(cur_func->u.func->field_name, name)) { + func = ((WASMFunctionInstanceCommon *)cur_func); + break; + } + } + } } - char *buffer_ = nullptr; + char *buffer_ = ((char *)malloc(path.size())); uint32_t buffer_for_wasm; + strncpy(buffer_, path.c_str(), path.size()); // use native address for accessing in runtime - buffer_for_wasm = wasm_runtime_module_malloc(module_inst, 100, reinterpret_cast(&buffer_)); + buffer_for_wasm = wasm_runtime_module_malloc(module_inst, path.size(), reinterpret_cast(&buffer_)); if (buffer_for_wasm != 0) { uint32 argv[2]; - strncpy(buffer_, path.c_str(), path.size()); // use native address for accessing in runtime argv[0] = buffer_for_wasm; // pass the buffer_ address for WASM space argv[1] = option; // the size of buffer_ auto res = wasm_runtime_call_wasm(exec_env, func, 2, argv); @@ -94,39 +114,58 @@ int WAMRInstance::invoke_fopen(uint32 fd, const std::string &path, uint32 option return -1; }; int WAMRInstance::invoke_fseek(uint32 fd, uint32 offset) { - if (!(func = wasm_runtime_lookup_function(module_inst, "fseek", "($i)i"))) { - LOGV(ERROR) << "The wasi open function is not found."; - return -1; - } - char *buffer_ = nullptr; - uint32_t buffer_for_wasm; + auto name = "fseek"; + if (!(func = wasm_runtime_lookup_function(module_inst, name, nullptr))) { + LOGV(ERROR) << "The wasi fopen function is not found."; + auto target_module = get_module_instance()->e; + for (int i = 0; i < target_module->function_count; i++) { + auto cur_func = &target_module->functions[i]; + if (cur_func->is_import_func) { + LOGV(DEBUG) << cur_func->u.func_import->field_name; + if (!strcmp(cur_func->u.func_import->field_name, name)) { - buffer_for_wasm = wasm_runtime_module_malloc(module_inst, 100, reinterpret_cast(&buffer_)); - if (buffer_for_wasm != 0) { - uint32 argv[2]; + func = ((WASMFunctionInstanceCommon *)cur_func); + break; + } + + } else { + LOGV(DEBUG) << cur_func->u.func->field_name; + + if (!strcmp(cur_func->u.func->field_name, name)) { + func = ((WASMFunctionInstanceCommon *)cur_func); + break; + } + } + } } - return -1; + uint32 argv[2] = {fd, offset}; + return wasm_runtime_call_wasm(exec_env, func, 2, argv); }; int WAMRInstance::invoke_preopen(uint32 fd, const std::string &path) { auto name = "__wasilibc_register_preopened_fd"; if (!(func = wasm_runtime_lookup_function(module_inst, name, nullptr))) { - LOGV(ERROR) << "The __wasilibc_register_preopened_fd function is not found."; + LOGV(ERROR) << "The wasi fopen function is not found."; auto target_module = get_module_instance()->e; for (int i = 0; i < target_module->function_count; i++) { auto cur_func = &target_module->functions[i]; if (cur_func->is_import_func) { LOGV(DEBUG) << cur_func->u.func_import->field_name; - if (!strcmp(cur_func->u.func_import->field_name, name)) + if (!strcmp(cur_func->u.func_import->field_name, name)) { + func = ((WASMFunctionInstanceCommon *)cur_func); + break; + } + } else { LOGV(DEBUG) << cur_func->u.func->field_name; - if (!strcmp(cur_func->u.func->field_name, name)) + if (!strcmp(cur_func->u.func->field_name, name)) { func = ((WASMFunctionInstanceCommon *)cur_func); + break; + } } } } - char *buffer_ = nullptr; uint32_t buffer_for_wasm; diff --git a/src/wamr_wasi_context.cpp b/src/wamr_wasi_context.cpp index ebcf0de..e29570b 100644 --- a/src/wamr_wasi_context.cpp +++ b/src/wamr_wasi_context.cpp @@ -12,11 +12,23 @@ void WAMRWASIContext::dump_impl(WASIContext *env) { map_dir.emplace_back(wamr->map_dir_[i]); } this->exit_code = env->exit_code; + for (auto [fd, res] : wamr->fd_map_) { + auto [path, flags, offset] = res; + auto dumped_res = std::make_tuple(path, flags, offset); + this->fd_map[fd] = dumped_res; + } } void WAMRWASIContext::restore_impl(WASIContext *env) { - for (auto [fd, res] : wamr->fd_map_) { + int r; + for (auto [fd, res] : this->fd_map) { // differ from path from file - wamr->invoke_fopen(fd, std::get<0>(res),std::get<1>(res)); - wamr->invoke_fseek(fd, std::get<2>(res)); + LOGV(INFO) << "fd: " << fd << " path: " << std::get<0>(res) << " flags: " << std::get<1>(res) + << " offset: " << std::get<2>(res); + while (true) { + r = wamr->invoke_fopen(std::get<0>(res), std::get<1>(res)); + if (r == fd) + break; + wamr->invoke_fseek(fd, std::get<2>(res)); + } } }; \ No newline at end of file From b0410befa81cd9dbac652aa2afbccc81b55affd7 Mon Sep 17 00:00:00 2001 From: victoryang00 Date: Fri, 25 Aug 2023 07:45:39 +0000 Subject: [PATCH 02/15] fix fd a little bit --- include/wamr.h | 1 + lib/wasm-micro-runtime | 2 +- src/checkpoint.cpp | 75 ++++++++++++++++++++------------------- src/wamr.cpp | 38 +++++++++++++++++--- src/wamr_wasi_context.cpp | 12 +++---- test/CMakeLists.txt | 4 +-- test/read-file.c | 1 + 7 files changed, 82 insertions(+), 51 deletions(-) diff --git a/include/wamr.h b/include/wamr.h index b14694e..d9c08da 100644 --- a/include/wamr.h +++ b/include/wamr.h @@ -58,6 +58,7 @@ class WAMRInstance { int invoke_main(); int invoke_fopen(std::string &path, uint32 option); + int invoke_frenumber(uint32 fd, uint32 to); int invoke_fseek(uint32 fd, uint32 offset); int invoke_preopen(uint32 fd, const std::string &path); ~WAMRInstance(); diff --git a/lib/wasm-micro-runtime b/lib/wasm-micro-runtime index 36ee6c2..72c06b5 160000 --- a/lib/wasm-micro-runtime +++ b/lib/wasm-micro-runtime @@ -1 +1 @@ -Subproject commit 36ee6c2353fed83b7297a3171ff7dcd360d1d11e +Subproject commit 72c06b5c507a01645909a67e9d7320b045cbbcf0 diff --git a/src/checkpoint.cpp b/src/checkpoint.cpp index b84955f..db0abd5 100644 --- a/src/checkpoint.cpp +++ b/src/checkpoint.cpp @@ -20,28 +20,30 @@ constinit std::vector> as; constinit std::mutex as_mtx; /**fopen, fseek*/ void insert_fd(int fd, const char *path, int flags, int offset) { - printf("\n #insert_fd(fd,filename,flags, offset) %d %s %d %d \n\n",fd, path,flags, offset); - - if (wamr->fd_map_.find(fd) != wamr->fd_map_.end()) { - LOGV(ERROR) << "fd already exist" << fd; - if(offset == 0) { - // fOpen call - std::string curPath; - int curFlags; - int curOffset; - std::tie(curPath, curFlags, curOffset) = wamr->fd_map_[fd]; - wamr->fd_map_[fd] = std::make_tuple(std::string(path), flags, curOffset); - } else { - // fSeek call - std::string curPath; - int curFlags; - int curOffset; - std::tie(curPath, curFlags, curOffset) = wamr->fd_map_[fd]; - wamr->fd_map_[fd] = std::make_tuple(curPath, curFlags, offset); - } - - } else - wamr->fd_map_.insert(std::make_pair(fd, std::make_tuple(std::string(path), flags, offset))); + if (fd > 2) { + printf("\n #insert_fd(fd,filename,flags, offset) %d %s %d %d \n\n", fd, path, flags, offset); + + if (wamr->fd_map_.find(fd) != wamr->fd_map_.end()) { + LOGV(ERROR) << "fd already exist" << fd; + if (offset == 0) { + // fOpen call + std::string curPath; + int curFlags; + int curOffset; + std::tie(curPath, curFlags, curOffset) = wamr->fd_map_[fd]; + wamr->fd_map_[fd] = std::make_tuple(std::string(path), flags, curOffset); + } else { + // fSeek call + std::string curPath; + int curFlags; + int curOffset; + std::tie(curPath, curFlags, curOffset) = wamr->fd_map_[fd]; + wamr->fd_map_[fd] = std::make_tuple(curPath, curFlags, offset); + } + + } else + wamr->fd_map_.insert(std::make_pair(fd, std::make_tuple(std::string(path), flags, offset))); + } } /* update fd->offset**/ @@ -51,25 +53,25 @@ void remove_fd(int fd) { if (wamr->fd_map_.find(fd) != wamr->fd_map_.end()) wamr->fd_map_.erase(fd); else - LOGV(ERROR)<< "fd not found" << fd; -} -void insert_socket(int fd){ - + LOGV(ERROR) << "fd not found" << fd; } +void insert_socket(int fd) {} void serialize_to_file(WASMExecEnv *instance) { /** Sounds like AoT/JIT is in this?*/ // Note: insert fd - std::ifstream stdoutput; stdoutput.open("output.txt"); - std:string current_str; + std::ifstream stdoutput; + stdoutput.open("output.txt"); +std: + string current_str; std::string fd_output; std::string filename_output; std::string flags_output; - - if(stdoutput.is_open()) { - while(stdoutput.good()) { + + if (stdoutput.is_open()) { + while (stdoutput.good()) { stdoutput >> current_str; - if(current_str == "fopen_test(fd,filename,flags)") { - stdoutput >> fd_output; + if (current_str == "fopen_test(fd,filename,flags)") { + stdoutput >> fd_output; stdoutput >> filename_output; stdoutput >> flags_output; insert_fd(std::stoi(fd_output), filename_output.c_str(), std::stoi(flags_output), 0); @@ -79,7 +81,7 @@ void serialize_to_file(WASMExecEnv *instance) { stdoutput.close(); // - std::cout<<"dasfasdfasf"<< re.str()<<"dasfasdfasf\n"; + std::cout << "dasfasdfasf" << re.str() << "dasfasdfasf\n"; auto cluster = wasm_exec_env_get_cluster(instance); if (bh_list_length(&cluster->exec_env_list) > 1) { auto elem = (WASMExecEnv *)bh_list_first_elem(&cluster->exec_env_list); @@ -154,8 +156,7 @@ int main(int argc, char *argv[]) { options.add_options()("t,target", "The webassembly file to execute", cxxopts::value()->default_value("./test/counter.wasm"))( "j,jit", "Whether the jit mode or interp mode", cxxopts::value()->default_value("false"))( - "d,dir", "The directory list exposed to WAMR", - cxxopts::value>()->default_value("./"))( + "d,dir", "The directory list exposed to WAMR", cxxopts::value>()->default_value("./"))( "m,map_dir", "The mapped directory list exposed to WAMRe", cxxopts::value>()->default_value(""))( "e,env", "The environment list exposed to WAMR", @@ -182,7 +183,7 @@ int main(int argc, char *argv[]) { wamr = new WAMRInstance(target.c_str(), is_jit); wamr->set_wasi_args(dir, map_dir, env, arg, addr, ns_pool); wamr->instantiate(); - freopen("output.txt","w",stdout); + freopen("output.txt", "w", stdout); #ifndef MVVM_DEBUG // Define the sigaction structure diff --git a/src/wamr.cpp b/src/wamr.cpp index 12b90cd..3f24c94 100644 --- a/src/wamr.cpp +++ b/src/wamr.cpp @@ -4,6 +4,7 @@ #include "wamr.h" #include "thread_manager.h" +#include "wasm_export.h" #include "wasm_interp.h" #include @@ -19,6 +20,7 @@ WAMRInstance::WAMRInstance(const char *wasm_path, bool is_jit) : is_jit(is_jit) wasm_args.running_mode = RunningMode::Mode_Interp; else wasm_args.running_mode = RunningMode::Mode_LLVM_JIT; + // static char global_heap_buf[512 * 1024];// what is this? // wasm_args.mem_alloc_type = Alloc_With_Pool; // wasm_args.mem_alloc_option.pool.heap_buf = global_heap_buf; // wasm_args.mem_alloc_option.pool.heap_size = sizeof(global_heap_buf); @@ -98,23 +100,51 @@ int WAMRInstance::invoke_fopen(std::string &path, uint32 option) { } } } - char *buffer_ = ((char *)malloc(path.size())); + char *buffer_ = nullptr; uint32_t buffer_for_wasm; - strncpy(buffer_, path.c_str(), path.size()); // use native address for accessing in runtime - buffer_for_wasm = wasm_runtime_module_malloc(module_inst, path.size(), reinterpret_cast(&buffer_)); + buffer_for_wasm = wasm_runtime_module_malloc(module_inst, path.size(), (void **)&buffer_); if (buffer_for_wasm != 0) { uint32 argv[2]; argv[0] = buffer_for_wasm; // pass the buffer_ address for WASM space argv[1] = option; // the size of buffer_ + strncpy(buffer_, path.c_str(), path.size()); // use native address for accessing in runtime auto res = wasm_runtime_call_wasm(exec_env, func, 2, argv); wasm_runtime_module_free(module_inst, buffer_for_wasm); return res; } return -1; }; +int WAMRInstance::invoke_frenumber(uint32 fd, uint32 to) { + auto name = "__wasi_fd_renumber"; + if (!(func = wasm_runtime_lookup_function(module_inst, name, nullptr))) { + LOGV(ERROR) << "The wasi fopen function is not found."; + auto target_module = get_module_instance()->e; + for (int i = 0; i < target_module->function_count; i++) { + auto cur_func = &target_module->functions[i]; + if (cur_func->is_import_func) { + LOGV(DEBUG) << cur_func->u.func_import->field_name; + if (!strcmp(cur_func->u.func_import->field_name, name)) { + + func = ((WASMFunctionInstanceCommon *)cur_func); + break; + } + + } else { + LOGV(DEBUG) << cur_func->u.func->field_name; + + if (!strcmp(cur_func->u.func->field_name, name)) { + func = ((WASMFunctionInstanceCommon *)cur_func); + break; + } + } + } + } + uint32 argv[2] = {fd, to}; + return wasm_runtime_call_wasm(exec_env, func, 2, argv); +}; int WAMRInstance::invoke_fseek(uint32 fd, uint32 offset) { - auto name = "fseek"; + auto name = "__wasi_fd_seek"; if (!(func = wasm_runtime_lookup_function(module_inst, name, nullptr))) { LOGV(ERROR) << "The wasi fopen function is not found."; auto target_module = get_module_instance()->e; diff --git a/src/wamr_wasi_context.cpp b/src/wamr_wasi_context.cpp index e29570b..2a2ec08 100644 --- a/src/wamr_wasi_context.cpp +++ b/src/wamr_wasi_context.cpp @@ -23,12 +23,10 @@ void WAMRWASIContext::restore_impl(WASIContext *env) { for (auto [fd, res] : this->fd_map) { // differ from path from file LOGV(INFO) << "fd: " << fd << " path: " << std::get<0>(res) << " flags: " << std::get<1>(res) - << " offset: " << std::get<2>(res); - while (true) { - r = wamr->invoke_fopen(std::get<0>(res), std::get<1>(res)); - if (r == fd) - break; - wamr->invoke_fseek(fd, std::get<2>(res)); - } + << " offset: " << std::get<2>(res); + r = wamr->invoke_fopen(std::get<0>(res), std::get<1>(res)); + if (r != fd) + wamr->invoke_frenumber(r, fd); + wamr->invoke_fseek(fd, std::get<2>(res)); } }; \ No newline at end of file diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 56466b0..9ad5415 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -6,7 +6,7 @@ endif() set(WAMR_DIR ${PROJECT_SOURCE_DIR}/lib/wasm-micro-runtime/) enable_testing() function(wamr_app input) - add_custom_target(${input}_wamr ALL COMMAND ${HOST_PREFIX}clang -Wno-implicit-function-declaration -Wno-int-conversion --target=wasm32-unknown-wasi -g -pthread -Wl,--allow-undefined-file=${WAMR_DIR}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt --sysroot=${HOST_PREFIX}/../share/wasi-sysroot ${CMAKE_CURRENT_SOURCE_DIR}/${input}.c -o ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm) + add_custom_target(${input}_wamr ALL COMMAND ${HOST_PREFIX}clang -Wno-implicit-function-declaration -Wno-int-conversion --target=wasm32-unknown-wasi -g -pthread -Wl,--export=malloc -Wl,--export=free -Wl,--allow-undefined-file=${WAMR_DIR}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt --sysroot=${HOST_PREFIX}/../share/wasi-sysroot ${CMAKE_CURRENT_SOURCE_DIR}/${input}.c -o ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm) add_test(NAME ${input}_checkpoint COMMAND ${CMAKE_CURRENT_BINARY_DIR}/../MVVM_checkpoint --target ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm) # add_test(NAME ${input}_restore COMMAND ${CMAKE_CURRENT_BINARY_DIR}/../MVVM_restore --target ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm) endfunction() @@ -17,7 +17,7 @@ wamr_app(read-file) wamr_app(gups) function(wamr_socket_app input) - add_custom_target(${input}_wamr ALL COMMAND ${HOST_PREFIX}clang --target=wasm32-unknown-wasi -g -pthread -Wl,--allow-undefined-file=${WAMR_DIR}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt --sysroot=${HOST_PREFIX}/../share/wasi-sysroot ${WAMR_DIR}/samples/socket-api/wasm-app-prefix/src/wasm-app-build/libsocket_wasi_ext.a -I${WAMR_DIR}/core/iwasm/libraries/lib-socket/inc/ ${CMAKE_CURRENT_SOURCE_DIR}/${input}.c -o ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm) + add_custom_target(${input}_wamr ALL COMMAND ${HOST_PREFIX}clang --target=wasm32-unknown-wasi -g -pthread -Wl,--export=malloc -Wl,--export=free -Wl,--allow-undefined-file=${WAMR_DIR}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt --sysroot=${HOST_PREFIX}/../share/wasi-sysroot ${WAMR_DIR}/samples/socket-api/wasm-app-prefix/src/wasm-app-build/libsocket_wasi_ext.a -I${WAMR_DIR}/core/iwasm/libraries/lib-socket/inc/ ${CMAKE_CURRENT_SOURCE_DIR}/${input}.c -o ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm) add_test(NAME ${input}_checkpoint COMMAND ${CMAKE_CURRENT_BINARY_DIR}/../MVVM_checkpoint --target ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm) # add_test(NAME ${input}_restore COMMAND ${CMAKE_CURRENT_BINARY_DIR}/../MVVM_restore --target ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm) endfunction() diff --git a/test/read-file.c b/test/read-file.c index bdaf628..487dec9 100644 --- a/test/read-file.c +++ b/test/read-file.c @@ -79,4 +79,5 @@ int main() { fclose(file1); fclose(file2); fclose(file3); + __wasi_fd_renumber(1,1); } \ No newline at end of file From 74bc2eb697f28b469bc443b8f8806d9ef7b92580 Mon Sep 17 00:00:00 2001 From: victoryang00 Date: Sat, 16 Sep 2023 04:58:28 +0000 Subject: [PATCH 03/15] fix fd a little bit --- src/checkpoint.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/checkpoint.cpp b/src/checkpoint.cpp index db0abd5..eb8a38a 100644 --- a/src/checkpoint.cpp +++ b/src/checkpoint.cpp @@ -16,8 +16,8 @@ WAMRInstance *wamr = nullptr; std::ostringstream re{}; auto writer = FwriteStream("test.bin"); -constinit std::vector> as; -constinit std::mutex as_mtx; +std::vector> as; +std::mutex as_mtx; /**fopen, fseek*/ void insert_fd(int fd, const char *path, int flags, int offset) { if (fd > 2) { From c70c9e9d030d07078a2607344efa72f7945aeb60 Mon Sep 17 00:00:00 2001 From: kikispace Date: Mon, 9 Oct 2023 15:08:41 -0700 Subject: [PATCH 04/15] add strace --- lib/wasm-micro-runtime | 2 +- src/wamr.cpp | 2 +- src/wamr_wasi_context.cpp | 7 +++++++ test/read-file.c | 2 ++ 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/wasm-micro-runtime b/lib/wasm-micro-runtime index 72c06b5..da95bab 160000 --- a/lib/wasm-micro-runtime +++ b/lib/wasm-micro-runtime @@ -1 +1 @@ -Subproject commit 72c06b5c507a01645909a67e9d7320b045cbbcf0 +Subproject commit da95babb0197dd37a8b227bf9e76ee7ec500414a diff --git a/src/wamr.cpp b/src/wamr.cpp index 3f24c94..ffa32fc 100644 --- a/src/wamr.cpp +++ b/src/wamr.cpp @@ -116,7 +116,7 @@ int WAMRInstance::invoke_fopen(std::string &path, uint32 option) { return -1; }; int WAMRInstance::invoke_frenumber(uint32 fd, uint32 to) { - auto name = "__wasi_fd_renumber"; + auto name = "fd_renumber"; if (!(func = wasm_runtime_lookup_function(module_inst, name, nullptr))) { LOGV(ERROR) << "The wasi fopen function is not found."; auto target_module = get_module_instance()->e; diff --git a/src/wamr_wasi_context.cpp b/src/wamr_wasi_context.cpp index 2a2ec08..105c19b 100644 --- a/src/wamr_wasi_context.cpp +++ b/src/wamr_wasi_context.cpp @@ -3,6 +3,7 @@ // #include "wamr_wasi_context.h" #include "wamr.h" +#include extern WAMRInstance *wamr; void WAMRWASIContext::dump_impl(WASIContext *env) { for (int i = 0; i < wamr->dir_.size(); i++) { @@ -20,6 +21,12 @@ void WAMRWASIContext::dump_impl(WASIContext *env) { } void WAMRWASIContext::restore_impl(WASIContext *env) { int r; + // std::string stdin_fd= "/dev/stdin"; + // wamr->invoke_fopen(stdin_fd, 0); + // std::string stdout_fd= "/dev/stdout"; + // wamr->invoke_fopen(stdout_fd, 0); + // std::string stderr_fd= "/dev/stderr"; + // wamr->invoke_fopen(stderr_fd, 0); for (auto [fd, res] : this->fd_map) { // differ from path from file LOGV(INFO) << "fd: " << fd << " path: " << std::get<0>(res) << " flags: " << std::get<1>(res) diff --git a/test/read-file.c b/test/read-file.c index 487dec9..484ce5b 100644 --- a/test/read-file.c +++ b/test/read-file.c @@ -65,6 +65,8 @@ int main() { size_t len2 = strlen(line2); fwrite(line1, sizeof(char), len1, file3); + //ntwritten + // checkpoint: lib_wasi_wrapper fwrite system record // volatile int c = 0; From 7fa267b348d4e28207c2ec279d709449fd9eac84 Mon Sep 17 00:00:00 2001 From: kikispace Date: Mon, 9 Oct 2023 18:01:25 -0700 Subject: [PATCH 05/15] Add all strace --- lib/wasm-micro-runtime | 2 +- src/wamr.cpp | 2 +- src/wamr_wasi_context.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/wasm-micro-runtime b/lib/wasm-micro-runtime index da95bab..339151e 160000 --- a/lib/wasm-micro-runtime +++ b/lib/wasm-micro-runtime @@ -1 +1 @@ -Subproject commit da95babb0197dd37a8b227bf9e76ee7ec500414a +Subproject commit 339151ef2dd96ae38f19c7c1bfff608b7f47d4c1 diff --git a/src/wamr.cpp b/src/wamr.cpp index ffa32fc..3f24c94 100644 --- a/src/wamr.cpp +++ b/src/wamr.cpp @@ -116,7 +116,7 @@ int WAMRInstance::invoke_fopen(std::string &path, uint32 option) { return -1; }; int WAMRInstance::invoke_frenumber(uint32 fd, uint32 to) { - auto name = "fd_renumber"; + auto name = "__wasi_fd_renumber"; if (!(func = wasm_runtime_lookup_function(module_inst, name, nullptr))) { LOGV(ERROR) << "The wasi fopen function is not found."; auto target_module = get_module_instance()->e; diff --git a/src/wamr_wasi_context.cpp b/src/wamr_wasi_context.cpp index 105c19b..e296842 100644 --- a/src/wamr_wasi_context.cpp +++ b/src/wamr_wasi_context.cpp @@ -32,7 +32,7 @@ void WAMRWASIContext::restore_impl(WASIContext *env) { LOGV(INFO) << "fd: " << fd << " path: " << std::get<0>(res) << " flags: " << std::get<1>(res) << " offset: " << std::get<2>(res); r = wamr->invoke_fopen(std::get<0>(res), std::get<1>(res)); - if (r != fd) + if (fd != 0 && fd != 1 && fd != 2) wamr->invoke_frenumber(r, fd); wamr->invoke_fseek(fd, std::get<2>(res)); } From f9d29c0caedad666172edb01c62a20fe5d765d3d Mon Sep 17 00:00:00 2001 From: victoryang00 Date: Mon, 16 Oct 2023 18:01:58 +0000 Subject: [PATCH 06/15] fix ret value --- lib/wasm-micro-runtime | 2 +- src/wamr.cpp | 4 ++-- src/wamr_wasi_context.cpp | 2 +- test/read-file.c | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/wasm-micro-runtime b/lib/wasm-micro-runtime index 339151e..774cbc3 160000 --- a/lib/wasm-micro-runtime +++ b/lib/wasm-micro-runtime @@ -1 +1 @@ -Subproject commit 339151ef2dd96ae38f19c7c1bfff608b7f47d4c1 +Subproject commit 774cbc3883f55192c8ee2e138c15001d39189039 diff --git a/src/wamr.cpp b/src/wamr.cpp index 3f24c94..8a57ee1 100644 --- a/src/wamr.cpp +++ b/src/wamr.cpp @@ -109,9 +109,9 @@ int WAMRInstance::invoke_fopen(std::string &path, uint32 option) { argv[0] = buffer_for_wasm; // pass the buffer_ address for WASM space argv[1] = option; // the size of buffer_ strncpy(buffer_, path.c_str(), path.size()); // use native address for accessing in runtime - auto res = wasm_runtime_call_wasm(exec_env, func, 2, argv); + wasm_runtime_call_wasm(exec_env, func, 2, argv); wasm_runtime_module_free(module_inst, buffer_for_wasm); - return res; + return ((int)argv[0]); } return -1; }; diff --git a/src/wamr_wasi_context.cpp b/src/wamr_wasi_context.cpp index e296842..105c19b 100644 --- a/src/wamr_wasi_context.cpp +++ b/src/wamr_wasi_context.cpp @@ -32,7 +32,7 @@ void WAMRWASIContext::restore_impl(WASIContext *env) { LOGV(INFO) << "fd: " << fd << " path: " << std::get<0>(res) << " flags: " << std::get<1>(res) << " offset: " << std::get<2>(res); r = wamr->invoke_fopen(std::get<0>(res), std::get<1>(res)); - if (fd != 0 && fd != 1 && fd != 2) + if (r != fd) wamr->invoke_frenumber(r, fd); wamr->invoke_fseek(fd, std::get<2>(res)); } diff --git a/test/read-file.c b/test/read-file.c index 484ce5b..7cd2396 100644 --- a/test/read-file.c +++ b/test/read-file.c @@ -81,5 +81,5 @@ int main() { fclose(file1); fclose(file2); fclose(file3); - __wasi_fd_renumber(1,1); + __wasi_fd_renumber(3,10); } \ No newline at end of file From 83919a434eb9d2355233b8a09174217e711a92a7 Mon Sep 17 00:00:00 2001 From: kikispace Date: Mon, 16 Oct 2023 18:29:54 -0700 Subject: [PATCH 07/15] fixed invoke_socket* --- include/wamr.h | 36 ++++++++ include/wamr_export.h | 8 ++ lib/wasm-micro-runtime | 2 +- src/checkpoint.cpp | 39 +++++++++ src/restore.cpp | 7 ++ src/wamr.cpp | 172 +++++++++++++++++++++++++++++++++++++- src/wamr_wasi_context.cpp | 8 ++ 7 files changed, 270 insertions(+), 2 deletions(-) diff --git a/include/wamr.h b/include/wamr.h index d9c08da..a03345f 100644 --- a/include/wamr.h +++ b/include/wamr.h @@ -12,6 +12,32 @@ #include "wamr_read_write.h" #include "wasm_runtime.h" + +struct WasiSockOpenData { + uint32_t poolfd; + int af; + int socktype; + uint32_t* sockfd; +}; + +struct WasiSockSendToData { + uint32_t sock; + const iovec_app1_t* si_data; + uint32 si_data_len; + uint16_t si_flags; + const __wasi_addr_t* dest_addr; + uint32* so_data_len; +}; + +struct WasiSockRecvFromData { + uint32_t sock; + iovec_app1_t* ri_data; + uint32 ri_data_len; + uint16_t ri_flags; + __wasi_addr_t* src_addr; + uint32* ro_data_len; +}; + class WAMRInstance { WASMExecEnv *exec_env{}; WASMExecEnv *cur_env{}; @@ -28,6 +54,11 @@ class WAMRInstance { std::vector ns_pool_; std::map> fd_map_; // add offset to pair->tuple, 3rd param 'int' + + WasiSockOpenData sock_open_data; + WasiSockSendToData sock_sendto_data; + WasiSockRecvFromData sock_recvfrom_data; + bool is_jit; char *buffer{}; char error_buf[128]{}; @@ -61,6 +92,11 @@ class WAMRInstance { int invoke_frenumber(uint32 fd, uint32 to); int invoke_fseek(uint32 fd, uint32 offset); int invoke_preopen(uint32 fd, const std::string &path); + int invoke_socket(uint32 domain, const std::string &path); + int invoke_(uint32 domain, const std::string &path); + int invoke_sock_sendto(uint32_t sock, const iovec_app1_t* si_data, uint32 si_data_len, uint16_t si_flags, const __wasi_addr_t* dest_addr, uint32* so_data_len); + int invoke_sock_recvfrom(uint32_t sock, iovec_app1_t* ri_data, uint32 ri_data_len, uint16_t ri_flags, __wasi_addr_t* src_addr, uint32* ro_data_len); + int invoke_sock_open(uint32_t poolfd, int af, int socktype, uint32_t* sockfd); ~WAMRInstance(); }; #endif // MVVM_WAMR_H diff --git a/include/wamr_export.h b/include/wamr_export.h index eae7682..15078bc 100644 --- a/include/wamr_export.h +++ b/include/wamr_export.h @@ -7,8 +7,16 @@ extern "C" { #endif #ifdef MVVM_WASI #include "wasm_runtime.h" +typedef struct iovec_app1 { + uint32 buf_offset; + uint32 buf_len; +} iovec_app1_t; + void serialize_to_file(struct WASMExecEnv *); #endif +void insert_sock_send_to_data(uint32_t sock, const iovec_app1_t* si_data, uint32 si_data_len, uint16_t si_flags, const __wasi_addr_t* dest_addr, uint32* so_data_len); +void insert_sock_open_data(uint32_t, int, int, uint32_t *); +void insert_sock_recv_from_data(uint32_t sock, iovec_app1_t* ri_data, uint32 ri_data_len, uint16_t ri_flags, __wasi_addr_t* src_addr, uint32* ro_data_len); void insert_fd(int, char const *, int, int); void remove_fd(int); void insert_socket(char const *, int); diff --git a/lib/wasm-micro-runtime b/lib/wasm-micro-runtime index 339151e..6128c89 160000 --- a/lib/wasm-micro-runtime +++ b/lib/wasm-micro-runtime @@ -1 +1 @@ -Subproject commit 339151ef2dd96ae38f19c7c1bfff608b7f47d4c1 +Subproject commit 6128c894fb5d324b1fd2d67eb6d2c9afb37c18a2 diff --git a/src/checkpoint.cpp b/src/checkpoint.cpp index eb8a38a..9adc186 100644 --- a/src/checkpoint.cpp +++ b/src/checkpoint.cpp @@ -18,6 +18,45 @@ std::ostringstream re{}; auto writer = FwriteStream("test.bin"); std::vector> as; std::mutex as_mtx; + +void insert_sock_open_data(uint32_t poolfd, int af, int socktype, uint32_t* sockfd) { + if (wamr->sock_open_data.poolfd == 0) { + wamr->sock_open_data.poolfd = poolfd; + wamr->sock_open_data.af = af; + wamr->sock_open_data.socktype = socktype; + wamr->sock_open_data.sockfd = sockfd; + } else { + LOGV(ERROR) << "sock_open_data already exist"; + } +} + +void insert_sock_send_to_data(uint32_t sock, const iovec_app1_t* si_data, uint32 si_data_len, uint16_t si_flags, const __wasi_addr_t* dest_addr, uint32* so_data_len) { + if(wamr->sock_sendto_data.sock == 0) { + wamr->sock_sendto_data.sock = sock; + wamr->sock_sendto_data.si_data = si_data; + wamr->sock_sendto_data.si_data_len = si_data_len; + wamr->sock_sendto_data.si_flags = si_flags; + wamr->sock_sendto_data.dest_addr = dest_addr; + wamr->sock_sendto_data.so_data_len = so_data_len; + } else { + LOGV(ERROR) << "sock_sendto_data already exist"; + } +} + +void insert_sock_recv_from_data(uint32_t sock, iovec_app1_t* ri_data, uint32 ri_data_len, uint16_t ri_flags, __wasi_addr_t* src_addr, uint32* ro_data_len) { + if(wamr->sock_recvfrom_data.sock == 0) { + wamr->sock_recvfrom_data.sock = sock; + wamr->sock_recvfrom_data.ri_data = ri_data; + wamr->sock_recvfrom_data.ri_data_len = ri_data_len; + wamr->sock_recvfrom_data.ri_flags = ri_flags; + wamr->sock_recvfrom_data.src_addr = src_addr; + wamr->sock_recvfrom_data.ro_data_len = ro_data_len; + } else { + LOGV(ERROR) << "sock_recvfrom_data already exist"; + } +} + + /**fopen, fseek*/ void insert_fd(int fd, const char *path, int flags, int offset) { if (fd > 2) { diff --git a/src/restore.cpp b/src/restore.cpp index c117807..f7523d4 100644 --- a/src/restore.cpp +++ b/src/restore.cpp @@ -13,6 +13,13 @@ auto reader = FreadStream("test.bin"); WAMRInstance *wamr = nullptr; +void insert_sock_open_data(uint32_t poolfd, int af, int socktype, uint32_t *sockfd) {} + +void insert_sock_send_to_data(uint32_t sock, const iovec_app1_t *si_data, uint32 si_data_len, uint16_t si_flags, + const __wasi_addr_t *dest_addr, uint32 *so_data_len) {} + +void insert_sock_recv_from_data(uint32_t sock, iovec_app1_t *ri_data, uint32 ri_data_len, uint16_t ri_flags, + __wasi_addr_t *src_addr, uint32 *ro_data_len) {} void insert_fd(int fd, const char *path, int flags, int offset){}; void remove_fd(int fd) {} void serialize_to_file(WASMExecEnv *instance) {} diff --git a/src/wamr.cpp b/src/wamr.cpp index 3f24c94..7cdc7e5 100644 --- a/src/wamr.cpp +++ b/src/wamr.cpp @@ -3,6 +3,7 @@ // #include "wamr.h" +#include "platform_common.h" #include "thread_manager.h" #include "wasm_export.h" #include "wasm_interp.h" @@ -143,6 +144,174 @@ int WAMRInstance::invoke_frenumber(uint32 fd, uint32 to) { uint32 argv[2] = {fd, to}; return wasm_runtime_call_wasm(exec_env, func, 2, argv); }; +int WAMRInstance::invoke_sock_open(uint32_t poolfd, int af, int socktype, uint32_t* sockfd) { + auto name = "__wasi_sock_open"; + if (!(func = wasm_runtime_lookup_function(module_inst, name, nullptr))) { + LOGV(ERROR) << "The wasi fopen function is not found."; + auto target_module = get_module_instance()->e; + for (int i = 0; i < target_module->function_count; i++) { + auto cur_func = &target_module->functions[i]; + if (cur_func->is_import_func) { + LOGV(DEBUG) << cur_func->u.func_import->field_name; + if (!strcmp(cur_func->u.func_import->field_name, name)) { + + func = ((WASMFunctionInstanceCommon *)cur_func); + break; + } + + } else { + LOGV(DEBUG) << cur_func->u.func->field_name; + + if (!strcmp(cur_func->u.func->field_name, name)) { + func = ((WASMFunctionInstanceCommon *)cur_func); + break; + } + } + } + } + void *buffer_ = nullptr; + uint32_t buffer_for_wasm; + + buffer_for_wasm = wasm_runtime_module_malloc(module_inst, sizeof(uint32), reinterpret_cast(&buffer_)); + if (buffer_for_wasm != 0) { + memcpy(buffer_, ((void*)sockfd), sizeof(uint32)); // use native address for accessing in runtime + uint32 argv[4] = {poolfd, static_cast(af), static_cast(socktype),buffer_for_wasm}; + auto res = wasm_runtime_call_wasm(exec_env, func, 4, argv); + wasm_runtime_module_free(module_inst, buffer_for_wasm); + return res; + } + return -1; + +} + +int WAMRInstance::invoke_sock_sendto(uint32_t sock, const iovec_app1_t* si_data, uint32 si_data_len, uint16_t si_flags, const __wasi_addr_t* dest_addr, uint32* so_data_len) { + auto name = "__wasi_sock_send_to"; + if (!(func = wasm_runtime_lookup_function(module_inst, name, nullptr))) { + LOGV(ERROR) << "The wasi fopen function is not found."; + auto target_module = get_module_instance()->e; + for (int i = 0; i < target_module->function_count; i++) { + auto cur_func = &target_module->functions[i]; + if (cur_func->is_import_func) { + LOGV(DEBUG) << cur_func->u.func_import->field_name; + if (!strcmp(cur_func->u.func_import->field_name, name)) { + + func = ((WASMFunctionInstanceCommon *)cur_func); + break; + } + + } else { + LOGV(DEBUG) << cur_func->u.func->field_name; + + if (!strcmp(cur_func->u.func->field_name, name)) { + func = ((WASMFunctionInstanceCommon *)cur_func); + break; + } + } + } + } + + void *si_data_ = nullptr; + uint32_t si_data_for_wasm; + void *dest_addr_ = nullptr; + uint32_t dest_addr_for_wasm; + void *so_data_len_ = nullptr; + uint32_t so_data_len_for_wasm; + int res = -1; + + si_data_for_wasm = + wasm_runtime_module_malloc(module_inst, sizeof(iovec_app1_t), reinterpret_cast(&si_data_)); + if (si_data_for_wasm != 0) { + memcpy(si_data_, ((void *)si_data), sizeof(iovec_app1_t)); // use native address for accessing in runtime + + dest_addr_for_wasm = + wasm_runtime_module_malloc(module_inst, sizeof(__wasi_addr_t), reinterpret_cast(&dest_addr_)); + if (dest_addr_for_wasm != 0) { + memcpy(dest_addr_, ((void *)dest_addr), sizeof(__wasi_addr_t)); // use native address for accessing in runtime + + so_data_len_for_wasm = + wasm_runtime_module_malloc(module_inst, sizeof(uint32), reinterpret_cast(&so_data_len_)); + if (so_data_len_for_wasm != 0) { + memcpy(so_data_len_, ((void *)so_data_len), + sizeof(uint32)); // use native address for accessing in runtime + uint32 argv[6] = {sock, si_data_for_wasm, si_data_len, + si_flags, dest_addr_for_wasm, so_data_len_for_wasm}; + + wasm_runtime_call_wasm(exec_env, func, 6, argv); + res = argv[0]; + wasm_runtime_module_free(module_inst, si_data_for_wasm); + } + wasm_runtime_module_free(module_inst, dest_addr_for_wasm); + } + wasm_runtime_module_free(module_inst, so_data_len_for_wasm); + return res; + } + return -1; +} + +int WAMRInstance::invoke_sock_recvfrom(uint32_t sock, iovec_app1_t* ri_data, uint32 ri_data_len, uint16_t ri_flags, __wasi_addr_t* src_addr, uint32* ro_data_len) { + auto name = "__wasi_sock_recv_from"; + if (!(func = wasm_runtime_lookup_function(module_inst, name, nullptr))) { + LOGV(ERROR) << "The wasi fopen function is not found."; + auto target_module = get_module_instance()->e; + for (int i = 0; i < target_module->function_count; i++) { + auto cur_func = &target_module->functions[i]; + if (cur_func->is_import_func) { + LOGV(DEBUG) << cur_func->u.func_import->field_name; + if (!strcmp(cur_func->u.func_import->field_name, name)) { + + func = ((WASMFunctionInstanceCommon *)cur_func); + break; + } + + } else { + LOGV(DEBUG) << cur_func->u.func->field_name; + + if (!strcmp(cur_func->u.func->field_name, name)) { + func = ((WASMFunctionInstanceCommon *)cur_func); + break; + } + } + } + } + + void *ri_data_ = nullptr; + uint32_t ri_data_for_wasm; + void *src_addr_ = nullptr; + uint32_t src_addr_for_wasm; + void *ro_data_len_ = nullptr; + uint32_t ro_data_len_for_wasm; + int res = -1; + + ri_data_for_wasm = + wasm_runtime_module_malloc(module_inst, sizeof(iovec_app1_t), reinterpret_cast(&ri_data_)); + if (ri_data_for_wasm != 0) { + memcpy(ri_data_, ((void *)ri_data), sizeof(iovec_app1_t)); // use native address for accessing in runtime + + src_addr_for_wasm = + wasm_runtime_module_malloc(module_inst, sizeof(__wasi_addr_t), reinterpret_cast(&src_addr_)); + if (src_addr_for_wasm != 0) { + memcpy(src_addr_, ((void *)src_addr), sizeof(__wasi_addr_t)); // use native address for accessing in runtime + + ro_data_len_for_wasm = + wasm_runtime_module_malloc(module_inst, sizeof(uint32), reinterpret_cast(&ro_data_len_)); + if (ro_data_len_for_wasm != 0) { + memcpy(ro_data_len_, ((void *)ro_data_len), + sizeof(uint32)); // use native address for accessing in runtime + uint32 argv[6] = {sock, ri_data_for_wasm, ri_data_len, + ri_flags, src_addr_for_wasm, ro_data_len_for_wasm}; + + wasm_runtime_call_wasm(exec_env, func, 6, argv); + res = argv[0]; + wasm_runtime_module_free(module_inst, ri_data_for_wasm); + } + wasm_runtime_module_free(module_inst, src_addr_for_wasm); + } + wasm_runtime_module_free(module_inst, ro_data_len_for_wasm); + return res; + } + return -1; +} + int WAMRInstance::invoke_fseek(uint32 fd, uint32 offset) { auto name = "__wasi_fd_seek"; if (!(func = wasm_runtime_lookup_function(module_inst, name, nullptr))) { @@ -205,7 +374,8 @@ int WAMRInstance::invoke_preopen(uint32 fd, const std::string &path) { strncpy(buffer_, path.c_str(), path.size()); // use native address for accessing in runtime argv[0] = fd; // pass the buffer_ address for WASM space argv[1] = buffer_for_wasm; // the size of buffer_ - auto res = wasm_runtime_call_wasm(exec_env, func, 2, argv); + wasm_runtime_call_wasm(exec_env, func, 2, argv); + int res = argv[0]; wasm_runtime_module_free(module_inst, buffer_for_wasm); return res; } diff --git a/src/wamr_wasi_context.cpp b/src/wamr_wasi_context.cpp index e296842..14c8808 100644 --- a/src/wamr_wasi_context.cpp +++ b/src/wamr_wasi_context.cpp @@ -35,5 +35,13 @@ void WAMRWASIContext::restore_impl(WASIContext *env) { if (fd != 0 && fd != 1 && fd != 2) wamr->invoke_frenumber(r, fd); wamr->invoke_fseek(fd, std::get<2>(res)); + wamr->invoke_sock_open(wamr->sock_open_data.poolfd, wamr->sock_open_data.af, wamr->sock_open_data.socktype, + wamr->sock_open_data.sockfd); + wamr->invoke_sock_sendto(wamr->sock_sendto_data.sock, wamr->sock_sendto_data.si_data, + wamr->sock_sendto_data.si_data_len, wamr->sock_sendto_data.si_flags, + wamr->sock_sendto_data.dest_addr, wamr->sock_sendto_data.so_data_len); + wamr->invoke_sock_recvfrom(wamr->sock_recvfrom_data.sock, wamr->sock_recvfrom_data.ri_data, + wamr->sock_recvfrom_data.ri_data_len, wamr->sock_recvfrom_data.ri_flags, + wamr->sock_recvfrom_data.src_addr, wamr->sock_recvfrom_data.ro_data_len); } }; \ No newline at end of file From 9df086ac723613e776d7f1b00f1c66e88470ccc4 Mon Sep 17 00:00:00 2001 From: kikispace Date: Thu, 19 Oct 2023 14:01:07 -0700 Subject: [PATCH 08/15] 1. add read-file to output.txt 2. move the record and replay to wamr_export 3. reader writer seperately --- lib/wasm-micro-runtime | 2 +- src/checkpoint.cpp | 98 +++++++----------------------------------- src/restore.cpp | 32 +++++++------- src/wamr_export.cpp | 85 ++++++++++++++++++++++++++++++++++++ test/read-file.c | 7 ++- 5 files changed, 123 insertions(+), 101 deletions(-) create mode 100644 src/wamr_export.cpp diff --git a/lib/wasm-micro-runtime b/lib/wasm-micro-runtime index 6128c89..972bf93 160000 --- a/lib/wasm-micro-runtime +++ b/lib/wasm-micro-runtime @@ -1 +1 @@ -Subproject commit 6128c894fb5d324b1fd2d67eb6d2c9afb37c18a2 +Subproject commit 972bf938f6bbd23350c6f301257b4733c05dd1ac diff --git a/src/checkpoint.cpp b/src/checkpoint.cpp index 9adc186..ecda655 100644 --- a/src/checkpoint.cpp +++ b/src/checkpoint.cpp @@ -15,91 +15,14 @@ WAMRInstance *wamr = nullptr; std::ostringstream re{}; -auto writer = FwriteStream("test.bin"); +FwriteStream *writer; std::vector> as; std::mutex as_mtx; -void insert_sock_open_data(uint32_t poolfd, int af, int socktype, uint32_t* sockfd) { - if (wamr->sock_open_data.poolfd == 0) { - wamr->sock_open_data.poolfd = poolfd; - wamr->sock_open_data.af = af; - wamr->sock_open_data.socktype = socktype; - wamr->sock_open_data.sockfd = sockfd; - } else { - LOGV(ERROR) << "sock_open_data already exist"; - } -} - -void insert_sock_send_to_data(uint32_t sock, const iovec_app1_t* si_data, uint32 si_data_len, uint16_t si_flags, const __wasi_addr_t* dest_addr, uint32* so_data_len) { - if(wamr->sock_sendto_data.sock == 0) { - wamr->sock_sendto_data.sock = sock; - wamr->sock_sendto_data.si_data = si_data; - wamr->sock_sendto_data.si_data_len = si_data_len; - wamr->sock_sendto_data.si_flags = si_flags; - wamr->sock_sendto_data.dest_addr = dest_addr; - wamr->sock_sendto_data.so_data_len = so_data_len; - } else { - LOGV(ERROR) << "sock_sendto_data already exist"; - } -} - -void insert_sock_recv_from_data(uint32_t sock, iovec_app1_t* ri_data, uint32 ri_data_len, uint16_t ri_flags, __wasi_addr_t* src_addr, uint32* ro_data_len) { - if(wamr->sock_recvfrom_data.sock == 0) { - wamr->sock_recvfrom_data.sock = sock; - wamr->sock_recvfrom_data.ri_data = ri_data; - wamr->sock_recvfrom_data.ri_data_len = ri_data_len; - wamr->sock_recvfrom_data.ri_flags = ri_flags; - wamr->sock_recvfrom_data.src_addr = src_addr; - wamr->sock_recvfrom_data.ro_data_len = ro_data_len; - } else { - LOGV(ERROR) << "sock_recvfrom_data already exist"; - } -} - - -/**fopen, fseek*/ -void insert_fd(int fd, const char *path, int flags, int offset) { - if (fd > 2) { - printf("\n #insert_fd(fd,filename,flags, offset) %d %s %d %d \n\n", fd, path, flags, offset); - - if (wamr->fd_map_.find(fd) != wamr->fd_map_.end()) { - LOGV(ERROR) << "fd already exist" << fd; - if (offset == 0) { - // fOpen call - std::string curPath; - int curFlags; - int curOffset; - std::tie(curPath, curFlags, curOffset) = wamr->fd_map_[fd]; - wamr->fd_map_[fd] = std::make_tuple(std::string(path), flags, curOffset); - } else { - // fSeek call - std::string curPath; - int curFlags; - int curOffset; - std::tie(curPath, curFlags, curOffset) = wamr->fd_map_[fd]; - wamr->fd_map_[fd] = std::make_tuple(curPath, curFlags, offset); - } - - } else - wamr->fd_map_.insert(std::make_pair(fd, std::make_tuple(std::string(path), flags, offset))); - } -} - -/* update fd->offset**/ -void insert_fd_fseek(); -/**fclose */ -void remove_fd(int fd) { - if (wamr->fd_map_.find(fd) != wamr->fd_map_.end()) - wamr->fd_map_.erase(fd); - else - LOGV(ERROR) << "fd not found" << fd; -} -void insert_socket(int fd) {} void serialize_to_file(WASMExecEnv *instance) { /** Sounds like AoT/JIT is in this?*/ // Note: insert fd std::ifstream stdoutput; - stdoutput.open("output.txt"); std: string current_str; std::string fd_output; @@ -139,7 +62,7 @@ void serialize_to_file(WASMExecEnv *instance) { as.emplace_back(a); as.back().get()->cur_count = cur_count; if (as.size() == all_count) { - struct_pack::serialize_to(writer, as); + struct_pack::serialize_to(*writer, as); LOGV(INFO) << "serialize to file" << cur_count << " " << all_count << "\n"; } as_mtx.unlock(); @@ -150,7 +73,7 @@ void serialize_to_file(WASMExecEnv *instance) { dump(a, instance); as.emplace_back(a); as.back().get()->cur_count = 0; - struct_pack::serialize_to(writer, as); + struct_pack::serialize_to(*writer, as); } exit(0); @@ -205,7 +128,18 @@ int main(int argc, char *argv[]) { "n,ns_pool", "The ns lookup pool exposed to WAMR", cxxopts::value>()->default_value(""))("h,help", "The value for epoch value", cxxopts::value()->default_value("false")); - + auto removeExtension = [](std::string &filename) { + size_t dotPos = filename.find_last_of('.'); + std::string res; + if (dotPos != std::string::npos) { + // Extract the substring before the period + res = filename.substr(0, dotPos); + } else { + // If there's no period in the string, it means there's no extension. + LOGV(ERROR) << "No extension found."; + } + return res; + }; auto result = options.parse(argc, argv); if (result["help"].as()) { std::cout << options.help() << std::endl; @@ -219,10 +153,10 @@ int main(int argc, char *argv[]) { auto arg = result["arg"].as>(); auto addr = result["addr"].as>(); auto ns_pool = result["ns_pool"].as>(); + writer = new FwriteStream((removeExtension(target) + ".bin").c_str()); wamr = new WAMRInstance(target.c_str(), is_jit); wamr->set_wasi_args(dir, map_dir, env, arg, addr, ns_pool); wamr->instantiate(); - freopen("output.txt", "w", stdout); #ifndef MVVM_DEBUG // Define the sigaction structure diff --git a/src/restore.cpp b/src/restore.cpp index f7523d4..8bcab63 100644 --- a/src/restore.cpp +++ b/src/restore.cpp @@ -11,24 +11,9 @@ #include #include -auto reader = FreadStream("test.bin"); +FreadStream *reader; WAMRInstance *wamr = nullptr; -void insert_sock_open_data(uint32_t poolfd, int af, int socktype, uint32_t *sockfd) {} - -void insert_sock_send_to_data(uint32_t sock, const iovec_app1_t *si_data, uint32 si_data_len, uint16_t si_flags, - const __wasi_addr_t *dest_addr, uint32 *so_data_len) {} - -void insert_sock_recv_from_data(uint32_t sock, iovec_app1_t *ri_data, uint32 ri_data_len, uint16_t ri_flags, - __wasi_addr_t *src_addr, uint32 *ro_data_len) {} -void insert_fd(int fd, const char *path, int flags, int offset){}; -void remove_fd(int fd) {} void serialize_to_file(WASMExecEnv *instance) {} -void insert_socket(char const *, int){}; -void remove_socket(char const *){}; -void insert_lock(char const *, int){}; -void insert_sem(char const *, int){}; -void remove_lock(char const *){}; -void remove_sem(char const *){}; int main(int argc, char **argv) { cxxopts::Options options("MVVM", "Migratable Velocity Virtual Machine, to ship the VM state to another machine"); options.add_options()("t,target", "The webassembly file to execute", @@ -36,6 +21,18 @@ int main(int argc, char **argv) { "j,jit", "Whether the jit mode or interp mode", cxxopts::value()->default_value("false"))( "h,help", "The value for epoch value", cxxopts::value()->default_value("false")); // Can first discover from the wasi context. + auto removeExtension = [](std::string &filename) { + size_t dotPos = filename.find_last_of('.'); + std::string res; + if (dotPos != std::string::npos) { + // Extract the substring before the period + res = filename.substr(0, dotPos); + } else { + // If there's no period in the string, it means there's no extension. + LOGV(ERROR) << "No extension found."; + } + return res; + }; auto result = options.parse(argc, argv); if (result["help"].as()) { @@ -43,8 +40,9 @@ int main(int argc, char **argv) { exit(0); } auto target = result["target"].as(); + reader = new FreadStream((removeExtension(target) + ".bin").c_str()); wamr = new WAMRInstance(target.c_str(), false); - auto a = struct_pack::deserialize>>(reader).value(); + auto a = struct_pack::deserialize>>(*reader).value(); wamr->instantiate(); wamr->recover(&a); return 0; diff --git a/src/wamr_export.cpp b/src/wamr_export.cpp new file mode 100644 index 0000000..54d02d2 --- /dev/null +++ b/src/wamr_export.cpp @@ -0,0 +1,85 @@ +// +// Created by victoryang00 on 10/19/23. +// + +#include "wamr_block_addr.h" +#include "wamr.h" +extern WAMRInstance *wamr; + + +void insert_sock_open_data(uint32_t poolfd, int af, int socktype, uint32_t* sockfd) { + if (wamr->sock_open_data.poolfd == 0) { + wamr->sock_open_data.poolfd = poolfd; + wamr->sock_open_data.af = af; + wamr->sock_open_data.socktype = socktype; + wamr->sock_open_data.sockfd = sockfd; + } else { + LOGV(ERROR) << "sock_open_data already exist"; + } +} + +void insert_sock_send_to_data(uint32_t sock, const iovec_app1_t* si_data, uint32 si_data_len, uint16_t si_flags, const __wasi_addr_t* dest_addr, uint32* so_data_len) { + if(wamr->sock_sendto_data.sock == 0) { + wamr->sock_sendto_data.sock = sock; + wamr->sock_sendto_data.si_data = si_data; + wamr->sock_sendto_data.si_data_len = si_data_len; + wamr->sock_sendto_data.si_flags = si_flags; + wamr->sock_sendto_data.dest_addr = dest_addr; + wamr->sock_sendto_data.so_data_len = so_data_len; + } else { + LOGV(ERROR) << "sock_sendto_data already exist"; + } +} + +void insert_sock_recv_from_data(uint32_t sock, iovec_app1_t* ri_data, uint32 ri_data_len, uint16_t ri_flags, __wasi_addr_t* src_addr, uint32* ro_data_len) { + if(wamr->sock_recvfrom_data.sock == 0) { + wamr->sock_recvfrom_data.sock = sock; + wamr->sock_recvfrom_data.ri_data = ri_data; + wamr->sock_recvfrom_data.ri_data_len = ri_data_len; + wamr->sock_recvfrom_data.ri_flags = ri_flags; + wamr->sock_recvfrom_data.src_addr = src_addr; + wamr->sock_recvfrom_data.ro_data_len = ro_data_len; + } else { + LOGV(ERROR) << "sock_recvfrom_data already exist"; + } +} + + +/**fopen, fseek*/ +void insert_fd(int fd, const char *path, int flags, int offset) { + if (fd > 2) { + printf("\n #insert_fd(fd,filename,flags, offset) %d %s %d %d \n\n", fd, path, flags, offset); + + if (wamr->fd_map_.find(fd) != wamr->fd_map_.end()) { + LOGV(ERROR) << "fd already exist" << fd; + if (offset == 0) { + // fOpen call + std::string curPath; + int curFlags; + int curOffset; + std::tie(curPath, curFlags, curOffset) = wamr->fd_map_[fd]; + wamr->fd_map_[fd] = std::make_tuple(std::string(path), flags, curOffset); + } else { + // fSeek call + std::string curPath; + int curFlags; + int curOffset; + std::tie(curPath, curFlags, curOffset) = wamr->fd_map_[fd]; + wamr->fd_map_[fd] = std::make_tuple(curPath, curFlags, offset); + } + + } else + wamr->fd_map_.insert(std::make_pair(fd, std::make_tuple(std::string(path), flags, offset))); + } +} + +/* update fd->offset**/ +void insert_fd_fseek(); +/**fclose */ +void remove_fd(int fd) { + if (wamr->fd_map_.find(fd) != wamr->fd_map_.end()) + wamr->fd_map_.erase(fd); + else + LOGV(ERROR) << "fd not found" << fd; +} +void insert_socket(int fd) {} diff --git a/test/read-file.c b/test/read-file.c index 7cd2396..d5a6ec9 100644 --- a/test/read-file.c +++ b/test/read-file.c @@ -8,11 +8,16 @@ #include #include +FILE *a_file = NULL; + FILE *fopen_test(const char *restrict filename, const char *restrict mode) { FILE *f; int fd; int flags; + if(!a_file){ + a_file = fopen("./test.txt","a"); + } /* Check for valid initial mode character */ if (!strchr("rwa", *mode)) { @@ -27,7 +32,7 @@ FILE *fopen_test(const char *restrict filename, const char *restrict mode) #else // WASI libc ignores the mode parameter anyway, so skip the varargs. fd = __wasilibc_open_nomode(filename, flags); - printf("\n fopen_test(fd,filename,flags) %d %s %d \n\n",fd,filename,flags); + fprintf(a_file,"\n fopen_test(fd,filename,flags) %d %s %d \n\n",fd,filename,flags); #endif if (fd < 0) return 0; #ifdef __wasilibc_unmodified_upstream // WASI has no syscall From 951dada1cc6a45d10f7acbff0c57b05b56028141 Mon Sep 17 00:00:00 2001 From: kikispace Date: Mon, 23 Oct 2023 17:13:49 -0700 Subject: [PATCH 09/15] update new --- CMakeLists.txt | 1 - include/wamr.h | 39 ++-------- include/wamr_export.h | 29 +++++--- include/wamr_wasi_context.h | 41 +++++++++++ lib/wasm-micro-runtime | 2 +- src/wamr.cpp | 27 +++---- src/wamr_export.cpp | 140 +++++++++++++++++++++++++++--------- src/wamr_wasi_context.cpp | 32 ++++++--- 8 files changed, 213 insertions(+), 98 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0b6b094..c782513 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -96,6 +96,5 @@ target_link_libraries(MVVM_restore fmt::fmt cxxopts::cxxopts vmlib -lm -ldl -lpt target_link_libraries(MVVM_checkpoint fmt::fmt cxxopts::cxxopts vmlib -lm -ldl -lpthread ${LLVM_LLDB_LIB} ${LLVM_AVAILABLE_LIBS}) add_definitions(-DMVVM_INTERP=1) add_definitions(-DMVVM_DEBUG=1) -add_definitions(-DMVVM_WASI=1) add_definitions(-DCXXOPTS_NO_RTTI=1) add_subdirectory(test) \ No newline at end of file diff --git a/include/wamr.h b/include/wamr.h index a03345f..2bc5a56 100644 --- a/include/wamr.h +++ b/include/wamr.h @@ -12,32 +12,6 @@ #include "wamr_read_write.h" #include "wasm_runtime.h" - -struct WasiSockOpenData { - uint32_t poolfd; - int af; - int socktype; - uint32_t* sockfd; -}; - -struct WasiSockSendToData { - uint32_t sock; - const iovec_app1_t* si_data; - uint32 si_data_len; - uint16_t si_flags; - const __wasi_addr_t* dest_addr; - uint32* so_data_len; -}; - -struct WasiSockRecvFromData { - uint32_t sock; - iovec_app1_t* ri_data; - uint32 ri_data_len; - uint16_t ri_flags; - __wasi_addr_t* src_addr; - uint32* ro_data_len; -}; - class WAMRInstance { WASMExecEnv *exec_env{}; WASMExecEnv *cur_env{}; @@ -54,10 +28,7 @@ class WAMRInstance { std::vector ns_pool_; std::map> fd_map_; // add offset to pair->tuple, 3rd param 'int' - - WasiSockOpenData sock_open_data; - WasiSockSendToData sock_sendto_data; - WasiSockRecvFromData sock_recvfrom_data; + std::map socket_fd_map_; bool is_jit; char *buffer{}; @@ -94,9 +65,11 @@ class WAMRInstance { int invoke_preopen(uint32 fd, const std::string &path); int invoke_socket(uint32 domain, const std::string &path); int invoke_(uint32 domain, const std::string &path); - int invoke_sock_sendto(uint32_t sock, const iovec_app1_t* si_data, uint32 si_data_len, uint16_t si_flags, const __wasi_addr_t* dest_addr, uint32* so_data_len); - int invoke_sock_recvfrom(uint32_t sock, iovec_app1_t* ri_data, uint32 ri_data_len, uint16_t ri_flags, __wasi_addr_t* src_addr, uint32* ro_data_len); - int invoke_sock_open(uint32_t poolfd, int af, int socktype, uint32_t* sockfd); + int invoke_sock_sendto(uint32_t sock, const iovec_app_t *si_data, uint32 si_data_len, uint16_t si_flags, + const __wasi_addr_t *dest_addr, uint32 *so_data_len); + int invoke_sock_recvfrom(uint32_t sock, iovec_app_t *ri_data, uint32 ri_data_len, uint16_t ri_flags, + __wasi_addr_t *src_addr, uint32 *ro_data_len); + int invoke_sock_open(uint32_t poolfd, int af, int socktype, uint32_t *sockfd); ~WAMRInstance(); }; #endif // MVVM_WAMR_H diff --git a/include/wamr_export.h b/include/wamr_export.h index 15078bc..f22cb04 100644 --- a/include/wamr_export.h +++ b/include/wamr_export.h @@ -1,25 +1,36 @@ // // Created by victoryang00 on 6/17/23. // +#include "wasm_runtime.h" #ifdef __cplusplus extern "C" { #endif -#ifdef MVVM_WASI -#include "wasm_runtime.h" -typedef struct iovec_app1 { + +#if !defined(MVVM_WASI) +#define MVVM_WASI +struct SocketAddrPool { + uint16 ip4[4]; + uint16 ip6[8]; + bool is_4; + uint16 port; +}; +typedef struct iovec_app { uint32 buf_offset; uint32 buf_len; -} iovec_app1_t; - +} iovec_app_t; void serialize_to_file(struct WASMExecEnv *); + #endif -void insert_sock_send_to_data(uint32_t sock, const iovec_app1_t* si_data, uint32 si_data_len, uint16_t si_flags, const __wasi_addr_t* dest_addr, uint32* so_data_len); -void insert_sock_open_data(uint32_t, int, int, uint32_t *); -void insert_sock_recv_from_data(uint32_t sock, iovec_app1_t* ri_data, uint32 ri_data_len, uint16_t ri_flags, __wasi_addr_t* src_addr, uint32* ro_data_len); +void insert_sock_send_to_data(uint32_t sock, const iovec_app_t *si_data, uint32 si_data_len, uint16_t si_flags, + const __wasi_addr_t *dest_addr, uint32 *so_data_len); +void insert_sock_open_data(uint32_t, int, int, uint32_t); +void insert_sock_recv_from_data(uint32_t sock, iovec_app_t *ri_data, uint32 ri_data_len, uint16_t ri_flags, + __wasi_addr_t *src_addr, uint32 *ro_data_len); void insert_fd(int, char const *, int, int); void remove_fd(int); -void insert_socket(char const *, int); +void insert_socket(int, int, int, int); +void update_socket_fd_address(int, struct SocketAddrPool *); void remove_socket(char const *); // see whether there's socket maintainance impl in wasi? void insert_lock(char const *, int); void insert_sem(char const *, int); diff --git a/include/wamr_wasi_context.h b/include/wamr_wasi_context.h index be9bd44..e827076 100644 --- a/include/wamr_wasi_context.h +++ b/include/wamr_wasi_context.h @@ -6,8 +6,10 @@ #define MVVM_WAMR_WASI_CONTEXT_H #include "logging.h" +#include "wamr_export.h" #include "wamr_serializer.h" #include "wasm_runtime.h" +#include "wasmtime_ssp.h" #include #include #include @@ -28,8 +30,47 @@ struct WAMRAddrPool { bool is_4; uint8 mask; }; +struct WAMRWasiAddr { + WAMRAddrPool ip; + uint16 port; +}; +struct WasiSockOpenData { + uint32 poolfd; + int af; + int socktype; + uint32 sockfd; +}; + +struct WasiSockSendToData { + uint32 sock; + iovec_app_t si_data; + uint32 si_data_len; + uint16_t si_flags; + WAMRWasiAddr dest_addr; + uint32 so_data_len; +}; + +struct WasiSockRecvFromData { + uint32_t sock; + iovec_app_t ri_data; + uint32 ri_data_len; + uint16_t ri_flags; + WAMRWasiAddr src_addr; + uint32 ro_data_len; +}; + +struct SocketMetaData { + int domain{}; + int type{}; + int protocol{}; + SocketAddrPool socketAddress{}; + WasiSockOpenData socketOpenData{}; + WasiSockSendToData socketSentToData{}; + WasiSockRecvFromData socketRecvFromData{}; +}; struct WAMRWASIContext { std::map> fd_map; + std::map socket_fd_map; std::vector dir; std::vector map_dir; WAMRArgvEnvironValues argv_environ; diff --git a/lib/wasm-micro-runtime b/lib/wasm-micro-runtime index 972bf93..4f1c282 160000 --- a/lib/wasm-micro-runtime +++ b/lib/wasm-micro-runtime @@ -1 +1 @@ -Subproject commit 972bf938f6bbd23350c6f301257b4733c05dd1ac +Subproject commit 4f1c282b18244ff7679cc0ac4f1fb67909c56c08 diff --git a/src/wamr.cpp b/src/wamr.cpp index 436e696..86fb0d1 100644 --- a/src/wamr.cpp +++ b/src/wamr.cpp @@ -144,7 +144,8 @@ int WAMRInstance::invoke_frenumber(uint32 fd, uint32 to) { uint32 argv[2] = {fd, to}; return wasm_runtime_call_wasm(exec_env, func, 2, argv); }; -int WAMRInstance::invoke_sock_open(uint32_t poolfd, int af, int socktype, uint32_t* sockfd) { + +int WAMRInstance::invoke_sock_open(uint32_t poolfd, int af, int socktype, uint32_t *sockfd) { auto name = "__wasi_sock_open"; if (!(func = wasm_runtime_lookup_function(module_inst, name, nullptr))) { LOGV(ERROR) << "The wasi fopen function is not found."; @@ -174,17 +175,17 @@ int WAMRInstance::invoke_sock_open(uint32_t poolfd, int af, int socktype, uint32 buffer_for_wasm = wasm_runtime_module_malloc(module_inst, sizeof(uint32), reinterpret_cast(&buffer_)); if (buffer_for_wasm != 0) { - memcpy(buffer_, ((void*)sockfd), sizeof(uint32)); // use native address for accessing in runtime - uint32 argv[4] = {poolfd, static_cast(af), static_cast(socktype),buffer_for_wasm}; + memcpy(buffer_, ((void *)sockfd), sizeof(uint32)); // use native address for accessing in runtime + uint32 argv[4] = {poolfd, static_cast(af), static_cast(socktype), buffer_for_wasm}; auto res = wasm_runtime_call_wasm(exec_env, func, 4, argv); wasm_runtime_module_free(module_inst, buffer_for_wasm); return res; } return -1; - } -int WAMRInstance::invoke_sock_sendto(uint32_t sock, const iovec_app1_t* si_data, uint32 si_data_len, uint16_t si_flags, const __wasi_addr_t* dest_addr, uint32* so_data_len) { +int WAMRInstance::invoke_sock_sendto(uint32_t sock, const iovec_app_t *si_data, uint32 si_data_len, uint16_t si_flags, + const __wasi_addr_t *dest_addr, uint32 *so_data_len) { auto name = "__wasi_sock_send_to"; if (!(func = wasm_runtime_lookup_function(module_inst, name, nullptr))) { LOGV(ERROR) << "The wasi fopen function is not found."; @@ -219,21 +220,22 @@ int WAMRInstance::invoke_sock_sendto(uint32_t sock, const iovec_app1_t* si_data, int res = -1; si_data_for_wasm = - wasm_runtime_module_malloc(module_inst, sizeof(iovec_app1_t), reinterpret_cast(&si_data_)); + wasm_runtime_module_malloc(module_inst, sizeof(iovec_app_t), reinterpret_cast(&si_data_)); if (si_data_for_wasm != 0) { - memcpy(si_data_, ((void *)si_data), sizeof(iovec_app1_t)); // use native address for accessing in runtime + memcpy(si_data_, ((void *)si_data), sizeof(iovec_app_t)); // use native address for accessing in runtime dest_addr_for_wasm = wasm_runtime_module_malloc(module_inst, sizeof(__wasi_addr_t), reinterpret_cast(&dest_addr_)); if (dest_addr_for_wasm != 0) { - memcpy(dest_addr_, ((void *)dest_addr), sizeof(__wasi_addr_t)); // use native address for accessing in runtime + memcpy(dest_addr_, ((void *)dest_addr), + sizeof(__wasi_addr_t)); // use native address for accessing in runtime so_data_len_for_wasm = wasm_runtime_module_malloc(module_inst, sizeof(uint32), reinterpret_cast(&so_data_len_)); if (so_data_len_for_wasm != 0) { memcpy(so_data_len_, ((void *)so_data_len), sizeof(uint32)); // use native address for accessing in runtime - uint32 argv[6] = {sock, si_data_for_wasm, si_data_len, + uint32 argv[6] = {sock, si_data_for_wasm, si_data_len, si_flags, dest_addr_for_wasm, so_data_len_for_wasm}; wasm_runtime_call_wasm(exec_env, func, 6, argv); @@ -248,7 +250,8 @@ int WAMRInstance::invoke_sock_sendto(uint32_t sock, const iovec_app1_t* si_data, return -1; } -int WAMRInstance::invoke_sock_recvfrom(uint32_t sock, iovec_app1_t* ri_data, uint32 ri_data_len, uint16_t ri_flags, __wasi_addr_t* src_addr, uint32* ro_data_len) { +int WAMRInstance::invoke_sock_recvfrom(uint32_t sock, iovec_app_t *ri_data, uint32 ri_data_len, uint16_t ri_flags, + __wasi_addr_t *src_addr, uint32 *ro_data_len) { auto name = "__wasi_sock_recv_from"; if (!(func = wasm_runtime_lookup_function(module_inst, name, nullptr))) { LOGV(ERROR) << "The wasi fopen function is not found."; @@ -283,9 +286,9 @@ int WAMRInstance::invoke_sock_recvfrom(uint32_t sock, iovec_app1_t* ri_data, uin int res = -1; ri_data_for_wasm = - wasm_runtime_module_malloc(module_inst, sizeof(iovec_app1_t), reinterpret_cast(&ri_data_)); + wasm_runtime_module_malloc(module_inst, sizeof(iovec_app_t), reinterpret_cast(&ri_data_)); if (ri_data_for_wasm != 0) { - memcpy(ri_data_, ((void *)ri_data), sizeof(iovec_app1_t)); // use native address for accessing in runtime + memcpy(ri_data_, ((void *)ri_data), sizeof(iovec_app_t)); // use native address for accessing in runtime src_addr_for_wasm = wasm_runtime_module_malloc(module_inst, sizeof(__wasi_addr_t), reinterpret_cast(&src_addr_)); diff --git a/src/wamr_export.cpp b/src/wamr_export.cpp index 54d02d2..1093313 100644 --- a/src/wamr_export.cpp +++ b/src/wamr_export.cpp @@ -2,48 +2,69 @@ // Created by victoryang00 on 10/19/23. // -#include "wamr_block_addr.h" #include "wamr.h" +#include "wamr_block_addr.h" +#include "wamr_wasi_context.h" extern WAMRInstance *wamr; +void insert_sock_open_data(uint32_t poolfd, int af, int socktype, uint32_t sockfd) { + SocketMetaData newSocketData{}; + newSocketData = wamr->socket_fd_map_[sockfd]; -void insert_sock_open_data(uint32_t poolfd, int af, int socktype, uint32_t* sockfd) { - if (wamr->sock_open_data.poolfd == 0) { - wamr->sock_open_data.poolfd = poolfd; - wamr->sock_open_data.af = af; - wamr->sock_open_data.socktype = socktype; - wamr->sock_open_data.sockfd = sockfd; - } else { - LOGV(ERROR) << "sock_open_data already exist"; - } -} + WasiSockOpenData openData; + openData.poolfd = poolfd; + openData.af = af; + openData.socktype = socktype; + openData.sockfd = sockfd; -void insert_sock_send_to_data(uint32_t sock, const iovec_app1_t* si_data, uint32 si_data_len, uint16_t si_flags, const __wasi_addr_t* dest_addr, uint32* so_data_len) { - if(wamr->sock_sendto_data.sock == 0) { - wamr->sock_sendto_data.sock = sock; - wamr->sock_sendto_data.si_data = si_data; - wamr->sock_sendto_data.si_data_len = si_data_len; - wamr->sock_sendto_data.si_flags = si_flags; - wamr->sock_sendto_data.dest_addr = dest_addr; - wamr->sock_sendto_data.so_data_len = so_data_len; - } else { - LOGV(ERROR) << "sock_sendto_data already exist"; - } + newSocketData.socketOpenData = openData; + wamr->socket_fd_map_.insert(std::make_pair(sockfd, newSocketData)); } -void insert_sock_recv_from_data(uint32_t sock, iovec_app1_t* ri_data, uint32 ri_data_len, uint16_t ri_flags, __wasi_addr_t* src_addr, uint32* ro_data_len) { - if(wamr->sock_recvfrom_data.sock == 0) { - wamr->sock_recvfrom_data.sock = sock; - wamr->sock_recvfrom_data.ri_data = ri_data; - wamr->sock_recvfrom_data.ri_data_len = ri_data_len; - wamr->sock_recvfrom_data.ri_flags = ri_flags; - wamr->sock_recvfrom_data.src_addr = src_addr; - wamr->sock_recvfrom_data.ro_data_len = ro_data_len; - } else { - LOGV(ERROR) << "sock_recvfrom_data already exist"; - } +void insert_sock_send_to_data(uint32_t sock, const iovec_app_t *si_data, uint32 si_data_len, uint16_t si_flags, + const __wasi_addr_t *dest_addr, uint32 *so_data_len) { + SocketMetaData newSocketData{}; + newSocketData = wamr->socket_fd_map_[sock]; + + // WasiSockSendToData sentToData{}; + // sentToData.sock = sock; + // std::memcpy(&sentToData.si_data, si_data, sizeof(iovec_app_t)); + // sentToData.si_data_len = si_data_len; + // sentToData.si_flags = si_flags; + // // sentToData.dest_addr = dest_addr; //翻译 + // if (dest_addr) { + // if (dest_addr->kind == 0) { // ip4 + // sentToData.dest_addr.ip = WAMRAddrPool { + // .ip4 = {dest_addr->addr[0], dest_addr->addr[1], dest_addr->addr[2], + // dest_addr->addr[3]}, + // .is_4 = true, + // .mask = 0, + // }; + // } else { // ip6 + // } + // } + // sentToData.so_data_len = *so_data_len; + + // newSocketData.socketSentToData = sentToData; + // wamr->socket_fd_map_.insert(std::make_pair(sock, newSocketData)); } +void insert_sock_recv_from_data(uint32_t sock, iovec_app_t *ri_data, uint32 ri_data_len, uint16_t ri_flags, + __wasi_addr_t *src_addr, uint32 *ro_data_len) { + // SocketMetaData newSocketData{}; + // newSocketData = wamr->socket_fd_map_[sock]; + + // WasiSockRecvFromData recvFromData{}; + // recvFromData.sock = sock; + // recvFromData.ri_data = ri_data; + // recvFromData.ri_data_len = ri_data_len; + // recvFromData.ri_flags = ri_flags; + // recvFromData.src_addr = src_addr; + // recvFromData.ro_data_len = ro_data_len; + + // newSocketData.socketSentToData = recvFromData; + // wamr->socket_fd_map_.insert(std::make_pair(sock, newSocketData)); +} /**fopen, fseek*/ void insert_fd(int fd, const char *path, int flags, int offset) { @@ -82,4 +103,55 @@ void remove_fd(int fd) { else LOGV(ERROR) << "fd not found" << fd; } -void insert_socket(int fd) {} + +/* + create fd-socketmetadata map and store the "domain", "type", "protocol" value +*/ +void insert_socket(int fd, int domain, int type, int protocol) { + printf("\n #insert_socket(fd, domain, type, protocol) %d %d %d %d \n\n", fd, domain, type, protocol); + + if (wamr->socket_fd_map_.find(fd) != wamr->socket_fd_map_.end()) { + LOGV(ERROR) << "socket_fd already exist" << fd; + } else { + SocketMetaData metaData{}; + metaData.domain = domain; + metaData.type = type; + metaData.protocol = protocol; + wamr->socket_fd_map_.insert(std::make_pair(fd, metaData)); + } +} + +void update_socket_fd_address(int fd, SocketAddrPool *address) { + printf("\n #update_socket_fd_address(fd, address) %d \n\n", fd); + + if (wamr->socket_fd_map_.find(fd) == wamr->socket_fd_map_.end()) { + // note: ? fd here is not same as insert_socket? + // set default value + insert_socket(fd, 0, 0, 0); + } + + SocketMetaData metaData{}; + metaData.domain = wamr->socket_fd_map_[fd].domain; + metaData.type = wamr->socket_fd_map_[fd].type; + metaData.protocol = wamr->socket_fd_map_[fd].protocol; + + metaData.socketAddress.port = address->port; + if (address->is_4) { + metaData.socketAddress.is_4 = true; + metaData.socketAddress.ip4[0] = address->ip4[0]; + metaData.socketAddress.ip4[1] = address->ip4[1]; + metaData.socketAddress.ip4[2] = address->ip4[2]; + metaData.socketAddress.ip4[3] = address->ip4[3]; + + } else { + metaData.socketAddress.ip6[0] = address->ip6[0]; + metaData.socketAddress.ip6[1] = address->ip6[1]; + metaData.socketAddress.ip6[2] = address->ip6[2]; + metaData.socketAddress.ip6[3] = address->ip6[3]; + metaData.socketAddress.ip6[4] = address->ip6[4]; + metaData.socketAddress.ip6[5] = address->ip6[5]; + metaData.socketAddress.ip6[6] = address->ip6[6]; + metaData.socketAddress.ip6[7] = address->ip6[7]; + } + wamr->socket_fd_map_[fd] = metaData; +} diff --git a/src/wamr_wasi_context.cpp b/src/wamr_wasi_context.cpp index b5cffda..1470da6 100644 --- a/src/wamr_wasi_context.cpp +++ b/src/wamr_wasi_context.cpp @@ -18,6 +18,11 @@ void WAMRWASIContext::dump_impl(WASIContext *env) { auto dumped_res = std::make_tuple(path, flags, offset); this->fd_map[fd] = dumped_res; } + + for (auto [fd, socketMetaData] : wamr->socket_fd_map_) { + SocketMetaData socketMetaDataCopy = socketMetaData; + this->socket_fd_map[fd] = socketMetaDataCopy; + } } void WAMRWASIContext::restore_impl(WASIContext *env) { int r; @@ -35,13 +40,24 @@ void WAMRWASIContext::restore_impl(WASIContext *env) { if (r != fd) wamr->invoke_frenumber(r, fd); wamr->invoke_fseek(fd, std::get<2>(res)); - wamr->invoke_sock_open(wamr->sock_open_data.poolfd, wamr->sock_open_data.af, wamr->sock_open_data.socktype, - wamr->sock_open_data.sockfd); - wamr->invoke_sock_sendto(wamr->sock_sendto_data.sock, wamr->sock_sendto_data.si_data, - wamr->sock_sendto_data.si_data_len, wamr->sock_sendto_data.si_flags, - wamr->sock_sendto_data.dest_addr, wamr->sock_sendto_data.so_data_len); - wamr->invoke_sock_recvfrom(wamr->sock_recvfrom_data.sock, wamr->sock_recvfrom_data.ri_data, - wamr->sock_recvfrom_data.ri_data_len, wamr->sock_recvfrom_data.ri_flags, - wamr->sock_recvfrom_data.src_addr, wamr->sock_recvfrom_data.ro_data_len); + } + + for (auto [fd, socketMetaData] : this->socket_fd_map) { + LOGV(INFO) << "fd: " << fd << " SocketMetaData[domain]: " << socketMetaData.domain + // << " SocketMetaData[socketAddress]: " << socketMetaData.socketAddress + << " SocketMetaData[protocol]: " << socketMetaData.protocol + << " SocketMetaData[type]: " << socketMetaData.type; + uint32 tmp_sock_fd; + wamr->invoke_sock_open(socketMetaData.socketOpenData.poolfd, socketMetaData.socketOpenData.af, + socketMetaData.socketOpenData.socktype, &tmp_sock_fd); + // if socket不一样 renumber + // wamr->invoke_sock_sendto(socketMetaData.socketSentToData.sock, &socketMetaData.socketSentToData.si_data, + // socketMetaData.socketSentToData.si_data_len, socketMetaData.socketSentToData.si_flags, + // socketMetaData.socketSentToData.dest_addr,//翻译 + // socketMetaData.socketSentToData.so_data_len); + // wamr->invoke_sock_recvfrom( + // socketMetaData.socketRecvFromData.sock, socketMetaData.socketRecvFromData.ri_data, + // socketMetaData.socketRecvFromData.ri_data_len, socketMetaData.socketRecvFromData.ri_flags, + // &socketMetaData.socketRecvFromData.src_addr, socketMetaData.socketRecvFromData.ro_data_len); } }; \ No newline at end of file From 535a7d391a1ca02335c6b468b44484cf58a636db Mon Sep 17 00:00:00 2001 From: Yiwei Yang <40686366+victoryang00@users.noreply.github.com> Date: Wed, 25 Oct 2023 14:27:51 -0700 Subject: [PATCH 10/15] fix compilation --- .github/workflows/build-macos.yml | 2 +- .github/workflows/build-ubuntu.yml | 2 +- .github/workflows/build-windows.yml | 2 +- .gitignore | 3 +- bench/CMakeLists.txt | 54 ++++++++++++++-------------- include/wamr_module_instance.h | 24 ++++++------- include/wamr_module_instance_extra.h | 13 +++---- include/wamr_wasi_context.h | 2 +- include/wamr_wasi_nn_context.h | 40 +++++++++++++++++---- lib/wasm-micro-runtime | 2 +- src/checkpoint.cpp | 52 +++++++++++++-------------- src/wamr.cpp | 1 - src/wamr_wasi_context.cpp | 12 +++---- src/wamr_wasi_nn_context.cpp | 8 +++++ 14 files changed, 126 insertions(+), 91 deletions(-) create mode 100644 src/wamr_wasi_nn_context.cpp diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index 87cdce6..780134a 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v2 - name: Install Dependency - run: brew install cxxopts fmt llvm@14 + run: brew install cxxopts fmt llvm@14 google-benchmark - name: Checkout run: git submodule update --init diff --git a/.github/workflows/build-ubuntu.yml b/.github/workflows/build-ubuntu.yml index 2478f9f..5384bdb 100644 --- a/.github/workflows/build-ubuntu.yml +++ b/.github/workflows/build-ubuntu.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v2 - name: Install Dependency - run: sudo apt install libssl-dev gcc-13 g++-13 libprotobuf-c-dev libprotobuf-dev libgrpc-dev libfmt-dev llvm-14-dev libedit-dev libcxxopts-dev libpfm4-dev liblldb-dev + run: sudo apt install libssl-dev gcc-13 g++-13 libprotobuf-c-dev libprotobuf-dev libgrpc-dev libfmt-dev llvm-14-dev libedit-dev libcxxopts-dev libpfm4-dev liblldb-dev libgoogle-benchmark-dev - name: Checkout run: git submodule update --init diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index f8f6d77..f3a6ff4 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -34,7 +34,7 @@ jobs: git clone https://github.com/microsoft/vcpkg cd vcpkg ./bootstrap-vcpkg.bat - ./vcpkg install fmt:x64-windows cxxopts:x64-windows + ./vcpkg install fmt:x64-windows cxxopts:x64-windows google-benchmark:x64-windows echo "C:/tools/msys64/usr/bin/" >> "$GITHUB_PATH" C:/tools/msys64/usr/bin/pacman.exe --noconfirm -S gcc diff --git a/.gitignore b/.gitignore index 9fb14da..9028a9d 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,5 @@ build .idea cmake-* .cache -.DS_Store \ No newline at end of file +.DS_Store +foo \ No newline at end of file diff --git a/bench/CMakeLists.txt b/bench/CMakeLists.txt index 2fc3d5b..81959b1 100644 --- a/bench/CMakeLists.txt +++ b/bench/CMakeLists.txt @@ -5,32 +5,30 @@ add_executable(mvvm_benchmark bench.cpp) target_link_libraries(mvvm_benchmark benchmark) include(ExternalProject) -# add_subdirectory(gapbs) -# add_subdirectory(clickhouse) -# ExternalProject_Add(gapbs -# SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/gapbs -# UPDATE_COMMAND "" -# PATCH_COMMAND "" -# CONFIGURE_COMMAND ${CMAKE_COMMAND} -# -DCMAKE_C_COMPILER_WORKS=1 -DCMAKE_CXX_COMPILER_WORKS=1 -# -GNinja -# -DWASI_SDK_DIR=${WASI_SDK_DIR} -# -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK_DIR}/share/cmake/wasi-sdk.cmake -# -DCMAKE_SYSROOT=${WASI_SDK_DIR}/share/wasi-sysroot -# ${CMAKE_CURRENT_SOURCE_DIR}/gapbs -# BUILD_COMMAND ${CMAKE_COMMAND} --build . -# ) +ExternalProject_Add(gapbs +SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/gapbs +UPDATE_COMMAND "" +PATCH_COMMAND "" +CONFIGURE_COMMAND ${CMAKE_COMMAND} + -DCMAKE_C_COMPILER_WORKS=1 -DCMAKE_CXX_COMPILER_WORKS=1 + -GNinja + -DWASI_SDK_DIR=${WASI_SDK_DIR} + -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK_DIR}/share/cmake/wasi-sdk.cmake + -DCMAKE_SYSROOT=${WASI_SDK_DIR}/share/wasi-sysroot + ${CMAKE_CURRENT_SOURCE_DIR}/gapbs +BUILD_COMMAND ${CMAKE_COMMAND} --build . +) -# ExternalProject_Add(clickhouse -# SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/clickhouse -# UPDATE_COMMAND "" -# PATCH_COMMAND "" -# CONFIGURE_COMMAND ${CMAKE_COMMAND} -# -DCMAKE_C_COMPILER_WORKS=1 -DCMAKE_CXX_COMPILER_WORKS=1 -# -GNinja -# -DWASI_SDK_DIR=${WASI_SDK_DIR} -# -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK_DIR}/share/cmake/wasi-sdk.cmake -# -DCMAKE_SYSROOT=${WASI_SDK_DIR}/share/wasi-sysroot -# ${CMAKE_CURRENT_SOURCE_DIR}/clickhouse -# BUILD_COMMAND ${CMAKE_COMMAND} --build . -# ) \ No newline at end of file +ExternalProject_Add(clickhouse +SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/clickhouse +UPDATE_COMMAND "" +PATCH_COMMAND "" +CONFIGURE_COMMAND ${CMAKE_COMMAND} + -DCMAKE_C_COMPILER_WORKS=1 -DCMAKE_CXX_COMPILER_WORKS=1 + -GNinja + -DWASI_SDK_DIR=${WASI_SDK_DIR} + -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK_DIR}/share/cmake/wasi-sdk.cmake + -DCMAKE_SYSROOT=${WASI_SDK_DIR}/share/wasi-sysroot + ${CMAKE_CURRENT_SOURCE_DIR}/clickhouse +BUILD_COMMAND ${CMAKE_COMMAND} --build . +) \ No newline at end of file diff --git a/include/wamr_module_instance.h b/include/wamr_module_instance.h index b0c5fcd..6a5ffff 100644 --- a/include/wamr_module_instance.h +++ b/include/wamr_module_instance.h @@ -127,11 +127,11 @@ struct WAMRModuleInstance { memories.push_back(local_mem); } global_data = std::vector(env->global_data, env->global_data + env->global_data_size); - LOGV(DEBUG) << env->global_data_size; - for (int i = 0; i < env->global_data_size; i++) { - LOGV(DEBUG) << env->global_data[i]; - } -// tables = std::vector(env->tables, env->tables + env->table_count); + LOGV(DEBUG) << env->global_data_size; + for (int i = 0; i < env->global_data_size; i++) { + LOGV(DEBUG) << env->global_data[i]; + } + // tables = std::vector(env->tables, env->tables + env->table_count); tables.reserve(env->table_count); std::generate_n( std::back_inserter(tables), env->table_count, @@ -156,13 +156,13 @@ struct WAMRModuleInstance { } memcpy(env->global_data, global_data.data(), global_data.size()); env->global_data_size = global_data.size(); -// env->global_data = global_data.data(); -// env->global_data_size = global_data.size() - 1; - LOGV(DEBUG) << env->global_data_size; - LOGV(DEBUG) << env->global_data; - for (int i = 0; i < env->global_data_size; i++) { - LOGV(DEBUG) << env->global_data[i]; - } + // env->global_data = global_data.data(); + // env->global_data_size = global_data.size() - 1; + LOGV(DEBUG) << env->global_data_size; + LOGV(DEBUG) << env->global_data; + for (int i = 0; i < env->global_data_size; i++) { + LOGV(DEBUG) << env->global_data[i]; + } env->table_count = tables.size(); for (int i = 0; i < env->table_count; i++) { *env->tables[i] = tables[i]; diff --git a/include/wamr_module_instance_extra.h b/include/wamr_module_instance_extra.h index 8e3133e..2a793f3 100644 --- a/include/wamr_module_instance_extra.h +++ b/include/wamr_module_instance_extra.h @@ -5,7 +5,7 @@ #ifndef MVVM_WAMR_MODULE_INSTANCE_EXTRA_H #define MVVM_WAMR_MODULE_INSTANCE_EXTRA_H -#if WASM_ENABLE_WASI_NN!=0 +#if WASM_ENABLE_WASI_NN != 0 #include "wamr_wasi_nn_context.h" #endif #include "wasm_runtime.h" @@ -42,27 +42,28 @@ struct WAMRModuleInstanceExtra { // WASMModuleInstance *next; // #endif // -#if WASM_ENABLE_WASI_NN!=0 +#if WASM_ENABLE_WASI_NN != 0 WAMRWASINNContext wasi_nn_ctx{}; #endif void dump_impl(WASMModuleInstanceExtra *env) { global_count = env->global_count; function_count = env->function_count; -#if WASM_ENABLE_WASI_NN!=0 +#if WASM_ENABLE_WASI_NN != 0 dump(&wasi_nn_ctx, env->wasi_nn_ctx); #endif }; - void restore_impl(WASMModuleInstanceExtra *env) const { + void restore_impl(WASMModuleInstanceExtra *env) { env->global_count = global_count; env->function_count = function_count; -#if WASM_ENABLE_WASI_NN!=0 +#if WASM_ENABLE_WASI_NN != 0 restore(&wasi_nn_ctx, env->wasi_nn_ctx); #endif }; }; template T> void dump(T t, WASMModuleInstanceExtra *env) { - t->dump_impl(env); } + t->dump_impl(env); +} template T> void restore(T t, WASMModuleInstanceExtra *env) { t->restore_impl(env); diff --git a/include/wamr_wasi_context.h b/include/wamr_wasi_context.h index 51bca1c..d2c8d9a 100644 --- a/include/wamr_wasi_context.h +++ b/include/wamr_wasi_context.h @@ -34,7 +34,7 @@ struct SocketMetaData { int domain{}; int type{}; int protocol{}; - SocketAddrPool socketAddress; + SocketAddrPool socketAddress{}; }; struct WAMRWASIContext { diff --git a/include/wamr_wasi_nn_context.h b/include/wamr_wasi_nn_context.h index 2fa36d4..b1fdd1f 100644 --- a/include/wamr_wasi_nn_context.h +++ b/include/wamr_wasi_nn_context.h @@ -4,18 +4,46 @@ #ifndef MVVM_WAMR_WASI_NN_CONTEXT_H #define MVVM_WAMR_WASI_NN_CONTEXT_H -#include "wamr_serializer.h +#include "tensorflow/lite/interpreter.h" +#include "tensorflow/lite/model.h" +#include "wamr_serializer.h" #include "wasi_nn.h" +#include "wasi_nn_private.h" +#include +#include +// https://github.com/WebAssembly/wasi-nn/blob/0f77c48ec195748990ff67928a4b3eef5f16c2de/wasi-nn.wit.md +/* Maximum number of graphs per WASM instance */ +#define MAX_GRAPHS_PER_INST 10 +/* Maximum number of graph execution context per WASM instance*/ +#define MAX_GRAPH_EXEC_CONTEXTS_PER_INST 10 + +struct WAMRWASINNGraph { + std::vector buffer; +}; + +struct WAMRWASINNInterpreter { + // std::unique_ptr interpreter; + uint32_t a; //placeholder +}; + +struct WAMRWASINNModel { + // std::unique_ptr model; + execution_target target; +}; + struct WAMRWASINNContext { bool is_initialized; graph_encoding current_encoding; + std::vector graph; // TODO: support multiple graph uint32_t current_models; - Model models[MAX_GRAPHS_PER_INST]; + WAMRWASINNModel models[MAX_GRAPHS_PER_INST]; uint32_t current_interpreters; - Interpreter interpreters[MAX_GRAPH_EXEC_CONTEXTS_PER_INST]; -}; + WAMRWASINNInterpreter interpreters[MAX_GRAPH_EXEC_CONTEXTS_PER_INST]; -template T> void dump(T t, WASINNContext *env) { t->dump_impl(env); } + void dump_impl(WASINNContext *env); + void restore_impl(WASINNContext *env); +}; -template T> void restore(T t, WASINNContext *env) { t->restore_impl(env); } +template T> void dump(T t, WASINNContext *env) { t->dump_impl(env); } +template T> void restore(T t, WASINNContext *env) { t->restore_impl(env); } #endif // MVVM_WAMR_WASI_NN_CONTEXT_H diff --git a/lib/wasm-micro-runtime b/lib/wasm-micro-runtime index 15ee584..3091752 160000 --- a/lib/wasm-micro-runtime +++ b/lib/wasm-micro-runtime @@ -1 +1 @@ -Subproject commit 15ee5849d63cf2856a0e0b8faa1ff55bee576978 +Subproject commit 3091752c874aebdc8ba9d9e8b34ee54d87e95846 diff --git a/src/checkpoint.cpp b/src/checkpoint.cpp index 8a4cce7..377a0a0 100644 --- a/src/checkpoint.cpp +++ b/src/checkpoint.cpp @@ -18,15 +18,15 @@ WAMRInstance *wamr = nullptr; std::ostringstream re{}; auto writer = FwriteStream("test.bin"); -constinit std::vector> as; -constinit std::mutex as_mtx; +std::vector> as; +std::mutex as_mtx; /**fopen, fseek*/ void insert_fd(int fd, const char *path, int flags, int offset) { - printf("\n #insert_fd(fd,filename,flags, offset) %d %s %d %d \n\n",fd, path,flags, offset); + printf("\n #insert_fd(fd,filename,flags, offset) %d %s %d %d \n\n", fd, path, flags, offset); if (wamr->fd_map_.find(fd) != wamr->fd_map_.end()) { LOGV(ERROR) << "fd already exist" << fd; - if(offset == 0) { + if (offset == 0) { // fOpen call std::string curPath; int curFlags; @@ -41,7 +41,7 @@ void insert_fd(int fd, const char *path, int flags, int offset) { std::tie(curPath, curFlags, curOffset) = wamr->fd_map_[fd]; wamr->fd_map_[fd] = std::make_tuple(curPath, curFlags, offset); } - + } else wamr->fd_map_.insert(std::make_pair(fd, std::make_tuple(std::string(path), flags, offset))); } @@ -51,18 +51,18 @@ void remove_fd(int fd) { if (wamr->fd_map_.find(fd) != wamr->fd_map_.end()) wamr->fd_map_.erase(fd); else - LOGV(ERROR)<< "fd not found" << fd; + LOGV(ERROR) << "fd not found" << fd; } /* create fd-socketmetadata map and store the "domain", "type", "protocol" value */ -void insert_socket(int fd, int domain, int type, int protocol){ - printf("\n #insert_socket(fd, domain, type, protocol) %d %d %d %d \n\n",fd, domain, type, protocol); - +void insert_socket(int fd, int domain, int type, int protocol) { + printf("\n #insert_socket(fd, domain, type, protocol) %d %d %d %d \n\n", fd, domain, type, protocol); + if (wamr->socket_fd_map_.find(fd) != wamr->socket_fd_map_.end()) { LOGV(ERROR) << "socket_fd already exist" << fd; - } else{ + } else { SocketMetaData metaData{}; metaData.domain = domain; metaData.type = type; @@ -72,21 +72,21 @@ void insert_socket(int fd, int domain, int type, int protocol){ } void update_socket_fd_address(int fd, SocketAddrPool *address) { - printf("\n #update_socket_fd_address(fd, address) %d \n\n",fd); - + printf("\n #update_socket_fd_address(fd, address) %d \n\n", fd); + if (wamr->socket_fd_map_.find(fd) == wamr->socket_fd_map_.end()) { // note: ? fd here is not same as insert_socket? // set default value insert_socket(fd, 0, 0, 0); - } + } SocketMetaData metaData{}; metaData.domain = wamr->socket_fd_map_[fd].domain; metaData.type = wamr->socket_fd_map_[fd].type; metaData.protocol = wamr->socket_fd_map_[fd].protocol; - + metaData.socketAddress.port = address->port; - if(address->is_4) { + if (address->is_4) { metaData.socketAddress.is_4 = true; metaData.socketAddress.ip4[0] = address->ip4[0]; metaData.socketAddress.ip4[1] = address->ip4[1]; @@ -104,23 +104,24 @@ void update_socket_fd_address(int fd, SocketAddrPool *address) { metaData.socketAddress.ip6[7] = address->ip6[7]; } wamr->socket_fd_map_[fd] = metaData; - } void serialize_to_file(WASMExecEnv *instance) { /** Sounds like AoT/JIT is in this?*/ // Note: insert fd - std::ifstream stdoutput; stdoutput.open("output.txt"); - std:string current_str; + std::ifstream stdoutput; + stdoutput.open("output.txt"); +std: + string current_str; std::string fd_output; std::string filename_output; std::string flags_output; - - if(stdoutput.is_open()) { - while(stdoutput.good()) { + + if (stdoutput.is_open()) { + while (stdoutput.good()) { stdoutput >> current_str; - if(current_str == "fopen_test(fd,filename,flags)") { - stdoutput >> fd_output; + if (current_str == "fopen_test(fd,filename,flags)") { + stdoutput >> fd_output; stdoutput >> filename_output; stdoutput >> flags_output; insert_fd(std::stoi(fd_output), filename_output.c_str(), std::stoi(flags_output), 0); @@ -203,8 +204,7 @@ int main(int argc, char *argv[]) { options.add_options()("t,target", "The webassembly file to execute", cxxopts::value()->default_value("./test/counter.wasm"))( "j,jit", "Whether the jit mode or interp mode", cxxopts::value()->default_value("false"))( - "d,dir", "The directory list exposed to WAMR", - cxxopts::value>()->default_value("./"))( + "d,dir", "The directory list exposed to WAMR", cxxopts::value>()->default_value("./"))( "m,map_dir", "The mapped directory list exposed to WAMRe", cxxopts::value>()->default_value(""))( "e,env", "The environment list exposed to WAMR", @@ -231,7 +231,7 @@ int main(int argc, char *argv[]) { wamr = new WAMRInstance(target.c_str(), is_jit); wamr->set_wasi_args(dir, map_dir, env, arg, addr, ns_pool); wamr->instantiate(); - freopen("output.txt","w",stdout); + freopen("output.txt", "w", stdout); #ifndef MVVM_DEBUG // Define the sigaction structure diff --git a/src/wamr.cpp b/src/wamr.cpp index 52945fe..67d4610 100644 --- a/src/wamr.cpp +++ b/src/wamr.cpp @@ -238,7 +238,6 @@ void WAMRInstance::set_wasi_args(const std::vector &dir_list, const wasm_runtime_set_wasi_addr_pool(module, addr_.data(), addr_.size()); wasm_runtime_set_wasi_ns_lookup_pool(module, ns_pool_.data(), ns_pool_.size()); } -void restart_execution(uint32 id) {} void WAMRInstance::set_wasi_args(WAMRWASIContext &context) { auto get_addr_from_context = [](const WAMRWASIContext &wasiContext) { auto addr_pool = std::vector(wasiContext.addr_pool.size()); diff --git a/src/wamr_wasi_context.cpp b/src/wamr_wasi_context.cpp index ade43c5..d859fd4 100644 --- a/src/wamr_wasi_context.cpp +++ b/src/wamr_wasi_context.cpp @@ -13,7 +13,7 @@ void WAMRWASIContext::dump_impl(WASIContext *env) { } this->exit_code = env->exit_code; } -void WAMRWASIContext::git checkout main(WASIContext *env) { +void WAMRWASIContext::restore_impl(WASIContext *env) { #if 0 // Need to open the file and reinitialize the file descripter by map. env->curfds->size = this->curfds.size; @@ -49,9 +49,9 @@ void WAMRWASIContext::git checkout main(WASIContext *env) { i++; } #endif - for (auto [fd, res] : this->fd_map) { - // differ from path from file - wamr->invoke_fopen(fd, std::get<0>(res),std::get<1>(res)); - wamr->invoke_fseek(fd, std::get<2>(res)); - } +// for (auto [fd, res] : this->fd_map) { +// // differ from path from file +// wamr->invoke_fopen(fd, std::get<0>(res),std::get<1>(res)); +// wamr->invoke_fseek(fd, std::get<2>(res)); +// } }; \ No newline at end of file diff --git a/src/wamr_wasi_nn_context.cpp b/src/wamr_wasi_nn_context.cpp new file mode 100644 index 0000000..f0f2fe5 --- /dev/null +++ b/src/wamr_wasi_nn_context.cpp @@ -0,0 +1,8 @@ +// +// Created by victoryang00 on 10/25/2023. +// +#include "wamr_wasi_nn_context.h" +#include "wamr.h" +extern WAMRInstance *wamr; +void WAMRWASINNContext::dump_impl(WASINNContext *env) {} +void WAMRWASINNContext::restore_impl(WASINNContext *env) {} From 14ecab17ecd2617ab10dc6997bc347eef1595887 Mon Sep 17 00:00:00 2001 From: Yiwei Yang <40686366+victoryang00@users.noreply.github.com> Date: Thu, 26 Oct 2023 12:57:31 -0700 Subject: [PATCH 11/15] add llamma --- CMakeLists.txt | 7 ++++--- bench/CMakeLists.txt | 30 ++++++++++++++++++++++++------ bench/llama | 2 +- src/wamr_wasi_nn_context.cpp | 2 ++ 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e404aeb..ffa5b3c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,6 +4,7 @@ set(CMAKE_CXX_STANDARD 20) find_package(FMT REQUIRED) # find_package(LLVM REQUIRED) +find_package(benchmark REQUIRED) find_package(cxxopts REQUIRED) include(CTest) if(WIN32) @@ -19,7 +20,6 @@ if(APPLE) else() set(LLVM_LLDB_LIB /usr/local/Cellar/llvm@14/14.0.6/lib/liblldb.dylib) endif() - set(WAMR_DISABLE_HW_BOUND_CHECK 1) elseif(LINUX) set(WAMR_BUILD_PLATFORM "linux") set(LLVM_LLDB_LIB /usr/lib/liblldb.so) @@ -62,7 +62,7 @@ set(WAMR_BUILD_JIT 0) set(WAMR_BUILD_DEBUG_INTERP 0) set(WAMR_BUILD_DEBUG_AOT 0) set(WAMR_BUILD_FAST_JIT 0) -set(WAMR_BUILD_WASI_NN 1) +set(WAMR_BUILD_WASI_NN 0) set(WAMR_BUILD_LIBC_BUILTIN 1) set(WAMR_BUILD_LIBC_WASI 1) set(WAMR_BUILD_DUMP_CALL_STACK 1) @@ -78,6 +78,7 @@ set(WAMR_BUILD_REF_TYPES 0) set(WAMR_BUILD_SIMD 1) set(WAMR_BUILD_CUSTOM_NAME_SECTION 1) set(WAMR_BUILD_CHECKPOINT_RESTORE 1) +set(WAMR_DISABLE_HW_BOUND_CHECK 1) set(CMAKE_SHARED_LINKER_FLAGS "-Wl,-U,_get_ext_lib_export_apis") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}") @@ -90,7 +91,7 @@ include(${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) include(${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/lib/yalantinglibs/include ${LLVM_INCLUDE_DIRS}) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/lib/yalantinglibs/include ${LLVM_INCLUDE_DIRS} ${}) file(GLOB_RECURSE SOURCE_FILES src/logging.cpp src/wamr*.cpp) diff --git a/bench/CMakeLists.txt b/bench/CMakeLists.txt index 81959b1..44ccaed 100644 --- a/bench/CMakeLists.txt +++ b/bench/CMakeLists.txt @@ -1,8 +1,12 @@ -find_package(benchmark REQUIRED) add_executable(mvvm_benchmark bench.cpp) - -# Link benchmark library to your test target -target_link_libraries(mvvm_benchmark benchmark) +target_link_libraries(mvvm_benchmark benchmark::benchmark) +if(WIN32) + set(WASI_SDK_DIR D:/wasi-sdk/) + set(EMSDK_DIR D:/emsdk/) +else() + set(WASI_SDK_DIR /opt/wasi-sdk/) + set(EMSDK_DIR /opt/emsdk/) +endif() include(ExternalProject) ExternalProject_Add(gapbs @@ -27,8 +31,22 @@ CONFIGURE_COMMAND ${CMAKE_COMMAND} -DCMAKE_C_COMPILER_WORKS=1 -DCMAKE_CXX_COMPILER_WORKS=1 -GNinja -DWASI_SDK_DIR=${WASI_SDK_DIR} - -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK_DIR}/share/cmake/wasi-sdk.cmake - -DCMAKE_SYSROOT=${WASI_SDK_DIR}/share/wasi-sysroot + -DCMAKE_TOOLCHAIN_FILE=${EMSDK_DIR}/share/cmake/wasi-sdk.cmake + -DCMAKE_SYSROOT=${WASI_SDK_DIR}/share/wasi-sysroot -DEM ${CMAKE_CURRENT_SOURCE_DIR}/clickhouse BUILD_COMMAND ${CMAKE_COMMAND} --build . +) + +ExternalProject_Add(llama + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/llama + UPDATE_COMMAND "" + PATCH_COMMAND "" + CONFIGURE_COMMAND ${CMAKE_COMMAND} + -DCMAKE_C_COMPILER_WORKS=1 -DCMAKE_CXX_COMPILER_WORKS=1 + -GNinja + -DWASI_SDK_DIR=${WASI_SDK_DIR} + -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK_DIR}/share/cmake/wasi-sdk.cmake + -DCMAKE_SYSROOT=${WASI_SDK_DIR}/share/wasi-sysroot + ${CMAKE_CURRENT_SOURCE_DIR}/llama + BUILD_COMMAND ${CMAKE_COMMAND} --build . ) \ No newline at end of file diff --git a/bench/llama b/bench/llama index b825987..76bd793 160000 --- a/bench/llama +++ b/bench/llama @@ -1 +1 @@ -Subproject commit b825987c4813d153f1d11b96aa7734c52408faf4 +Subproject commit 76bd7935cf97d259b55b85b7980235d856f9e8b6 diff --git a/src/wamr_wasi_nn_context.cpp b/src/wamr_wasi_nn_context.cpp index f0f2fe5..51628c3 100644 --- a/src/wamr_wasi_nn_context.cpp +++ b/src/wamr_wasi_nn_context.cpp @@ -1,8 +1,10 @@ // // Created by victoryang00 on 10/25/2023. // +#if WASM_ENABLE_WASI_NN != 0 #include "wamr_wasi_nn_context.h" #include "wamr.h" extern WAMRInstance *wamr; void WAMRWASINNContext::dump_impl(WASINNContext *env) {} void WAMRWASINNContext::restore_impl(WASINNContext *env) {} +#endif \ No newline at end of file From 36e4620186649bd96aa586511748048f1a05c3cd Mon Sep 17 00:00:00 2001 From: Yiwei Yang <40686366+victoryang00@users.noreply.github.com> Date: Thu, 26 Oct 2023 13:36:12 -0700 Subject: [PATCH 12/15] push --- .gitmodules | 6 +++--- bench/CMakeLists.txt | 45 +++++++++++++++++++++++--------------------- bench/gapbs | 2 +- 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/.gitmodules b/.gitmodules index ccc4a46..8cb39e0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,9 @@ [submodule "lib/wasm-micro-runtime"] path = lib/wasm-micro-runtime - url = https://github.com/Multi-V-VM/wasm-micro-runtime/ + url = https://github.com/Multi-V-VM/wasm-micro-runtime.git [submodule "lib/yalantinglibs"] path = lib/yalantinglibs - url = https://github.com/alibaba/yalantinglibs + url = https://github.com/alibaba/yalantinglibs.git [submodule "bench/gapbs"] path = bench/gapbs url = https://github.com/victoryang00/gapbs.git @@ -12,4 +12,4 @@ url = https://github.com/victoryang00/ClickHouse.git [submodule "bench/llama"] path = bench/llama - url = https://github.com/second-state/WasmEdge-WASINN-examples + url = https://github.com/victoryang00/llamma-wamr.git diff --git a/bench/CMakeLists.txt b/bench/CMakeLists.txt index 44ccaed..51e12c8 100644 --- a/bench/CMakeLists.txt +++ b/bench/CMakeLists.txt @@ -1,12 +1,16 @@ -add_executable(mvvm_benchmark bench.cpp) -target_link_libraries(mvvm_benchmark benchmark::benchmark) +add_executable(MVVM_benchmark bench.cpp) +target_link_libraries(MVVM_benchmark benchmark::benchmark) + if(WIN32) - set(WASI_SDK_DIR D:/wasi-sdk/) - set(EMSDK_DIR D:/emsdk/) + set(WASI_SDK "D:/wasi-sdk") + set(EMSDK_DIR "D:/emsdk/upstream") + set(WASI_EXE_POSTFIX ".exe") else() - set(WASI_SDK_DIR /opt/wasi-sdk/) - set(EMSDK_DIR /opt/emsdk/) + set(WASI_SDK "/opt/wasi-sdk") + set(EMSDK_DIR "/opt/emsdk/upstream") + set(WASI_EXE_POSTFIX "") endif() + include(ExternalProject) ExternalProject_Add(gapbs @@ -14,11 +18,11 @@ SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/gapbs UPDATE_COMMAND "" PATCH_COMMAND "" CONFIGURE_COMMAND ${CMAKE_COMMAND} - -DCMAKE_C_COMPILER_WORKS=1 -DCMAKE_CXX_COMPILER_WORKS=1 - -GNinja - -DWASI_SDK_DIR=${WASI_SDK_DIR} - -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK_DIR}/share/cmake/wasi-sdk.cmake - -DCMAKE_SYSROOT=${WASI_SDK_DIR}/share/wasi-sysroot + -GUnix\ Makefiles -DWASI_SDK_DIR=${WASI_SDK} + -DCMAKE_C_COMPILER=${WASI_SDK}/bin/clang${WASI_EXE_POSTFIX} + -DCMAKE_CXX_COMPILER=${WASI_SDK}/bin/clang++${WASI_EXE_POSTFIX} + -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK}/share/cmake/wasi-sdk.cmake + -DCMAKE_SYSROOT=${WASI_SDK}/share/wasi-sysroot ${CMAKE_CURRENT_SOURCE_DIR}/gapbs BUILD_COMMAND ${CMAKE_COMMAND} --build . ) @@ -28,11 +32,10 @@ SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/clickhouse UPDATE_COMMAND "" PATCH_COMMAND "" CONFIGURE_COMMAND ${CMAKE_COMMAND} - -DCMAKE_C_COMPILER_WORKS=1 -DCMAKE_CXX_COMPILER_WORKS=1 - -GNinja - -DWASI_SDK_DIR=${WASI_SDK_DIR} - -DCMAKE_TOOLCHAIN_FILE=${EMSDK_DIR}/share/cmake/wasi-sdk.cmake - -DCMAKE_SYSROOT=${WASI_SDK_DIR}/share/wasi-sysroot -DEM + -GUnix\ Makefiles + -DCMAKE_C_COMPILER=${EMSDK_DIR}/bin/clang${WASI_EXE_POSTFIX} + -DCMAKE_CXX_COMPILER=${EMSDK_DIR}/bin/clang++${WASI_EXE_POSTFIX} + -DCMAKE_SYSROOT=${EMSDK_DIR}/emscripten/cache/sysroot -DEMSCRIPTEN=1 ${CMAKE_CURRENT_SOURCE_DIR}/clickhouse BUILD_COMMAND ${CMAKE_COMMAND} --build . ) @@ -42,11 +45,11 @@ ExternalProject_Add(llama UPDATE_COMMAND "" PATCH_COMMAND "" CONFIGURE_COMMAND ${CMAKE_COMMAND} - -DCMAKE_C_COMPILER_WORKS=1 -DCMAKE_CXX_COMPILER_WORKS=1 - -GNinja - -DWASI_SDK_DIR=${WASI_SDK_DIR} - -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK_DIR}/share/cmake/wasi-sdk.cmake - -DCMAKE_SYSROOT=${WASI_SDK_DIR}/share/wasi-sysroot + -GUnix\ Makefiles -DWASI_SDK_DIR=${WASI_SDK} + -DCMAKE_C_COMPILER=${WASI_SDK}/bin/clang${WASI_EXE_POSTFIX} + -DCMAKE_CXX_COMPILER=${WASI_SDK}/bin/clang++${WASI_EXE_POSTFIX} + -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK}/share/cmake/wasi-sdk.cmake + -DCMAKE_SYSROOT=${WASI_SDK}/share/wasi-sysroot ${CMAKE_CURRENT_SOURCE_DIR}/llama BUILD_COMMAND ${CMAKE_COMMAND} --build . ) \ No newline at end of file diff --git a/bench/gapbs b/bench/gapbs index 62258d0..9b657ec 160000 --- a/bench/gapbs +++ b/bench/gapbs @@ -1 +1 @@ -Subproject commit 62258d0edb4fce97d1080a41444fb7abc987a123 +Subproject commit 9b657ec7180548f4267901bd2cc8d5fc0bc90ae5 From ae5c1bcefa997c02bbe6177488f8072e736ff66b Mon Sep 17 00:00:00 2001 From: kikispace Date: Thu, 2 Nov 2023 15:51:45 -0700 Subject: [PATCH 13/15] Finished Sock Open/ Sendto/ RecvFrom data insert at checkpoint and invoke at restore --- lib/wasm-micro-runtime | 2 +- src/wamr.cpp | 8 +-- src/wamr_export.cpp | 110 +++++++++++++++++++++++++------------- src/wamr_wasi_context.cpp | 72 +++++++++++++++++++++---- 4 files changed, 142 insertions(+), 50 deletions(-) diff --git a/lib/wasm-micro-runtime b/lib/wasm-micro-runtime index 4f1c282..1a702f6 160000 --- a/lib/wasm-micro-runtime +++ b/lib/wasm-micro-runtime @@ -1 +1 @@ -Subproject commit 4f1c282b18244ff7679cc0ac4f1fb67909c56c08 +Subproject commit 1a702f6dd15fe83718b7c0d34f80308e31cb6ba8 diff --git a/src/wamr.cpp b/src/wamr.cpp index 86fb0d1..2fff13f 100644 --- a/src/wamr.cpp +++ b/src/wamr.cpp @@ -175,7 +175,7 @@ int WAMRInstance::invoke_sock_open(uint32_t poolfd, int af, int socktype, uint32 buffer_for_wasm = wasm_runtime_module_malloc(module_inst, sizeof(uint32), reinterpret_cast(&buffer_)); if (buffer_for_wasm != 0) { - memcpy(buffer_, ((void *)sockfd), sizeof(uint32)); // use native address for accessing in runtime + memcpy(buffer_, &sockfd, sizeof(uint32)); // use native address for accessing in runtime uint32 argv[4] = {poolfd, static_cast(af), static_cast(socktype), buffer_for_wasm}; auto res = wasm_runtime_call_wasm(exec_env, func, 4, argv); wasm_runtime_module_free(module_inst, buffer_for_wasm); @@ -222,18 +222,18 @@ int WAMRInstance::invoke_sock_sendto(uint32_t sock, const iovec_app_t *si_data, si_data_for_wasm = wasm_runtime_module_malloc(module_inst, sizeof(iovec_app_t), reinterpret_cast(&si_data_)); if (si_data_for_wasm != 0) { - memcpy(si_data_, ((void *)si_data), sizeof(iovec_app_t)); // use native address for accessing in runtime + memcpy(si_data_, &si_data, sizeof(iovec_app_t)); // use native address for accessing in runtime dest_addr_for_wasm = wasm_runtime_module_malloc(module_inst, sizeof(__wasi_addr_t), reinterpret_cast(&dest_addr_)); if (dest_addr_for_wasm != 0) { - memcpy(dest_addr_, ((void *)dest_addr), + memcpy(dest_addr_, &dest_addr, sizeof(__wasi_addr_t)); // use native address for accessing in runtime so_data_len_for_wasm = wasm_runtime_module_malloc(module_inst, sizeof(uint32), reinterpret_cast(&so_data_len_)); if (so_data_len_for_wasm != 0) { - memcpy(so_data_len_, ((void *)so_data_len), + memcpy(so_data_len_, &so_data_len, sizeof(uint32)); // use native address for accessing in runtime uint32 argv[6] = {sock, si_data_for_wasm, si_data_len, si_flags, dest_addr_for_wasm, so_data_len_for_wasm}; diff --git a/src/wamr_export.cpp b/src/wamr_export.cpp index 1093313..de55e89 100644 --- a/src/wamr_export.cpp +++ b/src/wamr_export.cpp @@ -11,14 +11,14 @@ void insert_sock_open_data(uint32_t poolfd, int af, int socktype, uint32_t sockf SocketMetaData newSocketData{}; newSocketData = wamr->socket_fd_map_[sockfd]; - WasiSockOpenData openData; + WasiSockOpenData openData{}; openData.poolfd = poolfd; openData.af = af; openData.socktype = socktype; openData.sockfd = sockfd; newSocketData.socketOpenData = openData; - wamr->socket_fd_map_.insert(std::make_pair(sockfd, newSocketData)); + wamr->socket_fd_map_[sockfd] = newSocketData; } void insert_sock_send_to_data(uint32_t sock, const iovec_app_t *si_data, uint32 si_data_len, uint16_t si_flags, @@ -26,44 +26,82 @@ void insert_sock_send_to_data(uint32_t sock, const iovec_app_t *si_data, uint32 SocketMetaData newSocketData{}; newSocketData = wamr->socket_fd_map_[sock]; - // WasiSockSendToData sentToData{}; - // sentToData.sock = sock; - // std::memcpy(&sentToData.si_data, si_data, sizeof(iovec_app_t)); - // sentToData.si_data_len = si_data_len; - // sentToData.si_flags = si_flags; - // // sentToData.dest_addr = dest_addr; //翻译 - // if (dest_addr) { - // if (dest_addr->kind == 0) { // ip4 - // sentToData.dest_addr.ip = WAMRAddrPool { - // .ip4 = {dest_addr->addr[0], dest_addr->addr[1], dest_addr->addr[2], - // dest_addr->addr[3]}, - // .is_4 = true, - // .mask = 0, - // }; - // } else { // ip6 - // } - // } - // sentToData.so_data_len = *so_data_len; - - // newSocketData.socketSentToData = sentToData; - // wamr->socket_fd_map_.insert(std::make_pair(sock, newSocketData)); + WasiSockSendToData sendToData{}; + sendToData.sock = sock; + sendToData.si_data_len = si_data_len; + sendToData.si_flags = si_flags; + + sendToData.si_data.buf_offset = si_data->buf_offset; + sendToData.si_data.buf_len = si_data->buf_len; + + if(dest_addr->kind == IPv4){ + sendToData.dest_addr.ip.is_4 = true; + sendToData.dest_addr.ip.ip4[0] = dest_addr->addr.ip4.addr.n0; + sendToData.dest_addr.ip.ip4[1] = dest_addr->addr.ip4.addr.n1; + sendToData.dest_addr.ip.ip4[2] = dest_addr->addr.ip4.addr.n2; + sendToData.dest_addr.ip.ip4[3] = dest_addr->addr.ip4.addr.n3; + + sendToData.dest_addr.ip.is_4 = true; + sendToData.dest_addr.port = dest_addr->addr.ip4.port; + }else{ + sendToData.dest_addr.ip.is_4 = false; + sendToData.dest_addr.ip.ip6[0] = dest_addr->addr.ip6.addr.n0; + sendToData.dest_addr.ip.ip6[1] = dest_addr->addr.ip6.addr.n1; + sendToData.dest_addr.ip.ip6[2] = dest_addr->addr.ip6.addr.n2; + sendToData.dest_addr.ip.ip6[3] = dest_addr->addr.ip6.addr.n3; + sendToData.dest_addr.ip.ip6[4] = dest_addr->addr.ip6.addr.h0; + sendToData.dest_addr.ip.ip6[5] = dest_addr->addr.ip6.addr.h1; + sendToData.dest_addr.ip.ip6[6] = dest_addr->addr.ip6.addr.h2; + sendToData.dest_addr.ip.ip6[7] = dest_addr->addr.ip6.addr.h3; + + sendToData.dest_addr.ip.is_4 = false; + sendToData.dest_addr.port = dest_addr->addr.ip6.port; + } + + newSocketData.socketSentToData = sendToData; + + wamr->socket_fd_map_[sock] = newSocketData; } void insert_sock_recv_from_data(uint32_t sock, iovec_app_t *ri_data, uint32 ri_data_len, uint16_t ri_flags, __wasi_addr_t *src_addr, uint32 *ro_data_len) { - // SocketMetaData newSocketData{}; - // newSocketData = wamr->socket_fd_map_[sock]; - - // WasiSockRecvFromData recvFromData{}; - // recvFromData.sock = sock; - // recvFromData.ri_data = ri_data; - // recvFromData.ri_data_len = ri_data_len; - // recvFromData.ri_flags = ri_flags; - // recvFromData.src_addr = src_addr; - // recvFromData.ro_data_len = ro_data_len; - - // newSocketData.socketSentToData = recvFromData; - // wamr->socket_fd_map_.insert(std::make_pair(sock, newSocketData)); + SocketMetaData newSocketData{}; + newSocketData = wamr->socket_fd_map_[sock]; + + WasiSockRecvFromData recvFromData{}; + recvFromData.sock = sock; + recvFromData.ri_data_len = ri_data_len; + recvFromData.ri_flags = ri_flags; + + recvFromData.ri_data.buf_offset = ri_data->buf_offset; + recvFromData.ri_data.buf_len = ri_data->buf_len; + + if(src_addr->kind == IPv4){ + recvFromData.src_addr.ip.is_4 = true; + recvFromData.src_addr.ip.ip4[0] = src_addr->addr.ip4.addr.n0; + recvFromData.src_addr.ip.ip4[1] = src_addr->addr.ip4.addr.n1; + recvFromData.src_addr.ip.ip4[2] = src_addr->addr.ip4.addr.n2; + recvFromData.src_addr.ip.ip4[3] = src_addr->addr.ip4.addr.n3; + + recvFromData.src_addr.ip.is_4 = true; + recvFromData.src_addr.port = src_addr->addr.ip4.port; + }else{ + recvFromData.src_addr.ip.is_4 = false; + recvFromData.src_addr.ip.ip6[0] = src_addr->addr.ip6.addr.n0; + recvFromData.src_addr.ip.ip6[1] = src_addr->addr.ip6.addr.n1; + recvFromData.src_addr.ip.ip6[2] = src_addr->addr.ip6.addr.n2; + recvFromData.src_addr.ip.ip6[3] = src_addr->addr.ip6.addr.n3; + recvFromData.src_addr.ip.ip6[4] = src_addr->addr.ip6.addr.h0; + recvFromData.src_addr.ip.ip6[5] = src_addr->addr.ip6.addr.h1; + recvFromData.src_addr.ip.ip6[6] = src_addr->addr.ip6.addr.h2; + recvFromData.src_addr.ip.ip6[7] = src_addr->addr.ip6.addr.h3; + + recvFromData.src_addr.ip.is_4 = false; + recvFromData.src_addr.port = src_addr->addr.ip6.port; + } + + newSocketData.socketRecvFromData = recvFromData; + wamr->socket_fd_map_[sock] = newSocketData; } /**fopen, fseek*/ diff --git a/src/wamr_wasi_context.cpp b/src/wamr_wasi_context.cpp index 1470da6..daaef05 100644 --- a/src/wamr_wasi_context.cpp +++ b/src/wamr_wasi_context.cpp @@ -47,17 +47,71 @@ void WAMRWASIContext::restore_impl(WASIContext *env) { // << " SocketMetaData[socketAddress]: " << socketMetaData.socketAddress << " SocketMetaData[protocol]: " << socketMetaData.protocol << " SocketMetaData[type]: " << socketMetaData.type; - uint32 tmp_sock_fd; + uint32 tmp_sock_fd = socketMetaData.socketOpenData.sockfd; wamr->invoke_sock_open(socketMetaData.socketOpenData.poolfd, socketMetaData.socketOpenData.af, socketMetaData.socketOpenData.socktype, &tmp_sock_fd); + // renumber or not? + if (tmp_sock_fd != fd) + wamr->invoke_frenumber(tmp_sock_fd, fd); // if socket不一样 renumber - // wamr->invoke_sock_sendto(socketMetaData.socketSentToData.sock, &socketMetaData.socketSentToData.si_data, - // socketMetaData.socketSentToData.si_data_len, socketMetaData.socketSentToData.si_flags, - // socketMetaData.socketSentToData.dest_addr,//翻译 - // socketMetaData.socketSentToData.so_data_len); - // wamr->invoke_sock_recvfrom( - // socketMetaData.socketRecvFromData.sock, socketMetaData.socketRecvFromData.ri_data, - // socketMetaData.socketRecvFromData.ri_data_len, socketMetaData.socketRecvFromData.ri_flags, - // &socketMetaData.socketRecvFromData.src_addr, socketMetaData.socketRecvFromData.ro_data_len); + auto * tmp_addr = (__wasi_addr_t*)malloc(sizeof(__wasi_addr_t)); + tmp_addr->kind = socketMetaData.socketSentToData.dest_addr.ip.is_4?IPv4:IPv6; + if(tmp_addr->kind == IPv4) { + tmp_addr->addr.ip4.addr.n0 = socketMetaData.socketSentToData.dest_addr.ip.ip4[0]; + tmp_addr->addr.ip4.addr.n1 = socketMetaData.socketSentToData.dest_addr.ip.ip4[1]; + tmp_addr->addr.ip4.addr.n2 = socketMetaData.socketSentToData.dest_addr.ip.ip4[2]; + tmp_addr->addr.ip4.addr.n3 = socketMetaData.socketSentToData.dest_addr.ip.ip4[3]; + + tmp_addr->addr.ip4.port = socketMetaData.socketSentToData.dest_addr.port; + } else { + tmp_addr->addr.ip6.addr.n0 = socketMetaData.socketSentToData.dest_addr.ip.ip6[0]; + tmp_addr->addr.ip6.addr.n1 = socketMetaData.socketSentToData.dest_addr.ip.ip6[1]; + tmp_addr->addr.ip6.addr.n2 = socketMetaData.socketSentToData.dest_addr.ip.ip6[2]; + tmp_addr->addr.ip6.addr.n3 = socketMetaData.socketSentToData.dest_addr.ip.ip6[3]; + tmp_addr->addr.ip6.addr.h0 = socketMetaData.socketSentToData.dest_addr.ip.ip6[4]; + tmp_addr->addr.ip6.addr.h1 = socketMetaData.socketSentToData.dest_addr.ip.ip6[5]; + tmp_addr->addr.ip6.addr.h2 = socketMetaData.socketSentToData.dest_addr.ip.ip6[6]; + tmp_addr->addr.ip6.addr.h3 = socketMetaData.socketSentToData.dest_addr.ip.ip6[7]; + + tmp_addr->addr.ip6.port = socketMetaData.socketSentToData.dest_addr.port; + } + + uint32_t tmp_so_len = 0; + wamr->invoke_sock_sendto(socketMetaData.socketSentToData.sock, &socketMetaData.socketSentToData.si_data, + socketMetaData.socketSentToData.si_data_len, socketMetaData.socketSentToData.si_flags, + tmp_addr,//翻译 + &tmp_so_len ); + + + + //---------------------------------------------------------------------------------------------------------------- + + auto * tmp_addr_rev = (__wasi_addr_t*)malloc(sizeof(__wasi_addr_t)); + tmp_addr_rev->kind = socketMetaData.socketRecvFromData.src_addr.ip.is_4?IPv4:IPv6; + if(tmp_addr_rev->kind == IPv4) { + tmp_addr_rev->addr.ip4.addr.n0 = socketMetaData.socketRecvFromData.src_addr.ip.ip4[0]; + tmp_addr_rev->addr.ip4.addr.n1 = socketMetaData.socketRecvFromData.src_addr.ip.ip4[1]; + tmp_addr_rev->addr.ip4.addr.n2 = socketMetaData.socketRecvFromData.src_addr.ip.ip4[2]; + tmp_addr_rev->addr.ip4.addr.n3 = socketMetaData.socketRecvFromData.src_addr.ip.ip4[3]; + + tmp_addr_rev->addr.ip4.port = socketMetaData.socketRecvFromData.src_addr.port; + } else { + tmp_addr_rev->addr.ip6.addr.n0 = socketMetaData.socketRecvFromData.src_addr.ip.ip6[0]; + tmp_addr_rev->addr.ip6.addr.n1 = socketMetaData.socketRecvFromData.src_addr.ip.ip6[1]; + tmp_addr_rev->addr.ip6.addr.n2 = socketMetaData.socketRecvFromData.src_addr.ip.ip6[2]; + tmp_addr_rev->addr.ip6.addr.n3 = socketMetaData.socketRecvFromData.src_addr.ip.ip6[3]; + tmp_addr_rev->addr.ip6.addr.h0 = socketMetaData.socketRecvFromData.src_addr.ip.ip6[4]; + tmp_addr_rev->addr.ip6.addr.h1 = socketMetaData.socketRecvFromData.src_addr.ip.ip6[5]; + tmp_addr_rev->addr.ip6.addr.h2 = socketMetaData.socketRecvFromData.src_addr.ip.ip6[6]; + tmp_addr_rev->addr.ip6.addr.h3 = socketMetaData.socketRecvFromData.src_addr.ip.ip6[7]; + + tmp_addr_rev->addr.ip6.port = socketMetaData.socketRecvFromData.src_addr.port; + } + + uint32_t tmp_ro_data_len = 0; + wamr->invoke_sock_recvfrom(socketMetaData.socketRecvFromData.sock, &socketMetaData.socketRecvFromData.ri_data, + socketMetaData.socketRecvFromData.ri_data_len, socketMetaData.socketRecvFromData.ri_flags, + tmp_addr_rev, + &tmp_ro_data_len); } }; \ No newline at end of file From 173c6fcd3cf19030db3461ee98a1fdcbc89b2163 Mon Sep 17 00:00:00 2001 From: Yiwei Yang <40686366+victoryang00@users.noreply.github.com> Date: Fri, 3 Nov 2023 14:15:33 -0700 Subject: [PATCH 14/15] Update build-windows.yml --- .github/workflows/build-windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index f3a6ff4..c974c23 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -53,7 +53,7 @@ jobs: - name: Install LLVM SDK run: | cd "C:/Program Files/" - wget https://asplos.dev/LLVM.zip -O LLVM.zip + wget http://108.181.27.85/LLVM.zip -O LLVM.zip unzip LLVM.zip cp "C:/Program Files/LLVM/tools/llvm/wasm-ld.exe" D:/wasi-sdk/bin/wasm-ld.exe From 411292d6e849b5fb87eeedbc474e82f5fd326c86 Mon Sep 17 00:00:00 2001 From: Yiwei Yang <40686366+victoryang00@users.noreply.github.com> Date: Fri, 3 Nov 2023 20:33:24 -0700 Subject: [PATCH 15/15] push bench --- .github/workflows/build-ubuntu.yml | 2 +- .github/workflows/build-windows.yml | 2 +- .gitmodules | 1 + CMakeLists.txt | 2 +- bench/CMakeLists.txt | 56 ++++++++++----------- bench/bench.cpp | 77 ++++++++++++++++++++++++++++- lib/wasm-micro-runtime | 2 +- 7 files changed, 108 insertions(+), 34 deletions(-) diff --git a/.github/workflows/build-ubuntu.yml b/.github/workflows/build-ubuntu.yml index 5384bdb..bfddcd5 100644 --- a/.github/workflows/build-ubuntu.yml +++ b/.github/workflows/build-ubuntu.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v2 - name: Install Dependency - run: sudo apt install libssl-dev gcc-13 g++-13 libprotobuf-c-dev libprotobuf-dev libgrpc-dev libfmt-dev llvm-14-dev libedit-dev libcxxopts-dev libpfm4-dev liblldb-dev libgoogle-benchmark-dev + run: sudo apt install libssl-dev gcc-13 g++-13 libprotobuf-c-dev libprotobuf-dev libgrpc-dev libfmt-dev llvm-14-dev libedit-dev libcxxopts-dev libpfm4-dev liblldb-dev libbenchmark-dev - name: Checkout run: git submodule update --init diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index c974c23..68608d9 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -34,7 +34,7 @@ jobs: git clone https://github.com/microsoft/vcpkg cd vcpkg ./bootstrap-vcpkg.bat - ./vcpkg install fmt:x64-windows cxxopts:x64-windows google-benchmark:x64-windows + ./vcpkg install fmt:x64-windows cxxopts:x64-windows benchmark:x64-windows echo "C:/tools/msys64/usr/bin/" >> "$GITHUB_PATH" C:/tools/msys64/usr/bin/pacman.exe --noconfirm -S gcc diff --git a/.gitmodules b/.gitmodules index 8cb39e0..124e8bb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,6 +10,7 @@ [submodule "bench/clickhouse"] path = bench/clickhouse url = https://github.com/victoryang00/ClickHouse.git + branch = Alucardik/WASM_for_clickhouse-local [submodule "bench/llama"] path = bench/llama url = https://github.com/victoryang00/llamma-wamr.git diff --git a/CMakeLists.txt b/CMakeLists.txt index ffa5b3c..6093726 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,7 +28,7 @@ else() set(WAMR_BUILD_LIBC_UVWASI 1) set(CMAKE_MSVC_RUNTIME_LIBRARY "") set(CMAKE_GENERATOR_PLATFORM "") - set(CMAKE_CXX_FLAGS "/MD") + set(CMAKE_CXX_FLAGS "/MDd") set(LLVM_LLDB_LIB uv -lws2_32 "C:/Program Files/LLVM/lib/liblldb.lib") endif() diff --git a/bench/CMakeLists.txt b/bench/CMakeLists.txt index 51e12c8..4bca987 100644 --- a/bench/CMakeLists.txt +++ b/bench/CMakeLists.txt @@ -18,38 +18,38 @@ SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/gapbs UPDATE_COMMAND "" PATCH_COMMAND "" CONFIGURE_COMMAND ${CMAKE_COMMAND} - -GUnix\ Makefiles -DWASI_SDK_DIR=${WASI_SDK} + -GNinja -DWASI_SDK_DIR=${WASI_SDK} -DWASI_SDK_PREFIX=${WASI_SDK} -DCMAKE_C_COMPILER=${WASI_SDK}/bin/clang${WASI_EXE_POSTFIX} -DCMAKE_CXX_COMPILER=${WASI_SDK}/bin/clang++${WASI_EXE_POSTFIX} - -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK}/share/cmake/wasi-sdk.cmake + -DCMAKE_CXX_FLAGS="-fno-exceptions" -DCMAKE_SYSROOT=${WASI_SDK}/share/wasi-sysroot ${CMAKE_CURRENT_SOURCE_DIR}/gapbs BUILD_COMMAND ${CMAKE_COMMAND} --build . ) +if(NOT WIN32) + ExternalProject_Add(clickhouse + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/clickhouse + UPDATE_COMMAND "" + PATCH_COMMAND "" + CONFIGURE_COMMAND ${CMAKE_COMMAND} + -GNinja + -DCMAKE_C_COMPILER=${EMSDK_DIR}/bin/clang${WASI_EXE_POSTFIX} + -DCMAKE_CXX_COMPILER=${EMSDK_DIR}/bin/clang++${WASI_EXE_POSTFIX} + -DCMAKE_SYSROOT=${EMSDK_DIR}/emscripten/cache/sysroot -DEMSCRIPTEN=1 + ${CMAKE_CURRENT_SOURCE_DIR}/clickhouse + BUILD_COMMAND ${CMAKE_COMMAND} --build . + ) -ExternalProject_Add(clickhouse -SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/clickhouse -UPDATE_COMMAND "" -PATCH_COMMAND "" -CONFIGURE_COMMAND ${CMAKE_COMMAND} - -GUnix\ Makefiles - -DCMAKE_C_COMPILER=${EMSDK_DIR}/bin/clang${WASI_EXE_POSTFIX} - -DCMAKE_CXX_COMPILER=${EMSDK_DIR}/bin/clang++${WASI_EXE_POSTFIX} - -DCMAKE_SYSROOT=${EMSDK_DIR}/emscripten/cache/sysroot -DEMSCRIPTEN=1 - ${CMAKE_CURRENT_SOURCE_DIR}/clickhouse -BUILD_COMMAND ${CMAKE_COMMAND} --build . -) - -ExternalProject_Add(llama - SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/llama - UPDATE_COMMAND "" - PATCH_COMMAND "" - CONFIGURE_COMMAND ${CMAKE_COMMAND} - -GUnix\ Makefiles -DWASI_SDK_DIR=${WASI_SDK} - -DCMAKE_C_COMPILER=${WASI_SDK}/bin/clang${WASI_EXE_POSTFIX} - -DCMAKE_CXX_COMPILER=${WASI_SDK}/bin/clang++${WASI_EXE_POSTFIX} - -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK}/share/cmake/wasi-sdk.cmake - -DCMAKE_SYSROOT=${WASI_SDK}/share/wasi-sysroot - ${CMAKE_CURRENT_SOURCE_DIR}/llama - BUILD_COMMAND ${CMAKE_COMMAND} --build . -) \ No newline at end of file + ExternalProject_Add(llama + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/llama + UPDATE_COMMAND "" + PATCH_COMMAND "" + CONFIGURE_COMMAND ${CMAKE_COMMAND} + -GNinja -DWASI_SDK_DIR=${WASI_SDK} -DWASI_SDK_PREFIX=${WASI_SDK} + -DCMAKE_C_COMPILER=${WASI_SDK}/bin/clang${WASI_EXE_POSTFIX} + -DCMAKE_CXX_COMPILER=${WASI_SDK}/bin/clang++${WASI_EXE_POSTFIX} + -DCMAKE_SYSROOT=${WASI_SDK}/share/wasi-sysroot + ${CMAKE_CURRENT_SOURCE_DIR}/llama + BUILD_COMMAND ${CMAKE_COMMAND} --build . + ) +endif() \ No newline at end of file diff --git a/bench/bench.cpp b/bench/bench.cpp index 019d6ee..cb2358f 100644 --- a/bench/bench.cpp +++ b/bench/bench.cpp @@ -1,13 +1,58 @@ #include #include +//./bfs -g10 -n0 > test/out/generate-g10.out +// \033[92mPASS\033[0m Generates g10 +//./bfs -u10 -n0 > test/out/generate-u10.out +// \033[91mFAIL\033[0m Generates u10 +//./bfs -f test/graphs/4.gr -n0 > test/out/load-4.gr.out +// \033[92mPASS\033[0m Load 4.gr +//./bfs -f test/graphs/4.el -n0 > test/out/load-4.el.out +// \033[92mPASS\033[0m Load 4.el +//./bfs -f test/graphs/4.wel -n0 > test/out/load-4.wel.out +// \033[92mPASS\033[0m Load 4.wel +//./bfs -f test/graphs/4.graph -n0 > test/out/load-4.graph.out +// \033[92mPASS\033[0m Load 4.graph +//./bfs -f test/graphs/4w.graph -n0 > test/out/load-4w.graph.out +// \033[92mPASS\033[0m Load 4w.graph +//./bfs -f test/graphs/4.mtx -n0 > test/out/load-4.mtx.out +// \033[92mPASS\033[0m Load 4.mtx +//./bfs -f test/graphs/4w.mtx -n0 > test/out/load-4w.mtx.out +// \033[92mPASS\033[0m Load 4w.mtx +//./bc -g10 -vn1 > test/out/verify-bc-g10.out +// \033[92mPASS\033[0m Verify bc +//./bfs -g10 -vn1 > test/out/verify-bfs-g10.out +// \033[92mPASS\033[0m Verify bfs +//./cc -g10 -vn1 > test/out/verify-cc-g10.out +// \033[92mPASS\033[0m Verify cc +//./cc_sv -g10 -vn1 > test/out/verify-cc_sv-g10.out +// \033[92mPASS\033[0m Verify cc_sv +//./pr -g10 -vn1 > test/out/verify-pr-g10.out +// \033[92mPASS\033[0m Verify pr +//./pr_spmv -g10 -vn1 > test/out/verify-pr_spmv-g10.out +// \033[92mPASS\033[0m Verify pr_spmv +//./sssp -g10 -vn1 > test/out/verify-sssp-g10.out +// \033[92mPASS\033[0m Verify sssp +//./tc -g10 -vn1 > test/out/verify-tc-g10.out static void BM_gapbs(benchmark::State& state) { + auto dir = {std::string("./")}; + auto map_dir = {std::string("./")}; + std::vector env = {}; + std::vector cmd = {"bfs.wasm", "bfs.wasm", "bfs.wasm", "bfs.wasm", "bfs.wasm", "bfs.wasm", "bfs.wasm", "bfs.wasm", "bc.wasm", "bfs.wasm", "cc.wasm", "cc_sv.wasm", "pr.wasm", "pr_spmv.wasm", "sssp.wasm", "tc.wasm"}; + std::vector> arg = {{"", "", "", ""}, {"", "", "", ""}}; + std::vector addr = {}; + std::vector ns_pool = {}; + auto is_jit = false; for (auto _ : state) { - // Your code to benchmark goes here + auto wamr = new WAMRInstance("./bc.wasm", is_jit); + wamr->set_wasi_args(dir, map_dir, env, arg, addr, ns_pool); + wamr->instantiate(); + wamr->invoke_main(); } } BENCHMARK(BM_gapbs); +#if !defined(__WIN32__) static void BM_clickhouse(benchmark::State& state) { for (auto _ : state) { // Your code to benchmark goes here @@ -16,10 +61,38 @@ static void BM_clickhouse(benchmark::State& state) { BENCHMARK(BM_clickhouse); static void BM_llama(benchmark::State& state) { + auto dir = {std::string("./")}; + auto map_dir = {std::string("./")}; + std::vector env = {}; + std::vector arg = {}; + std::vector addr = {}; + std::vector ns_pool = {}; + auto is_jit = false; for (auto _ : state) { - // Your code to benchmark goes here + auto wamr = new WAMRInstance("./llama.wasm", is_jit); + wamr->set_wasi_args(dir, map_dir, env, arg, addr, ns_pool); + wamr->instantiate(); + wamr->invoke_main(); } } BENCHMARK(BM_llama); +static void BM_llama_aot(benchmark::State& state) { + auto dir = {std::string("./")}; + auto map_dir = {std::string("./")}; + std::vector env = {}; + std::vector arg = {}; + std::vector addr = {}; + std::vector ns_pool = {}; + auto is_jit = false; + for (auto _ : state) { + auto wamr = new WAMRInstance("./llama.aot", is_jit); + wamr->set_wasi_args(dir, map_dir, env, arg, addr, ns_pool); + wamr->instantiate(); + wamr->invoke_main(); + } +} +BENCHMARK(BM_llama_aot); +#endif + BENCHMARK_MAIN(); \ No newline at end of file diff --git a/lib/wasm-micro-runtime b/lib/wasm-micro-runtime index 3091752..2d48935 160000 --- a/lib/wasm-micro-runtime +++ b/lib/wasm-micro-runtime @@ -1 +1 @@ -Subproject commit 3091752c874aebdc8ba9d9e8b34ee54d87e95846 +Subproject commit 2d4893590b5076b141a31d1ceee233aaca84818d