refactor codebase and make libthread working. Wrong number for mutex …

…and mulithread and segfault in hdastar.

Signed-off-by: victoryang00 <[email protected]>

artifact/bench_orb_slam3.py

@@ -6,7 +6,7 @@
    "rgbd_tum",
 ]
 arg = [
-    [ "./ORBvoc.txt","./TUM1.yaml","./","./associations/fr2_desk.txt"],
+    [ "./ORBvoc.txt","./TUM1.yaml","./","./associations/fr1_xyz.txt"],
 ]
 
 

artifact/ckpt_restore_latency.py

@@ -1 +1,4 @@
-T
+# for thread cound to 1, 2, 4, 8, 16
+import os
+import common_util.
+def get_

artifact/test_fd.py

@@ -9,7 +9,10 @@
 def run_fd_once(funcs):
     for i in funcs:
         os.system(f"LOGV=0 ./MVVM_checkpoint -t ./test/read-file.aot -f {i}")
-        os.system(f"LOGV=0 ./MVVM_restore -t ./test/read-file.aot -c 100000000")
+        os.system(f"LOGV=0 ./MVVM_restore -t ./test/read-file.aot -c 10000")
+
+def get_fd_merge_latency():
+    pass
 
 
 if __name__ == "__main__":

artifact/test_sock.py

@@ -53,6 +53,11 @@ def run_tcp_migrate_once(funcs):
         os.system(f"LOGV=0 ./MVVM_checkpoint -t ./test/client.aot -c {i}")
         os.system(f"LOGV=0 ./MVVM_restore -t ./test/client.aot")
 
+def get_tcp_latency():
+    pass
+def get_tcp_bandwidth():
+    pass
+
 if __name__ == "__main__":
     funcs = ["socket", "sendto", "recvfrom", "close"]
     func_idxs = [common_util.get_func_index(x) for x in funcs]

bench/CMakeLists.txt

@@ -18,7 +18,6 @@ ExternalProject_Add(gapbs
         -DCMAKE_BUILD_TYPE=Release
         -GNinja -DWASI_SDK_DIR=${WASI_SDK} -DWASI_SDK_PREFIX=${WASI_SDK}
         -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK}/share/cmake/wasi-sdk-pthread.cmake
-        -DCMAKE_EXE_LINKER_FLAGS="-Wl,--export-all -Wl,--allow-undefined-file=${WASI_SDK}/share/wasi-sysroot/share/wasm32-wasi-threads/undefined-symbols.txt"
         -DCMAKE_SYSROOT=${WASI_SDK}/share/wasi-sysroot -DCMAKE_C_COMPILER_WORK=1 -DCMAKE_CXX_COMPILER_WORK=1
         ${CMAKE_CURRENT_SOURCE_DIR}/gapbs
         BUILD_COMMAND ${CMAKE_COMMAND} --build .
@@ -59,7 +58,6 @@ ExternalProject_Add(gapbs-native
         -DCMAKE_BUILD_TYPE=Release
         -GNinja -DWASI_SDK_DIR=${WASI_SDK} -DWASI_SDK_PREFIX=${WASI_SDK}
         -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK}/share/cmake/wasi-sdk-pthread.cmake
-        -DCMAKE_EXE_LINKER_FLAGS="-Wl,--export-all -Wl,--allow-undefined-file=${WASI_SDK}/share/wasi-sysroot/share/wasm32-wasi-threads/undefined-symbols.txt"
         -DCMAKE_SYSROOT=${WASI_SDK}/share/wasi-sysroot -DCMAKE_C_COMPILER_WORK=1 -DCMAKE_CXX_COMPILER_WORK=1
         ${CMAKE_CURRENT_SOURCE_DIR}/redis
         BUILD_COMMAND     ${CMAKE_COMMAND} --build .
@@ -84,7 +82,6 @@ ExternalProject_Add(ftp
         -DCMAKE_BUILD_TYPE=Release
         -GNinja -DWASI_SDK_DIR=${WASI_SDK} -DWASI_SDK_PREFIX=${WASI_SDK}
         -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK}/share/cmake/wasi-sdk.cmake
-        -DCMAKE_EXE_LINKER_FLAGS="-Wl,--export-all -Wl,--allow-undefined-file=${WASI_SDK}/share/wasi-sysroot/share/wasm32-wasi-threads/undefined-symbols.txt"
         -DCMAKE_SYSROOT=${WASI_SDK}/share/wasi-sysroot -DCMAKE_C_COMPILER_WORK=1 -DCMAKE_CXX_COMPILER_WORK=1
         ${CMAKE_CURRENT_SOURCE_DIR}/ftp
         BUILD_COMMAND ${CMAKE_COMMAND} --build .
@@ -98,7 +95,6 @@ ExternalProject_Add(nas
         -DCMAKE_BUILD_TYPE=Release
         -GNinja -DWASI_SDK_DIR=${WASI_SDK} -DWASI_SDK_PREFIX=${WASI_SDK}
         -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK}/share/cmake/wasi-sdk.cmake
-        -DCMAKE_EXE_LINKER_FLAGS="-Wl,--export-all -Wl,--allow-undefined-file=${WASI_SDK}/share/wasi-sysroot/share/wasm32-wasi-threads/undefined-symbols.txt"
         -DCMAKE_SYSROOT=${WASI_SDK}/share/wasi-sysroot -DCMAKE_C_COMPILER_WORK=1 -DCMAKE_CXX_COMPILER_WORK=1
         ${CMAKE_CURRENT_SOURCE_DIR}/nas
         BUILD_COMMAND ${CMAKE_COMMAND} --build .
@@ -146,9 +142,8 @@ ExternalProject_Add(hdastar
         PATCH_COMMAND ""
         CONFIGURE_COMMAND ${CMAKE_COMMAND}
         -GNinja -DWASI_SDK_DIR=${WASI_SDK} -DWASI_SDK_PREFIX=${WASI_SDK}
-        -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK}/share/cmake/wasi-sdk.cmake
+        -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK}/share/cmake/wasi-sdk-pthread.cmake
         -DCMAKE_BUILD_TYPE=Release
-        -DCMAKE_EXE_LINKER_FLAGS="-Wl,--export-all -Wl,--allow-undefined-file=${WASI_SDK}/share/wasi-sysroot/share/wasm32-wasi-threads/undefined-symbols.txt"
         -DCMAKE_SYSROOT=${WASI_SDK}/share/wasi-sysroot -DCMAKE_C_COMPILER_WORK=1 -DCMAKE_CXX_COMPILER_WORK=1
         ${CMAKE_CURRENT_SOURCE_DIR}/hdastar
         BUILD_COMMAND ${CMAKE_COMMAND} --build .
@@ -207,6 +202,7 @@ if (WAMR_BUILD_AOT)
     bench_wamr_app(mg)
     bench_wamr_app(sp)
     bench_wamr_app(linpack)
+    bench_wamr_app(rgbd_tum)
     bench_wamr_app(llama)
     bench_wamr_app(hdastar)
 endif ()

include/wamr.h

@@ -46,21 +46,24 @@ class WAMRInstance {
     std::map<int, std::tuple<std::string, std::vector<std::tuple<int, int, fd_op>>>> fd_map_{};
     // add offset to pair->tuple, 3rd param 'int'
     std::map<int, int> new_sock_map_{};
-    std::map<int, SocketMetaData, std::greater<int>> socket_fd_map_{};
+    std::map<int, SocketMetaData, std::greater<>> socket_fd_map_{};
     SocketAddrPool local_addr{};
     // lwcp is LightWeight CheckPoint
     std::map<ssize_t, int> lwcp_list;
     size_t ready = 0;
     std::mutex as_mtx;
     std::vector<struct sync_op_t> sync_ops;
-    bool should_snapshot_socket{};
+    bool should_snapshot{};
+    WASMMemoryInstance **tmp_buf;
+    uint32 tmp_buf_size;
     void replay_sync_ops(bool, wasm_exec_env_t);
     void register_tid_map();
     std::vector<struct sync_op_t>::iterator sync_iter;
     std::mutex sync_op_mutex;
     std::condition_variable sync_op_cv;
     std::map<ssize_t, ssize_t> tid_map;
-    ssize_t cur_thread;
+    std::map<ssize_t, size_t> tid_start_arg_map;
+    size_t cur_thread;
     bool is_jit{};
     bool is_aot{};
     char error_buf[128]{};
@@ -76,11 +79,9 @@ class WAMRInstance {
     void recover(std::vector<std::unique_ptr<WAMRExecEnv>> *execEnv);
     bool load_wasm_binary(const char *wasm_path, char **buffer_ptr);
     bool get_int3_addr();
-    bool get_switch_addr();
     bool replace_int3_with_nop();
     bool replace_mfence_with_nop();
     bool replace_nop_with_int3();
-    bool replace_switch_with_nop();
     std::chrono::time_point<std::chrono::high_resolution_clock> time;
     std::vector<long long> latencies;
     WASMFunction *get_func();
@@ -90,10 +91,10 @@ class WAMRInstance {
     AOTFunctionInstance *get_func(int index);
 #endif
     WASMExecEnv *get_exec_env();
-    WASMModuleInstance *get_module_instance();
+    WASMModuleInstance *get_module_instance() const;
 
 #if WASM_ENABLE_AOT != 0
-    AOTModule *get_module();
+    AOTModule *get_module() const;
 #endif
 
     void set_wasi_args(WAMRWASIContext &addrs);

include/wamr_exec_env.h

@@ -13,130 +13,20 @@
 #include <vector>
 
 struct WAMRExecEnv { // multiple
-    /* Next thread's exec env of a WASM module instance. we can get the previous exec env outside layer */
-    //    struct WASMExecEnv *next;
-    //
-    //    /* Previous thread's exec env of a WASM module instance. */
-    //    struct WASMExecEnv *prev;
-    uint32 cur_count{};
-
-    /* Note: field module_inst, argv_buf, native_stack_boundary,
-       susƒend_flags, aux_stack_boundary, aux_stack_bottom, and
-       native_symbol are used by AOTed code, don't change the
-       places of them */
-
-    /* The WASM module instance of current thread */
+    /** We need to put Module Inst at the top. */
     WAMRModuleInstance module_inst{};
-
-    // #if WASM_ENABLE_AOT != 0
-    //     uint32 *argv_buf;
-    // #endif
-
-    /* The boundary of native stack. When runtime detects that native
-       frame may overrun this boundary, it throws stack overflow
-       exception. */
-    //    uint8 *native_stack_boundary;
-
-    /* Used to terminate or suspend current thread
-        bit 0: need to terminate
-        bit 1: need to suspend
-        bit 2: need to go into breakpoint
-        bit 3: return from pthread_exit */
+    /* The thread id of wasm interpreter for current thread. */
+    uint32 cur_count{};
     uint32 flags;
 
     /* Auxiliary stack boundary */
     uint32 aux_boundary;
     /* Auxiliary stack bottom */
     uint32 aux_bottom;
-
-    // #if WASM_ENABLE_AOT != 0
-    //     /* Native symbol list, reserved */
-    //     void **native_symbol;
-    // #endif
-
-    /*
-     * The lowest stack pointer value observed.
-     * Assumption: native stack grows to the lower address.
-     */
-    //    uint8 *native_stack_top_min;
-
-    // #if WASM_ENABLE_FAST_JIT != 0
-    //     /**
-    //      * Cache for
-    //      * - jit native operations in 32-bit target which hasn't 64-bit
-    //      *   int/float registers, mainly for the operations of double and int64,
-    //      *   such as F64TOI64, F32TOI64, I64 MUL/REM, and so on.
-    //      * - SSE instructions.
-    //      **/
-    //     uint64 jit_cache[2];
-    // #endif
-
-    // #if WASM_ENABLE_THREAD_MGR != 0
-    //     /* thread return value */
-    //     void *thread_ret_value;
-    //
-    //     /* Must be provided by thread library */
-    //     void *(*thread_start_routine)(void *);
-    //     void *thread_arg;
-    //
-    //     /* pointer to the cluster */
-    //     WASMCluster *cluster;
-    //
-    //     /* used to support debugger */
-    //     korp_mutex wait_lock;
-    //     korp_cond wait_cond;
-    //     /* the count of threads which are joining current thread */
-    //     uint32 wait_count;
-    //
-    //     /* whether current thread is detached */
-    //     bool thread_is_detached;
-    // #endif
-
-    // #if WASM_ENABLE_DEBUG_INTERP != 0
-    //     WASMCurrentEnvStatus *current_status;
-    // #endif
-
-    /* attachment for native function */
-    //    void *attachment;
-
-    //    void *user_data;
-
-    /* Current interpreter frame of current thread */
+    /* Auxiliary stack top */
     std::vector<std::unique_ptr<WAMRInterpFrame>> frames;
 
-    /* The native thread handle of current thread */
-    //    korp_tid handle;
-
-    // #ifdef OS_ENABLE_HW_BOUND_CHECK
-    //     WASMJmpBuf *jmpbuf_stack_top;
-    //     /* One guard page for the exception check */
-    //     uint8 *exce_check_guard_page;
-    // #endif
-
-    // #if WASM_ENABLE_MEMORY_PROFILING != 0
-    //     uint32 max_wasm_stack_used;
-    // #endif
-
-    /* The WASM stack size */
-    //    uint32 wasm_stack_size;
-    /* The WASM stack of current thread */
-    //    union {
-    //        uint64 __make_it_8_byte_aligned_;
-    //
-    //        struct {
-    //            /* The top boundary of the stack. */
-    //            uint8 *top_boundary;
-    //
-    //            /* Top cell index which is free. */
-    //            uint8 *top;
-    //
-    //            /* The WASM stack. */
-    //            uint8 bottom[1];
-    //        } s;
-    //    } wasm_stack;
-
     void dump_impl(WASMExecEnv *env);
-
     void restore_impl(WASMExecEnv *env);
 };
 

include/wamr_export.h

@@ -42,8 +42,13 @@ void set_tcp();
 int get_sock_fd(int);
 void insert_sync_op(wasm_exec_env_t exec_env, const uint32 *mutex, enum sync_op locking);
 void restart_execution(uint32 targs);
+void insert_tid_start_arg(ssize_t, size_t);
 extern int pthread_create_wrapper(wasm_exec_env_t exec_env, uint32 *thread, const void *attr, uint32 elem_index,
                                   uint32 arg);
+extern int32 pthread_mutex_lock_wrapper(wasm_exec_env_t, uint32 *);
+extern int32 pthread_mutex_unlock_wrapper(wasm_exec_env_t, uint32 *);
+extern int32 pthread_mutex_init_wrapper(wasm_exec_env_t, uint32 *, void *);
+extern int32 thread_spawn_wrapper(wasm_exec_env_t exec_env, uint32 start_arg);
 #endif
 void serialize_to_file(struct WASMExecEnv *);
 #endif

include/wamr_interp_frame.h

@@ -14,26 +14,10 @@ struct WAMRInterpFrame {
     uint32 size{};
     /* Instruction pointer of the bytecode array.  */
     uint32 ip{};
-
-    // #if WASM_ENABLE_FAST_JIT != 0
-    //     uint8 *jitted_return_addr;
-    // #endif
     int32 function_index{};
-
-    // #if WASM_ENABLE_PERF_PROFILING != 0
-    //     uint64 time_started;
-    // #endif
-
-    // #if WASM_ENABLE_FAST_INTERP != 0
-    //     /* Return offset of the first return value of current frame,
-    //    the callee will put return values here continuously */
-    //     uint32 ret_offset;
-    //     uint32 *lp;
-    //     uint32 operand[1];
-    // #else
     /* Operand stack top pointer of the current frame. The bottom of
        the stack is the next cell after the last local variable. */
-    uint32 sp{}; // all the sp that can be restart
+    uint32 sp{};
 
     /*
      * Frame data, the layout is:
@@ -42,8 +26,6 @@ struct WAMRInterpFrame {
      *  csp_bottom to csp_boundary: wasm label stack
      *  jit spill cache: only available for fast jit
      */
-    // #endif
-
     std::vector<uint32> stack_frame;
 
     void dump_impl(WASMInterpFrame *env);

include/wamr_memory_instance.h

@@ -40,11 +40,6 @@ struct WAMRMemoryInstance {
         max_page_count = env->max_page_count;
         memory_data.resize(env->memory_data_size);
         memcpy(memory_data.data(), env->memory_data, env->memory_data_size);
-        //        LOGV(DEBUG)<< "memory_data_size:" << env->memory_data_size;
-        //        for (int i = 0; i < 100; ++i) {
-        //            LOGV(DEBUG)<< "memory_data:" <<i << memory_data[i];
-        //            LOGV(DEBUG)<< "memory_data:" <<i << env->memory_data[i];
-        //        }
         heap_data = std::vector<uint8>(env->heap_data, env->heap_data_end);
     };
     void restore_impl(WASMMemoryInstance *env) {
@@ -58,11 +53,6 @@ struct WAMRMemoryInstance {
         env->memory_data = (uint8 *)malloc(env->memory_data_size);
         memcpy(env->memory_data, memory_data.data(), env->memory_data_size);
         env->memory_data_end = env->memory_data + memory_data.size();
-        //        LOGV(DEBUG)<< "memory_data_size:" << env->memory_data_size;
-        //        for (int i = 0; i < 100; ++i) {
-        //            LOGV(DEBUG)<< "memory_data:" <<i << memory_data[i];
-        //            LOGV(DEBUG)<< "memory_data:" <<i << env->memory_data[i];
-        //        }
         env->heap_data = (uint8 *)malloc(heap_data.size());
         memcpy(env->heap_data, heap_data.data(), heap_data.size());
         env->heap_data_end = env->heap_data + heap_data.size();