diff --git a/artifact/bench_windows.py b/artifact/bench_windows.py index 6a58409..ae6c258 100644 --- a/artifact/bench_windows.py +++ b/artifact/bench_windows.py @@ -3,6 +3,7 @@ import numpy as np import matplotlib.pyplot as plt from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor, TimeoutError cmd = [ "linpack", @@ -33,7 +34,7 @@ arg = [ [], ["stories110M.bin", "-z", "tokenizer.bin", "-t", "0.0"], - ["./ORBvoc.txt,", "./TUM3.yaml", "./", "./associations/fr1_xyz.txt"], + ["./ORBvoc.txt", "./TUM3.yaml", "./", "./associations/fr1_xyz.txt"], [], [], [], @@ -71,25 +72,7 @@ def run_mvvm(): lines = output.split("\n") for line in lines: if line.__contains__("Execution time:"): - exec_time = line.split(" ")[-2] - print(exec, exec_time) - results.append((exec, exec_time)) # discover 4 aot_variant - return results - - -def run_wamr(): - results = [] - results1 = [] - for _ in range(common_util.trial): - for i in range(len(cmd)): - aot = cmd[i] + "-pure.aot" - results1.append(common_util.run(aot, arg[i], envs[i])) - # print the results - for exec, output in results1: - lines = output.split("\n") - for line in lines: - if line.__contains__("Execution time:"): - exec_time = line.split(" ")[-2] + exec_time = line.split()[-2] print(exec, exec_time) results.append((exec, exec_time)) # discover 4 aot_variant return results @@ -101,25 +84,39 @@ def run_native(): for _ in range(common_util.trial): for i in range(len(cmd)): aot = cmd[i] - results1.append( - common_util.run_native(aot, folder[i], arg[i], envs[i]), - ) + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(lambda: results.append(common_util.run_native(aot, folder[i], arg[i], envs[i]))) + try: + result = future.result(timeout=100) # 5 seconds timeout + except TimeoutError: + print("Operation timed out") + # results1.append( + # common_util.run_native(aot, folder[i], arg[i], envs[i]), + # ) for exec, output in results1: print(exec, output) lines = output.split("\n") for line in lines: if line.__contains__("elapsed"): - # Split the time string into minutes, seconds, and milliseconds - minutes, seconds = line.split(" ")[2].replace("elapsed", "").split(":") - seconds, milliseconds = seconds.split(".") - - # Convert each part to seconds (note that milliseconds are converted and added as a fraction of a second) - total_seconds = ( - int(minutes) * 60 + int(seconds) + int(milliseconds) / 1000 - ) - - print(total_seconds) - exec_time = total_seconds + try: + minutes, seconds = line.split()[2].replace("elapsed", "").split(":") + seconds, milliseconds = seconds.split(".") + + # Convert each part to seconds (note that milliseconds are converted and added as a fraction of a second) + total_seconds = ( + int(minutes) * 60 + int(seconds) + int(milliseconds) / 1000 + ) + + print(total_seconds) + exec_time = total_seconds + except: + try: + from datetime import datetime + + time_object = datetime.strptime(line.split()[2].replace("elapsed", ""), "%H:%M:%S").time() + print(time_object) + except: + exec_time = float(line.split()[0].replace("user", "")) results.append((exec, exec_time)) return results @@ -130,7 +127,7 @@ def write_to_csv(filename): with open(filename, "a+", newline="") as csvfile: writer = csv.writer(csvfile) # Optionally write headers - writer.writerow(["name", "mvvm", "wamr", "native"]) + writer.writerow(["name", "mvvm", "native"]) # Write the data for idx, row in enumerate(mvvm_results): @@ -138,7 +135,6 @@ def write_to_csv(filename): [ row[0], row[1], - wamr_results[idx][1], native_results[idx][1], ] ) @@ -150,14 +146,14 @@ def read_from_csv(filename): next(reader) results = [] for row in reader: - results.append((row[0], float(row[1]), float(row[2]), float(row[3]))) + results.append((row[0], float(row[1]), float(row[2]))) return results # print the results def plot(result, file_name="windows.pdf"): workloads = defaultdict(list) - for workload, mvvm, wamr, native in result: + for workload, mvvm, native in result: workloads[ workload.replace("OMP_NUM_THREADS=", "") .replace("-g15", "") @@ -172,18 +168,16 @@ def plot(result, file_name="windows.pdf"): .replace("./associations/fr1_xyz.txt", "") .replace("./", "") .strip() - ].append((mvvm, wamr, native)) + ].append((mvvm, native)) # Calculate the medians and standard deviations for each workload statistics = {} for workload, times in workloads.items(): - mvvms, wamr,native = zip(*times) + mvvms, native = zip(*times) statistics[workload] = { "mvvm_median": np.median(mvvms), - "wamr_median": np.median(wamr), "native_median": np.median(native), "mvvm_std": np.std(mvvms), - "wamr_std": np.std(wamr), "native_std": np.std(native), } font = {"size": 14} @@ -193,7 +187,7 @@ def plot(result, file_name="windows.pdf"): # Plotting fig, ax = plt.subplots(figsize=(15, 7)) # Define the bar width and positions - bar_width = 0.7 / 3 + bar_width = 0.7 /2 index = np.arange(len(statistics)) # Plot the bars for each workload @@ -212,15 +206,6 @@ def plot(result, file_name="windows.pdf"): ) ax.bar( index[i] + bar_width, - stats["wamr_median"], - bar_width, - yerr=stats["wamr_std"], - capsize=5, - color="red", - label="WAMR" if i == 0 else "", - ) - ax.bar( - index[i] + 2 * bar_width, stats["native_median"], bar_width, yerr=stats["native_std"], @@ -230,7 +215,7 @@ def plot(result, file_name="windows.pdf"): ) # Labeling and formatting ax.set_ylabel("Time(s)") - ax.set_xticks(index + bar_width / 2) + ax.set_xticks(index + bar_width ) ticklabel = (x.replace("a=b", "") for x in list(statistics.keys())) ax.set_xticklabels(ticklabel, fontsize=10) ax.legend() @@ -243,13 +228,13 @@ def plot(result, file_name="windows.pdf"): if __name__ == "__main__": - mvvm_results = run_mvvm() - wamr_results = run_wamr() - native_results = run_native() + # mvvm_results = run_mvvm() + # wamr_results = run_wamr() + # native_results = run_native() # print the results - print(mvvm_results) - print(wamr_results) - print(native_results) + mvvm_results=run_mvvm() + # wamr_results=run_wamr() + native_results=run_native() write_to_csv("windows.csv") - # results = read_from_csv("windows.csv") - # plot(results) + results = read_from_csv("windows.csv") + plot(results) diff --git a/artifact/result/ckpt_restore_latency_whole.pdf b/artifact/result/ckpt_restore_latency_whole.pdf index 2e3bec2..51a2842 100644 Binary files a/artifact/result/ckpt_restore_latency_whole.pdf and b/artifact/result/ckpt_restore_latency_whole.pdf differ diff --git a/artifact/result/latency.pdf b/artifact/result/latency.pdf deleted file mode 100644 index 45daf3f..0000000 Binary files a/artifact/result/latency.pdf and /dev/null differ diff --git a/artifact/result/windows.csv b/artifact/result/windows.csv index 0434222..817eea9 100644 --- a/artifact/result/windows.csv +++ b/artifact/result/windows.csv @@ -1,111 +1,81 @@ -name,mvvm,wamr,native -a=b linpack.aot,65.257584,23.685475,29.01 -OMP_NUM_THREADS=1 llama.aot stories110M.bin -z tokenizer.bin -t 0.0,48.162543,23.685475,13.02 -"a=b rgbd_tum.aot ./ORBvoc.txt, ./TUM3.yaml ./ ./associations/fr1_xyz.txt",872.773619,23.685475,0.008 -OMP_NUM_THREADS=1 bt.aot,872.773619,23.685475,31.071 -OMP_NUM_THREADS=1 cg.aot,22.757516,23.685475,12.033 -OMP_NUM_THREADS=1 ep.aot,0.000539,23.685475,0.004 -OMP_NUM_THREADS=1 ft.aot,31.111255,23.685475,31.005 -OMP_NUM_THREADS=1 lu.aot,0.024766,23.685475,0.004 -OMP_NUM_THREADS=1 mg.aot,20.22196,23.685475,10.052 -OMP_NUM_THREADS=1 sp.aot,20.22196,23.685475,0.06 -a=b redis.aot,20.22196,23.685475,0.004 -a=b linpack.aot,21.222971,23.071749,29.094 -OMP_NUM_THREADS=1 llama.aot stories110M.bin -z tokenizer.bin -t 0.0,21.222971,23.071749,13.023 -"a=b rgbd_tum.aot ./ORBvoc.txt, ./TUM3.yaml ./ ./associations/fr1_xyz.txt",825.783301,23.071749,0.004 -OMP_NUM_THREADS=1 bt.aot,825.783301,23.071749,31.049 -OMP_NUM_THREADS=1 cg.aot,23.273768,23.071749,12.012 -OMP_NUM_THREADS=1 ep.aot,0.000604,23.071749,0.002 -OMP_NUM_THREADS=1 ft.aot,34.869603,23.071749,29.077 -OMP_NUM_THREADS=1 lu.aot,0.027421,23.071749,0.003 -OMP_NUM_THREADS=1 mg.aot,21.568194,23.071749,10.008 -OMP_NUM_THREADS=1 sp.aot,21.568194,23.071749,0.059 -a=b redis.aot,21.568194,23.071749,0.002 -a=b linpack.aot,24.285947,22.907635,29.091 -OMP_NUM_THREADS=1 llama.aot stories110M.bin -z tokenizer.bin -t 0.0,24.285947,22.907635,13.009 -"a=b rgbd_tum.aot ./ORBvoc.txt, ./TUM3.yaml ./ ./associations/fr1_xyz.txt",861.7128,22.907635,0.004 -OMP_NUM_THREADS=1 bt.aot,861.7128,22.907635,31.005 -OMP_NUM_THREADS=1 cg.aot,24.943927,22.907635,11.097 -OMP_NUM_THREADS=1 ep.aot,0.000552,22.907635,0.002 -OMP_NUM_THREADS=1 ft.aot,35.43775,22.907635,30.044 -OMP_NUM_THREADS=1 lu.aot,0.024445,22.907635,0.003 -OMP_NUM_THREADS=1 mg.aot,21.484104,22.907635,10.008 -OMP_NUM_THREADS=1 sp.aot,21.484104,22.907635,0.057 -a=b redis.aot,21.484104,22.907635,0.002 -a=b linpack.aot,21.571797,23.432488,28.08 -OMP_NUM_THREADS=1 llama.aot stories110M.bin -z tokenizer.bin -t 0.0,21.571797,23.432488,13.009 -"a=b rgbd_tum.aot ./ORBvoc.txt, ./TUM3.yaml ./ ./associations/fr1_xyz.txt",852.130692,23.432488,0.004 -OMP_NUM_THREADS=1 bt.aot,852.130692,23.432488,31.013 -OMP_NUM_THREADS=1 cg.aot,24.996387,23.432488,12.023 -OMP_NUM_THREADS=1 ep.aot,0.00054,23.432488,0.002 -OMP_NUM_THREADS=1 ft.aot,34.067451,23.432488,29.015 -OMP_NUM_THREADS=1 lu.aot,0.029147,23.432488,0.003 -OMP_NUM_THREADS=1 mg.aot,21.092533,23.432488,10.02 -OMP_NUM_THREADS=1 sp.aot,21.092533,23.432488,0.058 -a=b redis.aot,21.092533,23.432488,0.002 -a=b linpack.aot,21.317692,22.04948,29.099 -OMP_NUM_THREADS=1 llama.aot stories110M.bin -z tokenizer.bin -t 0.0,21.317692,22.04948,13.014 -"a=b rgbd_tum.aot ./ORBvoc.txt, ./TUM3.yaml ./ ./associations/fr1_xyz.txt",855.494118,22.04948,0.004 -OMP_NUM_THREADS=1 bt.aot,855.494118,22.04948,32.011 -OMP_NUM_THREADS=1 cg.aot,24.755341,22.04948,11.085 -OMP_NUM_THREADS=1 ep.aot,0.000491,22.04948,0.002 -OMP_NUM_THREADS=1 ft.aot,33.525148,22.04948,29.02 -OMP_NUM_THREADS=1 lu.aot,0.023048,22.04948,0.003 -OMP_NUM_THREADS=1 mg.aot,20.931011,22.04948,9.081 -OMP_NUM_THREADS=1 sp.aot,20.931011,22.04948,0.061 -a=b redis.aot,20.931011,22.04948,0.002 -a=b linpack.aot,21.744898,22.17191,30.086 -OMP_NUM_THREADS=1 llama.aot stories110M.bin -z tokenizer.bin -t 0.0,21.744898,22.17191,13.074 -"a=b rgbd_tum.aot ./ORBvoc.txt, ./TUM3.yaml ./ ./associations/fr1_xyz.txt",842.886785,22.17191,0.004 -OMP_NUM_THREADS=1 bt.aot,842.886785,22.17191,30.073 -OMP_NUM_THREADS=1 cg.aot,24.598304,22.17191,12.029 -OMP_NUM_THREADS=1 ep.aot,0.000423,22.17191,0.002 -OMP_NUM_THREADS=1 ft.aot,33.236581,22.17191,29.073 -OMP_NUM_THREADS=1 lu.aot,0.025789,22.17191,0.003 -OMP_NUM_THREADS=1 mg.aot,21.231102,22.17191,10.051 -OMP_NUM_THREADS=1 sp.aot,21.231102,22.17191,0.059 -a=b redis.aot,21.231102,22.17191,0.002 -a=b linpack.aot,21.170858,21.879022,30.022 -OMP_NUM_THREADS=1 llama.aot stories110M.bin -z tokenizer.bin -t 0.0,21.170858,21.879022,13.035 -"a=b rgbd_tum.aot ./ORBvoc.txt, ./TUM3.yaml ./ ./associations/fr1_xyz.txt",841.635191,21.879022,0.004 -OMP_NUM_THREADS=1 bt.aot,841.635191,21.879022,31.097 -OMP_NUM_THREADS=1 cg.aot,24.747761,21.879022,12.036 -OMP_NUM_THREADS=1 ep.aot,0.000422,21.879022,0.002 -OMP_NUM_THREADS=1 ft.aot,33.843813,21.879022,29.097 -OMP_NUM_THREADS=1 lu.aot,0.02354,21.879022,0.003 -OMP_NUM_THREADS=1 mg.aot,21.320099,21.879022,10.008 -OMP_NUM_THREADS=1 sp.aot,21.320099,21.879022,0.057 -a=b redis.aot,21.320099,21.879022,0.002 -a=b linpack.aot,22.022136,21.401152,28.093 -OMP_NUM_THREADS=1 llama.aot stories110M.bin -z tokenizer.bin -t 0.0,22.022136,21.401152,13.033 -"a=b rgbd_tum.aot ./ORBvoc.txt, ./TUM3.yaml ./ ./associations/fr1_xyz.txt",849.786345,21.401152,0.004 -OMP_NUM_THREADS=1 bt.aot,849.786345,21.401152,30.032 -OMP_NUM_THREADS=1 cg.aot,24.168857,21.401152,11.067 -OMP_NUM_THREADS=1 ep.aot,0.000439,21.401152,0.002 -OMP_NUM_THREADS=1 ft.aot,34.130708,21.401152,27.094 -OMP_NUM_THREADS=1 lu.aot,0.026111,21.401152,0.003 -OMP_NUM_THREADS=1 mg.aot,22.90108,21.401152,9.086 -OMP_NUM_THREADS=1 sp.aot,22.90108,21.401152,0.056 -a=b redis.aot,22.90108,21.401152,0.002 -a=b linpack.aot,23.213674,21.673556,28.047 -OMP_NUM_THREADS=1 llama.aot stories110M.bin -z tokenizer.bin -t 0.0,23.213674,21.673556,12.089 -"a=b rgbd_tum.aot ./ORBvoc.txt, ./TUM3.yaml ./ ./associations/fr1_xyz.txt",857.563863,21.673556,0.004 -OMP_NUM_THREADS=1 bt.aot,857.563863,21.673556,29.03 -OMP_NUM_THREADS=1 cg.aot,25.684547,21.673556,11.056 -OMP_NUM_THREADS=1 ep.aot,0.000533,21.673556,0.002 -OMP_NUM_THREADS=1 ft.aot,36.856358,21.673556,27.056 -OMP_NUM_THREADS=1 lu.aot,0.025263,21.673556,0.003 -OMP_NUM_THREADS=1 mg.aot,21.693797,21.673556,10.053 -OMP_NUM_THREADS=1 sp.aot,21.693797,21.673556,0.06 -a=b redis.aot,21.693797,21.673556,0.002 -a=b linpack.aot,23.564979,23.490303,29.065 -OMP_NUM_THREADS=1 llama.aot stories110M.bin -z tokenizer.bin -t 0.0,23.564979,23.490303,13.028 -"a=b rgbd_tum.aot ./ORBvoc.txt, ./TUM3.yaml ./ ./associations/fr1_xyz.txt",865.143924,23.490303,0.004 -OMP_NUM_THREADS=1 bt.aot,865.143924,23.490303,30.027 -OMP_NUM_THREADS=1 cg.aot,25.359335,23.490303,12.018 -OMP_NUM_THREADS=1 ep.aot,0.000653,23.490303,0.002 -OMP_NUM_THREADS=1 ft.aot,35.618984,23.490303,27.087 -OMP_NUM_THREADS=1 lu.aot,0.029986,23.490303,0.003 -OMP_NUM_THREADS=1 mg.aot,22.961563,23.490303,10.016 -OMP_NUM_THREADS=1 sp.aot,22.961563,23.490303,0.06 -a=b redis.aot,22.961563,23.490303,0.002 +name,mvvm,native +a=b linpack.aot,36.875728,25.089 +a=b rgbd_tum.aot ./ORBvoc.txt ./TUM3.yaml ./ ./associations/fr1_xyz.txt,760.961321,12.049 +OMP_NUM_THREADS=1 cg.aot,20.553871,96.003 +OMP_NUM_THREADS=1 ep.aot,0.000519,28.055 +OMP_NUM_THREADS=1 ft.aot,27.5285,10.057 +OMP_NUM_THREADS=1 lu.aot,0.023669,0.002 +OMP_NUM_THREADS=1 mg.aot,18.439463,24.097 +a=b redis.aot,34.938116,0.003 +a=b linpack.aot,36.463317,9.017 +a=b rgbd_tum.aot ./ORBvoc.txt ./TUM3.yaml ./ ./associations/fr1_xyz.txt,742.233683,0.051 +OMP_NUM_THREADS=1 cg.aot,21.159921,0.051 +OMP_NUM_THREADS=1 ep.aot,0.000514,26.001 +OMP_NUM_THREADS=1 ft.aot,27.590321,12.054 +OMP_NUM_THREADS=1 lu.aot,0.024134,97.0 +OMP_NUM_THREADS=1 mg.aot,18.545828,29.005 +a=b redis.aot,34.812165,10.071 +a=b linpack.aot,36.913629,0.002 +a=b rgbd_tum.aot ./ORBvoc.txt ./TUM3.yaml ./ ./associations/fr1_xyz.txt,748.680853,25.043 +OMP_NUM_THREADS=1 cg.aot,20.662478,0.003 +OMP_NUM_THREADS=1 ep.aot,0.000424,9.002 +OMP_NUM_THREADS=1 ft.aot,27.951032,0.051 +OMP_NUM_THREADS=1 lu.aot,0.022966,0.051 +OMP_NUM_THREADS=1 mg.aot,18.514203,25.003 +a=b redis.aot,34.982459,12.043 +a=b linpack.aot,36.834715,88.071 +a=b rgbd_tum.aot ./ORBvoc.txt ./TUM3.yaml ./ ./associations/fr1_xyz.txt,755.615254,27.05 +OMP_NUM_THREADS=1 cg.aot,20.635818,10.048 +OMP_NUM_THREADS=1 ep.aot,0.000407,0.003 +OMP_NUM_THREADS=1 ft.aot,27.45233,23.062 +OMP_NUM_THREADS=1 lu.aot,0.023021,0.003 +OMP_NUM_THREADS=1 mg.aot,18.777736,9.004 +a=b redis.aot,34.910377,0.053 +a=b linpack.aot,36.670241,0.053 +a=b rgbd_tum.aot ./ORBvoc.txt ./TUM3.yaml ./ ./associations/fr1_xyz.txt,754.313282,25.04 +OMP_NUM_THREADS=1 cg.aot,20.773552,12.042 +OMP_NUM_THREADS=1 ep.aot,0.000427,87.076 +OMP_NUM_THREADS=1 ft.aot,27.558167,27.045 +OMP_NUM_THREADS=1 lu.aot,0.023717,10.042 +OMP_NUM_THREADS=1 mg.aot,18.574712,0.002 +a=b redis.aot,34.847002,23.079 +a=b linpack.aot,37.00937,0.003 +a=b rgbd_tum.aot ./ORBvoc.txt ./TUM3.yaml ./ ./associations/fr1_xyz.txt,799.227787,9.003 +OMP_NUM_THREADS=1 cg.aot,24.116978,0.052 +OMP_NUM_THREADS=1 ep.aot,0.000501,0.052 +OMP_NUM_THREADS=1 ft.aot,28.097015,25.016 +OMP_NUM_THREADS=1 lu.aot,0.022781,12.041 +OMP_NUM_THREADS=1 mg.aot,18.495219,89.088 +a=b redis.aot,35.798685,27.058 +a=b linpack.aot,36.814429,10.046 +a=b rgbd_tum.aot ./ORBvoc.txt ./TUM3.yaml ./ ./associations/fr1_xyz.txt,743.609241,0.002 +OMP_NUM_THREADS=1 cg.aot,21.472738,24.009 +OMP_NUM_THREADS=1 ep.aot,0.000507,0.003 +OMP_NUM_THREADS=1 ft.aot,27.901536,9.011 +OMP_NUM_THREADS=1 lu.aot,0.022625,0.052 +OMP_NUM_THREADS=1 mg.aot,18.491945,0.052 +a=b redis.aot,35.126543,26.035 +a=b linpack.aot,37.132155,12.028 +a=b rgbd_tum.aot ./ORBvoc.txt ./TUM3.yaml ./ ./associations/fr1_xyz.txt,762.204546,83.044 +OMP_NUM_THREADS=1 cg.aot,23.295101,27.082 +OMP_NUM_THREADS=1 ep.aot,0.000545,10.056 +OMP_NUM_THREADS=1 ft.aot,37.508713,0.002 +OMP_NUM_THREADS=1 lu.aot,0.024291,27.004 +OMP_NUM_THREADS=1 mg.aot,19.495638,0.003 +a=b redis.aot,36.892017,9.007 +a=b linpack.aot,19.805608,0.052 +a=b rgbd_tum.aot ./ORBvoc.txt ./TUM3.yaml ./ ./associations/fr1_xyz.txt,785.371629,7.09 +OMP_NUM_THREADS=1 cg.aot,20.722234,25.097 +OMP_NUM_THREADS=1 ep.aot,0.000427,12.047 +OMP_NUM_THREADS=1 ft.aot,27.942502,86.088 +OMP_NUM_THREADS=1 lu.aot,0.023177,27.04 +OMP_NUM_THREADS=1 mg.aot,18.482471,10.044 +a=b redis.aot,35.381083,0.002 +a=b linpack.aot,37.148041,24.065 +a=b rgbd_tum.aot ./ORBvoc.txt ./TUM3.yaml ./ ./associations/fr1_xyz.txt,757.206547,0.003 +OMP_NUM_THREADS=1 cg.aot,20.886888,9.018 +OMP_NUM_THREADS=1 ep.aot,0.000421,0.053 +OMP_NUM_THREADS=1 ft.aot,27.930926,0.017 +OMP_NUM_THREADS=1 lu.aot,0.022977,25.085 +OMP_NUM_THREADS=1 mg.aot,18.825622,12.038 +a=b redis.aot,37.090249,86.066 diff --git a/artifact/result/windows.pdf b/artifact/result/windows.pdf index 28e7d03..f2f8e9c 100644 Binary files a/artifact/result/windows.pdf and b/artifact/result/windows.pdf differ