Skip to content

Commit

Permalink
fix mac result
Browse files Browse the repository at this point in the history
  • Loading branch information
@victoryang00
victoryang00 committed Feb 29, 2024
1 parent ba34ef9 commit b96c877
Show file tree
Hide file tree
Showing 4 changed files with 120 additions and 124 deletions.
227 changes: 108 additions & 119 deletions artifact/bench_mac.py
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
import csv
import common_util
from multiprocessing import Pool

import numpy as np
import matplotlib.pyplot as plt
from collections import defaultdict

cmd = [
"linpack",
"llama",
"rgbd_tum",
"bfs",
"bc",
"bfs",
"cc",
Expand All @@ -18,17 +17,14 @@
"tc",
"bt",
"cg",
"ep",
"ft",
"lu",
"mg",
"sp",
"redis",
"hdastar",
]
folder = [
"linpack",
"llama",
"ORB_SLAM2",
"gapbs",
"gapbs",
Expand All @@ -38,43 +34,35 @@
"gapbs",
"gapbs",
"gapbs",
"gapbs",
"nas",
"nas",
"nas",
"nas",
"nas",
"nas",
"nas",
"redis",
"hdastar",
]
arg = [
[],
["stories110M.bin", "-z", "tokenizer.bin", "-t", "0.0"],
["./ORBvoc.txt,", "./TUM3.yaml", "./", "./associations/fr1_xyz.txt"],
["-f", "./road.sg", "-n300"],
["-g20", "-vn300"],
["-g20", "-vn300"],
["-f", "./road.sg", "-n300"],
["-g20", "-vn300"],
["-g20", "-vn300"],
["-g20", "-vn300"],
["-g20", "-vn300"],
["-g20", "-n300"],
[],
["-g20", "-vn300"],
["-g20", "-n1"],
[],
[],
[],
[],
[],
[],
[],
["maze-6404.txt", "8"],
]
envs = [
"a=b",
"OMP_NUM_THREADS=4",
"a=b",
"OMP_NUM_THREADS=4",
"OMP_NUM_THREADS=4",
Expand All @@ -90,162 +78,163 @@
"OMP_NUM_THREADS=4",
"OMP_NUM_THREADS=4",
"OMP_NUM_THREADS=4",
"OMP_NUM_THREADS=4",
"OMP_NUM_THREADS=4",
"a=b",
"a=b",
]

pool = Pool(processes=1)


def run_mvvm():
results = []
results1 = []
for _ in range(common_util.trial):
for i in range(len(cmd)):
for j in range(len(common_util.aot_variant)):
for env in common_util.list_of_arg:
aot = cmd[i] + common_util.aot_variant[j]
results1.append(
pool.apply_async(common_util.run, (aot, arg[i], env))
)
aot = cmd[i] + ".aot"
results1.append(common_util.run(aot, arg[i], envs[i]))
# print the results
results1 = [x.get() for x in results1]

for exec, output in results1:
lines = output.split("\n")
for line in lines:
if line.__contains__("Execution time:"):
exec_time = line.split(" ")[-2]
exec_time = line.split()[-2]
print(exec, exec_time)
results.append((exec , exec_time)) # discover 4 aot_variant
results.append((exec, exec_time)) # discover 4 aot_variant
return results



def run_hcontainer():
def run_native():
results = []
results1 = []
for _ in range(common_util.trial):
for env in common_util.envs:
for i in range(len(cmd)):
aot = cmd[i]
results1.append(
pool.apply_async(
common_util.run_hcontainer,
(aot, "linpack", arg[i], env),
)
)
# print the results
results1 = [x.get() for x in results1]
for exec, output in results1:
print(exec, output)
lines = output.split("\n")
for line in lines:
if line.__contains__("user"):
exec_time = float(line.split(" ")[0].replace("user", ""))
results.append((exec, exec_time))
return results

def run_qemu_x86_64():
results= []
results1 = []
for _ in range(common_util.trial):
for i in range(len(cmd)):
aot = cmd[i]
results1.append(
pool.apply_async(
common_util.run_qemu_x86_64,
(aot, "linpack", arg[i], envs[i]),
)
common_util.run_native(aot, folder[i], arg[i], envs[i]),
)
# print the results
results1 = [x.get() for x in results1]
for exec, output in results1:
print(exec, output)
lines = output.split("\n")
for line in lines:
if line.__contains__("user"):
exec_time = float(line.split(" ")[0].replace("user", ""))
try:
exec_time = float(line.split()[0].replace("user", ""))
except:
exec_time = float(line.split()[2].replace("user", ""))
print(exec, exec_time)
results.append((exec, exec_time))
return results

def run_qemu_aarch64():
results = []
results1 = []
for _ in range(common_util.trial):
for i in range(len(cmd)):
aot = cmd[i]
results1.append(
pool.apply_async(
common_util.run_qemu_aarch64,
(aot, "linpack", arg[i], envs[i]),
)
)
results1 = [x.get() for x in results1]
for exec, output in results1:
print(exec, output)
lines = output.split("\n")
for line in lines:
if line.__contains__("user"):
exec_time = float(line.split(" ")[0].replace("user", ""))
results.append((exec, exec_time))

return results


def run_native():
results = []
results1 = []
for _ in range(common_util.trial):
for i in range(len(cmd)):
aot = cmd[i]
results1.append(
pool.apply_async(
common_util.run_native,
(aot, "linpack", arg[i], envs[i]),
)
)
results1 = [x.get() for x in results1]
for exec, output in results1:
print(exec, output)
lines = output.split("\n")
for line in lines:
if line.__contains__("user"):
exec_time = float(line.split(" ")[0].replace("user", ""))
results.append((exec, exec_time))
return results

def write_to_csv(filename):
# 'data' is a list of tuples, e.g., [(checkpoint_result_0, checkpoint_result_1, restore_result_2), ...]

with open(filename, "a+", newline="") as csvfile:
writer = csv.writer(csvfile)
# Optionally write headers
writer.writerow(
["name", "mvvm", "hcontainer", "qemu_x86_64", "qemu_aach64", "native"]
)
writer.writerow(["name", "mvvm", "native"])

# Write the data
for idx, row in enumerate(mvvm_results):
writer.writerow(
[
row[0],
row[1],
hcontainer_results[idx][1],
qemu_x86_64_results[idx][1],
qemu_aarch64_results[idx][1],
native_results[idx][1],
]
)


def read_from_csv(filename):
with open(filename, "r") as csvfile:
reader = csv.reader(csvfile)
next(reader)
results = []
for row in reader:
results.append((row[0], float(row[1]), float(row[2])))
return results

# print the results
def plot(result, file_name="mac.pdf"):
workloads = defaultdict(list)
for workload, mvvm, native in result:
workloads[
workload.replace("OMP_NUM_THREADS=", "")
.replace("-g15", "")
.replace("-n300", "")
.replace(" -f ", "")
.replace("-vn300", "")
.replace("maze-6404.txt", "")
.replace("stories110M.bin", "")
.replace("-z tokenizer.bin -t 0.0", "")
.replace("ORBvoc.txt", "")
.replace("TUM3.yaml", "")
.replace("./associations/fr1_xyz.txt", "")
.replace("./", "")
.strip()
].append((mvvm, native))

# Calculate the medians and standard deviations for each workload
statistics = {}
for workload, times in workloads.items():
mvvms, native = zip(*times)
statistics[workload] = {
"mvvm_median": np.median(mvvms),
"native_median": np.median(native),
"mvvm_std": np.std(mvvms),
"native_std": np.std(native),
}
font = {"size": 14}

# using rc function
plt.rc("font", **font)
# Plotting
fig, ax = plt.subplots(figsize=(15, 7))
# Define the bar width and positions
bar_width = 0.7 / 2
index = np.arange(len(statistics))

# Plot the bars for each workload
# for i, (workload, stats) in enumerate(statistics.items()):
# ax.bar(index[i], stats['mvvm_median'], bar_width, yerr=stats['mvvm_std'], capsize=5, label=f'mvvm')
# ax.bar(index[i] + bar_width, stats['wamr_median'], bar_width, yerr=stats['wamr_std'], capsize=5, label=f'wamr')
for i, (workload, stats) in enumerate(statistics.items()):
ax.bar(
index[i],
stats["mvvm_median"],
bar_width,
yerr=stats["mvvm_std"],
capsize=5,
color="blue",
label="MVVM" if i == 0 else "",
)
ax.bar(
index[i] + bar_width,
stats["native_median"],
bar_width,
yerr=stats["native_std"],
capsize=5,
color="green",
label="Native" if i == 0 else "",
)
# Labeling and formatting
ax.set_ylabel("Time(s)")
ax.set_xticks(index + bar_width )
ticklabel = (x.replace("a=b", "") for x in list(statistics.keys()))
ax.set_xticklabels(ticklabel, fontsize=10)
ax.legend()

# Show the plot
plt.tight_layout()
# plt.show()
plt.savefig(file_name)
# %%



if __name__ == "__main__":
mvvm_results = run_mvvm()
native_results = run_native()
qemu_x86_64_results = run_qemu_x86_64()
# print the results
qemu_aarch64_results = run_qemu_aarch64()
hcontainer_results = run_hcontainer()

write_to_csv("llama_result.csv")
write_to_csv("mac.csv")
results = read_from_csv("mac.csv")
plot(results, "mac.pdf")
12 changes: 12 additions & 0 deletions artifact/result/mac.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
name,mvvm,native
a=b linpack.aot,20.722633,20.57
"a=b rgbd_tum.aot ./ORBvoc.txt, ./TUM3.yaml ./ ./associations/fr1_xyz.txt",1679.497309,6.014
OMP_NUM_THREADS=4 bc.aot -g20 -vn300,583.025857,129.39
OMP_NUM_THREADS=4 sssp.aot -g20 -vn300,726.861742,31.63
OMP_NUM_THREADS=4 bt.aot,231.107716,55.55
OMP_NUM_THREADS=4 cg.aot,87.744345,90.79
OMP_NUM_THREADS=4 ft.aot,110.019402,84.02
OMP_NUM_THREADS=4 lu.aot,0.103716,0.01
OMP_NUM_THREADS=4 mg.aot,79.522478,5.17
OMP_NUM_THREADS=4 sp.aot,46.774105,24.89
a=b redis.aot,86.2,18.78
Binary file added artifact/result/mac.pdf
View file
Binary file not shown.
5 changes: 0 additions & 5 deletions bench/hdastar_gen.py
View file

This file was deleted.

0 comments on commit b96c877

Please sign in to comment.