Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Break up main into separate files and added CPU timeline #15

Open
wants to merge 43 commits into
base: paper-2.2
Choose a base branch
from
Open
Changes from 4 commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
62bc7d5
Break up main into separate files
lauren45983 Nov 4, 2020
3320b21
Added image of graph
lauren45983 Nov 4, 2020
fb56690
Coded the trial conditions in the data so that we can compare data
lauren45983 Nov 6, 2020
3e64bc7
Moved over the inter trial analysis into their own file
lauren45983 Nov 13, 2020
16bbb30
Hotfix for missing check_qs
charmoniumQ Nov 17, 2020
150c828
added desktop sponza results
Hyjale Nov 17, 2020
4f47126
Hotfix for missing check_qs (part 2)
charmoniumQ Nov 17, 2020
cd31a71
Merge branch 'improved-script' of https://github.com/ILLIXR/results i…
Hyjale Nov 17, 2020
1ee3c6d
Merge branch 'improved-script' of https://github.com/ILLIXR/results i…
Hyjale Nov 17, 2020
e30373b
Updated README.md to be up to date with changes in results
lauren45983 Nov 18, 2020
f350a77
Merge branch 'improved-script' of https://github.com/ILLIXR/results i…
lauren45983 Nov 18, 2020
1918d91
Added documentation
charmoniumQ Nov 18, 2020
4728022
Remove ILLIXR
charmoniumQ Nov 18, 2020
88e376f
Update docs
charmoniumQ Nov 18, 2020
1158d7a
Update docs
charmoniumQ Nov 18, 2020
21c1cb1
Removed commented out code that is now in per/inter trial analysis
lauren45983 Nov 20, 2020
2e344c2
Merge branch 'improved-script' of https://github.com/ILLIXR/results i…
lauren45983 Nov 20, 2020
2ffe8c8
Added timers for duration of each function
lauren45983 Nov 20, 2020
f7751a2
Removed obsolete code
charmoniumQ Nov 20, 2020
03c3c7b
Merge branch 'improved-script' of github.com:ILLIXR/results into impr…
charmoniumQ Nov 20, 2020
566382d
Automated all graphs except power
JeffreyZh4ng Nov 21, 2020
d032989
Merge
JeffreyZh4ng Nov 21, 2020
d8bbe38
Added files that I forgot about
JeffreyZh4ng Nov 22, 2020
f813279
ISCA results v1
mhuzai Nov 22, 2020
a409855
Add missing results
mhuzai Nov 22, 2020
12e17dc
Power and MTP done
JeffreyZh4ng Nov 22, 2020
72ab5f5
Lotta new graphs
JeffreyZh4ng Nov 23, 2020
d38695b
Graph nits, pretty much finalized
JeffreyZh4ng Nov 23, 2020
d4fe726
Many more graphs and style changes
JeffreyZh4ng Nov 25, 2020
fc37b9f
Final push
JeffreyZh4ng Nov 25, 2020
9938463
Added some code comments
JeffreyZh4ng Nov 25, 2020
c4195ff
Added Named Tuple Documentation
lauren45983 Dec 2, 2020
5a493a7
Merge branch 'improved-script' of https://github.com/ILLIXR/results i…
lauren45983 Dec 2, 2020
8138c56
Merge branch 'improved-script' of github.com:ILLIXR/results into impr…
charmoniumQ Dec 2, 2020
32993a8
Moved plot graphs into inter trial analysis
lauren45983 Dec 4, 2020
e681806
Merge branch 'improved-script' of https://github.com/ILLIXR/results i…
lauren45983 Dec 4, 2020
cebf870
Ammend previous commit
lauren45983 Dec 4, 2020
7cdfc7d
Merged fps/frequency graphs into the inter_trial_analysis
charmoniumQ Dec 6, 2020
fe0eaea
Added cpu_timer3 data
charmoniumQ Dec 18, 2020
9c69790
Update README.md
charmoniumQ Dec 19, 2020
bc9f556
Update README.md
charmoniumQ Dec 19, 2020
a24f67c
Update README.md
charmoniumQ Dec 19, 2020
f2f8584
wip
charmoniumQ Apr 8, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
115 changes: 115 additions & 0 deletions analysis/inter_trial_analysis.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
from util import PerTrialData
import pandas as pd
from typing import List, Dict
from tqdm import tqdm

def analysis(trials: List[PerTrialData], replaced_names: Dict[str,str]) -> None:
populate_fps(trials, replaced_names)
populate_cpu(trials, replaced_names)
populate_gpu(trials, replaced_names)
populate_power(trials, replaced_names)
populate_mtp(trials, replaced_names)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There seems to be a lot of similar code in these populate_x functions. Try to see if there is some way of creating a generic populate function so we don't have to have so much repeating code.


def populate_fps(trials: List[PerTrialData], replaced_names: Dict[str,str]) -> None:
account_names = trials[0].ts.index.levels[0]
ignore_list = ['opencv', 'Runtime', 'camera_cvtfmt', 'app_gpu1', 'app_gpu2', 'hologram', 'timewarp_gl gpu', 'app']
account_list = [name for name in account_names if name not in ignore_list]
account_list.append('app')
account_list = [replaced_names[name] if name in replaced_names else name for name in account_list]
data_frame = pd.DataFrame()
data_frame["Components"] = account_list

for trial in tqdm(trials):
account_names = trial.ts.index.levels[0]

values = []
ignore_list = ['opencv', 'Runtime', 'camera_cvtfmt', 'app_gpu1', 'app_gpu2', 'hologram', 'timewarp_gl gpu', 'app']
for idx, name in enumerate(account_names):
if name in ignore_list:
continue

values.append(trial.summaries["period_mean"][name])
values.append(trial.summaries["period_mean"]['app'])

data_frame[trial.conditions.application + '-'+ trial.conditions.machine] = values
data_frame.to_csv('../output/fps.csv', index=False)

def populate_cpu(trials: List[PerTrialData], replaced_names: Dict[str,str]) -> None:
account_names = trials[0].ts.index.levels[0]
ignore_list = ['opencv', 'Runtime', 'camera_cvtfmt', 'app_gpu1', 'app_gpu2', 'hologram', 'timewarp_gl gpu', 'app']
account_list = [name for name in account_names if name not in ignore_list]
account_list.append('app')
account_list = [replaced_names[name] if name in replaced_names else name for name in account_list]
account_list.insert(0, "Run Name")
data_frame = pd.DataFrame([], columns=account_list)

for trial in tqdm(trials):
account_names = trial.ts.index.levels[0]

values = {"Run Name": trial.conditions.application + '-'+ trial.conditions.machine}
for idx, name in enumerate(account_names):
if name in ignore_list:
continue

formatted_name = replaced_names[name] if name in replaced_names else name
values.update({formatted_name: trial.summaries["cpu_time_duration_sum"][name]})
values.update({"Application": trial.summaries["cpu_time_duration_sum"]['app']})

data_frame = data_frame.append(values, ignore_index=True, sort=False)
# from IPython import embed; embed()

data_frame.to_csv('../output/cpu.csv', index=False)

def populate_gpu(trials: List[PerTrialData], replaced_names: Dict[str,str]) -> None:
account_names = trials[0].ts.index.levels[0]
account_list = ['app_gpu1', 'app_gpu2', 'hologram', 'timewarp_gl gpu']
account_list = [replaced_names[name] if name in replaced_names else name for name in account_list]
account_list.insert(0, "Run Name")
data_frame = pd.DataFrame([], columns=account_list)

for trial in tqdm(trials):
account_names = trial.ts.index.levels[0]

values = {"Run Name": trial.conditions.application + '-'+ trial.conditions.machine}
name_list = ['app_gpu1', 'app_gpu2', 'hologram', 'timewarp_gl gpu']
for idx, name in enumerate(name_list):

formatted_name = replaced_names[name] if name in replaced_names else name
values.update({formatted_name: trial.summaries["gpu_time_duration_sum"][name]})

data_frame = data_frame.append(values, ignore_index=True, sort=False)
# from IPython import embed; embed()

data_frame.to_csv('../output/gpu.csv', index=False)

def populate_power(trials: List[PerTrialData], replaced_names: Dict[str,str]) -> None:
account_names = trials[0].ts.index.levels[0]
account_list = ['CPU Power', 'GPU Power', 'DDR Power', 'SOC Power', 'SYS Power']
account_list.insert(0, "Run Name")
data_frame = pd.DataFrame([], columns=account_list)

for trial in tqdm(trials):
account_names = trial.ts.index.levels[0]

if len(trial.power_data) == 4:
gpu_power = trial.power_data[0]
cpu_time = trial.power_data[1]
cpu_energy = trial.power_data[2]
ddr_energy = trial.power_data[3]

cpu_power = cpu_energy / cpu_time
ddr_power = ddr_energy / cpu_time
values = {"Run Name": trial.conditions.application + '-'+ trial.conditions.machine, 'GPU Power': gpu_power, 'CPU Power': cpu_power, 'DDR Power': ddr_power}
data_frame = data_frame.append(values, ignore_index=True, sort=False)
else:
values = {"Run Name": trial.conditions.application + '-'+ trial.conditions.machine, 'GPU Power': trial.power_data[1], 'DDR Power': trial.power_data[2], 'CPU Power': trial.power_data[3], 'SOC Power': trial.power_data[4], 'SYS Power': trial.power_data[5]}
data_frame = data_frame.append(values, ignore_index=True, sort=False)

# from IPython import embed; embed()

data_frame.to_csv('../output/power.csv', index=False)

def populate_mtp(trials: List[PerTrialData], replaced_names: Dict[str,str]) -> None:
for trial in tqdm(trials):
trial.mtp.to_csv(trial.output_path / "mtp.csv", index=False)

185 changes: 31 additions & 154 deletions analysis/main.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@

import yaml
import collections
import itertools
from pathlib import Path
@@ -17,19 +17,10 @@
import pandas as pd
import numpy as np
import subprocess

T = TypeVar("T")
V = TypeVar("V")
def it_concat(its: Iterable[Iterable[T]]) -> Iterable[T]:
return itertools.chain.from_iterable(its)

def list_concat(lsts: Iterable[List[T]]) -> List[T]:
return list(itertools.chain.from_iterable(lsts))

def dict_concat(dcts: Iterable[Dict[T, V]], **kwargs) -> Dict[T, V]:
return dict(it_concat(dct.items() for dct in dcts), **kwargs)

from util import list_concat, it_concat, dict_concat, is_int, is_float, TrialConditions, PerTrialData
verify_integrity = True
from per_trial_analysis import analysis as per_trial_analysis
from inter_trial_analysis import analysis as inter_trial_analysis

def read_illixr_table(metrics_path: Path, table_name: str, index_cols: List[str]) -> pd.DataFrame:
db_path = metrics_path / (table_name + ".sqlite")
@@ -41,12 +32,6 @@ def read_illixr_table(metrics_path: Path, table_name: str, index_cols: List[str]
.sort_index()
)

def is_int(string: str) -> bool:
return bool(re.match(r"^\d+$", string))

def is_float(string: str) -> bool:
return bool(re.match(r"^\d+.\d+(e[+-]\d+)?$", string))

def read_illixr_csv(metrics_path: Path, record_name: str, index_cols: List[str], other_cols: List[str]) -> pd.DataFrame:
cols = index_cols + other_cols
pattern = re.compile(
@@ -598,138 +583,16 @@ def get_data_cached(metrics_path: Path) -> Tuple[Any]:
'hologram': 'Hologram',
}

def populate_fps(data_frame, name_list, csv_name):
metrics_path = Path("..") / f"{name_list[0]}"
ts, summaries, switchboard_topic_stop, thread_ids, warnings_log, power_data, m2p = get_data_cached(metrics_path)
account_names = ts.index.levels[0]
ignore_list = ['opencv', 'Runtime', 'camera_cvtfmt', 'app_gpu1', 'app_gpu2', 'hologram', 'timewarp_gl gpu', 'app']
account_list = [name for name in account_names if name not in ignore_list]
account_list.append('app')
account_list = [replaced_names[name] if name in replaced_names else name for name in account_list]
data_frame["Components"] = account_list

for run_name in tqdm(name_list):
metrics_path = Path("..") / f"{run_name}"
ts, summaries, switchboard_topic_stop, thread_ids, warnings_log, power_data, m2p = get_data_cached(metrics_path)
account_names = ts.index.levels[0]

values = []
ignore_list = ['opencv', 'Runtime', 'camera_cvtfmt', 'app_gpu1', 'app_gpu2', 'hologram', 'timewarp_gl gpu', 'app']
for idx, name in enumerate(account_names):
if name in ignore_list:
continue

values.append(summaries["period_mean"][name])
values.append(summaries["period_mean"]['app'])

data_frame[run_name] = values
data_frame.to_csv(csv_name, index=False)


def populate_cpu(data_frame, name_list, csv_name):
metrics_path = Path("..") / f"{name_list[0]}"
ts, summaries, switchboard_topic_stop, thread_ids, warnings_log, power_data, m2p = get_data_cached(metrics_path)
account_names = ts.index.levels[0]
ignore_list = ['opencv', 'Runtime', 'camera_cvtfmt', 'app_gpu1', 'app_gpu2', 'hologram', 'timewarp_gl gpu', 'app']
account_list = [name for name in account_names if name not in ignore_list]
account_list.append('app')
account_list = [replaced_names[name] if name in replaced_names else name for name in account_list]
account_list.insert(0, "Run Name")
data_frame = pd.DataFrame([], columns=account_list)

for run_name in tqdm(name_list):
metrics_path = Path("..") / f"{run_name}"
ts, summaries, switchboard_topic_stop, thread_ids, warnings_log, power_data, m2p = get_data_cached(metrics_path)
account_names = ts.index.levels[0]

values = {"Run Name": run_name}
ignore_list = ['opencv', 'Runtime', 'camera_cvtfmt', 'app_gpu1', 'app_gpu2', 'hologram', 'timewarp_gl gpu', 'app']
for idx, name in enumerate(account_names):
if name in ignore_list:
continue

formatted_name = replaced_names[name] if name in replaced_names else name
values.update({formatted_name: summaries["cpu_time_duration_sum"][name]})
values.update({"Application": summaries["cpu_time_duration_sum"]['app']})

data_frame = data_frame.append(values, ignore_index=True, sort=False)
# from IPython import embed; embed()

data_frame.to_csv(csv_name, index=False)

# Components on the X
# Each run on the Y
#populate_cpu(cpu_spreadsheet, sponza_list + materials_list + platformer_list + demo_list, "cpu_spreadsheet.csv")

def populate_gpu(data_frame, name_list, csv_name):
metrics_path = Path("..") / f"{name_list[0]}"
ts, summaries, switchboard_topic_stop, thread_ids, warnings_log, power_data, m2p = get_data_cached(metrics_path)
account_names = ts.index.levels[0]
account_list = ['app_gpu1', 'app_gpu2', 'hologram', 'timewarp_gl gpu']
account_list = [replaced_names[name] if name in replaced_names else name for name in account_list]
account_list.insert(0, "Run Name")
data_frame = pd.DataFrame([], columns=account_list)

for run_name in tqdm(name_list):
metrics_path = Path("..") / f"{run_name}"
ts, summaries, switchboard_topic_stop, thread_ids, warnings_log, power_data, m2p = get_data_cached(metrics_path)
account_names = ts.index.levels[0]

values = {"Run Name": run_name}
name_list = ['app_gpu1', 'app_gpu2', 'hologram', 'timewarp_gl gpu']
for idx, name in enumerate(name_list):

formatted_name = replaced_names[name] if name in replaced_names else name
values.update({formatted_name: summaries["gpu_time_duration_sum"][name]})

data_frame = data_frame.append(values, ignore_index=True, sort=False)
# from IPython import embed; embed()

data_frame.to_csv(csv_name, index=False)

# Components on the X
# Each run on the Y
#populate_gpu(gpu_spreadsheet, sponza_list + materials_list + platformer_list + demo_list, "gpu_spreadsheet.csv")

def populate_power(data_frame, name_list, csv_name):
metrics_path = Path("..") / f"{name_list[0]}"
ts, summaries, switchboard_topic_stop, thread_ids, warnings_log, power_data, m2p = get_data_cached(metrics_path)
account_names = ts.index.levels[0]
account_list = ['CPU Power', 'GPU Power', 'DDR Power', 'SOC Power', 'SYS Power']
account_list.insert(0, "Run Name")
data_frame = pd.DataFrame([], columns=account_list)

for run_name in tqdm(name_list):
metrics_path = Path("..") / f"{run_name}"
ts, summaries, switchboard_topic_stop, thread_ids, warnings_log, power_data, m2p = get_data_cached(metrics_path)
account_names = ts.index.levels[0]

if len(power_data) == 4:
gpu_power = power_data[0]
cpu_time = power_data[1]
cpu_energy = power_data[2]
ddr_energy = power_data[3]

cpu_power = cpu_energy / cpu_time
ddr_power = ddr_energy / cpu_time
values = {"Run Name": run_name, 'GPU Power': gpu_power, 'CPU Power': cpu_power, 'DDR Power': ddr_power}
data_frame = data_frame.append(values, ignore_index=True, sort=False)
else:
values = {"Run Name": run_name, 'GPU Power': power_data[1], 'DDR Power': power_data[2], 'CPU Power': power_data[3], 'SOC Power': power_data[4], 'SYS Power': power_data[5]}
data_frame = data_frame.append(values, ignore_index=True, sort=False)

# from IPython import embed; embed()

data_frame.to_csv(csv_name, index=False)

#populate_power(power_spreadsheet, sponza_list + materials_list + platformer_list + demo_list, "power_spreadsheet.csv")

def populate_mtp(name_list: List[str]) -> None:
for run_name in tqdm(name_list):
metrics_path = Path("..") / f"{run_name}"
ts, summaries, switchboard_topic_stop, thread_ids, warnings_log, power_data, mtp = get_data_cached(metrics_path)
mtp.to_csv(metrics_path / f"{run_name}-mtp.csv", index=False)

#populate_mtp(sponza_list + materials_list + platformer_list + demo_list)


@@ -759,21 +622,20 @@ def write_graphs(
ax.legend()
ax.set_title("Wall-Time Duration by Component")
fig.savefig(metrics_path / "wall_time_durations.png")
#populate_fps(fps_spreadsheet_sponza, sponza_list, "sponza_fps.csv")
#populate_fps(fps_spreadsheet_materials, materials_list, "materials_fps.csv")
#populate_fps(fps_spreadsheet_platformer, platformer_list, "platformer_fps.csv")
#populate_fps(fps_spreadsheet_demo, demo_list, "demo_fps.csv")

populate_fps(fps_spreadsheet_sponza, sponza_list, "sponza_fps.csv")
populate_fps(fps_spreadsheet_materials, materials_list, "materials_fps.csv")
populate_fps(fps_spreadsheet_platformer, platformer_list, "platformer_fps.csv")
populate_fps(fps_spreadsheet_demo, demo_list, "demo_fps.csv")

populate_cpu(cpu_spreadsheet, sponza_list + materials_list + platformer_list + demo_list, "cpu_spreadsheet.csv")
populate_gpu(gpu_spreadsheet, sponza_list + materials_list + platformer_list + demo_list, "gpu_spreadsheet.csv")
populate_power(power_spreadsheet, sponza_list + materials_list + platformer_list + demo_list, "power_spreadsheet.csv")
populate_mtp(sponza_list + materials_list + platformer_list + demo_list)
#populate_cpu(cpu_spreadsheet, sponza_list + materials_list + platformer_list + demo_list, "cpu_spreadsheet.csv")
#populate_gpu(gpu_spreadsheet, sponza_list + materials_list + platformer_list + demo_list, "gpu_spreadsheet.csv")
#populate_power(power_spreadsheet, sponza_list + materials_list + platformer_list + demo_list, "power_spreadsheet.csv")
#populate_mtp(sponza_list + materials_list + platformer_list + demo_list)

write_graphs(
sponza_list + materials_list + platformer_list + demo_list,
['opencv', 'Runtime', 'camera_cvtfmt', 'app_gpu1', 'app_gpu2', 'hologram', 'timewarp_gl gpu'],
)
#write_graphs(
# sponza_list + materials_list + platformer_list + demo_list,
# ['opencv', 'Runtime', 'camera_cvtfmt', 'app_gpu1', 'app_gpu2', 'hologram', 'timewarp_gl gpu'],
#)

# # Stacked Energy Graphs
# if len(power_data) == 3:
@@ -897,3 +759,18 @@ def write_graphs(

# # import IPython; IPython.embed()
# # print(summaries["cpu_time_duration_sum"].to_csv())

trials: List[PerTrialData] = []
for metrics_path in Path("../metrics").iterdir():
with (metrics_path / "trial_conditions.yaml").open() as f:
conditions: Dict[str, str] = yaml.safe_load(f)
conditions_obj = TrialConditions(**conditions)
ts, summaries, switchboard_topic_stop, thread_ids, warnings_log, power_data, mtp = get_data(metrics_path)
output_path = Path("../output") / metrics_path.name
output_path.mkdir(exist_ok=True, parents=True)
trial = PerTrialData(ts = ts, summaries = summaries, thread_ids = thread_ids, output_path = output_path, switchboard_topic_stop = switchboard_topic_stop, mtp = mtp, warnings_log = warnings_log, conditions = conditions_obj, power_data = power_data)
trials.append(trial)
per_trial_analysis(trial)
inter_trial_analysis(trials, replaced_names)


Loading