From c56c375340c10b3f3bb74a582f8bf0b02da3ac09 Mon Sep 17 00:00:00 2001 From: Samuli Leivo Date: Fri, 8 Nov 2024 08:46:59 +0200 Subject: [PATCH] Add keywords and tools for power measurement Prevent power measurement errors failing any tests Signed-off-by: Samuli Leivo --- Robot-Framework/config/variables.robot | 2 + Robot-Framework/lib/parse_power_data.py | 48 +++++++ .../resources/power_meas_keywords.resource | 121 ++++++++++++++++++ .../test-suites/gui-tests/__init__.robot | 3 +- .../performance-tests/performance.robot | 5 +- flake.lock | 2 +- 6 files changed, 177 insertions(+), 4 deletions(-) create mode 100644 Robot-Framework/lib/parse_power_data.py create mode 100644 Robot-Framework/resources/power_meas_keywords.resource diff --git a/Robot-Framework/config/variables.robot b/Robot-Framework/config/variables.robot index 7badfbe7..986b98cb 100644 --- a/Robot-Framework/config/variables.robot +++ b/Robot-Framework/config/variables.robot @@ -44,6 +44,8 @@ Set Variables Set Global Variable ${ADMIN_VM} admin-vm Set Global Variable @{VMS} ${GUI_VM} ${CHROME_VM} ${GALA_VM} ${ZATHURA_VM} ${COMMS_VM} ${BUSINESS_VM} ${ADMIN_VM} + Run Keyword And Ignore Error Set Global Variable ${RPI_IP_ADDRESS} ${config['addresses']['measurement_agent']['device_ip_address']} + IF $BUILD_ID != '${EMPTY}' ${config}= Read Config ../config/${BUILD_ID}.json Set Global Variable ${JOB} ${config['Job']} diff --git a/Robot-Framework/lib/parse_power_data.py b/Robot-Framework/lib/parse_power_data.py new file mode 100644 index 00000000..29f8822b --- /dev/null +++ b/Robot-Framework/lib/parse_power_data.py @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: 2022-2024 Technology Innovation Institute (TII) +# SPDX-License-Identifier: Apache-2.0 + +import pandas as pd +import logging +import matplotlib.pyplot as plt + + +def extract_time_interval(csv_file, start_time, end_time): + columns = ['time', 'power'] + data = pd.read_csv(csv_file, names=columns) + interval = data.query("{} < time < {}".format(start_time, end_time)) + interval.to_csv('power_interval.csv', index=False) + return + +def generate_graph(csv_file, test_name): + data = pd.read_csv(csv_file) + start_time = data['time'].values[0] + end_time = data['time'].values[data.index.max()] + plt.figure(figsize=(20, 10)) + plt.set_loglevel('WARNING') + + # Show only hh-mm-ss part of the time at x-axis ticks + data['time'] = data['time'].str[11:19] + + plt.ticklabel_format(axis='y', style='plain') + plt.plot(data['time'], data['power'], marker='o', linestyle='-', color='b') + plt.yticks(fontsize=14) + + # Show full timestamps of the beginning and the end of the plotted time interval + plt.suptitle(f'Device power consumption {start_time} - {end_time}', fontsize=18, fontweight='bold') + + plt.title(f'During "{test_name}"', loc='center', fontweight="bold", fontsize=16) + plt.ylabel('Power (mW)', fontsize=16) + plt.grid(True) + plt.xticks(data['time'], rotation=45, fontsize=14) + + # Set maximum for tick number + plt.locator_params(axis='x', nbins=40) + + plt.savefig(f'../test-suites/power_test.png') + return + +def mean_power(csv_file): + columns = ['time', 'power'] + data = pd.read_csv(csv_file, names=columns) + mean_value = data['power'].mean() + return mean_value diff --git a/Robot-Framework/resources/power_meas_keywords.resource b/Robot-Framework/resources/power_meas_keywords.resource new file mode 100644 index 00000000..412b317d --- /dev/null +++ b/Robot-Framework/resources/power_meas_keywords.resource @@ -0,0 +1,121 @@ +# SPDX-FileCopyrightText: 2022-2024 Technology Innovation Institute (TII) +# SPDX-License-Identifier: Apache-2.0 + +*** Settings *** +Resource ../config/variables.robot +Library ../lib/parse_power_data.py +Library SSHLibrary +Library DateTime + +*** Variables *** +${SSH_MEASUREMENT} ${EMPTY} +${start_timestamp} ${EMPTY} + +*** Keywords *** + +Check variable availability + ${value}= Get Variable Value ${RPI_IP_ADDRESS} + IF """${value}""" != 'None' + RETURN ${True} + ELSE + RETURN ${False} + END + +Start power measurement + [Documentation] Connect to the measurement agent and run script to start collecting measurement results + [Arguments] ${id}=power_data ${timeout}=200 + ${availability} Check variable availability + IF ${availability}==False + Log To Console Power measurement agent IP address not defined. Ignoring all power measurement related keywords. + Set Global Variable ${SSH_MEASUREMENT} ${EMPTY} + RETURN + END + ${status} ${connection} Run Keyword And Ignore Error Connect to measurement agent + IF '${status}'!='PASS' + Set Global Variable ${SSH_MEASUREMENT} ${EMPTY} + Log To Console Power measurement agent not found. Ignoring all power measurement related keywords. + RETURN + END + # Multiple logging processes not allowed (for now) + Stop recording power + Start recording power ${id} ${timeout} + +Connect to measurement agent + [Documentation] Set up SSH connection to the measurement agent + [Arguments] ${IP}=${RPI_IP_ADDRESS} ${PORT}=22 ${target_output}=ghaf@raspberrypi + Log To Console Connecting to measurement agent + ${connection}= Open Connection ${IP} port=${PORT} prompt=\$ timeout=15 + ${output}= Login username=${LOGIN_PI} password=${PASSWORD_PI} + Should Contain ${output} ${target_output} + Set Global Variable ${SSH_MEASUREMENT} ${connection} + RETURN ${SSH_MEASUREMENT} + +Start recording power + [Arguments] ${file_name} ${timeout} + Log To Console Starting to record power measurements + Run Keyword And Ignore Error Execute Command nohup python /home/ghaf/ghaf/ghaf-power-measurement/measure_power.py ${file_name}.csv ${timeout} > output.log 2>&1 & timeout=3 + +Stop recording power + IF $SSH_MEASUREMENT=='${EMPTY}' + Log To Console No connection to power measurement device. Ignoring all power measurement related keywords. + RETURN + END + Log To Console Stopping power recording + Run Keyword And Ignore Error Execute Command pkill python timeout=3 + +Get power record + [Arguments] ${file_name}=power_data.csv + IF $SSH_MEASUREMENT=='${EMPTY}' + Log To Console No connection to power measurement device. Ignoring all power measurement related keywords. + RETURN + END + Run Keyword And Ignore Error Connect to measurement agent + Run Keyword And Ignore Error SSHLibrary.Get File /home/ghaf/ghaf/power_data/${file_name} ../../../power_measurements/ + +Save power measurement interval + [Documentation] Extract measurement data within given time interval + [Arguments] ${file_name} ${start_time} ${end_time} + IF $SSH_MEASUREMENT=='${EMPTY}' + Log To Console No connection to power measurement device. Ignoring all power measurement related keywords. + RETURN + END + Log To Console Extract power data from given time interval + ${time_interval} DateTime.Subtract Date From Date ${end_time} ${start_time} exclude_millis=True + IF ${time_interval} < 0 + Log To Console Invalid timestamp critera for extracting power data + RETURN + END + Run Keyword And Ignore Error Extract time interval ../../../power_measurements/${file_name} ${start_time} ${end_time} + +Generate power plot + [Documentation] Extract power data from start_timestamp to current time. + ... Plot power vs time and save to png file. + [Arguments] ${id} ${test_name} + IF $SSH_MEASUREMENT=='${EMPTY}' + Log To Console No connection to power measurement device. Ignoring all power measurement related keywords. + RETURN + END + ${end_timestamp} Get current timestamp + Switch Connection ${SSH_MEASUREMENT} + Run Keyword And Ignore Error Get power record ${id}.csv + Run Keyword And Ignore Error Save power measurement interval ${id}.csv '${start_timestamp}' '${end_timestamp}' + Run Keyword And Ignore Error Generate graph power_interval.csv ${test_name} + Run Keyword And Ignore Error Log Power plot HTML + +Set start timestamp + ${current_time} DateTime.Get Current Date exclude_millis=yes + Set Global Variable ${start_timestamp} ${current_time} + Log To Console ${start_timestamp} + +Get current timestamp + ${current_time} DateTime.Get Current Date exclude_millis=yes + RETURN ${current_time} + +Log average power + [Arguments] ${file_name} + IF $SSH_MEASUREMENT=='${EMPTY}' + Log To Console No connection to power measurement device. Ignoring all power measurement related keywords. + RETURN + END + ${keyword_status} ${mean_P} Run Keyword And Ignore Error Mean power ${file_name} + # TODO: With the statistics tools of performance testing also average power values could be plotted and monitored diff --git a/Robot-Framework/test-suites/gui-tests/__init__.robot b/Robot-Framework/test-suites/gui-tests/__init__.robot index 6eabae41..6f07430b 100644 --- a/Robot-Framework/test-suites/gui-tests/__init__.robot +++ b/Robot-Framework/test-suites/gui-tests/__init__.robot @@ -7,6 +7,7 @@ Resource ../../resources/ssh_keywords.resource Resource ../../resources/serial_keywords.resource Resource ../../resources/gui_keywords.resource Resource ../../resources/common_keywords.resource +Resource ../../resources/power_meas_keywords.resource Library ../../lib/gui_testing.py Suite Setup Common Setup Suite Teardown Common Teardown @@ -21,7 +22,7 @@ Common Setup IF ${port_22_is_available} == False FAIL Failed because port 22 of device was not available, tests can not be run. END - Connect + Connect to ghaf host IF "Lenovo" in "${DEVICE}" Verify service status range=15 service=microvm@gui-vm.service expected_status=active expected_state=running Connect to netvm diff --git a/Robot-Framework/test-suites/performance-tests/performance.robot b/Robot-Framework/test-suites/performance-tests/performance.robot index 2b906a2a..20bb5756 100644 --- a/Robot-Framework/test-suites/performance-tests/performance.robot +++ b/Robot-Framework/test-suites/performance-tests/performance.robot @@ -9,6 +9,7 @@ Resource ../../resources/device_control.resource Resource ../../resources/serial_keywords.resource Resource ../../config/variables.robot Resource ../../resources/performance_keywords.resource +Resource ../../resources/power_meas_keywords.resource Library ../../lib/output_parser.py Library ../../lib/parse_perfbench.py Library ../../lib/PerformanceDataProcessing.py ${DEVICE} ${BUILD_ID} ${JOB} @@ -38,7 +39,7 @@ nvpmodel check test END CPU One thread test - [Documentation] Run a CPU benchmark using Sysbench with a duration of 10 seconds and a SINGLE thread. + [Documentation] Run a CPU benchmark using Sysbench with a duration of 10 seconds and a SINGLE thread. ... The benchmark records to csv CPU events per second, events per thread, and latency data. ... Create visual plots to represent these metrics comparing to previous tests. [Tags] cpu SP-T61-1 nuc orin-agx orin-nx lenovo-x1 @@ -351,7 +352,7 @@ Perf-Bench test Common Setup Set Variables ${DEVICE} Run Keyword If "${DEVICE_IP_ADDRESS}" == "NONE" Get ethernet IP address - Connect + Connect to ghaf host LenovoX1 Setup [Documentation] Reboot LenovoX1 diff --git a/flake.lock b/flake.lock index 00db58a8..0326b7a4 100644 --- a/flake.lock +++ b/flake.lock @@ -58,4 +58,4 @@ }, "root": "root", "version": 7 -} +} \ No newline at end of file