[ci] Test stepping function of tvla in CI

This commit modifies `test_general_nonleaking_project()` to run TVLA with `--number-of-steps 10` and compare the produced result with the checked-in numerical value, rather than just checking for leakage. Signed-off-by: Vladimir Rozic <[email protected]>
lowRISC · Aug 15, 2023 · 7965169 · 7965169
1 parent 8a8578b
commit 7965169
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 9 deletions.
diff --git a/cw/tvla.py b/cw/tvla.py
@@ -817,12 +817,12 @@ def run_tvla(ctx: typer.Context):
     if save_to_disk_ttest:
         if num_steps > 1:
             log.info("Saving T-test Step")
-            np.savez('tmp/ttest-step.npy',
-                     ttest_step=ttest_step,
-                     trace_end_vec=trace_end_vec,
-                     rnd_list=rnd_list,
-                     byte_list=byte_list,
-                     single_trace=traces[1])
+            np.savez_compressed('tmp/ttest-step.npy',
+                                ttest_step=ttest_step,
+                                trace_end_vec=trace_end_vec,
+                                rnd_list=rnd_list,
+                                byte_list=byte_list,
+                                single_trace=traces[1])
         else:
             log.info("Saving T-test")
             np.save('tmp/ttest.npy', ttest_trace)

diff --git a/test/data/tvla_general/ttest-step-golden.npy.npz b/test/data/tvla_general/ttest-step-golden.npy.npz
diff --git a/test/tvla_test.py b/test/tvla_test.py
@@ -28,12 +28,25 @@ def ttest_significant(ttest_trace) -> bool:
     return np.any(abs_max > threshold)
 
 
+def ttest_compare_results(expected, received, delta) -> bool:
+    """Determine if the numerical values of two ttest traces match"""
+    nan_match = np.all(np.isnan(expected) == np.isnan(received))
+    numbers_match = np.all(np.logical_or(np.isnan(expected), np.abs(expected - received) < delta))
+    return nan_match and numbers_match
+
+
 def test_general_nonleaking_project():
     project_path = TestDataPath('tvla_general/ci_opentitan_simple_kmac.cwp')
     tvla = TvlaCmd(Args(['--project-file', str(project_path),
-                        '--mode', 'kmac', '--save-to-disk-ttest', 'run-tvla'])).run()
-    assert not ttest_significant(np.load('tmp/ttest.npy')), (
-           f"{tvla} did find significant leakage, which is unexpected")
+                         '--mode', 'kmac', '--save-to-disk-ttest',
+                         '--number-of-steps', '10', 'run-tvla'])).run()
+    expected_path = TestDataPath('tvla_general/ttest-step-golden.npy.npz')
+    expected_file = np.load(str(expected_path))
+    expected_trace = expected_file['ttest_step']
+    received_file = np.load('tmp/ttest-step.npy.npz')
+    received_trace = received_file['ttest_step']
+    assert ttest_compare_results(expected_trace, received_trace, 0.001), (
+           f"{tvla} generated ttest_step values that don't match the expected ones")
 
 
 def test_general_leaking_histogram():