From 3abf517690bb1997c9f8c8c27b4637aa80a8e4b4 Mon Sep 17 00:00:00 2001
From: Vladimir Rozic <vrozic@lowrisc.org>
Date: Fri, 6 Oct 2023 17:36:53 +0100
Subject: [PATCH] [tvla] Enable trace filtering for TVLA-general

Tvla has a mechanism for filtering out outlier traces from the
project. At the moment, this filtering is applied only to
aes specific TVLA. This commit enables filtering for general TVLA
as well.
As a result of applied filtering, the expected result of the CI
test test_general_aes_nonleaking_project() has changed. This
commit also updates the expected result of this test, in order
to leave the repository in a working state.

Signed-off-by: Vladimir Rozic <vrozic@lowrisc.org>
---
 cw/tvla.py                                    | 39 ++++++++-----------
 .../ttest-step-golden-aes.npy.npz             |  4 +-
 2 files changed, 19 insertions(+), 24 deletions(-)

diff --git a/cw/tvla.py b/cw/tvla.py
index 4b9b403b6..4d5f87082 100755
--- a/cw/tvla.py
+++ b/cw/tvla.py
@@ -483,28 +483,23 @@ def run_tvla(ctx: typer.Context):
                                                          trace_start] + 0.5) * trace_resolution
                     traces = traces.astype('uint16')
 
-                if general_test is False:
-                    # Filter out noisy traces.
-                    log.info("Filtering Traces")
-
-                    # Get the mean and standard deviation.
-                    mean = traces.mean(axis=0)
-                    std = traces.std(axis=0)
-
-                    # Define upper and lower limits.
-                    max_trace = mean + num_sigmas * std
-                    min_trace = mean - num_sigmas * std
-
-                    # Filtering of converted traces (len = num_samples). traces_to_use itself can be
-                    # used to index the entire project file (len >= num_samples).
-                    traces_to_use = np.zeros(len(project.waves), dtype=bool)
-                    traces_to_use[trace_start:trace_end + 1] = np.all((traces >= min_trace) &
-                                                                      (traces <= max_trace), axis=1)
-                    traces = traces[traces_to_use[trace_start:trace_end + 1]]
-                else:
-                    # For now, don't perform any filtering when doing general fixed-vs-random TVLA.
-                    traces_to_use = np.zeros(len(project.waves), dtype=bool)
-                    traces_to_use[trace_start:trace_end + 1] = True
+                # Filter out noisy traces.
+                log.info("Filtering Traces")
+
+                # Get the mean and standard deviation.
+                mean = traces.mean(axis=0)
+                std = traces.std(axis=0)
+
+                # Define upper and lower limits.
+                max_trace = mean + num_sigmas * std
+                min_trace = mean - num_sigmas * std
+
+                # Filtering of converted traces (len = num_samples). traces_to_use itself can be
+                # used to index the entire project file (len >= num_samples).
+                traces_to_use = np.zeros(len(project.waves), dtype=bool)
+                traces_to_use[trace_start:trace_end + 1] = np.all((traces >= min_trace) &
+                                                                  (traces <= max_trace), axis=1)
+                traces = traces[traces_to_use[trace_start:trace_end + 1]]
 
                 if i_step == 0:
                     # Keep a single trace to create the figures.
diff --git a/test/data/tvla_general/ttest-step-golden-aes.npy.npz b/test/data/tvla_general/ttest-step-golden-aes.npy.npz
index 3b83a9ce1..4c2a6ddbf 100644
--- a/test/data/tvla_general/ttest-step-golden-aes.npy.npz
+++ b/test/data/tvla_general/ttest-step-golden-aes.npy.npz
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:197ee6315dd31ce1155e0a59a6404b786b9e9d1f1b9a126101ea00b712e2de84
-size 187056
+oid sha256:a3d093bbe7ed6a87bd19860c0f2b2fa6b577170c0e66b0d839836e5d4f18c9c9
+size 185790