proper support for complex16u (2xuint8) signals (#772)

* proper support for complex16 (2xint8) signals
  * fix Modulator: give correct iq type
  * fix View: Use general min/max and not from signal
  * Remove Padding

* fix requirements.txt (skip PyQt5.14.2 on Windows)

* fix noise area drawing

* fix demod view range for FSK and PSK

* Autofit signal in ModulatorDialog.py

* optimize Unittests

data/azure-pipelines.yml

@@ -252,6 +252,8 @@ jobs:
 
   - script: |
       brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/8d748e26ccc9afc8ea0d0201ae234fda35de721e/Formula/boost.rb
+      brew reinstall https://raw.githubusercontent.com/Homebrew/homebrew-core/a806a621ed3722fb580a58000fb274a2f2d86a6d/Formula/icu4c.rb
+      brew link icu4c --force
       cd /tmp
       wget https://github.com/libusb/libusb/releases/download/v1.0.22/libusb-1.0.22.tar.bz2
       tar xf libusb-1.0.22.tar.bz2

data/requirements.txt

@@ -1,5 +1,6 @@
 numpy>=1.9; sys_platform != 'win32'
 numpy>=1.9,!=1.16.0; sys_platform == 'win32'
-pyqt5
+pyqt5; sys_platform != 'win32'
+pyqt5!=5.14.2; sys_platform == 'win32'
 psutil
 cython

src/urh/controller/dialogs/ModulatorDialog.py

@@ -41,16 +41,6 @@ def __init__(self, modulators, tree_model=None, parent=None):
 
         self.modulators = modulators
 
-        for graphic_view in (self.ui.gVCarrier, self.ui.gVData):
-            graphic_view.scene_y_min = -1
-            graphic_view.scene_y_max = 1
-            graphic_view.scene_x_zoom_stretch = 1.1
-
-        min_max_mod = IQArray.min_max_for_dtype(self.current_modulator.get_dtype())
-        self.ui.gVModulated.scene_y_min = min_max_mod[0]
-        self.ui.gVModulated.scene_y_max = min_max_mod[1]
-        self.ui.gVModulated.scene_x_zoom_stretch = 1.1
-
         self.set_ui_for_current_modulator()
 
         self.ui.cbShowDataBitsOnly.setText(self.tr("Show Only Data Sequence\n"))
@@ -132,8 +122,7 @@ def closeEvent(self, event: QCloseEvent):
 
     @property
     def current_modulator(self):
-        mod = self.modulators[self.ui.comboBoxCustomModulations.currentIndex()]
-        return mod
+        return self.modulators[self.ui.comboBoxCustomModulations.currentIndex()]
 
     def set_ui_for_current_modulator(self):
         index = self.ui.comboBoxModulationType.findText("*(" + self.current_modulator.modulation_type + ")",
@@ -188,7 +177,7 @@ def draw_data_bits(self):
         self.ui.gVData.update()
 
     def draw_modulated(self):
-        self.ui.gVModulated.plot_data(self.current_modulator.modulate(pause=0).imag.astype(numpy.float32))
+        self.ui.gVModulated.plot_data(self.current_modulator.modulate(pause=0).imag)
         if self.lock_samples_in_view:
             siv = self.ui.gVOriginalSignal.view_rect().width()
             self.adjust_samples_in_view(siv)
@@ -203,8 +192,8 @@ def draw_original_signal(self, start=0, end=-1):
         if end == -1:
             end = scene_manager.signal.num_samples
 
-        y = scene_manager.scene.sceneRect().y()
-        h = scene_manager.scene.sceneRect().height()
+        y = self.ui.gVOriginalSignal.view_rect().y()
+        h = self.ui.gVOriginalSignal.view_rect().height()
         self.ui.gVOriginalSignal.setSceneRect(start, y, end - start, h)
         self.ui.gVOriginalSignal.fitInView(self.ui.gVOriginalSignal.sceneRect())
         scene_manager.show_scene_section(start, end)

src/urh/controller/widgets/SignalFrame.py

@@ -731,6 +731,7 @@ def on_cb_signal_view_index_changed(self):
         else:
             self.ui.stackedWidget.setCurrentWidget(self.ui.pageSignal)
             self.ui.gvSignal.scene_type = self.ui.cbSignalView.currentIndex()
+            self.scene_manager.mod_type = self.signal.modulation_type
             self.ui.gvSignal.redraw_view(reinitialize=True)
             self.ui.labelRSSI.show()
 
@@ -1062,6 +1063,7 @@ def on_combobox_modulation_type_text_changed(self, txt: str):
 
             self.undo_stack.push(modulation_action)
 
+            self.scene_manager.mod_type = txt
             if self.ui.cbSignalView.currentIndex() == 1:
                 self.scene_manager.init_scene()
                 self.on_slider_y_scale_value_changed()

src/urh/cythonext/signal_functions.pyx

@@ -54,6 +54,31 @@ cdef get_numpy_dtype(iq cython_type):
         raise ValueError("dtype {} not supported for modulation".format(cython.typeof(cython_type)))
 
 cpdef modulate_c(uint8_t[:] bits, uint32_t samples_per_symbol, str modulation_type,
+                 float[:] parameters, uint16_t bits_per_symbol,
+                 float carrier_amplitude, float carrier_frequency, float carrier_phase, float sample_rate,
+                 uint32_t pause, uint32_t start, dtype=np.float32,
+                 float gauss_bt=0.5, float filter_width=1.0):
+
+    if dtype == np.int8:
+        return __modulate(
+            bits, samples_per_symbol, modulation_type, parameters, bits_per_symbol, carrier_amplitude,
+            carrier_frequency, carrier_phase, sample_rate, pause, start, <char>0, gauss_bt, filter_width
+        )
+    elif dtype == np.int16:
+        return __modulate(
+            bits, samples_per_symbol, modulation_type, parameters, bits_per_symbol, carrier_amplitude,
+            carrier_frequency, carrier_phase, sample_rate, pause, start, <short>0, gauss_bt, filter_width
+        )
+    elif dtype == np.float32:
+        return __modulate(
+            bits, samples_per_symbol, modulation_type, parameters, bits_per_symbol, carrier_amplitude,
+            carrier_frequency, carrier_phase, sample_rate, pause, start, <float>0.0, gauss_bt, filter_width
+        )
+    else:
+        raise ValueError("Unsupported dtype for modulation {}".format(dtype))
+
+
+cpdef __modulate(uint8_t[:] bits, uint32_t samples_per_symbol, str modulation_type,
                  float[:] parameters, uint16_t bits_per_symbol,
                  float carrier_amplitude, float carrier_frequency, float carrier_phase, float sample_rate,
                  uint32_t pause, uint32_t start, iq iq_type,
@@ -316,8 +341,8 @@ cpdef np.ndarray[np.float32_t, ndim=1] afp_demod(IQ samples, float noise_mag, st
     else:
         raise ValueError("Unsupported dtype")
 
-    # Atan2 liefert Werte im Bereich von -Pi bis Pi
-    # Wir nutzen die Magic Constant NOISE_FSK_PSK um Rauschen abzuschneiden
+    # Atan2 yields values from -Pi to Pi
+    # We use the Magic Constant NOISE_FSK_PSK to cut off noise
     noise_sqrd = noise_mag * noise_mag
     NOISE = get_noise_for_mod_type(mod_type)
     result[0] = NOISE

src/urh/settings.py

@@ -54,7 +54,7 @@ def get_qt_settings_filename():
 SELECTION_COLOR = QColor("darkblue")  # overwritten by system color (bin/urh)
 NOISE_COLOR = QColor("red")
 SELECTION_OPACITY = 1
-NOISE_OPACITY = 0.4
+NOISE_OPACITY = 0.33
 
 # SEPARATION COLORS
 ONES_AREA_COLOR = Qt.darkGreen

src/urh/signalprocessing/IQArray.py

@@ -189,13 +189,13 @@ def convert_to(self, target_dtype) -> np.ndarray:
     def from_file(filename: str):
         if filename.endswith(".complex16u") or filename.endswith(".cu8"):
             # two 8 bit unsigned integers
-            return IQArray(data=np.fromfile(filename, dtype=np.uint8))
+            return IQArray(IQArray(data=np.fromfile(filename, dtype=np.uint8)).convert_to(np.int8))
         elif filename.endswith(".complex16s") or filename.endswith(".cs8"):
             # two 8 bit signed integers
             return IQArray(data=np.fromfile(filename, dtype=np.int8))
         elif filename.endswith(".complex32u") or filename.endswith(".cu16"):
             # two 16 bit unsigned integers
-            return IQArray(data=np.fromfile(filename, dtype=np.uint16))
+            return IQArray(IQArray(data=np.fromfile(filename, dtype=np.uint16)).convert_to(np.int16))
         elif filename.endswith(".complex32s") or filename.endswith(".cs16"):
             # two 16 bit signed integers
             return IQArray(data=np.fromfile(filename, dtype=np.int16))

src/urh/signalprocessing/Modulator.py

@@ -220,9 +220,6 @@ def modulate(self, data=None, pause=0, start=0) -> IQArray:
         dtype = self.get_dtype()
         a = self.carrier_amplitude * IQArray.min_max_for_dtype(dtype)[1]
 
-        type_code = "b" if dtype == np.int8 else "h" if dtype == np.int16 else "f"
-        type_val = array.array(type_code, [0])[0]
-
         parameters = self.parameters
         if self.modulation_type == "ASK":
             parameters = array.array("f", [a*p/100 for p in parameters])
@@ -233,7 +230,7 @@ def modulate(self, data=None, pause=0, start=0) -> IQArray:
                                              self.modulation_type, parameters, self.bits_per_symbol,
                                              a, self.carrier_freq_hz,
                                              self.carrier_phase_deg * (np.pi / 180),
-                                             self.sample_rate, pause, start, type_val,
+                                             self.sample_rate, pause, start, dtype,
                                              self.gauss_bt, self.gauss_filter_width)
         return IQArray(result)
 

src/urh/signalprocessing/Signal.py

@@ -51,6 +51,8 @@ def __init__(self, filename: str, name="Signal", modulation: str = None, sample_
         self.noise_max_plot = 0
         self.block_protocol_update = False
 
+        self.iq_array = IQArray(None, np.int8, 1)
+
         self.wav_mode = filename.endswith(".wav")
         self.__changed = False
         if modulation is None:
@@ -281,19 +283,32 @@ def noise_threshold(self, value):
             self._qad = None
             self.clear_parameter_cache()
             self._noise_threshold = value
-            self.noise_min_plot = -value
-            self.noise_max_plot = value
+
+            middle = 0.5*sum(IQArray.min_max_for_dtype(self.iq_array.dtype))
+            a = self.max_amplitude * value / self.max_magnitude
+            self.noise_min_plot = middle - a
+            self.noise_max_plot = middle + a
             self.noise_threshold_changed.emit()
             if not self.block_protocol_update:
                 self.protocol_needs_update.emit()
 
+    @property
+    def max_magnitude(self):
+        mi, ma = IQArray.min_max_for_dtype(self.iq_array.dtype)
+        return (2 * max(mi**2, ma**2))**0.5
+
+    @property
+    def max_amplitude(self):
+        mi, ma = IQArray.min_max_for_dtype(self.iq_array.dtype)
+        return 0.5 * (ma - mi)
+
     @property
     def noise_threshold_relative(self):
-        return self.noise_threshold / (self.iq_array.maximum**2.0 + self.iq_array.minimum**2.0)**0.5
+        return self.noise_threshold / self.max_magnitude
 
     @noise_threshold_relative.setter
     def noise_threshold_relative(self, value: float):
-        self.noise_threshold = value * (self.iq_array.maximum**2.0 + self.iq_array.minimum**2.0)**0.5
+        self.noise_threshold = value * self.max_magnitude
 
     @property
     def qad(self):

src/urh/ui/painting/ContinuousSceneManager.py

@@ -10,8 +10,6 @@ def __init__(self, ring_buffer: RingBuffer, parent):
         self.__start = 0
         self.__end = 0
 
-        self.minimum, self.maximum = IQArray.min_max_for_dtype(self.ring_buffer.dtype)
-
     @property
     def plot_data(self):
         return self.ring_buffer.view_data.real

src/urh/ui/painting/LiveSceneManager.py

@@ -8,8 +8,6 @@ def __init__(self, data_array, parent):
         self.plot_data = data_array
         self.end = 0
 
-        self.minimum, self.maximum = IQArray.min_max_for_dtype(data_array.dtype)
-
     @property
     def num_samples(self):
         return self.end

src/urh/ui/painting/SceneManager.py

@@ -4,6 +4,7 @@
 from PyQt5.QtCore import QObject
 from PyQt5.QtGui import QPen, QColor
 from PyQt5.QtWidgets import QGraphicsPathItem
+from urh.signalprocessing.IQArray import IQArray
 
 from urh import settings
 from urh.cythonext import path_creator, util
@@ -16,10 +17,6 @@ def __init__(self, parent):
         self.scene = ZoomableScene()
         self.__plot_data = None  # type: np.ndarray
         self.line_item = self.scene.addLine(0, 0, 0, 0, QPen(settings.AXISCOLOR, 0))
-        self.minimum = float("nan")  # NaN = AutoDetect
-        self.maximum = float("nan")  # NaN = AutoDetect
-
-        self.padding = 1.25
 
     @property
     def plot_data(self):
@@ -66,24 +63,11 @@ def __limit_value(self, val: float) -> int:
     def show_full_scene(self):
         self.show_scene_section(0, self.num_samples)
 
-    def init_scene(self, apply_padding=True):
+    def init_scene(self):
         if self.num_samples == 0:
             return
 
-        if math.isnan(self.minimum) or math.isnan(self.maximum):
-            minimum, maximum = util.minmax(self.plot_data)
-        else:
-            minimum, maximum = self.minimum, self.maximum
-
-        padding = self.padding if apply_padding else 1
-
-        if abs(minimum) > abs(maximum):
-            minimum = -padding * abs(minimum)
-            maximum = -padding * minimum
-        else:
-            maximum = padding * abs(maximum)
-            minimum = -padding * maximum
-
+        minimum, maximum = IQArray.min_max_for_dtype(self.plot_data.dtype)
         self.scene.setSceneRect(0, minimum, self.num_samples, maximum - minimum)
         self.scene.setBackgroundBrush(settings.BGCOLOR)
 

src/urh/ui/painting/SignalSceneManager.py

@@ -1,3 +1,5 @@
+import math
+
 from urh.signalprocessing.Signal import Signal
 from urh.ui.painting.SceneManager import SceneManager
 
@@ -7,6 +9,7 @@ def __init__(self, signal: Signal, parent):
         super().__init__(parent)
         self.signal = signal
         self.scene_type = 0  # 0 = Analog Signal, 1 = QuadDemodView
+        self.mod_type = "ASK"
 
     def show_scene_section(self, x1: float, x2: float, subpath_ranges=None, colors=None):
         self.plot_data = self.signal.real_plot_data if self.scene_type == 0 else self.signal.qad
@@ -19,7 +22,9 @@ def init_scene(self):
         else:
             self.plot_data = self.signal.qad
 
-        super().init_scene(apply_padding=self.scene_type == 0)
+        super().init_scene()
+        if self.scene_type == 1 and (self.mod_type == "FSK" or self.mod_type == "PSK"):
+            self.scene.setSceneRect(0, -4, self.num_samples, 8)
 
         self.line_item.setLine(0, 0, 0, 0)  # Hide Axis
 

src/urh/ui/painting/SniffSceneManager.py

@@ -10,8 +10,6 @@ def __init__(self, data_array, parent, window_length=5 * 10**6):
         self.__end = 0
         self.window_length = window_length
 
-        self.minimum, self.maximum = IQArray.min_max_for_dtype(data_array.dtype)
-
     @property
     def plot_data(self):
         return self.data_array[self.__start:self.end]

src/urh/ui/painting/SpectrogramSceneManager.py

@@ -67,7 +67,7 @@ def show_full_scene(self):
         # after we know how wide the spectrogram actually is
         self.scene.setSceneRect(0, 0, x_pos, self.spectrogram.freq_bins)
 
-    def init_scene(self, apply_padding=True):
+    def init_scene(self):
         pass
 
     def eliminate(self):

src/urh/ui/views/ZoomAndDropableGraphicView.py

@@ -1,5 +1,8 @@
 from PyQt5.QtCore import pyqtSignal
 from PyQt5.QtGui import QDragEnterEvent, QDropEvent
+from urh.signalprocessing.IQArray import IQArray
+
+from urh.cythonext import util
 
 from urh.signalprocessing.ProtocolAnalyzer import ProtocolAnalyzer
 from urh.signalprocessing.Signal import Signal
@@ -44,9 +47,6 @@ def dropEvent(self, event: QDropEvent):
         if signal is None:
             return
 
-        self.draw_signal(signal, proto_analyzer)
-
-    def draw_signal(self, signal, proto_analyzer):
         if signal is None:
             return
 
@@ -60,6 +60,15 @@ def draw_signal(self, signal, proto_analyzer):
 
         self.signal_loaded.emit(self.proto_analyzer)
 
+    def auto_fit_view(self):
+        super().auto_fit_view()
+
+        plot_min, plot_max = util.minmax(self.signal.real_plot_data)
+        data_min, data_max = IQArray.min_max_for_dtype(self.signal.real_plot_data.dtype)
+        self.scale(1, (data_max - data_min) / (plot_max-plot_min))
+
+        self.centerOn(self.view_rect().x() + self.view_rect().width() / 2, self.y_center)
+
     def eliminate(self):
         # Do _not_ call eliminate() for self.signal and self.proto_analyzer
         # as these are references to the original data!