move to data format 4

spf/dataset/v4_data.py

@@ -2,19 +2,21 @@
 from numcodecs import Blosc
 
 v4rx_f64_keys = [
-    "system_timestamps",
-    "gps_timestamps",
-    "lat",
-    "long",
+    "system_timestamp",
+    "gps_timestamp",
+    "gps_lat",
+    "gps_long",
     "heading",
+]
+v4rx_2xf64_keys = [
     "avg_phase_diff",
-    "rssi",
-    "gain",
+    "rssis",
+    "gains",
 ]
 
 
 def v4rx_keys():
-    return v4rx_f64_keys + ["signal_matrix"]
+    return v4rx_f64_keys + v4rx_2xf64_keys + ["signal_matrix"]
 
 
 def v4rx_new_dataset(
@@ -44,8 +46,16 @@ def v4rx_new_dataset(
             receiver_z.create_dataset(
                 key,
                 shape=(timesteps,),
-                chunks=(1024 * chunk_size),
+                # chunks=(1024 * chunk_size),
+                dtype="float64",
+                # compressor=compressor,
+            )
+        for key in v4rx_2xf64_keys:
+            receiver_z.create_dataset(
+                key,
+                shape=(timesteps, 2),
+                # chunks=(1024 * chunk_size, 1),
                 dtype="float64",
-                compressor=compressor,
+                # compressor=compressor,
             )
     return z

spf/mavlink_radio_collection.py

@@ -9,7 +9,7 @@
 import yaml
 from pymavlink import mavutil
 
-from spf.data_collector import DroneDataCollector, FakeDroneDataCollector
+from spf.data_collector import DroneDataCollector, DroneDataCollectorRaw
 from spf.distance_finder.distance_finder_controller import DistanceFinderController
 from spf.gps.boundaries import franklin_safe  # crissy_boundary_convex
 from spf.mavlink.mavlink_controller import (
@@ -19,6 +19,38 @@
 )
 from spf.utils import is_pi
 
+
+def filenames_from_time_in_seconds(
+    time_in_seconds, temp_dir_name, yaml_config, data_version
+):
+    os.makedirs(temp_dir_name, exist_ok=True)
+    dt = datetime.fromtimestamp(time_in_seconds)
+    date_str = dt.strftime("%Y_%m_%d_%H_%M_%S")
+
+    output_files_prefix = (
+        f"rover_{date_str}_nRX{len(yaml_config['receivers'])}_{yaml_config['routine']}"
+    )
+    if args.tag != "":
+        output_files_prefix += f"_tag_{args.tag}"
+
+    filename_log = f"{temp_dir_name}/{output_files_prefix}.log.tmp"
+    filename_yaml = f"{temp_dir_name}/{output_files_prefix}.yaml.tmp"
+    if data_version == 3:
+        filename_data = f"{temp_dir_name}/{output_files_prefix}.npy.tmp"
+    elif data_version == 4:
+        filename_data = f"{temp_dir_name}/{output_files_prefix}.zarr.tmp"
+    else:
+        raise NotImplementedError
+    temp_filenames = {
+        "log": filename_log,
+        "yaml": filename_yaml,
+        "data": filename_data,
+    }
+    final_filenames = {k: v.replace(".tmp", "") for k, v in temp_filenames.items()}
+
+    return temp_filenames, final_filenames
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -73,7 +105,6 @@
     ),
 
     parser.add_argument("--fake-drone", action=argparse.BooleanOptionalAction)
-    parser.add_argument("--fake-radio", action=argparse.BooleanOptionalAction)
     parser.add_argument("--exit", action=argparse.BooleanOptionalAction)
     args = parser.parse_args()
 
@@ -108,32 +139,12 @@
         assert yaml_config["emitter"]["type"] == "sdr"
         yaml_config["emitter"]["tx-gain"] = args.tx_gain
 
-    def filenames_from_time_in_seconds(time_in_seconds, temp_dir_name, yaml_config):
-        os.makedirs(temp_dir_name, exist_ok=True)
-        dt = datetime.fromtimestamp(time_in_seconds)
-        date_str = dt.strftime("%Y_%m_%d_%H_%M_%S")
-
-        output_files_prefix = f"rover_{date_str}_nRX{len(yaml_config['receivers'])}_{yaml_config['routine']}"
-        if args.tag != "":
-            output_files_prefix += f"_tag_{args.tag}"
-
-        filename_log = f"{temp_dir_name}/{output_files_prefix}.log.tmp"
-        filename_yaml = f"{temp_dir_name}/{output_files_prefix}.yaml.tmp"
-        filename_npy = f"{temp_dir_name}/{output_files_prefix}.npy.tmp"
-        temp_filenames = {
-            "log": filename_log,
-            "yaml": filename_yaml,
-            "npy": filename_npy,
-        }
-        final_filenames = {k: v.replace(".tmp", "") for k, v in temp_filenames.items()}
-
-        return temp_filenames, final_filenames
-
     # setup filename
     # tmpdir = tempfile.TemporaryDirectory()
     # temp_dir_name = tmpdir.name
+
     temp_filenames, final_filenames = filenames_from_time_in_seconds(
-        run_started_at, args.temp, yaml_config
+        run_started_at, args.temp, yaml_config, data_version=yaml_config["data-version"]
     )
 
     logger = logging.getLogger(__name__)
@@ -189,18 +200,20 @@ def filenames_from_time_in_seconds(time_in_seconds, temp_dir_name, yaml_config):
             fake=True,
         )
 
-    if not args.fake_radio:
+    if yaml_config["data-version"] == 3:
         data_collector = DroneDataCollector(
-            data_filename=temp_filenames["npy"],
+            data_filename=temp_filenames["data"],
             yaml_config=yaml_config,
             position_controller=drone,
         )
-    else:
-        data_collector = FakeDroneDataCollector(
-            data_filename=temp_filenames["npy"],
+    elif yaml_config["data-version"] == 4:
+        data_collector = DroneDataCollectorRaw(
+            data_filename=temp_filenames["data"],
             yaml_config=yaml_config,
-            position_controller=None,
+            position_controller=drone,
         )
+    else:
+        raise NotImplementedError
 
     logging.info("MavRadioCollection: Radios online...")
     data_collector.radios_to_online()  # blocking
@@ -239,6 +252,6 @@ def check_exit():
     for k in temp_filenames:
         os.rename(temp_filenames[k], final_filenames[k])
 
-    if is_pi() and not args.fake_drone and not args.fake_radio:
+    if is_pi() and not args.fake_drone:
         time.sleep(5)
         subprocess.getoutput("sudo halt")

spf/rover_configs/rover_emitter_config_pi.yaml

@@ -48,4 +48,6 @@ routine: null
 
 drone-uri: serial
 
-dry-run: False
+dry-run: False
+
+data-version: 4

spf/rover_configs/rover_emitter_config_pi_simulator.yaml

@@ -48,4 +48,6 @@ routine: null
 
 drone-uri: udpin:127.0.0.1:14552
 
-dry-run: False
+dry-run: False
+
+data-version: 4

spf/rover_configs/rover_receiver_config_pi.yaml

@@ -51,4 +51,6 @@ routine: null
 
 drone-uri: serial
 
-dry-run: False
+dry-run: False
+
+data-version: 4

spf/rover_configs/rover_receiver_config_simulator.yaml

@@ -38,3 +38,6 @@ routine: null
 drone-uri: udpin:127.0.0.1:14532
 
 dry-run: False
+
+
+data-version: 4

spf/rover_configs/rover_receiver_config_simulator_docker.yaml

@@ -38,3 +38,5 @@ routine: null
 drone-uri: tcp:127.0.0.1:14591
 
 dry-run: False
+
+data-version: 4

spf/rover_configs/rover_single_receiver_config_pi.yaml

@@ -37,4 +37,7 @@ routine: null
 
 drone-uri: serial
 
-dry-run: False
+dry-run: False
+
+
+data-version: 4

spf/sdrpluto/sdr_controller.py

@@ -19,6 +19,7 @@
     beamformer_thetas,
     precompute_steering_vectors,
 )
+from spf.utils import random_signal_matrix
 
 # TODO close SDR on exit
 # import signal
@@ -182,19 +183,91 @@ def get_pplus(
     uri = get_uri(rx_config=rx_config, tx_config=tx_config, uri=uri)
     global pplus_online
     if uri not in pplus_online:
-        pplus_online[uri] = PPlus(rx_config=rx_config, tx_config=tx_config, uri=uri)
+        if "fake" in uri:
+            pplus_online[uri] = FakePPlus(
+                rx_config=rx_config, tx_config=tx_config, uri=uri
+            )
+        else:
+            pplus_online[uri] = PPlus(rx_config=rx_config, tx_config=tx_config, uri=uri)
     else:
         pplus_online[uri].set_config(rx_config=rx_config, tx_config=tx_config)
     logging.debug(f"{uri}: get_pplus PlutoPlus")
     return pplus_online[uri]
 
 
+class FakeSdr:
+    def set_buffer_size(self, buffer_size):
+        self.buffer_size = buffer_size
+
+    def rx(self):
+        return random_signal_matrix(2 * self.buffer_size).reshape(2, self.buffer_size)
+
+    def tx(self, _):
+        pass
+
+
+class FakePPlus:
+    def __init__(
+        self,
+        uri: str,
+        rx_config: ReceiverConfig = None,
+        tx_config: EmitterConfig = None,
+        phase_calibration=0.0,
+    ):
+        super(FakePPlus, self).__init__()
+        self.uri = uri
+
+        self.tx_config = None
+        self.rx_config = None
+        self.set_config(rx_config=rx_config, tx_config=tx_config)
+        self.sdr = FakeSdr()
+
+    def rssis(self):
+        return np.random.rand(2)
+
+    def gains(self):
+        return np.random.rand(2)
+
+    def set_config(
+        self, rx_config: ReceiverConfig = None, tx_config: EmitterConfig = None
+    ):
+        logging.debug(f"{self.uri}: set_config RX{str(rx_config)} TX{str(tx_config)})")
+        # RX should be setup like this
+        if rx_config is not None:
+            assert self.rx_config is None
+            self.rx_config = rx_config
+            self.sdr = FakeSdr()
+
+        # TX should be setup like this
+        if tx_config is not None:
+            assert self.tx_config is None
+            self.tx_config = tx_config
+
+    def close(self):
+        pass
+
+    def setup_rx_config(self):
+        self.sdr.set_buffer_size(self.rx_config.buffer_size)
+
+    def setup_tx_config(self):
+        pass
+
+    def close_rx(self):
+        pass
+
+    def close_tx(self):
+        pass
+
+    def check_for_freq_peak(self):
+        return True
+
+
 class PPlus:
     def __init__(
         self,
+        uri: str,
         rx_config: ReceiverConfig = None,
         tx_config: EmitterConfig = None,
-        uri: str = None,
         phase_calibration=0.0,
     ):
         super(PPlus, self).__init__()

tests/rover_config.yaml

@@ -15,7 +15,7 @@ distance-finder:
 #USB  2   (Radio A)  |     USB 1 (Radio B)
 receivers:
   #radio A
-  - receiver-port: 2
+  - receiver-uri: fake:A
     theta-in-pis: 0
     antenna-spacing-m: 0.05075 # 50.75 mm 
     nelements: 2
@@ -29,7 +29,7 @@ receivers:
     f-sampling: 16000000 # 16.0e6
     bandwidth: 300000 #3.0e5
   #radio B
-  - receiver-port: 1
+  - receiver-uri: fake:B
     theta-in-pis: 0.5
     antenna-spacing-m: 0.05075 # 50.75 mm 
     nelements: 2
@@ -52,3 +52,5 @@ routine: null
 drone-uri: tcp:127.0.0.1:14591
 
 dry-run: False
+
+data-version: 4

tests/test_config.yaml

@@ -15,7 +15,7 @@ distance-finder:
 #USB  2   (Radio A)  |     USB 1 (Radio B)
 receivers:
   #radio A
-  - receiver-port: 2
+  - receiver-uri: fake:A
     theta-in-pis: 0
     antenna-spacing-m: 0.05075 # 50.75 mm 
     nelements: 2
@@ -29,7 +29,7 @@ receivers:
     f-sampling: 16000000 # 16.0e6
     bandwidth: 300000 #3.0e5
   #radio B
-  - receiver-port: 1
+  - receiver-uri: fake:B
     theta-in-pis: 0.5
     antenna-spacing-m: 0.05075 # 50.75 mm 
     nelements: 2
@@ -51,4 +51,6 @@ routine: null
 
 drone-uri: serial
 
-dry-run: False
+dry-run: False
+
+data-version: 4