From 5bbf2453c4850d497119049a4c1e4e88cc687b6a Mon Sep 17 00:00:00 2001 From: Jason Detwiler Date: Tue, 17 Oct 2023 06:15:51 +0100 Subject: [PATCH 01/19] avoid error when running tests that don't use the tmpdir --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index f2cbdbf..aa8d49b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,7 +23,7 @@ def tmptestdir(): def pytest_sessionfinish(session, exitstatus): - if exitstatus == 0: + if exitstatus == 0 and os.path.exists(_tmptestdir): shutil.rmtree(_tmptestdir) From 92808428b1f3a5e0bb474c7758dc6d788820f379 Mon Sep 17 00:00:00 2001 From: Jason Detwiler Date: Tue, 17 Oct 2023 06:16:23 +0100 Subject: [PATCH 02/19] add option to return hexdump output as list of strings --- src/daq2lh5/orca/orca_packet.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/daq2lh5/orca/orca_packet.py b/src/daq2lh5/orca/orca_packet.py index a00c7cb..465cc5c 100644 --- a/src/daq2lh5/orca/orca_packet.py +++ b/src/daq2lh5/orca/orca_packet.py @@ -47,11 +47,9 @@ def hex_dump( as_short: bool = False, id_dict: dict = None, use_logging: bool = True, + return_output = False ) -> None: - dump_cmd = print # noqa: T202 - if use_logging: - dump_cmd = log.debug - + output = [] data_id = get_data_id(packet, shift=shift_data_id) n_words = get_n_words(packet) if id_dict is not None: @@ -62,9 +60,9 @@ def hex_dump( else: heading = f"data ID = {data_id}" if print_n_words: - dump_cmd(f"{heading}: {n_words} words") + output.append(f"{heading}: {n_words} words") else: - dump_cmd(f"{heading}:") + output.append(f"{heading}:") n_to_print = int(np.minimum(n_words, max_words)) pad = int(np.ceil(np.log10(n_to_print))) for i in range(n_to_print): @@ -76,4 +74,14 @@ def hex_dump( line += f" {packet[i]}" if as_short: line += f" {np.frombuffer(packet[i:i+1].tobytes(), dtype='uint16')}" - dump_cmd(line) + output.append(line) + + dump_cmd = print # noqa: T202 + if use_logging: + dump_cmd = log.debug + for line in output: + dump_cmd(line) + + if return_output: + return output + From 3c92275e8791cba3269d185fddaf9ecf5c2681f8 Mon Sep 17 00:00:00 2001 From: Jason Detwiler Date: Tue, 17 Oct 2023 06:16:43 +0100 Subject: [PATCH 03/19] add tests of orca_packet.py --- tests/orca/test_orca_packet.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/orca/test_orca_packet.py b/tests/orca/test_orca_packet.py index b821d42..8a695af 100644 --- a/tests/orca/test_orca_packet.py +++ b/tests/orca/test_orca_packet.py @@ -1,2 +1,11 @@ -def test_orca_packet_import(): - pass +from daq2lh5.orca import orca_packet + +def test_orca_packet_funcs(orca_stream): + packet = orca_stream.load_packet() + # These values are particular to the test orca file in legend-testdata and + # may need to be changed if that file is changed + assert orca_packet.is_short(packet) == False + assert orca_packet.get_data_id(packet) == 7 + assert orca_packet.get_n_words(packet) == 4 + assert orca_packet.hex_dump(packet, return_output=True)[-1] == '3 0x6286930b' + orca_stream.close_stream() # avoid warning that file is still open From 5c8883c816a2f8f9d353f529532eb380b02fce13 Mon Sep 17 00:00:00 2001 From: Jason Detwiler Date: Tue, 17 Oct 2023 09:13:17 +0100 Subject: [PATCH 04/19] add ability to load packets by index --- src/daq2lh5/orca/orca_streamer.py | 127 +++++++++++++++++++++++++++--- 1 file changed, 115 insertions(+), 12 deletions(-) diff --git a/src/daq2lh5/orca/orca_streamer.py b/src/daq2lh5/orca/orca_streamer.py index 3c69556..7b02725 100644 --- a/src/daq2lh5/orca/orca_streamer.py +++ b/src/daq2lh5/orca/orca_streamer.py @@ -32,30 +32,133 @@ class OrcaStreamer(DataStreamer): def __init__(self) -> None: super().__init__() self.in_stream = None + self.packet_locs = [] self.buffer = np.empty(1024, dtype="uint32") # start with a 4 kB packet buffer self.header = None self.header_decoder = OrcaHeaderDecoder() self.decoder_id_dict = {} # dict of data_id to decoder object self.rbl_id_dict = {} # dict of RawBufferLists for each data_id + def load_packet_header(self) -> np.uint32 | None: + """Loads the packet header at the current read location into the buffer + + and updates internal variables. + """ + pkt_hdr = self.buffer[:1] + n_bytes_read = self.in_stream.readinto(pkt_hdr) # buffer is at least 4 kB long + self.n_bytes_read += n_bytes_read + if n_bytes_read == 0: # EOF + return None + if n_bytes_read != 4: + raise RuntimeError(f"only got {n_bytes_read} bytes for packet header") + + # packet is valid. Can set the packet_id and log its location + self.packet_id += 1 + filepos = self.in_stream.tell() - n_bytes_read + if self.packet_id < len(self.packet_locs): + if self.packet_locs[self.packet_id] != filepos: + raise RuntimeError(f"filepos for packet {self.packet_id} was {filepos} but {self.packet_locs[self.packet_id]} was expected") + else: + if len(self.packet_locs) != self.packet_id: + raise RuntimeError(f"loaded packet {self.packet_id} after packet {len(self.packet_locs)-1}") + self.packet_locs.append(filepos) + + return pkt_hdr + + def skip_packet(self, n: int = 1) -> bool: + """Skip a packets without loading it into the internal buffer. + + Requires loading the header. Optionally skips n packets. + + Returns + ---------- + succeeded + returns False if reached EOF, otherwise returns true + """ + if self.in_stream is None: + raise RuntimeError("self.in_stream is None") + if not int(n) >= 0: + raise ValueError(f"n must be a non-negative int, can't be {n}") + n = int(n) + while n > 0: + pkt_hdr = self.load_packet_header() + if pkt_hdr is None: + return False + self.in_stream.seek((orca_packet.get_n_words(pkt_hdr) - 1) * 4, 1) + n -= 1 + return True + + + def build_packet_locs(self, saveloc=True) -> None: + loc = self.in_stream.tell() + if len(self.packet_locs) > 0: + self.in_stream.seek(self.packet_locs[-1]) + while self.skip_packet(): + pass # builds the rest of the packet_locs list + if saveloc: + self.in_stream.seek(loc) + + def count_packets(self, saveloc=True) -> None: + self.build_packet_locs(saveloc=saveloc) + return len(self.packet_locs) + + # TODO: need to correct for endianness? - def load_packet(self, skip_unknown_ids: bool = False) -> np.uint32 | None: + def load_packet(self, index: int = None, whence: int = 0, skip_unknown_ids: bool = False) -> np.uint32 | None: """Loads the next packet into the internal buffer. Returns packet as a :class:`numpy.uint32` view of the buffer (a slice), returns ``None`` at EOF. + + Parameters + ---------- + index + Optionally give an index of packet to skip to, relative to the + "whence" location. Can be positive or negative. If out-of-range for + the file, None will be returned. + whence + used when an index is supplied. Follows the file.seek() convention: + - whence = 0 (default) means index is relative to the beginning of + the file + - whence = 1 means index is relative to the current position in the + file + - whence = 2 means relative to the end of the file + + + Returns + ---------- + packet + a view of the internal buffer spanning the packet data (uint32 + ndarray). If you want to hold on to the packet data while you load + more packets, you can call copy() on the view to make a copy. + + """ if self.in_stream is None: raise RuntimeError("self.in_stream is None") - # read packet header - pkt_hdr = self.buffer[:1] - n_bytes_read = self.in_stream.readinto(pkt_hdr) # buffer is at least 4 kB long - self.n_bytes_read += n_bytes_read - if n_bytes_read == 0: - return None - if n_bytes_read != 4: - raise RuntimeError(f"only got {n_bytes_read} bytes for packet header") + if index is not None: + if whence not in [0,1,2]: + raise ValueError(f"whence can't be {whence}") + index = int(index) + # convert whence 1 or 2 to whence = 0 + if whence == 1: # index is relative to current position + index += self.packet_id-1 + elif whence == 2: # index is relative to end of file + self.build_packet_locs(saveloc=False) + index += len(self.packet_locs)-2 + if index < 0: + self.in_stream.seek(0) + return None + while index >= len(self.packet_locs): + if self.skip_packet() == False: + return None + self.in_stream.seek(self.packet_locs[index]) + + + # load packet header + pkt_hdr = self.load_packet_header() + if pkt_hdr is None: return None # if it's a short packet, we are done if orca_packet.is_short(pkt_hdr): @@ -69,7 +172,6 @@ def load_packet(self, skip_unknown_ids: bool = False) -> np.uint32 | None: not in self.decoder_id_dict ): self.in_stream.seek((n_words - 1) * 4, 1) - self.n_bytes_read += (n_words - 1) * 4 # well, we didn't really read it... return pkt_hdr # load into buffer, resizing as necessary @@ -204,15 +306,17 @@ def open_stream( """ self.set_in_stream(stream_name) + self.packet_id = -1 # read in the header packet = self.load_packet() + if packet is None: + raise RuntimeError(f"no orca data in file {stream_name}") if orca_packet.get_data_id(packet) != 0: raise RuntimeError( f"got data id {orca_packet.get_data_id(packet)} for header" ) - self.packet_id = 0 self.any_full |= self.header_decoder.decode_packet(packet, self.packet_id) self.header = self.header_decoder.header @@ -296,7 +400,6 @@ def read_packet(self) -> bool: packet = self.load_packet(skip_unknown_ids=True) if packet is None: return False - self.packet_id += 1 # look up the data id, decoder, and rbl data_id = orca_packet.get_data_id(packet, shift=False) From 3415ec9f4c501e6dfc04c23c27f5bbabee60fa9f Mon Sep 17 00:00:00 2001 From: Jason Detwiler Date: Tue, 17 Oct 2023 09:13:44 +0100 Subject: [PATCH 05/19] add start of tests for orca fc decoding --- tests/orca/test_orca_fc.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 tests/orca/test_orca_fc.py diff --git a/tests/orca/test_orca_fc.py b/tests/orca/test_orca_fc.py new file mode 100644 index 0000000..b9b4178 --- /dev/null +++ b/tests/orca/test_orca_fc.py @@ -0,0 +1,19 @@ +import pytest +from daq2lh5.orca import orca_packet + +@pytest.fixture(scope="module") +def fc_packets(orca_stream): + packets = [] + packets.append(orca_stream.load_packet(2).copy()) # config + packets.append(orca_stream.load_packet(3).copy()) # waveform + orca_stream.close_stream() # avoid warning that file is still open + return packets + + +def test_orfc_config_decoding(fc_packets): + config_packet = fc_packets[0] + assert config_packet is not None + +def test_orfc_waveform_decoding(fc_packets): + wf_packet = fc_packets[1] + assert wf_packet is not None From 2d17bb0d6d0f9eb05ec40440b4d3b73d34796fa3 Mon Sep 17 00:00:00 2001 From: Jason Detwiler Date: Tue, 17 Oct 2023 09:13:57 +0100 Subject: [PATCH 06/19] check packet counting function --- tests/orca/test_orca_packet.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/orca/test_orca_packet.py b/tests/orca/test_orca_packet.py index 8a695af..37e157c 100644 --- a/tests/orca/test_orca_packet.py +++ b/tests/orca/test_orca_packet.py @@ -1,11 +1,15 @@ from daq2lh5.orca import orca_packet def test_orca_packet_funcs(orca_stream): + # The values in this test are particular to the test orca file in + # legend-testdata and may need to be changed if that file is changed + + assert orca_stream.count_packets() == 72 + packet = orca_stream.load_packet() - # These values are particular to the test orca file in legend-testdata and - # may need to be changed if that file is changed assert orca_packet.is_short(packet) == False assert orca_packet.get_data_id(packet) == 7 assert orca_packet.get_n_words(packet) == 4 assert orca_packet.hex_dump(packet, return_output=True)[-1] == '3 0x6286930b' + orca_stream.close_stream() # avoid warning that file is still open From 24c65c877fd51a0d1c1b21423e1d0126f3ae17eb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Oct 2023 12:48:15 +0000 Subject: [PATCH 07/19] style: pre-commit fixes --- src/daq2lh5/orca/orca_packet.py | 3 +-- src/daq2lh5/orca/orca_streamer.py | 38 +++++++++++++++++-------------- tests/orca/test_orca_fc.py | 8 ++++--- tests/orca/test_orca_packet.py | 3 ++- 4 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/daq2lh5/orca/orca_packet.py b/src/daq2lh5/orca/orca_packet.py index 465cc5c..2b0c94f 100644 --- a/src/daq2lh5/orca/orca_packet.py +++ b/src/daq2lh5/orca/orca_packet.py @@ -47,7 +47,7 @@ def hex_dump( as_short: bool = False, id_dict: dict = None, use_logging: bool = True, - return_output = False + return_output=False, ) -> None: output = [] data_id = get_data_id(packet, shift=shift_data_id) @@ -84,4 +84,3 @@ def hex_dump( if return_output: return output - diff --git a/src/daq2lh5/orca/orca_streamer.py b/src/daq2lh5/orca/orca_streamer.py index 7b02725..0378271 100644 --- a/src/daq2lh5/orca/orca_streamer.py +++ b/src/daq2lh5/orca/orca_streamer.py @@ -47,7 +47,7 @@ def load_packet_header(self) -> np.uint32 | None: pkt_hdr = self.buffer[:1] n_bytes_read = self.in_stream.readinto(pkt_hdr) # buffer is at least 4 kB long self.n_bytes_read += n_bytes_read - if n_bytes_read == 0: # EOF + if n_bytes_read == 0: # EOF return None if n_bytes_read != 4: raise RuntimeError(f"only got {n_bytes_read} bytes for packet header") @@ -57,10 +57,14 @@ def load_packet_header(self) -> np.uint32 | None: filepos = self.in_stream.tell() - n_bytes_read if self.packet_id < len(self.packet_locs): if self.packet_locs[self.packet_id] != filepos: - raise RuntimeError(f"filepos for packet {self.packet_id} was {filepos} but {self.packet_locs[self.packet_id]} was expected") + raise RuntimeError( + f"filepos for packet {self.packet_id} was {filepos} but {self.packet_locs[self.packet_id]} was expected" + ) else: if len(self.packet_locs) != self.packet_id: - raise RuntimeError(f"loaded packet {self.packet_id} after packet {len(self.packet_locs)-1}") + raise RuntimeError( + f"loaded packet {self.packet_id} after packet {len(self.packet_locs)-1}" + ) self.packet_locs.append(filepos) return pkt_hdr @@ -82,29 +86,29 @@ def skip_packet(self, n: int = 1) -> bool: n = int(n) while n > 0: pkt_hdr = self.load_packet_header() - if pkt_hdr is None: + if pkt_hdr is None: return False self.in_stream.seek((orca_packet.get_n_words(pkt_hdr) - 1) * 4, 1) n -= 1 return True - def build_packet_locs(self, saveloc=True) -> None: loc = self.in_stream.tell() if len(self.packet_locs) > 0: self.in_stream.seek(self.packet_locs[-1]) while self.skip_packet(): - pass # builds the rest of the packet_locs list - if saveloc: + pass # builds the rest of the packet_locs list + if saveloc: self.in_stream.seek(loc) def count_packets(self, saveloc=True) -> None: self.build_packet_locs(saveloc=saveloc) return len(self.packet_locs) - # TODO: need to correct for endianness? - def load_packet(self, index: int = None, whence: int = 0, skip_unknown_ids: bool = False) -> np.uint32 | None: + def load_packet( + self, index: int = None, whence: int = 0, skip_unknown_ids: bool = False + ) -> np.uint32 | None: """Loads the next packet into the internal buffer. Returns packet as a :class:`numpy.uint32` view of the buffer (a slice), @@ -138,16 +142,16 @@ def load_packet(self, index: int = None, whence: int = 0, skip_unknown_ids: bool raise RuntimeError("self.in_stream is None") if index is not None: - if whence not in [0,1,2]: + if whence not in [0, 1, 2]: raise ValueError(f"whence can't be {whence}") index = int(index) # convert whence 1 or 2 to whence = 0 - if whence == 1: # index is relative to current position - index += self.packet_id-1 - elif whence == 2: # index is relative to end of file + if whence == 1: # index is relative to current position + index += self.packet_id - 1 + elif whence == 2: # index is relative to end of file self.build_packet_locs(saveloc=False) - index += len(self.packet_locs)-2 - if index < 0: + index += len(self.packet_locs) - 2 + if index < 0: self.in_stream.seek(0) return None while index >= len(self.packet_locs): @@ -155,10 +159,10 @@ def load_packet(self, index: int = None, whence: int = 0, skip_unknown_ids: bool return None self.in_stream.seek(self.packet_locs[index]) - # load packet header pkt_hdr = self.load_packet_header() - if pkt_hdr is None: return None + if pkt_hdr is None: + return None # if it's a short packet, we are done if orca_packet.is_short(pkt_hdr): diff --git a/tests/orca/test_orca_fc.py b/tests/orca/test_orca_fc.py index b9b4178..fc4a586 100644 --- a/tests/orca/test_orca_fc.py +++ b/tests/orca/test_orca_fc.py @@ -1,11 +1,12 @@ import pytest -from daq2lh5.orca import orca_packet + + @pytest.fixture(scope="module") def fc_packets(orca_stream): packets = [] - packets.append(orca_stream.load_packet(2).copy()) # config - packets.append(orca_stream.load_packet(3).copy()) # waveform + packets.append(orca_stream.load_packet(2).copy()) # config + packets.append(orca_stream.load_packet(3).copy()) # waveform orca_stream.close_stream() # avoid warning that file is still open return packets @@ -14,6 +15,7 @@ def test_orfc_config_decoding(fc_packets): config_packet = fc_packets[0] assert config_packet is not None + def test_orfc_waveform_decoding(fc_packets): wf_packet = fc_packets[1] assert wf_packet is not None diff --git a/tests/orca/test_orca_packet.py b/tests/orca/test_orca_packet.py index 37e157c..39c1ca3 100644 --- a/tests/orca/test_orca_packet.py +++ b/tests/orca/test_orca_packet.py @@ -1,5 +1,6 @@ from daq2lh5.orca import orca_packet + def test_orca_packet_funcs(orca_stream): # The values in this test are particular to the test orca file in # legend-testdata and may need to be changed if that file is changed @@ -10,6 +11,6 @@ def test_orca_packet_funcs(orca_stream): assert orca_packet.is_short(packet) == False assert orca_packet.get_data_id(packet) == 7 assert orca_packet.get_n_words(packet) == 4 - assert orca_packet.hex_dump(packet, return_output=True)[-1] == '3 0x6286930b' + assert orca_packet.hex_dump(packet, return_output=True)[-1] == "3 0x6286930b" orca_stream.close_stream() # avoid warning that file is still open From 5399a63e304d2d355d21a306eb42c9949939a432 Mon Sep 17 00:00:00 2001 From: Jason Detwiler Date: Wed, 18 Oct 2023 06:30:28 +0100 Subject: [PATCH 08/19] keep packet_id aligned with loc --- src/daq2lh5/orca/orca_streamer.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/daq2lh5/orca/orca_streamer.py b/src/daq2lh5/orca/orca_streamer.py index 0378271..bde8e2d 100644 --- a/src/daq2lh5/orca/orca_streamer.py +++ b/src/daq2lh5/orca/orca_streamer.py @@ -94,12 +94,15 @@ def skip_packet(self, n: int = 1) -> bool: def build_packet_locs(self, saveloc=True) -> None: loc = self.in_stream.tell() + pid = self.packet_id if len(self.packet_locs) > 0: self.in_stream.seek(self.packet_locs[-1]) + self.packet_id = len(self.packet_locs)-2 while self.skip_packet(): pass # builds the rest of the packet_locs list if saveloc: self.in_stream.seek(loc) + self.packet_id = pid def count_packets(self, saveloc=True) -> None: self.build_packet_locs(saveloc=saveloc) @@ -153,11 +156,13 @@ def load_packet( index += len(self.packet_locs) - 2 if index < 0: self.in_stream.seek(0) + self.packet_id = -1 return None while index >= len(self.packet_locs): if self.skip_packet() == False: return None self.in_stream.seek(self.packet_locs[index]) + self.packet_id = index-1 # load packet header pkt_hdr = self.load_packet_header() From d06ec9bff7eaf08701ddddca3d54ba92acf35e02 Mon Sep 17 00:00:00 2001 From: Jason Detwiler Date: Thu, 19 Oct 2023 03:23:58 +0100 Subject: [PATCH 09/19] switch to new file and add some more tests --- tests/orca/conftest.py | 2 +- tests/orca/test_orca_packet.py | 25 ++++++++++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/tests/orca/conftest.py b/tests/orca/conftest.py index 61e1163..1fc60f2 100644 --- a/tests/orca/conftest.py +++ b/tests/orca/conftest.py @@ -7,6 +7,6 @@ def orca_stream(lgnd_test_data): orstr = OrcaStreamer() orstr.open_stream( - lgnd_test_data.get_path("orca/fc/L200-comm-20220519-phy-geds.orca") + lgnd_test_data.get_path("orca/fc/l200-p02-r008-phy-20230113T174010Z.orca") ) return orstr diff --git a/tests/orca/test_orca_packet.py b/tests/orca/test_orca_packet.py index 39c1ca3..7027828 100644 --- a/tests/orca/test_orca_packet.py +++ b/tests/orca/test_orca_packet.py @@ -5,12 +5,31 @@ def test_orca_packet_funcs(orca_stream): # The values in this test are particular to the test orca file in # legend-testdata and may need to be changed if that file is changed - assert orca_stream.count_packets() == 72 + assert orca_stream.count_packets() == 911 packet = orca_stream.load_packet() assert orca_packet.is_short(packet) == False - assert orca_packet.get_data_id(packet) == 7 + assert orca_packet.get_data_id(packet) == 3 assert orca_packet.get_n_words(packet) == 4 - assert orca_packet.hex_dump(packet, return_output=True)[-1] == "3 0x6286930b" + assert orca_packet.hex_dump(packet, return_output=True)[-1] == '3 0x63c1977a' + + id_dict = orca_stream.header.get_id_to_decoder_name_dict() + seen = [] + for ii in range(100): + packet = orca_stream.load_packet(ii) + if packet is None: break + name = id_dict[orca_packet.get_data_id(packet)] + if ii == 0: assert name == 'OrcaHeaderDecoder' + if ii == 1: assert name == 'ORRunDecoderForRun' + if ii == 911: assert name == 'ORRunDecoderForRun' + if name not in seen: seen.append(name) + expected = [ + 'OrcaHeaderDecoder', + 'ORRunDecoderForRun', + 'ORFlashCamListenerConfigDecoder', + 'ORFlashCamListenerStatusDecoder', + 'ORFlashCamWaveformDecoder' + ] + assert seen == expected orca_stream.close_stream() # avoid warning that file is still open From 05b7fc178b32e786f3f1c20eb8263d275b12e8d4 Mon Sep 17 00:00:00 2001 From: Jason Detwiler Date: Wed, 18 Oct 2023 06:34:38 +0100 Subject: [PATCH 10/19] fix last-packet check --- tests/orca/test_orca_packet.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/orca/test_orca_packet.py b/tests/orca/test_orca_packet.py index 7027828..49eca80 100644 --- a/tests/orca/test_orca_packet.py +++ b/tests/orca/test_orca_packet.py @@ -19,9 +19,10 @@ def test_orca_packet_funcs(orca_stream): packet = orca_stream.load_packet(ii) if packet is None: break name = id_dict[orca_packet.get_data_id(packet)] + #if ii < 20: print(ii, name) if ii == 0: assert name == 'OrcaHeaderDecoder' if ii == 1: assert name == 'ORRunDecoderForRun' - if ii == 911: assert name == 'ORRunDecoderForRun' + if ii == 910: assert name == 'ORRunDecoderForRun' if name not in seen: seen.append(name) expected = [ 'OrcaHeaderDecoder', From eb7a844148ca0bfbcd011a3e0b6c4c7e6f1da62d Mon Sep 17 00:00:00 2001 From: Jason Detwiler Date: Thu, 19 Oct 2023 03:25:21 +0100 Subject: [PATCH 11/19] start to add status packet tests --- tests/orca/test_orca_fc.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/tests/orca/test_orca_fc.py b/tests/orca/test_orca_fc.py index fc4a586..ace31bd 100644 --- a/tests/orca/test_orca_fc.py +++ b/tests/orca/test_orca_fc.py @@ -5,17 +5,35 @@ @pytest.fixture(scope="module") def fc_packets(orca_stream): packets = [] - packets.append(orca_stream.load_packet(2).copy()) # config - packets.append(orca_stream.load_packet(3).copy()) # waveform + packets.append(orca_stream.load_packet(3).copy()) # config + packets.append(orca_stream.load_packet(4).copy()) # status + packets.append(orca_stream.load_packet(13).copy()) # waveform orca_stream.close_stream() # avoid warning that file is still open return packets -def test_orfc_config_decoding(fc_packets): +def test_orfc_config_decoding(orca_stream, fc_packets): config_packet = fc_packets[0] assert config_packet is not None + data_id = orca_packet.get_data_id(config_packet) + name = orca_stream.header.get_id_to_decoder_name_dict()[data_id] + assert name == 'ORFlashCamListenerConfigDecoder' -def test_orfc_waveform_decoding(fc_packets): - wf_packet = fc_packets[1] + +def test_orfc_status_decoding(orca_stream, fc_packets): + status_packet = fc_packets[1] + assert status_packet is not None + + data_id = orca_packet.get_data_id(status_packet) + name = orca_stream.header.get_id_to_decoder_name_dict()[data_id] + assert name == 'ORFlashCamListenerStatusDecoder' + + +def test_orfc_waveform_decoding(orca_stream, fc_packets): + wf_packet = fc_packets[2] assert wf_packet is not None + + data_id = orca_packet.get_data_id(wf_packet) + name = orca_stream.header.get_id_to_decoder_name_dict()[data_id] + assert name == 'ORFlashCamWaveformDecoder' From ab79de46b399a8ca980fbea9a75b4cdfea12a402 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 19 Oct 2023 02:25:49 +0000 Subject: [PATCH 12/19] style: pre-commit fixes --- src/daq2lh5/orca/orca_streamer.py | 4 ++-- tests/orca/test_orca_fc.py | 13 ++++++------- tests/orca/test_orca_packet.py | 29 +++++++++++++++++------------ 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/src/daq2lh5/orca/orca_streamer.py b/src/daq2lh5/orca/orca_streamer.py index bde8e2d..9ec15e1 100644 --- a/src/daq2lh5/orca/orca_streamer.py +++ b/src/daq2lh5/orca/orca_streamer.py @@ -97,7 +97,7 @@ def build_packet_locs(self, saveloc=True) -> None: pid = self.packet_id if len(self.packet_locs) > 0: self.in_stream.seek(self.packet_locs[-1]) - self.packet_id = len(self.packet_locs)-2 + self.packet_id = len(self.packet_locs) - 2 while self.skip_packet(): pass # builds the rest of the packet_locs list if saveloc: @@ -162,7 +162,7 @@ def load_packet( if self.skip_packet() == False: return None self.in_stream.seek(self.packet_locs[index]) - self.packet_id = index-1 + self.packet_id = index - 1 # load packet header pkt_hdr = self.load_packet_header() diff --git a/tests/orca/test_orca_fc.py b/tests/orca/test_orca_fc.py index ace31bd..a1cbf3f 100644 --- a/tests/orca/test_orca_fc.py +++ b/tests/orca/test_orca_fc.py @@ -1,13 +1,12 @@ import pytest - @pytest.fixture(scope="module") def fc_packets(orca_stream): packets = [] - packets.append(orca_stream.load_packet(3).copy()) # config - packets.append(orca_stream.load_packet(4).copy()) # status - packets.append(orca_stream.load_packet(13).copy()) # waveform + packets.append(orca_stream.load_packet(3).copy()) # config + packets.append(orca_stream.load_packet(4).copy()) # status + packets.append(orca_stream.load_packet(13).copy()) # waveform orca_stream.close_stream() # avoid warning that file is still open return packets @@ -18,7 +17,7 @@ def test_orfc_config_decoding(orca_stream, fc_packets): data_id = orca_packet.get_data_id(config_packet) name = orca_stream.header.get_id_to_decoder_name_dict()[data_id] - assert name == 'ORFlashCamListenerConfigDecoder' + assert name == "ORFlashCamListenerConfigDecoder" def test_orfc_status_decoding(orca_stream, fc_packets): @@ -27,7 +26,7 @@ def test_orfc_status_decoding(orca_stream, fc_packets): data_id = orca_packet.get_data_id(status_packet) name = orca_stream.header.get_id_to_decoder_name_dict()[data_id] - assert name == 'ORFlashCamListenerStatusDecoder' + assert name == "ORFlashCamListenerStatusDecoder" def test_orfc_waveform_decoding(orca_stream, fc_packets): @@ -36,4 +35,4 @@ def test_orfc_waveform_decoding(orca_stream, fc_packets): data_id = orca_packet.get_data_id(wf_packet) name = orca_stream.header.get_id_to_decoder_name_dict()[data_id] - assert name == 'ORFlashCamWaveformDecoder' + assert name == "ORFlashCamWaveformDecoder" diff --git a/tests/orca/test_orca_packet.py b/tests/orca/test_orca_packet.py index 49eca80..d93f25e 100644 --- a/tests/orca/test_orca_packet.py +++ b/tests/orca/test_orca_packet.py @@ -11,25 +11,30 @@ def test_orca_packet_funcs(orca_stream): assert orca_packet.is_short(packet) == False assert orca_packet.get_data_id(packet) == 3 assert orca_packet.get_n_words(packet) == 4 - assert orca_packet.hex_dump(packet, return_output=True)[-1] == '3 0x63c1977a' + assert orca_packet.hex_dump(packet, return_output=True)[-1] == "3 0x63c1977a" id_dict = orca_stream.header.get_id_to_decoder_name_dict() seen = [] for ii in range(100): packet = orca_stream.load_packet(ii) - if packet is None: break + if packet is None: + break name = id_dict[orca_packet.get_data_id(packet)] - #if ii < 20: print(ii, name) - if ii == 0: assert name == 'OrcaHeaderDecoder' - if ii == 1: assert name == 'ORRunDecoderForRun' - if ii == 910: assert name == 'ORRunDecoderForRun' - if name not in seen: seen.append(name) + # if ii < 20: print(ii, name) + if ii == 0: + assert name == "OrcaHeaderDecoder" + if ii == 1: + assert name == "ORRunDecoderForRun" + if ii == 910: + assert name == "ORRunDecoderForRun" + if name not in seen: + seen.append(name) expected = [ - 'OrcaHeaderDecoder', - 'ORRunDecoderForRun', - 'ORFlashCamListenerConfigDecoder', - 'ORFlashCamListenerStatusDecoder', - 'ORFlashCamWaveformDecoder' + "OrcaHeaderDecoder", + "ORRunDecoderForRun", + "ORFlashCamListenerConfigDecoder", + "ORFlashCamListenerStatusDecoder", + "ORFlashCamWaveformDecoder", ] assert seen == expected From d3bb5abeeba31ab7f232765c6bc2aef1f225c68a Mon Sep 17 00:00:00 2001 From: Jason Detwiler Date: Thu, 19 Oct 2023 03:42:26 +0100 Subject: [PATCH 13/19] eliminate unneccessary build_raw warnings (do late eval) --- src/daq2lh5/orca/orca_streamer.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/daq2lh5/orca/orca_streamer.py b/src/daq2lh5/orca/orca_streamer.py index 9ec15e1..7615380 100644 --- a/src/daq2lh5/orca/orca_streamer.py +++ b/src/daq2lh5/orca/orca_streamer.py @@ -38,6 +38,7 @@ def __init__(self) -> None: self.header_decoder = OrcaHeaderDecoder() self.decoder_id_dict = {} # dict of data_id to decoder object self.rbl_id_dict = {} # dict of RawBufferLists for each data_id + self.missing_decoders = [] def load_packet_header(self) -> np.uint32 | None: """Loads the packet header at the current read location into the buffer @@ -353,9 +354,7 @@ def open_stream( name = id_to_dec_name_dict[data_id] if name not in instantiated_decoders: if name not in globals(): - log.warning( - f"no implementation of {name}, corresponding packets will be skipped" - ) + self.missing_decoders.append(data_id) continue decoder = globals()[name] instantiated_decoders[name] = decoder(header=self.header) @@ -415,6 +414,12 @@ def read_packet(self) -> bool: log.debug( f"packet {self.packet_id}: data_id = {data_id}, decoder = {'None' if data_id not in self.decoder_id_dict else type(self.decoder_id_dict[data_id]).__name__}" ) + if data_id in self.missing_decoders: + name = self.header.get_id_to_decoder_name_dict(shift_data_id=False)[data_id] + log.warning( + f"no implementation of {name}, packets were skipped" + ) + continue if data_id in self.rbl_id_dict: break From 5e196c138a69c1a351ab18c457148ce64dcb472d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 19 Oct 2023 02:45:14 +0000 Subject: [PATCH 14/19] style: pre-commit fixes --- src/daq2lh5/orca/orca_streamer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/daq2lh5/orca/orca_streamer.py b/src/daq2lh5/orca/orca_streamer.py index 7615380..fef598f 100644 --- a/src/daq2lh5/orca/orca_streamer.py +++ b/src/daq2lh5/orca/orca_streamer.py @@ -415,10 +415,10 @@ def read_packet(self) -> bool: f"packet {self.packet_id}: data_id = {data_id}, decoder = {'None' if data_id not in self.decoder_id_dict else type(self.decoder_id_dict[data_id]).__name__}" ) if data_id in self.missing_decoders: - name = self.header.get_id_to_decoder_name_dict(shift_data_id=False)[data_id] - log.warning( - f"no implementation of {name}, packets were skipped" - ) + name = self.header.get_id_to_decoder_name_dict(shift_data_id=False)[ + data_id + ] + log.warning(f"no implementation of {name}, packets were skipped") continue if data_id in self.rbl_id_dict: break From 430506eeb0191d870292bc2b51dcc4d6c1e7f09f Mon Sep 17 00:00:00 2001 From: Jason Detwiler Date: Mon, 23 Oct 2023 09:58:25 -0700 Subject: [PATCH 15/19] add missing include --- tests/orca/test_orca_fc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/orca/test_orca_fc.py b/tests/orca/test_orca_fc.py index a1cbf3f..7ad0f9a 100644 --- a/tests/orca/test_orca_fc.py +++ b/tests/orca/test_orca_fc.py @@ -1,4 +1,5 @@ import pytest +from daq2lh5.orca import orca_packet @pytest.fixture(scope="module") From 4359b8f04cda692519314e59ee66808b8bd14045 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 23 Oct 2023 16:58:50 +0000 Subject: [PATCH 16/19] style: pre-commit fixes --- tests/orca/test_orca_fc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/orca/test_orca_fc.py b/tests/orca/test_orca_fc.py index 7ad0f9a..99c74de 100644 --- a/tests/orca/test_orca_fc.py +++ b/tests/orca/test_orca_fc.py @@ -1,4 +1,5 @@ import pytest + from daq2lh5.orca import orca_packet From 33ace79a5f474dcb82a581af15c2d7e7a013eb7e Mon Sep 17 00:00:00 2001 From: Jason Detwiler Date: Mon, 23 Oct 2023 10:56:45 -0700 Subject: [PATCH 17/19] fixes for flake8 and dox --- src/daq2lh5/orca/orca_streamer.py | 13 ++++--------- tests/orca/test_orca_packet.py | 2 +- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/daq2lh5/orca/orca_streamer.py b/src/daq2lh5/orca/orca_streamer.py index fef598f..ad1c6ec 100644 --- a/src/daq2lh5/orca/orca_streamer.py +++ b/src/daq2lh5/orca/orca_streamer.py @@ -126,12 +126,9 @@ def load_packet( the file, None will be returned. whence used when an index is supplied. Follows the file.seek() convention: - - whence = 0 (default) means index is relative to the beginning of - the file - - whence = 1 means index is relative to the current position in the - file - - whence = 2 means relative to the end of the file - + whence = 0 (default) means index is relative to the beginning of the + file; whence = 1 means index is relative to the current position in + the file; whence = 2 means relative to the end of the file. Returns ---------- @@ -139,8 +136,6 @@ def load_packet( a view of the internal buffer spanning the packet data (uint32 ndarray). If you want to hold on to the packet data while you load more packets, you can call copy() on the view to make a copy. - - """ if self.in_stream is None: raise RuntimeError("self.in_stream is None") @@ -160,7 +155,7 @@ def load_packet( self.packet_id = -1 return None while index >= len(self.packet_locs): - if self.skip_packet() == False: + if not self.skip_packet(): return None self.in_stream.seek(self.packet_locs[index]) self.packet_id = index - 1 diff --git a/tests/orca/test_orca_packet.py b/tests/orca/test_orca_packet.py index d93f25e..6985c98 100644 --- a/tests/orca/test_orca_packet.py +++ b/tests/orca/test_orca_packet.py @@ -8,7 +8,7 @@ def test_orca_packet_funcs(orca_stream): assert orca_stream.count_packets() == 911 packet = orca_stream.load_packet() - assert orca_packet.is_short(packet) == False + assert orca_packet.is_short(packet) is False assert orca_packet.get_data_id(packet) == 3 assert orca_packet.get_n_words(packet) == 4 assert orca_packet.hex_dump(packet, return_output=True)[-1] == "3 0x63c1977a" From a4906fbf70957bac65b0950dbbc7092b83f57d9e Mon Sep 17 00:00:00 2001 From: Jason Detwiler Date: Tue, 24 Oct 2023 13:42:20 -0700 Subject: [PATCH 18/19] skip decoding status packets until pyfcutils is updated --- src/daq2lh5/orca/orca_flashcam.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/daq2lh5/orca/orca_flashcam.py b/src/daq2lh5/orca/orca_flashcam.py index 7b3080b..f94611d 100644 --- a/src/daq2lh5/orca/orca_flashcam.py +++ b/src/daq2lh5/orca/orca_flashcam.py @@ -326,6 +326,7 @@ def get_decoded_values(self, key: int = None) -> dict[str, Any]: def decode_packet( self, packet: OrcaPacket, packet_id: int, rbl: RawBufferLibrary ) -> bool: + return False # FIXME: skip decoding until pyfcutils is updated """Decode the ORCA FlashCam Status packet.""" # aliases for brevity if len(rbl) != 1: From 761e8029359a45de0839e78cf01095489bbe63db Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 24 Oct 2023 20:44:22 +0000 Subject: [PATCH 19/19] style: pre-commit fixes --- src/daq2lh5/orca/orca_flashcam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/daq2lh5/orca/orca_flashcam.py b/src/daq2lh5/orca/orca_flashcam.py index f94611d..d3871a0 100644 --- a/src/daq2lh5/orca/orca_flashcam.py +++ b/src/daq2lh5/orca/orca_flashcam.py @@ -326,7 +326,7 @@ def get_decoded_values(self, key: int = None) -> dict[str, Any]: def decode_packet( self, packet: OrcaPacket, packet_id: int, rbl: RawBufferLibrary ) -> bool: - return False # FIXME: skip decoding until pyfcutils is updated + return False # FIXME: skip decoding until pyfcutils is updated """Decode the ORCA FlashCam Status packet.""" # aliases for brevity if len(rbl) != 1: