From 49129cb641d7f7d3616b207bbc51bee24bf284a4 Mon Sep 17 00:00:00 2001 From: rpartzsch Date: Wed, 18 Dec 2024 10:49:13 +0100 Subject: [PATCH] MAINT: removed data parser class --- aidatlu/aidatlu_run.py | 12 +- aidatlu/main/data_parser.py | 307 +++++++++++++++++----------------- aidatlu/main/tlu.py | 4 +- aidatlu/test/software_test.py | 5 +- 4 files changed, 165 insertions(+), 163 deletions(-) diff --git a/aidatlu/aidatlu_run.py b/aidatlu/aidatlu_run.py index c2e1eaf..4008619 100644 --- a/aidatlu/aidatlu_run.py +++ b/aidatlu/aidatlu_run.py @@ -4,12 +4,12 @@ class AIDATLU: def __init__(self, config_path, clock_path): - print(" ---------------------------------------") - print(" _ ___ ___ _ _____ _ _ _ ") - print(" /_\ |_ _| \ /_\ |_ _| | | | | |") - print(" / _ \ | || |) / _ \ | | | |_| |_| |") - print(" /_/ \_\___|___/_/ \_\ |_| |____\___/ \n") - print(" ---------------------------------------") + print(r" ---------------------------------------") + print(r" _ ___ ___ _ _____ _ _ _ ") + print(r" /_\ |_ _| \ /_\ |_ _| | | | | |") + print(r" / _ \ | || |) / _ \ -- | | | |_| |_| |") + print(r" /_/ \_\___|___/_/ \_\ |_| |____\___/ ") + print(r" ---------------------------------------") print("tlu.help()\n") self.config_file = config_path diff --git a/aidatlu/main/data_parser.py b/aidatlu/main/data_parser.py index e32846e..da5fc7c 100644 --- a/aidatlu/main/data_parser.py +++ b/aidatlu/main/data_parser.py @@ -7,162 +7,165 @@ from aidatlu import logger -class DataParser: - def __init__(self) -> None: - self.log = logger.setup_main_logger(__class__.__name__) - self.features = np.dtype( - [ - ("eventnumber", "u4"), - ("timestamp", "u8"), - ("overflow", "u8"), - ("eventtype", "u4"), - ("input1", "bool"), - ("input2", "bool"), - ("input3", "bool"), - ("input4", "bool"), - ("input5", "bool"), - ("input6", "bool"), - ("sc1", "u4"), - ("sc2", "u4"), - ("sc3", "u4"), - ("sc4", "u4"), - ("sc5", "u4"), - ("sc6", "u4"), - ] - ) - self.raw_features = np.dtype([("raw", "u4")]) - - def interpret_data( - self, - filepath_in: str | Path, - filepath_out: str | Path, - chunk_size: int = 1000000, - ) -> None: - """Interprets raw tlu data. The data is interpreted in chunksizes. - The data is parsed form filepath_in to filepath_out. - An event consists of six consecutive raw data entries the last entry should be always 0. - The raw data is sliced and the last data entry checked for corrupted data. - - Args: - filepath_in (str | Path): raw data file path as string or Path object - filepath_out (str | Path): output path of the interpreted data as string or Path object - """ - self.log.info("Interpreting Data") - chunk_size = chunk_size * 6 - with tb.open_file(filepath_in, "r") as in_file: - n_words = in_file.root.raw_data.shape[0] - self.conf = np.array(in_file.root.conf[:]) - - if n_words == 0: - self.log.warning("Data is empty. Skip analysis!") - return - - with tb.open_file( - filepath_out, mode="w", title="TLU_interpreted" - ) as out_file: - data_table = self._create_table( - out_file, name="interpreted_data", title="data", dtype=self.features +def interpret_data( + filepath_in: str | Path, + filepath_out: str | Path = None, + chunk_size: int = 1000000, +) -> None: + """Interprets raw tlu data. The data is interpreted in chunksizes. + The data is parsed form filepath_in to filepath_out. + An event consists of six consecutive raw data entries the last entry should be always 0. + The raw data is sliced and the last data entry checked for corrupted data. + + Args: + filepath_in (str | Path): raw data file path as string or Path object + filepath_out (str | Path): output path of the interpreted data as string or Path object + """ + log = logger.setup_main_logger("Data Interpreter") + features = np.dtype( + [ + ("eventnumber", "u4"), + ("timestamp", "u8"), + ("overflow", "u8"), + ("eventtype", "u4"), + ("input1", "bool"), + ("input2", "bool"), + ("input3", "bool"), + ("input4", "bool"), + ("input5", "bool"), + ("input6", "bool"), + ("sc1", "u4"), + ("sc2", "u4"), + ("sc3", "u4"), + ("sc4", "u4"), + ("sc5", "u4"), + ("sc6", "u4"), + ] + ) + raw_features = np.dtype([("raw", "u4")]) + + log.info("Interpreting Data") + chunk_size = chunk_size * 6 + + if filepath_out == None: + filepath_out = filepath_in + "_interpreted.h5" + + with tb.open_file(filepath_in, "r") as in_file: + n_words = in_file.root.raw_data.shape[0] + conf = np.array(in_file.root.conf[:]) + + if n_words == 0: + log.warning("Data is empty. Skip analysis!") + return + + with tb.open_file(filepath_out, mode="w", title="TLU_interpreted") as out_file: + data_table = _create_table( + out_file, name="interpreted_data", title="data", dtype=features + ) + for chunk in tqdm(range(0, n_words, chunk_size)): + chunk_offset = chunk + stop = chunk_offset + chunk_size + if chunk + chunk_size > n_words: + stop = n_words + table = in_file.root.raw_data[chunk_offset:stop] + raw_data = np.array(table[:], dtype=raw_features) + data = _transform_data( + raw_data["raw"][::6], + raw_data["raw"][1::6], + raw_data["raw"][2::6], + raw_data["raw"][3::6], + raw_data["raw"][4::6], + raw_data["raw"][5::6], + log, + features, ) - for chunk in tqdm(range(0, n_words, chunk_size)): - chunk_offset = chunk - stop = chunk_offset + chunk_size - if chunk + chunk_size > n_words: - stop = n_words - table = in_file.root.raw_data[chunk_offset:stop] - raw_data = np.array(table[:], dtype=self.raw_features) - data = self._transform_data( - raw_data["raw"][::6], - raw_data["raw"][1::6], - raw_data["raw"][2::6], - raw_data["raw"][3::6], - raw_data["raw"][4::6], - raw_data["raw"][5::6], - ) - data_table.append(data) - - config = np.dtype( - [ - ("attribute", "S32"), - ("value", "S32"), - ] - ) - config_table = out_file.create_table( - out_file.root, - name="conf", - description=config, - ) - config_table.append(self.conf) - self.log.success('Data parsed from "%s" to "%s"' % (filepath_in, filepath_out)) - - def _create_table(self, out_file, name, title, dtype): - """Create hit table node for storage in out_file. - Copy configuration nodes from raw data file. - """ - table = out_file.create_table( - out_file.root, - name=name, - description=dtype, - title=title, - filters=tb.Filters(complib="blosc", complevel=5, fletcher32=False), - ) - - return table - - def _transform_data( - self, - w0: np.array, - w1: np.array, - w2: np.array, - w3: np.array, - w4: np.array, - w5: np.array, - ) -> np.array: - """Transforms raw data from the FIFO to a readable dataformat - - Args: - w0 (np.array): contains information which trigger input fired - w1 (np.array): contains timestamp information - w2 (np.array): trigger input information - w3 (np.array): eventnumber - w4 (np.array): trigger input information - w5 (np.array): this should always be 0. - - Returns: - np.array: array with columns - """ - if np.any(w5) != 0: - self.log.warning("Corrupted Data found") - - # Cast w0 to uint64 for concatenating 2 x 32bit to 64bit later - w0 = w0.astype(np.uint64) - - out_array = np.zeros(len(w3), dtype=self.features) - out_array["eventnumber"] = w3 - out_array["timestamp"] = (w0 & 0x0000FFFF << 32) + w1 - out_array["overflow"] = w0 & 0xFFFF - # TODO not sure what this is per. mode? - out_array["eventtype"] = (w0 >> 28) & 0xF - # Which trigger input produced the event. - out_array["input1"] = (w0 >> 16) & 0x1 - out_array["input2"] = (w0 >> 17) & 0x1 - out_array["input3"] = (w0 >> 18) & 0x1 - out_array["input4"] = (w0 >> 19) & 0x1 - out_array["input5"] = (w0 >> 20) & 0x1 - out_array["input6"] = (w0 >> 21) & 0x1 - # TODO not sure what these are prob. something from the DACs - out_array["sc1"] = (w2 >> 24) & 0xFF - out_array["sc2"] = (w2 >> 16) & 0xFF - out_array["sc3"] = (w2 >> 8) & 0xFF - out_array["sc4"] = w2 & 0xFF - out_array["sc5"] = (w4 >> 24) & 0xFF - out_array["sc6"] = (w4 >> 16) & 0xFF - return out_array + data_table.append(data) + + config = np.dtype( + [ + ("attribute", "S32"), + ("value", "S32"), + ] + ) + config_table = out_file.create_table( + out_file.root, + name="conf", + description=config, + ) + config_table.append(conf) + log.success('Data parsed from "%s" to "%s"' % (filepath_in, filepath_out)) + + +def _create_table(out_file, name, title, dtype): + """Create hit table node for storage in out_file. + Copy configuration nodes from raw data file. + """ + table = out_file.create_table( + out_file.root, + name=name, + description=dtype, + title=title, + filters=tb.Filters(complib="blosc", complevel=5, fletcher32=False), + ) + return table + + +def _transform_data( + w0: np.array, + w1: np.array, + w2: np.array, + w3: np.array, + w4: np.array, + w5: np.array, + log: logger, + features: np.dtype, +) -> np.array: + """Transforms raw data from the FIFO to a readable dataformat + + Args: + w0 (np.array): contains information which trigger input fired + w1 (np.array): contains timestamp information + w2 (np.array): trigger input information + w3 (np.array): eventnumber + w4 (np.array): trigger input information + w5 (np.array): this should always be 0. + log (logger): logging function + features (dtype): output array dtype structure + + Returns: + np.array: array with columns + """ + if np.any(w5) != 0: + log.warning("Corrupted Data found") + + # Cast w0 to uint64 for concatenating 2 x 32bit to 64bit later + w0 = w0.astype(np.uint64) + + out_array = np.zeros(len(w3), dtype=features) + out_array["eventnumber"] = w3 + out_array["timestamp"] = (w0 & 0x0000FFFF << 32) + w1 + out_array["overflow"] = w0 & 0xFFFF + # TODO not sure what this is per. mode? + out_array["eventtype"] = (w0 >> 28) & 0xF + # Which trigger input produced the event. + out_array["input1"] = (w0 >> 16) & 0x1 + out_array["input2"] = (w0 >> 17) & 0x1 + out_array["input3"] = (w0 >> 18) & 0x1 + out_array["input4"] = (w0 >> 19) & 0x1 + out_array["input5"] = (w0 >> 20) & 0x1 + out_array["input6"] = (w0 >> 21) & 0x1 + # Finer timestamp for each trigger input sampled with double data rate 640MHz clock (0.78125 ns) + out_array["sc1"] = (w2 >> 24) & 0xFF + out_array["sc2"] = (w2 >> 16) & 0xFF + out_array["sc3"] = (w2 >> 8) & 0xFF + out_array["sc4"] = w2 & 0xFF + out_array["sc5"] = (w4 >> 24) & 0xFF + out_array["sc6"] = (w4 >> 16) & 0xFF + return out_array if __name__ == "__main__": input_path = "../tlu_data/tlu_raw" + ".h5" output_path = "../tlu_data/tlu_interpreted" + ".h5" - data_parser = DataParser() - - data_parser.interpret_data(input_path, output_path) + interpret_data(input_path, output_path) diff --git a/aidatlu/main/tlu.py b/aidatlu/main/tlu.py index 4b84bdf..64573d1 100644 --- a/aidatlu/main/tlu.py +++ b/aidatlu/main/tlu.py @@ -17,7 +17,7 @@ from aidatlu.hardware.ioexpander_controller import IOControl from aidatlu.hardware.trigger_controller import TriggerLogic from aidatlu.main.config_parser import TLUConfigure -from aidatlu.main.data_parser import DataParser +import aidatlu.main.data_parser as DataParser class AidaTLU: @@ -41,7 +41,7 @@ def __init__(self, hw, config_path, clock_config_path) -> None: self.reset_configuration() self.config_parser = TLUConfigure(self, self.io_controller, config_path) - self.data_parser = DataParser() + self.data_parser = DataParser self.log.success("TLU initialized") diff --git a/aidatlu/test/software_test.py b/aidatlu/test/software_test.py index 14b42b6..93ef3e3 100644 --- a/aidatlu/test/software_test.py +++ b/aidatlu/test/software_test.py @@ -1,12 +1,11 @@ import numpy as np import tables as tb -from aidatlu.main.data_parser import DataParser +import aidatlu.main.data_parser as DataParser from aidatlu.main.config_parser import TLUConfigure def test_data_parser(): - data_parser = DataParser() - data_parser.interpret_data("raw_data_test.h5", "interpreted_data_test.h5") + DataParser.interpret_data("raw_data_test.h5", "interpreted_data_test.h5") def test_interpreted_data():