Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

MAINT: removed data parser class #10

Merged
merged 2 commits into from
Dec 18, 2024
Merged

MAINT: removed data parser class #10

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
49129cb
MAINT: removed data parser class
rpartzsch Dec 18, 2024
22d24c6
MAINT: import interpret_data() directly
rpartzsch Dec 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions aidatlu/aidatlu_run.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,13 @@

class AIDATLU:
def __init__(self, config_path, clock_path):
print(" ---------------------------------------")
print(" _ ___ ___ _ _____ _ _ _ ")
print(" /_\ |_ _| \ /_\ |_ _| | | | | |")
print(" / _ \ | || |) / _ \ | | | |_| |_| |")
print(" /_/ \_\___|___/_/ \_\ |_| |____\___/ \n")
print(" ---------------------------------------")
print(r"----------------------------------------")
print(r" _ ___ ___ _ _____ _ _ _ ")
print(r" /_\ |_ _| \ /_\ |_ _| | | | | |")
print(r" / _ \ | || |) / _ \ -- | | | |_| |_| |")
print(r" /_/ \_\___|___/_/ \_\ |_| |____\___/ ")
print(r" ")
print(r"----------------------------------------")
print("tlu.help()\n")

self.config_file = config_path
Expand Down
307 changes: 155 additions & 152 deletions aidatlu/main/data_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,162 +7,165 @@
from aidatlu import logger


class DataParser:
def __init__(self) -> None:
self.log = logger.setup_main_logger(__class__.__name__)
self.features = np.dtype(
[
("eventnumber", "u4"),
("timestamp", "u8"),
("overflow", "u8"),
("eventtype", "u4"),
("input1", "bool"),
("input2", "bool"),
("input3", "bool"),
("input4", "bool"),
("input5", "bool"),
("input6", "bool"),
("sc1", "u4"),
("sc2", "u4"),
("sc3", "u4"),
("sc4", "u4"),
("sc5", "u4"),
("sc6", "u4"),
]
)
self.raw_features = np.dtype([("raw", "u4")])

def interpret_data(
self,
filepath_in: str | Path,
filepath_out: str | Path,
chunk_size: int = 1000000,
) -> None:
"""Interprets raw tlu data. The data is interpreted in chunksizes.
The data is parsed form filepath_in to filepath_out.
An event consists of six consecutive raw data entries the last entry should be always 0.
The raw data is sliced and the last data entry checked for corrupted data.

Args:
filepath_in (str | Path): raw data file path as string or Path object
filepath_out (str | Path): output path of the interpreted data as string or Path object
"""
self.log.info("Interpreting Data")
chunk_size = chunk_size * 6
with tb.open_file(filepath_in, "r") as in_file:
n_words = in_file.root.raw_data.shape[0]
self.conf = np.array(in_file.root.conf[:])

if n_words == 0:
self.log.warning("Data is empty. Skip analysis!")
return

with tb.open_file(
filepath_out, mode="w", title="TLU_interpreted"
) as out_file:
data_table = self._create_table(
out_file, name="interpreted_data", title="data", dtype=self.features
def interpret_data(
filepath_in: str | Path,
filepath_out: str | Path = None,
chunk_size: int = 1000000,
) -> None:
"""Interprets raw tlu data. The data is interpreted in chunksizes.
The data is parsed form filepath_in to filepath_out.
An event consists of six consecutive raw data entries the last entry should be always 0.
The raw data is sliced and the last data entry checked for corrupted data.

Args:
filepath_in (str | Path): raw data file path as string or Path object
filepath_out (str | Path): output path of the interpreted data as string or Path object
"""
log = logger.setup_main_logger("Data Interpreter")
features = np.dtype(
[
("eventnumber", "u4"),
("timestamp", "u8"),
("overflow", "u8"),
("eventtype", "u4"),
("input1", "bool"),
("input2", "bool"),
("input3", "bool"),
("input4", "bool"),
("input5", "bool"),
("input6", "bool"),
("sc1", "u4"),
("sc2", "u4"),
("sc3", "u4"),
("sc4", "u4"),
("sc5", "u4"),
("sc6", "u4"),
]
)
raw_features = np.dtype([("raw", "u4")])

log.info("Interpreting Data")
chunk_size = chunk_size * 6

if filepath_out == None:
filepath_out = filepath_in + "_interpreted.h5"

with tb.open_file(filepath_in, "r") as in_file:
n_words = in_file.root.raw_data.shape[0]
conf = np.array(in_file.root.conf[:])

if n_words == 0:
log.warning("Data is empty. Skip analysis!")
return

with tb.open_file(filepath_out, mode="w", title="TLU_interpreted") as out_file:
data_table = _create_table(
out_file, name="interpreted_data", title="data", dtype=features
)
for chunk in tqdm(range(0, n_words, chunk_size)):
chunk_offset = chunk
stop = chunk_offset + chunk_size
if chunk + chunk_size > n_words:
stop = n_words
table = in_file.root.raw_data[chunk_offset:stop]
raw_data = np.array(table[:], dtype=raw_features)
data = _transform_data(
raw_data["raw"][::6],
raw_data["raw"][1::6],
raw_data["raw"][2::6],
raw_data["raw"][3::6],
raw_data["raw"][4::6],
raw_data["raw"][5::6],
log,
features,
)
for chunk in tqdm(range(0, n_words, chunk_size)):
chunk_offset = chunk
stop = chunk_offset + chunk_size
if chunk + chunk_size > n_words:
stop = n_words
table = in_file.root.raw_data[chunk_offset:stop]
raw_data = np.array(table[:], dtype=self.raw_features)
data = self._transform_data(
raw_data["raw"][::6],
raw_data["raw"][1::6],
raw_data["raw"][2::6],
raw_data["raw"][3::6],
raw_data["raw"][4::6],
raw_data["raw"][5::6],
)
data_table.append(data)

config = np.dtype(
[
("attribute", "S32"),
("value", "S32"),
]
)
config_table = out_file.create_table(
out_file.root,
name="conf",
description=config,
)
config_table.append(self.conf)
self.log.success('Data parsed from "%s" to "%s"' % (filepath_in, filepath_out))

def _create_table(self, out_file, name, title, dtype):
"""Create hit table node for storage in out_file.
Copy configuration nodes from raw data file.
"""
table = out_file.create_table(
out_file.root,
name=name,
description=dtype,
title=title,
filters=tb.Filters(complib="blosc", complevel=5, fletcher32=False),
)

return table

def _transform_data(
self,
w0: np.array,
w1: np.array,
w2: np.array,
w3: np.array,
w4: np.array,
w5: np.array,
) -> np.array:
"""Transforms raw data from the FIFO to a readable dataformat

Args:
w0 (np.array): contains information which trigger input fired
w1 (np.array): contains timestamp information
w2 (np.array): trigger input information
w3 (np.array): eventnumber
w4 (np.array): trigger input information
w5 (np.array): this should always be 0.

Returns:
np.array: array with columns
"""
if np.any(w5) != 0:
self.log.warning("Corrupted Data found")

# Cast w0 to uint64 for concatenating 2 x 32bit to 64bit later
w0 = w0.astype(np.uint64)

out_array = np.zeros(len(w3), dtype=self.features)
out_array["eventnumber"] = w3
out_array["timestamp"] = (w0 & 0x0000FFFF << 32) + w1
out_array["overflow"] = w0 & 0xFFFF
# TODO not sure what this is per. mode?
out_array["eventtype"] = (w0 >> 28) & 0xF
# Which trigger input produced the event.
out_array["input1"] = (w0 >> 16) & 0x1
out_array["input2"] = (w0 >> 17) & 0x1
out_array["input3"] = (w0 >> 18) & 0x1
out_array["input4"] = (w0 >> 19) & 0x1
out_array["input5"] = (w0 >> 20) & 0x1
out_array["input6"] = (w0 >> 21) & 0x1
# TODO not sure what these are prob. something from the DACs
out_array["sc1"] = (w2 >> 24) & 0xFF
out_array["sc2"] = (w2 >> 16) & 0xFF
out_array["sc3"] = (w2 >> 8) & 0xFF
out_array["sc4"] = w2 & 0xFF
out_array["sc5"] = (w4 >> 24) & 0xFF
out_array["sc6"] = (w4 >> 16) & 0xFF
return out_array
data_table.append(data)

config = np.dtype(
[
("attribute", "S32"),
("value", "S32"),
]
)
config_table = out_file.create_table(
out_file.root,
name="conf",
description=config,
)
config_table.append(conf)
log.success('Data parsed from "%s" to "%s"' % (filepath_in, filepath_out))


def _create_table(out_file, name, title, dtype):
"""Create hit table node for storage in out_file.
Copy configuration nodes from raw data file.
"""
table = out_file.create_table(
out_file.root,
name=name,
description=dtype,
title=title,
filters=tb.Filters(complib="blosc", complevel=5, fletcher32=False),
)
return table


def _transform_data(
w0: np.array,
w1: np.array,
w2: np.array,
w3: np.array,
w4: np.array,
w5: np.array,
log: logger,
features: np.dtype,
) -> np.array:
"""Transforms raw data from the FIFO to a readable dataformat

Args:
w0 (np.array): contains information which trigger input fired
w1 (np.array): contains timestamp information
w2 (np.array): trigger input information
w3 (np.array): eventnumber
w4 (np.array): trigger input information
w5 (np.array): this should always be 0.
log (logger): logging function
features (dtype): output array dtype structure

Returns:
np.array: array with columns
"""
if np.any(w5) != 0:
log.warning("Corrupted Data found")

# Cast w0 to uint64 for concatenating 2 x 32bit to 64bit later
w0 = w0.astype(np.uint64)

out_array = np.zeros(len(w3), dtype=features)
out_array["eventnumber"] = w3
out_array["timestamp"] = (w0 & 0x0000FFFF << 32) + w1
out_array["overflow"] = w0 & 0xFFFF
# TODO not sure what this is per. mode?
out_array["eventtype"] = (w0 >> 28) & 0xF
# Which trigger input produced the event.
out_array["input1"] = (w0 >> 16) & 0x1
out_array["input2"] = (w0 >> 17) & 0x1
out_array["input3"] = (w0 >> 18) & 0x1
out_array["input4"] = (w0 >> 19) & 0x1
out_array["input5"] = (w0 >> 20) & 0x1
out_array["input6"] = (w0 >> 21) & 0x1
# Finer timestamp for each trigger input sampled with double data rate 640MHz clock (0.78125 ns)
out_array["sc1"] = (w2 >> 24) & 0xFF
out_array["sc2"] = (w2 >> 16) & 0xFF
out_array["sc3"] = (w2 >> 8) & 0xFF
out_array["sc4"] = w2 & 0xFF
out_array["sc5"] = (w4 >> 24) & 0xFF
out_array["sc6"] = (w4 >> 16) & 0xFF
return out_array


if __name__ == "__main__":
input_path = "../tlu_data/tlu_raw" + ".h5"
output_path = "../tlu_data/tlu_interpreted" + ".h5"

data_parser = DataParser()

data_parser.interpret_data(input_path, output_path)
interpret_data(input_path, output_path)
15 changes: 4 additions & 11 deletions aidatlu/main/tlu.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import logging
import threading
import time
from datetime import datetime
Expand All @@ -17,7 +16,7 @@
from aidatlu.hardware.ioexpander_controller import IOControl
from aidatlu.hardware.trigger_controller import TriggerLogic
from aidatlu.main.config_parser import TLUConfigure
from aidatlu.main.data_parser import DataParser
from aidatlu.main.data_parser import interpret_data


class AidaTLU:
Expand All @@ -41,7 +40,6 @@ def __init__(self, hw, config_path, clock_config_path) -> None:

self.reset_configuration()
self.config_parser = TLUConfigure(self, self.io_controller, config_path)
self.data_parser = DataParser()

self.log.success("TLU initialized")

Expand Down Expand Up @@ -424,7 +422,7 @@ def run(self) -> None:
first_event = True
self.stop_condition = False
# prepare data handling and zmq connection
save_data, interpret_data = self.config_parser.get_data_handling()
save_data, interpret_data_bool = self.config_parser.get_data_handling()
self.zmq_address = self.config_parser.get_zmq_connection()
self.max_trigger, self.timeout = self.config_parser.get_stop_condition()

Expand Down Expand Up @@ -491,13 +489,8 @@ def run(self) -> None:

if save_data:
self.h5_file.close()
if interpret_data:
try:
self.data_parser.interpret_data(
self.raw_data_path, self.interpreted_data_path
)
except:
self.log.warning("Cannot interpret data.")
if interpret_data_bool:
interpret_data(self.raw_data_path, self.interpreted_data_path)
self.log.success("Run finished")


Expand Down
5 changes: 2 additions & 3 deletions aidatlu/test/software_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
import numpy as np
import tables as tb
from aidatlu.main.data_parser import DataParser
from aidatlu.main.data_parser import interpret_data
from aidatlu.main.config_parser import TLUConfigure


def test_data_parser():
data_parser = DataParser()
data_parser.interpret_data("raw_data_test.h5", "interpreted_data_test.h5")
interpret_data("raw_data_test.h5", "interpreted_data_test.h5")


def test_interpreted_data():
Expand Down
Loading