From cc25ea1b2035139c7079001fddc263443c877b2d Mon Sep 17 00:00:00 2001 From: AZangiabadi <79270542+AZangiabadi@users.noreply.github.com> Date: Mon, 16 Sep 2024 11:57:10 -0400 Subject: [PATCH 1/4] Update read_mib.py 1- Added capability to read 514x514 detector size. 2- Added binning option. 3- Edited parse_hdr function to read more info from the header file (it requires the .hdr file to exist). --- py4DSTEM/io/filereaders/read_mib.py | 88 ++++++++++++++++++++++------- 1 file changed, 67 insertions(+), 21 deletions(-) diff --git a/py4DSTEM/io/filereaders/read_mib.py b/py4DSTEM/io/filereaders/read_mib.py index 7456bd594..5e2ad87be 100644 --- a/py4DSTEM/io/filereaders/read_mib.py +++ b/py4DSTEM/io/filereaders/read_mib.py @@ -1,16 +1,18 @@ -# Read the mib file captured using the Merlin detector +# Read the mib file captured using the Merlin detector # Author: Tara Mishra, tara.matsci@gmail. +# Edited: Amir ZangiAbadi aazangiabadi@gmail.com 2024-09-16 # Based on the PyXEM load_mib module https://github.com/pyxem/pyxem/blob/563a3bb5f3233f46cd3e57f3cd6f9ddf7af55ad0/pyxem/utils/io_utils.py import numpy as np from py4DSTEM.datacube import DataCube +from py4DSTEM.preprocess import bin_data_diffraction import os def load_mib( file_path, mem="MEMMAP", - binfactor=1, + binfactor=2, reshape=True, flip=True, scan=(256, 256), @@ -21,16 +23,20 @@ def load_mib( The scan size is not encoded in the MIB metadata - by default it is set to (256,256), and can be modified by passing the keyword `scan`. + + binfactor n, reduces the data size by n^2 times. """ - assert binfactor == 1, "MIB does not support bin-on-load... yet?" + #assert binfactor == 1, "MIB does not support bin-on-load... yet?" # Get scan info from kwargs header = parse_hdr(file_path) - width = header["width"] - height = header["height"] + width = header["Detector width"] + height = header["Detector height"] width_height = width * height + scan = scan_size(file_path) + data = get_mib_memmap(file_path) depth = get_mib_depth(header, file_path) hdr_bits = get_hdr_bits(header) @@ -61,8 +67,11 @@ def load_mib( data = np.array(data) # Load entire dataset into RAM py4dstem_data = DataCube(data=data) - return py4dstem_data + if binfactor > 1: + py4dstem_data = bin_data_diffraction(py4dstem_data, binfactor, dtype=None) + + return py4dstem_data def manageHeader(fname): """Get necessary information from the header of the .mib file. @@ -100,6 +109,8 @@ def manageHeader(fname): sensorLayout = elements_in_header[7].strip() Timestamp = elements_in_header[9] shuttertime = float(elements_in_header[10]) + ScanX = scan_size(fname)[0] # It needs a .hdr file next to the .mib files, otherwise comment these two lines. + ScanY = scan_size(fname)[1] # It needs a .hdr file next to the .mib files if PixelDepthInFile == "R64": bitdepth = int(elements_in_header[18]) # RAW @@ -118,11 +129,31 @@ def manageHeader(fname): Timestamp, shuttertime, bitdepth, + ScanX, + ScanY, ) return hdr +def scan_size(path): + header_path = path[:-3] + "hdr" + result = {} + if os.path.exists(header_path): + + with open(header_path, encoding='UTF-8') as f: + for line in f: + k, v = line.split("\t", 1) + k = k.rstrip(':') + v = v.rstrip("\n") + result[k] = v + if 'ScanX' in result and 'ScanY' in result: + return (int(result['ScanY']), int(result['ScanX'])) + else: + print('Header file (.hdr) does not exist in this folder. The scan size will be assumed to be 256 by 256 pix') + + + def parse_hdr(fp): """Parse information from mib file header info from _manageHeader function. Parameters @@ -163,6 +194,7 @@ def parse_hdr(fp): 'data offset': int number of characters at the header. """ + hdr_info = {} read_hdr = manageHeader(fp) @@ -170,14 +202,23 @@ def parse_hdr(fp): # Set the array size of the chip if read_hdr[3] == "1x1": - hdr_info["width"] = 256 - hdr_info["height"] = 256 + hdr_info["Detector width"] = 256 + hdr_info["Detector height"] = 256 elif read_hdr[3] == "2x2": - hdr_info["width"] = 512 - hdr_info["height"] = 512 - + hdr_info["Detector width"] = 512 + hdr_info["Detector height"] = 512 + elif read_hdr[3] == "2x2G": #since the 512 pix chip is actually made of four 256 chips, there is a gap with a width of 2 pix between each chip. + hdr_info["Detector width"] = 514 + hdr_info["Detector height"] = 514 + + + hdr_info["scan size X"] = read_hdr[7] # Make sure .hdr file exit otherwise comment this line. + hdr_info["scan size Y"] = read_hdr[8] # Make sure .hdr file exit otherwise comment this line. + hdr_info["Assembly Size"] = read_hdr[3] + + # Set mib offset hdr_info["offset"] = read_hdr[0] # Set data-type @@ -280,19 +321,25 @@ def get_mib_depth(hdr_info, fp): # Define standard frame sizes for quad and single medipix chips if hdr_info["Assembly Size"] == "2x2": mib_file_size_dict = { - "1": 33536, - "6": 262912, - "12": 525056, - "24": 1049344, + "1": 33536, # 512*512/8 + header_size(or 768 bytes) + "6": 262912, # 512*512 + header_size(or 768 bytes) + "12": 525056, #512*512*2 + header_size(or 768 bytes) + "24": 1049344, #512*512*4 + header_size(or 768 bytes) } if hdr_info["Assembly Size"] == "1x1": mib_file_size_dict = { - "1": 8576, - "6": 65920, - "12": 131456, - "24": 262528, + "1": 8576, # 256*256/8 + header_size(or 384 bytes) + "6": 65920, # 256*256 + header_size(or 384 bytes) + "12": 131456, # 256*256*2 + header_size(or 384 bytes) + "24": 262528, # 256*256*4 + header_size(or 384 bytes) + } + if hdr_info["Assembly Size"] == "2x2G": # Quad chips and two pix spacing between chips + mib_file_size_dict = { + "1": 33536, # 514*514/8 + header_size(or 768 bytes) + "6": 264964, # 514*514 + header_size(or 768 bytes) + "12": 529160, # 514*514*2 + header_size(or 768 bytes) + "24": 1057552, # 514*514*4 + header_size(or 768 bytes) } - file_size = os.path.getsize(fp[:-3] + "mib") if hdr_info["raw"] == "R64": single_frame = mib_file_size_dict.get(str(hdr_info["Counter Depth (number)"])) @@ -351,5 +398,4 @@ def get_hdr_bits(hdr_info): hdr_multiplier = (int(data_length) / 8) ** -1 hdr_bits = int(hdr_info["data offset"] * hdr_multiplier) - return hdr_bits From e0afd95f1c76d2a9d6e7599e7a1c697f3c8cc09e Mon Sep 17 00:00:00 2001 From: AZangiabadi <79270542+AZangiabadi@users.noreply.github.com> Date: Mon, 16 Sep 2024 12:10:47 -0400 Subject: [PATCH 2/4] Update read_mib.py Added example codes for loading. --- py4DSTEM/io/filereaders/read_mib.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/py4DSTEM/io/filereaders/read_mib.py b/py4DSTEM/io/filereaders/read_mib.py index 5e2ad87be..57ace3f9b 100644 --- a/py4DSTEM/io/filereaders/read_mib.py +++ b/py4DSTEM/io/filereaders/read_mib.py @@ -1,8 +1,31 @@ -# Read the mib file captured using the Merlin detector +# Read the mib file captured using the Merlin detector (Quantum Detectors) # Author: Tara Mishra, tara.matsci@gmail. # Edited: Amir ZangiAbadi aazangiabadi@gmail.com 2024-09-16 # Based on the PyXEM load_mib module https://github.com/pyxem/pyxem/blob/563a3bb5f3233f46cd3e57f3cd6f9ddf7af55ad0/pyxem/utils/io_utils.py +""" +Example: + +file_path = '/Users/yourfilelocation/default.mib' + +# read the hdr (header) file, without loading the data. +print(py4DSTEM.io.filereaders.read_mib.parse_hdr(file_path)) + +# Read/load the mib file. Binning while loading will take a bit longer, but saves time later. +file_data = py4DSTEM.io.filereaders.read_mib.load_mib(file_path, mem='MEMMAP', binfactor=4, reshape=True, flip=True) + + +# Save your file in .h5 format and load it as .h5 to continue processing it. + py4DSTEM.save( + file_path[:-3] + "h5", + file_data + ) + +file_data = py4DSTEM.read( + file_path[:-3] + "h5" +) +""" + import numpy as np from py4DSTEM.datacube import DataCube from py4DSTEM.preprocess import bin_data_diffraction From 715b4c1709b7db269a8ea51c0809da0e8d1677cc Mon Sep 17 00:00:00 2001 From: AZangiabadi <79270542+AZangiabadi@users.noreply.github.com> Date: Sat, 21 Sep 2024 12:25:39 -0400 Subject: [PATCH 3/4] running black read_mib.py Saved by running black --- py4DSTEM/io/filereaders/read_mib.py | 69 ++++++++++++++++------------- 1 file changed, 39 insertions(+), 30 deletions(-) diff --git a/py4DSTEM/io/filereaders/read_mib.py b/py4DSTEM/io/filereaders/read_mib.py index 57ace3f9b..9d68da455 100644 --- a/py4DSTEM/io/filereaders/read_mib.py +++ b/py4DSTEM/io/filereaders/read_mib.py @@ -50,7 +50,7 @@ def load_mib( binfactor n, reduces the data size by n^2 times. """ - #assert binfactor == 1, "MIB does not support bin-on-load... yet?" + # assert binfactor == 1, "MIB does not support bin-on-load...yet?" # Get scan info from kwargs header = parse_hdr(file_path) @@ -96,6 +96,7 @@ def load_mib( return py4dstem_data + def manageHeader(fname): """Get necessary information from the header of the .mib file. Parameters @@ -132,8 +133,10 @@ def manageHeader(fname): sensorLayout = elements_in_header[7].strip() Timestamp = elements_in_header[9] shuttertime = float(elements_in_header[10]) - ScanX = scan_size(fname)[0] # It needs a .hdr file next to the .mib files, otherwise comment these two lines. - ScanY = scan_size(fname)[1] # It needs a .hdr file next to the .mib files + ScanX = scan_size(fname)[ + 0 + ] # It needs a .hdr file next to the .mib files, otherwise comment these two lines. + ScanY = scan_size(fname)[1] # It needs a .hdr file next to the .mib files if PixelDepthInFile == "R64": bitdepth = int(elements_in_header[18]) # RAW @@ -164,17 +167,18 @@ def scan_size(path): result = {} if os.path.exists(header_path): - with open(header_path, encoding='UTF-8') as f: + with open(header_path, encoding="UTF-8") as f: for line in f: k, v = line.split("\t", 1) - k = k.rstrip(':') + k = k.rstrip(":") v = v.rstrip("\n") result[k] = v - if 'ScanX' in result and 'ScanY' in result: - return (int(result['ScanY']), int(result['ScanX'])) + if "ScanX" in result and "ScanY" in result: + return (int(result["ScanY"]), int(result["ScanX"])) else: - print('Header file (.hdr) does not exist in this folder. The scan size will be assumed to be 256 by 256 pix') - + print( + "Header file (.hdr) does not exist in this folder. The scan size will be assumed to be 256 by 256 pix" + ) def parse_hdr(fp): @@ -217,7 +221,7 @@ def parse_hdr(fp): 'data offset': int number of characters at the header. """ - + hdr_info = {} read_hdr = manageHeader(fp) @@ -230,17 +234,20 @@ def parse_hdr(fp): elif read_hdr[3] == "2x2": hdr_info["Detector width"] = 512 hdr_info["Detector height"] = 512 - elif read_hdr[3] == "2x2G": #since the 512 pix chip is actually made of four 256 chips, there is a gap with a width of 2 pix between each chip. + elif ( + read_hdr[3] == "2x2G" + ): # since the 512 pix chip is actually made of four 256 chips, there is a gap with a width of 2 pix between each chip. hdr_info["Detector width"] = 514 hdr_info["Detector height"] = 514 - - - hdr_info["scan size X"] = read_hdr[7] # Make sure .hdr file exit otherwise comment this line. - hdr_info["scan size Y"] = read_hdr[8] # Make sure .hdr file exit otherwise comment this line. - - hdr_info["Assembly Size"] = read_hdr[3] + hdr_info["scan size X"] = read_hdr[ + 7 + ] # Make sure .hdr file exit otherwise comment this line. + hdr_info["scan size Y"] = read_hdr[ + 8 + ] # Make sure .hdr file exit otherwise comment this line. + hdr_info["Assembly Size"] = read_hdr[3] # Set mib offset hdr_info["offset"] = read_hdr[0] @@ -344,24 +351,26 @@ def get_mib_depth(hdr_info, fp): # Define standard frame sizes for quad and single medipix chips if hdr_info["Assembly Size"] == "2x2": mib_file_size_dict = { - "1": 33536, # 512*512/8 + header_size(or 768 bytes) - "6": 262912, # 512*512 + header_size(or 768 bytes) - "12": 525056, #512*512*2 + header_size(or 768 bytes) - "24": 1049344, #512*512*4 + header_size(or 768 bytes) + "1": 33536, # 512*512/8 + header_size(or 768 bytes) + "6": 262912, # 512*512 + header_size(or 768 bytes) + "12": 525056, # 512*512*2 + header_size(or 768 bytes) + "24": 1049344, # 512*512*4 + header_size(or 768 bytes) } if hdr_info["Assembly Size"] == "1x1": mib_file_size_dict = { - "1": 8576, # 256*256/8 + header_size(or 384 bytes) - "6": 65920, # 256*256 + header_size(or 384 bytes) - "12": 131456, # 256*256*2 + header_size(or 384 bytes) - "24": 262528, # 256*256*4 + header_size(or 384 bytes) + "1": 8576, # 256*256/8 + header_size(or 384 bytes) + "6": 65920, # 256*256 + header_size(or 384 bytes) + "12": 131456, # 256*256*2 + header_size(or 384 bytes) + "24": 262528, # 256*256*4 + header_size(or 384 bytes) } - if hdr_info["Assembly Size"] == "2x2G": # Quad chips and two pix spacing between chips + if ( + hdr_info["Assembly Size"] == "2x2G" + ): # Quad chips and two pix spacing between chips mib_file_size_dict = { - "1": 33536, # 514*514/8 + header_size(or 768 bytes) - "6": 264964, # 514*514 + header_size(or 768 bytes) - "12": 529160, # 514*514*2 + header_size(or 768 bytes) - "24": 1057552, # 514*514*4 + header_size(or 768 bytes) + "1": 33536, # 514*514/8 + header_size(or 768 bytes) + "6": 264964, # 514*514 + header_size(or 768 bytes) + "12": 529160, # 514*514*2 + header_size(or 768 bytes) + "24": 1057552, # 514*514*4 + header_size(or 768 bytes) } file_size = os.path.getsize(fp[:-3] + "mib") if hdr_info["raw"] == "R64": From 254dfc4b7e126c1f898b3562b25f768c7f7f80cc Mon Sep 17 00:00:00 2001 From: Stephanie Ribet Date: Sat, 21 Sep 2024 12:49:48 -0700 Subject: [PATCH 4/4] small updates for files without .hdr file --- py4DSTEM/io/filereaders/read_mib.py | 82 +++++++++++++---------------- 1 file changed, 38 insertions(+), 44 deletions(-) diff --git a/py4DSTEM/io/filereaders/read_mib.py b/py4DSTEM/io/filereaders/read_mib.py index 9d68da455..178a239ad 100644 --- a/py4DSTEM/io/filereaders/read_mib.py +++ b/py4DSTEM/io/filereaders/read_mib.py @@ -3,28 +3,6 @@ # Edited: Amir ZangiAbadi aazangiabadi@gmail.com 2024-09-16 # Based on the PyXEM load_mib module https://github.com/pyxem/pyxem/blob/563a3bb5f3233f46cd3e57f3cd6f9ddf7af55ad0/pyxem/utils/io_utils.py -""" -Example: - -file_path = '/Users/yourfilelocation/default.mib' - -# read the hdr (header) file, without loading the data. -print(py4DSTEM.io.filereaders.read_mib.parse_hdr(file_path)) - -# Read/load the mib file. Binning while loading will take a bit longer, but saves time later. -file_data = py4DSTEM.io.filereaders.read_mib.load_mib(file_path, mem='MEMMAP', binfactor=4, reshape=True, flip=True) - - -# Save your file in .h5 format and load it as .h5 to continue processing it. - py4DSTEM.save( - file_path[:-3] + "h5", - file_data - ) - -file_data = py4DSTEM.read( - file_path[:-3] + "h5" -) -""" import numpy as np from py4DSTEM.datacube import DataCube @@ -35,22 +13,38 @@ def load_mib( file_path, mem="MEMMAP", - binfactor=2, + binfactor=1, reshape=True, - flip=True, scan=(256, 256), - **kwargs, ): """ Read a MIB file and return as py4DSTEM DataCube. - The scan size is not encoded in the MIB metadata - by default it is - set to (256,256), and can be modified by passing the keyword `scan`. + Example: + `file_path = '/Users/yourfilelocation/default.mib'` - binfactor n, reduces the data size by n^2 times. - """ + read the hdr (header) file, without loading the data: + `print(py4DSTEM.io.filereaders.read_mib.parse_hdr(file_path))` + + load data: + `datacube = py4DSTEM.import_file(file_path)` - # assert binfactor == 1, "MIB does not support bin-on-load...yet?" + + Parameters + ---------- + mem: str ("memmap" or "ram") + load the data with numpy memmap or entirely into ram + file_path: str + location of data and .hdr file if included + binfactor: int + binning in reciprocal space of data + reshape: bool + if True, reshapes data in x and y to scan size + scan: 2-tuple + (x, y) scan size. This size is overwritten if there is a .hdr file + in the same folder + + """ # Get scan info from kwargs header = parse_hdr(file_path) @@ -58,7 +52,7 @@ def load_mib( height = header["Detector height"] width_height = width * height - scan = scan_size(file_path) + scan = scan_size(file_path, scan) data = get_mib_memmap(file_path) depth = get_mib_depth(header, file_path) @@ -85,6 +79,8 @@ def load_mib( if reshape: data = data.reshape(scan[0], scan[1], width, height) + else: + data = data[:, None, :, :] if mem == "RAM": data = np.array(data) # Load entire dataset into RAM @@ -133,10 +129,12 @@ def manageHeader(fname): sensorLayout = elements_in_header[7].strip() Timestamp = elements_in_header[9] shuttertime = float(elements_in_header[10]) - ScanX = scan_size(fname)[ - 0 - ] # It needs a .hdr file next to the .mib files, otherwise comment these two lines. - ScanY = scan_size(fname)[1] # It needs a .hdr file next to the .mib files + try: + ScanX = scan_size(fname)[0] + ScanY = scan_size(fname)[1] + except: + ScanX = None + ScanY = None if PixelDepthInFile == "R64": bitdepth = int(elements_in_header[18]) # RAW @@ -158,11 +156,10 @@ def manageHeader(fname): ScanX, ScanY, ) - return hdr -def scan_size(path): +def scan_size(path, scan): header_path = path[:-3] + "hdr" result = {} if os.path.exists(header_path): @@ -177,8 +174,9 @@ def scan_size(path): return (int(result["ScanY"]), int(result["ScanX"])) else: print( - "Header file (.hdr) does not exist in this folder. The scan size will be assumed to be 256 by 256 pix" + f"Header file (.hdr) does not exist in this folder. The scan size will be assumed to be {scan[0]} by {scan[1]} pix" ) + return scan def parse_hdr(fp): @@ -240,12 +238,8 @@ def parse_hdr(fp): hdr_info["Detector width"] = 514 hdr_info["Detector height"] = 514 - hdr_info["scan size X"] = read_hdr[ - 7 - ] # Make sure .hdr file exit otherwise comment this line. - hdr_info["scan size Y"] = read_hdr[ - 8 - ] # Make sure .hdr file exit otherwise comment this line. + hdr_info["scan size X"] = read_hdr[7] + hdr_info["scan size Y"] = read_hdr[8] hdr_info["Assembly Size"] = read_hdr[3]