From 77d56290d4c41f84df0c64065103eff809a0ae1c Mon Sep 17 00:00:00 2001 From: jschilli Date: Sat, 20 Jan 2024 17:55:04 +0100 Subject: [PATCH 1/4] Added support for v11.50 : Few modifications in the VMPdata_dtype_from_colIDs Added new headers VMPmodule_hdr_v2 Modified MPRfile initialization Includes squashed linting fixes by @ml-evs --- galvani/BioLogic.py | 87 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 71 insertions(+), 16 deletions(-) diff --git a/galvani/BioLogic.py b/galvani/BioLogic.py index 44b1d07..4e6624c 100644 --- a/galvani/BioLogic.py +++ b/galvani/BioLogic.py @@ -86,6 +86,7 @@ def fieldname_to_dtype(fieldname): "/F", "/mF", "/uF", + "/µF", "/C", "/Ohm", ) @@ -230,7 +231,7 @@ def MPTfileCSV(file_or_path): return mpt_csv, comments -VMPmodule_hdr = np.dtype( +VMPmodule_hdr_v1 = np.dtype( [ ("shortname", "S10"), ("longname", "S25"), @@ -240,15 +241,27 @@ def MPTfileCSV(file_or_path): ] ) +VMPmodule_hdr_v2 = np.dtype( + [ + ("shortname", "S10"), + ("longname", "S25"), + ("max points", " ?? 9: ("Ece/V", "/mA", "/mA", "/V", "/V", "= 11.50 + if hdr_bytes[35:39] == b"\xff\xff\xff\xff": + VMPmodule_hdr = VMPmodule_hdr_v2 + hdr_bytes += fileobj.read(VMPmodule_hdr_v2.itemsize - VMPmodule_hdr_v1.itemsize) + hdr = np.frombuffer(hdr_bytes, dtype=VMPmodule_hdr, count=1) hdr_dict = dict(((n, hdr[n][0]) for n in VMPmodule_hdr.names)) hdr_dict["offset"] = fileobj.tell() @@ -456,7 +498,11 @@ def read_VMP_modules(fileobj, read_module_data=True): current module: %s length read: %d length expected: %d""" - % (hdr_dict["longname"], len(hdr_dict["data"]), hdr_dict["length"]) + % ( + hdr_dict["longname"], + len(hdr_dict["data"]), + hdr_dict["length"], + ) ) yield hdr_dict else: @@ -494,6 +540,8 @@ def __init__(self, file_or_path): raise ValueError("Invalid magic for .mpr file: %s" % magic) modules = list(read_VMP_modules(mpr_file)) + mpr_file.close() # close the mpr.file after reading all of it + self.modules = modules (settings_mod,) = (m for m in modules if m["shortname"] == b"VMP Set ") (data_module,) = (m for m in modules if m["shortname"] == b"VMP data ") @@ -504,15 +552,22 @@ def __init__(self, file_or_path): n_columns = np.frombuffer(data_module["data"][4:5], dtype="u1").item() if data_module["version"] == 0: - column_types = np.frombuffer( - data_module["data"][5:], dtype="u1", count=n_columns - ) - remaining_headers = data_module["data"][5 + n_columns:100] - main_data = data_module["data"][100:] + # If EC-Lab version >= 11.50, column_types is [0 1 0 3 0 174...] instead of [1 3 174...] + if np.frombuffer(data_module["data"][5:6], dtype="u1").item(): + column_types = np.frombuffer(data_module["data"][5:], dtype="u1", count=n_columns) + remaining_headers = data_module["data"][5 + n_columns:100] + main_data = data_module["data"][100:] + else: + column_types = np.frombuffer( + data_module["data"][5:], dtype="u1", count=n_columns * 2 + ) + column_types = column_types[1::2] # suppressing zeros in column types array + # remaining headers should be empty except for bytes 5 + n_columns * 2 + # and 1006 which are sometimes == 1 + remaining_headers = data_module["data"][6 + n_columns * 2:1006] + main_data = data_module["data"][1007:] elif data_module["version"] in [2, 3]: - column_types = np.frombuffer( - data_module["data"][5:], dtype=" Date: Mon, 22 Jan 2024 14:25:34 +0100 Subject: [PATCH 2/4] added tests for v11.50 --- galvani/BioLogic.py | 16 ++++- tests/test_BioLogic.py | 101 +++++++++++++++++++++++++++- tests/testdata/v1150/v1150_CA.mpr | 3 + tests/testdata/v1150/v1150_CA.mpt | 3 + tests/testdata/v1150/v1150_CP.mpr | 3 + tests/testdata/v1150/v1150_CP.mpt | 3 + tests/testdata/v1150/v1150_GCPL.mpr | 3 + tests/testdata/v1150/v1150_GCPL.mpt | 3 + tests/testdata/v1150/v1150_GEIS.mpr | 3 + tests/testdata/v1150/v1150_GEIS.mpt | 3 + tests/testdata/v1150/v1150_MB.mpr | 3 + tests/testdata/v1150/v1150_MB.mpt | 3 + tests/testdata/v1150/v1150_OCV.mpr | 3 + tests/testdata/v1150/v1150_OCV.mpt | 3 + tests/testdata/v1150/v1150_PEIS.mpr | 3 + tests/testdata/v1150/v1150_PEIS.mpt | 3 + 16 files changed, 156 insertions(+), 3 deletions(-) create mode 100644 tests/testdata/v1150/v1150_CA.mpr create mode 100644 tests/testdata/v1150/v1150_CA.mpt create mode 100644 tests/testdata/v1150/v1150_CP.mpr create mode 100644 tests/testdata/v1150/v1150_CP.mpt create mode 100644 tests/testdata/v1150/v1150_GCPL.mpr create mode 100644 tests/testdata/v1150/v1150_GCPL.mpt create mode 100644 tests/testdata/v1150/v1150_GEIS.mpr create mode 100644 tests/testdata/v1150/v1150_GEIS.mpt create mode 100644 tests/testdata/v1150/v1150_MB.mpr create mode 100644 tests/testdata/v1150/v1150_MB.mpt create mode 100644 tests/testdata/v1150/v1150_OCV.mpr create mode 100644 tests/testdata/v1150/v1150_OCV.mpt create mode 100644 tests/testdata/v1150/v1150_PEIS.mpr create mode 100644 tests/testdata/v1150/v1150_PEIS.mpt diff --git a/galvani/BioLogic.py b/galvani/BioLogic.py index f20cc67..5d08108 100644 --- a/galvani/BioLogic.py +++ b/galvani/BioLogic.py @@ -48,6 +48,13 @@ def fieldname_to_dtype(fieldname): "|Z|/Ohm", "Re(Z)/Ohm", "-Im(Z)/Ohm", + "Re(M)", + "Im(M)", + "|M|", + "Re(Permittivity)", + "Im(Permittivity)", + "|Permittivity|", + "Tan(Delta)", ): return (fieldname, np.float_) elif fieldname in ( @@ -60,13 +67,13 @@ def fieldname_to_dtype(fieldname): "Capacity/mA.h", ): return (fieldname, np.float_) - elif fieldname in ("cycle number", "I Range", "Ns", "half cycle"): + elif fieldname in ("cycle number", "I Range", "Ns", "half cycle", "z cycle"): return (fieldname, np.int_) elif fieldname in ("dq/mA.h", "dQ/mA.h"): return ("dQ/mA.h", np.float_) elif fieldname in ("I/mA", "/mA"): return ("I/mA", np.float_) - elif fieldname in ("Ewe/V", "/V", "Ecell/V"): + elif fieldname in ("Ewe/V", "/V", "Ecell/V", ""): return ("Ewe/V", np.float_) elif fieldname.endswith( ( @@ -87,8 +94,13 @@ def fieldname_to_dtype(fieldname): "/mF", "/uF", "/µF", + "/nF", "/C", "/Ohm", + "/Ohm-1", + "/Ohm.cm", + "/mS/cm", + "/%", ) ): return (fieldname, np.float_) diff --git a/tests/test_BioLogic.py b/tests/test_BioLogic.py index db70c39..5f83b36 100644 --- a/tests/test_BioLogic.py +++ b/tests/test_BioLogic.py @@ -9,7 +9,7 @@ from datetime import date, datetime import numpy as np -from numpy.testing import assert_array_almost_equal, assert_array_equal +from numpy.testing import assert_array_almost_equal, assert_array_equal, assert_allclose import pytest from galvani import BioLogic, MPTfile, MPRfile @@ -209,6 +209,89 @@ def assert_field_exact(fieldname): except AttributeError: pass +def assert_MPR_matches_MPT_v2(mpr, mpt, comments): + """ + Asserts that the fields in the MPR.data ar the same as in the MPT. Modified from assert_MPR_matches_MPT. + Automatically converts dtype from MPT data to dtype from MPR data before comparing the columns. + Speficicity for EIS_indicators : these fields are valid only at f<100kHz so their values are replaced + by -1 or 0 at high frequency in the MPT file, this is not the case in the MPR.data. + + Parameters + ---------- + mpr : MPRfile + Data extracted with the MPRfile class. + mpt : np.array + Data extracted with MPTfile method. + + Returns + ------- + None. + + """ + + def assert_field_matches(fieldname): + EIS_quality_indicators = [ + "THD Ewe/%", + "NSD Ewe/%", + "NSR Ewe/%", + "|Ewe h2|/V", + "|Ewe h3|/V", + "|Ewe h4|/V", + "|Ewe h5|/V", + "|Ewe h6|/V", + "|Ewe h7|/V", + "THD I/%", + "NSD I/%", + "NSR I/%", + "|I h2|/A", + "|I h3|/A", + "|I h4|/A", + "|I h5|/A", + "|I h6|/A", + "|I h7|/A", + ] + + if fieldname in EIS_quality_indicators: # EIS quality indicators only valid for f < 100kHz + index_inf_100k = np.where(mpr.data["freq/Hz"] < 100000)[0] + assert_allclose( + mpr.data[index_inf_100k][fieldname], + mpt[index_inf_100k][fieldname].astype(mpr.data[fieldname].dtype), + ) + elif fieldname == "/V": + assert_allclose( + mpr.data[fieldname], + mpt["Ewe/V"].astype(mpr.data[fieldname].dtype), + ) + elif fieldname == "/mA": + assert_allclose( + mpr.data[fieldname], + mpt["I/mA"].astype(mpr.data[fieldname].dtype), + ) + elif fieldname == "dq/mA.h": + assert_allclose( + mpr.data[fieldname], + mpt["dQ/mA.h"].astype(mpr.data[fieldname].dtype), + ) + else: + assert_allclose( + mpr.data[fieldname], + mpt[fieldname].astype(mpr.data[fieldname].dtype), + ) + + def assert_field_exact(fieldname): + if fieldname in mpr.dtype.fields: + assert_array_equal(mpr.data[fieldname], mpt[fieldname]) + + for key in mpr.flags_dict.keys(): + assert_array_equal(mpr.get_flag(key), mpt[key]) + + for d in mpr.dtype.descr[1:]: + assert_field_matches(d[0]) + + try: + assert timestamp_from_comments(comments) == mpr.timestamp.replace(microsecond=0) + except AttributeError: + pass @pytest.mark.parametrize( "basename", @@ -252,3 +335,19 @@ def test_MPR6_matches_MPT6(testdata_dir): mpt, comments = MPTfile(os.path.join(testdata_dir, "bio_logic6.mpt")) mpr.data = mpr.data[:958] # .mpt file is incomplete assert_MPR_matches_MPT(mpr, mpt, comments) + +@pytest.mark.parametrize( + "basename_v1150", + ["v1150_CA", "v1150_CP", "v1150_GCPL", "v1150_GEIS", "v1150_MB", "v1150_OCV", "v1150_PEIS"], +) +def test_MPR_matches_MPT_v1150(testdata_dir, basename_v1150): + """Check the MPR parser against the MPT parser. + + Load a binary .mpr file and a text .mpt file which should contain + exactly the same data. Check that the loaded data actually match. + """ + binpath = os.path.join(testdata_dir, "v1150", basename_v1150 + ".mpr") + txtpath = os.path.join(testdata_dir, "v1150", basename_v1150 + ".mpt") + mpr = MPRfile(binpath) + mpt, comments = MPTfile(txtpath, encoding="latin1") + assert_MPR_matches_MPT_v2(mpr, mpt, comments) diff --git a/tests/testdata/v1150/v1150_CA.mpr b/tests/testdata/v1150/v1150_CA.mpr new file mode 100644 index 0000000..f0ecc09 --- /dev/null +++ b/tests/testdata/v1150/v1150_CA.mpr @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:264d359400e5b0e491cc4ec8acc1c34544ebf87fd42c891bac5570a8e1a8bb32 +size 20587 diff --git a/tests/testdata/v1150/v1150_CA.mpt b/tests/testdata/v1150/v1150_CA.mpt new file mode 100644 index 0000000..86ed181 --- /dev/null +++ b/tests/testdata/v1150/v1150_CA.mpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20de12cd1a157887aacd22c805d7f4e17e256c22aed0d4bc03c9912427df8297 +size 33463 diff --git a/tests/testdata/v1150/v1150_CP.mpr b/tests/testdata/v1150/v1150_CP.mpr new file mode 100644 index 0000000..82f0d42 --- /dev/null +++ b/tests/testdata/v1150/v1150_CP.mpr @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d57e12146bc7085cb2edfc2c5bad308fb0d9534d2c06e59262118a19b1867f9 +size 18291 diff --git a/tests/testdata/v1150/v1150_CP.mpt b/tests/testdata/v1150/v1150_CP.mpt new file mode 100644 index 0000000..e152a89 --- /dev/null +++ b/tests/testdata/v1150/v1150_CP.mpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbbdca1a6f57c0cd3291cb0604cd7b3d1cae78c3912c7fa2e32aee837f540dcb +size 18821 diff --git a/tests/testdata/v1150/v1150_GCPL.mpr b/tests/testdata/v1150/v1150_GCPL.mpr new file mode 100644 index 0000000..febc106 --- /dev/null +++ b/tests/testdata/v1150/v1150_GCPL.mpr @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f814e04bd102db6211d933aecd2fb4f30f31b0a871ea26cce6c4a8900d2a85e3 +size 23857 diff --git a/tests/testdata/v1150/v1150_GCPL.mpt b/tests/testdata/v1150/v1150_GCPL.mpt new file mode 100644 index 0000000..5761be5 --- /dev/null +++ b/tests/testdata/v1150/v1150_GCPL.mpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60ade6fe10af5ed71549d10a7f5abaed97694c2130e6f4365ffc85c24fa19c93 +size 61681 diff --git a/tests/testdata/v1150/v1150_GEIS.mpr b/tests/testdata/v1150/v1150_GEIS.mpr new file mode 100644 index 0000000..13f1dad --- /dev/null +++ b/tests/testdata/v1150/v1150_GEIS.mpr @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b476461e9e0ec33475a80b29f74c1863e3ca9468006ac28ed59208a319f1dd0 +size 28893 diff --git a/tests/testdata/v1150/v1150_GEIS.mpt b/tests/testdata/v1150/v1150_GEIS.mpt new file mode 100644 index 0000000..9e15ff7 --- /dev/null +++ b/tests/testdata/v1150/v1150_GEIS.mpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c9bbf6240769d0ab87b297858607b9367e4d27cf2f9a45c9b9dc0d14515211e +size 80576 diff --git a/tests/testdata/v1150/v1150_MB.mpr b/tests/testdata/v1150/v1150_MB.mpr new file mode 100644 index 0000000..eba7f10 --- /dev/null +++ b/tests/testdata/v1150/v1150_MB.mpr @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4828a32ddde25b4a95b30a8dc9e86500bf27583fe646f82bb4d81a970d307be0 +size 103897 diff --git a/tests/testdata/v1150/v1150_MB.mpt b/tests/testdata/v1150/v1150_MB.mpt new file mode 100644 index 0000000..0e1ab47 --- /dev/null +++ b/tests/testdata/v1150/v1150_MB.mpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c614c410aadb304b80f65afbac3ac8329dca84afd8609e908464d8de405d7917 +size 685857 diff --git a/tests/testdata/v1150/v1150_OCV.mpr b/tests/testdata/v1150/v1150_OCV.mpr new file mode 100644 index 0000000..d0d46ea --- /dev/null +++ b/tests/testdata/v1150/v1150_OCV.mpr @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b1948c874defcca8280b88e40da0cde0a4d44c01d7f807479c3d4b906581ca3 +size 16822 diff --git a/tests/testdata/v1150/v1150_OCV.mpt b/tests/testdata/v1150/v1150_OCV.mpt new file mode 100644 index 0000000..1d387cc --- /dev/null +++ b/tests/testdata/v1150/v1150_OCV.mpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:643011cc4ed832e02d2b05d29b8cc32476c4c579182b9ac4c6b4b05f1d75f90c +size 3361 diff --git a/tests/testdata/v1150/v1150_PEIS.mpr b/tests/testdata/v1150/v1150_PEIS.mpr new file mode 100644 index 0000000..47142b9 --- /dev/null +++ b/tests/testdata/v1150/v1150_PEIS.mpr @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b8be2f275c2a5a54068f714184dbdc1dde6f90c69bb4b22b278c7aa27d7c125 +size 28881 diff --git a/tests/testdata/v1150/v1150_PEIS.mpt b/tests/testdata/v1150/v1150_PEIS.mpt new file mode 100644 index 0000000..614084b --- /dev/null +++ b/tests/testdata/v1150/v1150_PEIS.mpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abe49b6a4ae412217b1ac792f6f64b69462a32bea38161c32f183050c7472a9b +size 80379 From 5cdc620f16c96850d3ae2470143505f37c4b67ae Mon Sep 17 00:00:00 2001 From: Chris Kerr Date: Sat, 3 Feb 2024 14:00:16 +0200 Subject: [PATCH 3/4] Fix flake8 lint --- tests/test_BioLogic.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/test_BioLogic.py b/tests/test_BioLogic.py index 5f83b36..0d8d111 100644 --- a/tests/test_BioLogic.py +++ b/tests/test_BioLogic.py @@ -209,12 +209,17 @@ def assert_field_exact(fieldname): except AttributeError: pass + def assert_MPR_matches_MPT_v2(mpr, mpt, comments): """ - Asserts that the fields in the MPR.data ar the same as in the MPT. Modified from assert_MPR_matches_MPT. - Automatically converts dtype from MPT data to dtype from MPR data before comparing the columns. - Speficicity for EIS_indicators : these fields are valid only at f<100kHz so their values are replaced - by -1 or 0 at high frequency in the MPT file, this is not the case in the MPR.data. + Asserts that the fields in the MPR.data ar the same as in the MPT. + + Modified from assert_MPR_matches_MPT. Automatically converts dtype from MPT data + to dtype from MPR data before comparing the columns. + + Special case for EIS_indicators: these fields are valid only at f<100kHz so their + values are replaced by -1 or 0 at high frequency in the MPT file, this is not the + case in the MPR data. Parameters ---------- @@ -293,6 +298,7 @@ def assert_field_exact(fieldname): except AttributeError: pass + @pytest.mark.parametrize( "basename", [ @@ -336,6 +342,7 @@ def test_MPR6_matches_MPT6(testdata_dir): mpr.data = mpr.data[:958] # .mpt file is incomplete assert_MPR_matches_MPT(mpr, mpt, comments) + @pytest.mark.parametrize( "basename_v1150", ["v1150_CA", "v1150_CP", "v1150_GCPL", "v1150_GEIS", "v1150_MB", "v1150_OCV", "v1150_PEIS"], From 4971f2b550ee964940b4c68e41a4c87cfcb05fc6 Mon Sep 17 00:00:00 2001 From: Chris Kerr Date: Sat, 3 Feb 2024 14:24:03 +0200 Subject: [PATCH 4/4] Apply review comments --- galvani/BioLogic.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/galvani/BioLogic.py b/galvani/BioLogic.py index 5d08108..65d580c 100644 --- a/galvani/BioLogic.py +++ b/galvani/BioLogic.py @@ -257,10 +257,10 @@ def MPTfileCSV(file_or_path): [ ("shortname", "S10"), ("longname", "S25"), - ("max points", "