From 5c68e2c18954dc9a0a8582cbfb9e18f2dd96f3a8 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Sun, 22 Aug 2021 23:30:37 -0700 Subject: [PATCH 1/3] Added test for to_hierarchical_dataframe for the icephys tables with and without VectorIndex --- tests/unit/test_icephys_metadata_tables.py | 57 ++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/unit/test_icephys_metadata_tables.py b/tests/unit/test_icephys_metadata_tables.py index d63e1ffb8..b7816af61 100644 --- a/tests/unit/test_icephys_metadata_tables.py +++ b/tests/unit/test_icephys_metadata_tables.py @@ -22,6 +22,8 @@ from pynwb.base import TimeSeriesReferenceVectorData from pynwb import NWBHDF5IO from hdmf.utils import docval, popargs +from pynwb.testing import create_icephys_testfile +from hdmf.common.hierarchicaltable import to_hierarchical_dataframe class ICEphysMetaTestBase(TestCase): @@ -889,6 +891,61 @@ def test_enforce_unique_id(self): with self.assertRaises(ValueError): cond.add_experimental_condition(repetitions=[0, ], id=np.int64(10)) + def test_to_hierarchical_dataframe(self): + # test that to_hierarchical_dataframe works for the icephys tables when only containing regular VectorData + nwbfile = create_icephys_testfile() + temp = to_hierarchical_dataframe(nwbfile.icephys_experimental_conditions) + expected_index = [ + (100000, 32.0, 10000, 'R1', 1000, 'StimType_1', 'T1', 100, 0), + (100000, 32.0, 10000, 'R1', 1000, 'StimType_1', 'T1', 100, 0), + (100000, 32.0, 10000, 'R1', 1000, 'StimType_1', 'T1', 101, 1), + (100000, 32.0, 10000, 'R1', 1000, 'StimType_1', 'T1', 101, 1), + (100000, 32.0, 10001, 'R2', 1001, 'StimType_2', 'T2', 102, 2), + (100000, 32.0, 10001, 'R2', 1001, 'StimType_2', 'T2', 102, 2), + (100000, 32.0, 10001, 'R2', 1001, 'StimType_2', 'T2', 102, 2), + (100000, 32.0, 10001, 'R2', 1002, 'StimType_3', 'T3', 103, 3), + (100000, 32.0, 10001, 'R2', 1002, 'StimType_3', 'T3', 103, 3), + (100000, 32.0, 10001, 'R2', 1002, 'StimType_3', 'T3', 103, 3), + (100001, 24.0, 10002, 'R1', 1003, 'StimType_1', 'T1', 104, 4), + (100001, 24.0, 10002, 'R1', 1003, 'StimType_1', 'T1', 104, 4), + (100001, 24.0, 10002, 'R1', 1003, 'StimType_1', 'T1', 105, 5), + (100001, 24.0, 10002, 'R1', 1003, 'StimType_1', 'T1', 105, 5), + (100001, 24.0, 10003, 'R2', 1004, 'StimType_2', 'T2', 106, 6), + (100001, 24.0, 10003, 'R2', 1004, 'StimType_2', 'T2', 106, 6), + (100001, 24.0, 10003, 'R2', 1004, 'StimType_2', 'T2', 106, 6), + (100001, 24.0, 10003, 'R2', 1005, 'StimType_3', 'T3', 107, 7), + (100001, 24.0, 10003, 'R2', 1005, 'StimType_3', 'T3', 107, 7), + (100001, 24.0, 10003, 'R2', 1005, 'StimType_3', 'T3', 107, 7)] + self.assertListEqual(temp.index.tolist(), expected_index) + + def test_to_hierarchical_dataframe_with_indexed_custom_column(self): + # test that to_hierarchical_dataframe works for the icephys tables when containing also VectorIndex columns + nwbfile = create_icephys_testfile() + nwbfile.icephys_experimental_conditions.add_column('newcol', 'abc', data=[1, 2, 3], index=[2, 3]) + temp = to_hierarchical_dataframe(nwbfile.icephys_experimental_conditions) + expected_index = [ + (100000, 32.0, (1, 2), 10000, 'R1', 1000, 'StimType_1', 'T1', 100, 0), + (100000, 32.0, (1, 2), 10000, 'R1', 1000, 'StimType_1', 'T1', 100, 0), + (100000, 32.0, (1, 2), 10000, 'R1', 1000, 'StimType_1', 'T1', 101, 1), + (100000, 32.0, (1, 2), 10000, 'R1', 1000, 'StimType_1', 'T1', 101, 1), + (100000, 32.0, (1, 2), 10001, 'R2', 1001, 'StimType_2', 'T2', 102, 2), + (100000, 32.0, (1, 2), 10001, 'R2', 1001, 'StimType_2', 'T2', 102, 2), + (100000, 32.0, (1, 2), 10001, 'R2', 1001, 'StimType_2', 'T2', 102, 2), + (100000, 32.0, (1, 2), 10001, 'R2', 1002, 'StimType_3', 'T3', 103, 3), + (100000, 32.0, (1, 2), 10001, 'R2', 1002, 'StimType_3', 'T3', 103, 3), + (100000, 32.0, (1, 2), 10001, 'R2', 1002, 'StimType_3', 'T3', 103, 3), + (100001, 24.0, (3,), 10002, 'R1', 1003, 'StimType_1', 'T1', 104, 4), + (100001, 24.0, (3,), 10002, 'R1', 1003, 'StimType_1', 'T1', 104, 4), + (100001, 24.0, (3,), 10002, 'R1', 1003, 'StimType_1', 'T1', 105, 5), + (100001, 24.0, (3,), 10002, 'R1', 1003, 'StimType_1', 'T1', 105, 5), + (100001, 24.0, (3,), 10003, 'R2', 1004, 'StimType_2', 'T2', 106, 6), + (100001, 24.0, (3,), 10003, 'R2', 1004, 'StimType_2', 'T2', 106, 6), + (100001, 24.0, (3,), 10003, 'R2', 1004, 'StimType_2', 'T2', 106, 6), + (100001, 24.0, (3,), 10003, 'R2', 1005, 'StimType_3', 'T3', 107, 7), + (100001, 24.0, (3,), 10003, 'R2', 1005, 'StimType_3', 'T3', 107, 7), + (100001, 24.0, (3,), 10003, 'R2', 1005, 'StimType_3', 'T3', 107, 7)] + self.assertListEqual(temp.index.tolist(), expected_index) + class NWBFileTests(TestCase): """ From 870a84f19cf582f450a7e2e254d8e7bb3b8e90e6 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Sun, 22 Aug 2021 23:35:30 -0700 Subject: [PATCH 2/3] Update minimimum requirment for HDMF to bug-fix version --- requirements-min.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-min.txt b/requirements-min.txt index f7adc6641..6ae932eeb 100644 --- a/requirements-min.txt +++ b/requirements-min.txt @@ -1,6 +1,6 @@ # minimum versions of package dependencies for installing PyNWB h5py==2.10 # support for selection of datasets with list of indices added in 2.10 -hdmf==3.1.1 +hdmf==3.1.2 numpy==1.16 pandas==1.0.5 python-dateutil==2.7 From 22eedc3e2a37f617bb2ca01dec86a282b227ee5a Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Sun, 22 Aug 2021 23:37:59 -0700 Subject: [PATCH 3/3] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 303ce8b3e..4bec59ef3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Bug fixes: - Add `environment-ros3.yml` to `MANIFEST.in` for inclusion in source distributions. @rly (#1398) +- Update HDMF minimum version to `3.1.2` to fix bug when converting the icephys metadata tables via `to_hierarchical_dataframe` when the tables contain indexed columns and add corresponding unit tests. @oruebel (#1404) ## PyNWB 2.0.0 (August 13, 2021)