From b79990621019d3d5567b4b3a5edadf3ec3aae637 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 19 Nov 2020 09:49:00 +0100 Subject: [PATCH 01/68] maint: drafting a new package structure --- docs/api.rst | 1 - sdcflows/{models => workflows/apply}/__init__.py | 0 sdcflows/workflows/{unwarp.py => apply/base.py} | 2 +- sdcflows/{models => workflows/apply}/tests/__init__.py | 0 sdcflows/workflows/base.py | 6 +++--- sdcflows/workflows/fit/__init__.py | 0 sdcflows/{models => workflows/fit}/fieldmap.py | 10 +++++----- sdcflows/{models => workflows/fit}/pepolar.py | 6 +++--- sdcflows/{models => workflows/fit}/syn.py | 2 +- sdcflows/workflows/fit/tests/__init__.py | 0 .../{models => workflows/fit}/tests/test_pepolar.py | 2 +- .../{models => workflows/fit}/tests/test_phdiff.py | 2 +- 12 files changed, 15 insertions(+), 16 deletions(-) rename sdcflows/{models => workflows/apply}/__init__.py (100%) rename sdcflows/workflows/{unwarp.py => apply/base.py} (98%) rename sdcflows/{models => workflows/apply}/tests/__init__.py (100%) create mode 100644 sdcflows/workflows/fit/__init__.py rename sdcflows/{models => workflows/fit}/fieldmap.py (97%) rename sdcflows/{models => workflows/fit}/pepolar.py (98%) rename sdcflows/{models => workflows/fit}/syn.py (99%) create mode 100644 sdcflows/workflows/fit/tests/__init__.py rename sdcflows/{models => workflows/fit}/tests/test_pepolar.py (97%) rename sdcflows/{models => workflows/fit}/tests/test_phdiff.py (97%) diff --git a/docs/api.rst b/docs/api.rst index 85eae4b4be..dbe8a0e76a 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -8,6 +8,5 @@ Information on specific functions, classes, and methods. api/sdcflows.fieldmaps api/sdcflows.interfaces - api/sdcflows.models api/sdcflows.viz api/sdcflows.workflows \ No newline at end of file diff --git a/sdcflows/models/__init__.py b/sdcflows/workflows/apply/__init__.py similarity index 100% rename from sdcflows/models/__init__.py rename to sdcflows/workflows/apply/__init__.py diff --git a/sdcflows/workflows/unwarp.py b/sdcflows/workflows/apply/base.py similarity index 98% rename from sdcflows/workflows/unwarp.py rename to sdcflows/workflows/apply/base.py index bd010265b8..312997b84b 100644 --- a/sdcflows/workflows/unwarp.py +++ b/sdcflows/workflows/apply/base.py @@ -22,7 +22,7 @@ def init_sdc_unwarp_wf(omp_nthreads, debug, name='sdc_unwarp_wf'): :graph2use: orig :simple_form: yes - from sdcflows.workflows.unwarp import init_sdc_unwarp_wf + from sdcflows.workflows.apply.base import init_sdc_unwarp_wf wf = init_sdc_unwarp_wf(omp_nthreads=8, debug=False) diff --git a/sdcflows/models/tests/__init__.py b/sdcflows/workflows/apply/tests/__init__.py similarity index 100% rename from sdcflows/models/tests/__init__.py rename to sdcflows/workflows/apply/tests/__init__.py diff --git a/sdcflows/workflows/base.py b/sdcflows/workflows/base.py index fc0f09a487..6e2dd92656 100644 --- a/sdcflows/workflows/base.py +++ b/sdcflows/workflows/base.py @@ -106,7 +106,7 @@ def init_sdc_estimate_wf(bids_fmaps, omp_nthreads=1, debug=False): ) # PEPOLAR path if "epi" in bids_fmaps: - from ..models.pepolar import init_3dQwarp_wf + from .fit.pepolar import init_3dQwarp_wf outputnode.inputs.method = "PEB/PEPOLAR (phase-encoding based / PE-POLARity)" @@ -117,7 +117,7 @@ def init_sdc_estimate_wf(bids_fmaps, omp_nthreads=1, debug=False): # FIELDMAP path elif "fieldmap" in bids_fmaps or "phasediff" in bids_fmaps: - from ..models.fieldmap import init_fmap_wf + from .fit.fieldmap import init_fmap_wf if "fieldmap" in bids_fmaps: fmap = bids_fmaps["fieldmap"][0] @@ -135,7 +135,7 @@ def init_sdc_estimate_wf(bids_fmaps, omp_nthreads=1, debug=False): # FIELDMAP-less path elif "syn" in bids_fmaps: - from ..models.syn import init_syn_sdc_wf + from .fit.syn import init_syn_sdc_wf outputnode.inputs.method = 'FLB ("fieldmap-less", SyN-based)' diff --git a/sdcflows/workflows/fit/__init__.py b/sdcflows/workflows/fit/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sdcflows/models/fieldmap.py b/sdcflows/workflows/fit/fieldmap.py similarity index 97% rename from sdcflows/models/fieldmap.py rename to sdcflows/workflows/fit/fieldmap.py index 1cf5c7d96a..4f0eb2e9ad 100644 --- a/sdcflows/models/fieldmap.py +++ b/sdcflows/workflows/fit/fieldmap.py @@ -56,7 +56,7 @@ def init_fmap_wf(omp_nthreads, mode="phasediff", name="fmap_wf"): :graph2use: orig :simple_form: yes - from sdcflows.models.fieldmap import init_fmap_wf + from sdcflows.workflows.fit.fieldmap import init_fmap_wf wf = init_fmap_wf(omp_nthreads=6) Parameters @@ -167,7 +167,7 @@ def init_magnitude_wf(omp_nthreads, name="magnitude_wf"): :graph2use: orig :simple_form: yes - from sdcflows.models.fieldmap import init_magnitude_wf + from sdcflows.workflows.fit.fieldmap import init_magnitude_wf wf = init_magnitude_wf(omp_nthreads=6) Parameters @@ -239,7 +239,7 @@ def init_fmap_postproc_wf( :graph2use: orig :simple_form: yes - from sdcflows.models.fieldmap import init_fmap_postproc_wf + from sdcflows.workflows.fit.fieldmap import init_fmap_postproc_wf wf = init_fmap_postproc_wf(omp_nthreads=6) Parameters @@ -336,7 +336,7 @@ def init_phdiff_wf(omp_nthreads, name="phdiff_wf"): :graph2use: orig :simple_form: yes - from sdcflows.models.fieldmap import init_phdiff_wf + from sdcflows.workflows.fit.fieldmap import init_phdiff_wf wf = init_phdiff_wf(omp_nthreads=1) Parameters @@ -367,7 +367,7 @@ def init_phdiff_wf(omp_nthreads, name="phdiff_wf"): """ from nipype.interfaces.fsl import PRELUDE - from ..interfaces.fmap import Phasediff2Fieldmap, PhaseMap2rads, SubtractPhases + from ...interfaces.fmap import Phasediff2Fieldmap, PhaseMap2rads, SubtractPhases workflow = Workflow(name=name) workflow.__desc__ = f"""\ diff --git a/sdcflows/models/pepolar.py b/sdcflows/workflows/fit/pepolar.py similarity index 98% rename from sdcflows/models/pepolar.py rename to sdcflows/workflows/fit/pepolar.py index 00546b520e..10948dd381 100644 --- a/sdcflows/models/pepolar.py +++ b/sdcflows/workflows/fit/pepolar.py @@ -31,7 +31,7 @@ def init_topup_wf(omp_nthreads=1, debug=False, name="pepolar_estimate_wf"): :graph2use: orig :simple_form: yes - from sdcflows.models.pepolar import init_topup_wf + from sdcflows.workflows.fit.pepolar import init_topup_wf wf = init_topup_wf() Parameters @@ -133,7 +133,7 @@ def init_3dQwarp_wf(omp_nthreads=1, name="pepolar_estimate_wf"): :graph2use: orig :simple_form: yes - from sdcflows.models.pepolar import init_3dQwarp_wf + from sdcflows.workflows.fit.pepolar import init_3dQwarp_wf wf = init_3dQwarp_wf() Parameters @@ -164,7 +164,7 @@ def init_3dQwarp_wf(omp_nthreads=1, name="pepolar_estimate_wf"): ) from niworkflows.interfaces.freesurfer import StructuralReference from niworkflows.func.util import init_enhance_and_skullstrip_bold_wf - from ..interfaces.utils import Flatten + from ...interfaces.utils import Flatten workflow = Workflow(name=name) workflow.__desc__ = f"""{_PEPOLAR_DESC} \ diff --git a/sdcflows/models/syn.py b/sdcflows/workflows/fit/syn.py similarity index 99% rename from sdcflows/models/syn.py rename to sdcflows/workflows/fit/syn.py index dc6e47d327..9f55d680a6 100644 --- a/sdcflows/models/syn.py +++ b/sdcflows/workflows/fit/syn.py @@ -63,7 +63,7 @@ def init_syn_sdc_wf(omp_nthreads, epi_pe=None, atlas_threshold=3, name="syn_sdc_ :graph2use: orig :simple_form: yes - from sdcflows.models.syn import init_syn_sdc_wf + from sdcflows.workflows.fit.syn import init_syn_sdc_wf wf = init_syn_sdc_wf( epi_pe="j", omp_nthreads=8) diff --git a/sdcflows/workflows/fit/tests/__init__.py b/sdcflows/workflows/fit/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sdcflows/models/tests/test_pepolar.py b/sdcflows/workflows/fit/tests/test_pepolar.py similarity index 97% rename from sdcflows/models/tests/test_pepolar.py rename to sdcflows/workflows/fit/tests/test_pepolar.py index eb09d6c4d6..b74887f552 100644 --- a/sdcflows/models/tests/test_pepolar.py +++ b/sdcflows/workflows/fit/tests/test_pepolar.py @@ -50,7 +50,7 @@ def test_topup_wf(tmpdir, datadir, workdir, outdir, epi_path): if outdir: from nipype.interfaces.afni import Automask - from ...interfaces.reportlets import FieldmapReportlet + from ....interfaces.reportlets import FieldmapReportlet pre_mask = pe.Node(Automask(dilate=1, outputtype="NIFTI_GZ"), name="pre_mask") diff --git a/sdcflows/models/tests/test_phdiff.py b/sdcflows/workflows/fit/tests/test_phdiff.py similarity index 97% rename from sdcflows/models/tests/test_phdiff.py rename to sdcflows/workflows/fit/tests/test_phdiff.py index 05c9b09b6b..530351b6aa 100644 --- a/sdcflows/models/tests/test_phdiff.py +++ b/sdcflows/workflows/fit/tests/test_phdiff.py @@ -43,7 +43,7 @@ def test_phdiff(tmpdir, datadir, workdir, outdir, fmap_path): ] if outdir: - from ...interfaces.reportlets import FieldmapReportlet + from ....interfaces.reportlets import FieldmapReportlet rep = pe.Node(FieldmapReportlet(reference_label="Magnitude"), "simple_report") rep.interface._always_run = True From f60c6fb570b609e114116c27d5b293d90a1625cf Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 19 Nov 2020 10:01:17 +0100 Subject: [PATCH 02/68] enh: remove ``enhance_and_skullstrip_bold`` from ``init_sdc_unwarp_wf`` Resolves: #93. --- sdcflows/workflows/apply/base.py | 83 +++++++++++++++++--------------- 1 file changed, 45 insertions(+), 38 deletions(-) diff --git a/sdcflows/workflows/apply/base.py b/sdcflows/workflows/apply/base.py index 312997b84b..9dcfd90654 100644 --- a/sdcflows/workflows/apply/base.py +++ b/sdcflows/workflows/apply/base.py @@ -4,11 +4,9 @@ from nipype.pipeline import engine as pe from nipype.interfaces import utility as niu from niworkflows.engine.workflows import LiterateWorkflow as Workflow -from niworkflows.interfaces.registration import ANTSApplyTransformsRPT -from niworkflows.func.util import init_enhance_and_skullstrip_bold_wf -def init_sdc_unwarp_wf(omp_nthreads, debug, name='sdc_unwarp_wf'): +def init_sdc_unwarp_wf(omp_nthreads, debug, name="sdc_unwarp_wf"): """ Apply the warping given by a displacements fieldmap. @@ -49,51 +47,60 @@ def init_sdc_unwarp_wf(omp_nthreads, debug, name='sdc_unwarp_wf'): ------- out_reference : :obj:`str` the ``in_reference`` after unwarping - out_reference_brain : :obj:`str` - the ``in_reference`` after unwarping and skullstripping - out_warp : :obj:`str` - the ``in_warp`` field is forwarded for compatibility out_mask : :obj:`str` mask of the unwarped input file + out_warp : :obj:`str` + the ``in_warp`` field is forwarded for compatibility """ + from niworkflows.interfaces.registration import ANTSApplyTransformsRPT workflow = Workflow(name=name) - inputnode = pe.Node(niu.IdentityInterface( - fields=['in_warp', 'in_reference', 'in_reference_mask']), - name='inputnode') - outputnode = pe.Node(niu.IdentityInterface( - fields=['out_reference', 'out_reference_brain', 'out_warp', 'out_mask']), - name='outputnode') + inputnode = pe.Node( + niu.IdentityInterface(fields=["in_warp", "in_reference", "in_reference_mask"]), + name="inputnode", + ) + outputnode = pe.Node( + niu.IdentityInterface( + fields=["out_reference", "out_warp", "out_mask"] + ), + name="outputnode", + ) - unwarp_reference = pe.Node(ANTSApplyTransformsRPT(dimension=3, - generate_report=False, - float=True, - interpolation='LanczosWindowedSinc'), - name='unwarp_reference') + unwarp_reference = pe.Node( + ANTSApplyTransformsRPT( + dimension=3, + generate_report=False, + float=True, + interpolation="LanczosWindowedSinc", + ), + name="unwarp_reference", + ) - unwarp_mask = pe.Node(ANTSApplyTransformsRPT( - dimension=3, generate_report=False, float=True, - interpolation='NearestNeighbor'), name='unwarp_mask') + unwarp_mask = pe.Node( + ANTSApplyTransformsRPT( + dimension=3, + generate_report=False, + float=True, + interpolation="NearestNeighbor", + ), + name="unwarp_mask", + ) - enhance_and_skullstrip_bold_wf = init_enhance_and_skullstrip_bold_wf(omp_nthreads=omp_nthreads, - pre_mask=True) + # fmt:off workflow.connect([ (inputnode, unwarp_reference, [ - ('in_warp', 'transforms'), - ('in_reference', 'reference_image'), - ('in_reference', 'input_image')]), + ("in_warp", "transforms"), + ("in_reference", "reference_image"), + ("in_reference", "input_image"), + ]), (inputnode, unwarp_mask, [ - ('in_warp', 'transforms'), - ('in_reference_mask', 'reference_image'), - ('in_reference_mask', 'input_image')]), - (unwarp_reference, enhance_and_skullstrip_bold_wf, [ - ('output_image', 'inputnode.in_file')]), - (unwarp_mask, enhance_and_skullstrip_bold_wf, [ - ('output_image', 'inputnode.pre_mask')]), - (inputnode, outputnode, [('in_warp', 'out_warp')]), - (unwarp_reference, outputnode, [('output_image', 'out_reference')]), - (enhance_and_skullstrip_bold_wf, outputnode, [ - ('outputnode.mask_file', 'out_mask'), - ('outputnode.skull_stripped_file', 'out_reference_brain')]), + ("in_warp", "transforms"), + ("in_reference_mask", "reference_image"), + ("in_reference_mask", "input_image"), + ]), + (inputnode, outputnode, [("in_warp", "out_warp")]), + (unwarp_reference, outputnode, [("output_image", "out_reference")]), + (unwarp_mask, outputnode, [("output_image", "out_mask")]), ]) + # fmt: on return workflow From 10ce88efbca57707a807ed2783e1701000fc7572 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 19 Nov 2020 10:52:42 +0100 Subject: [PATCH 03/68] maint: extensive cleanup of ``fmap`` interfaces --- docs/api.rst | 1 + sdcflows/interfaces/fmap.py | 522 +++++------------------------------ sdcflows/utils/__init__.py | 0 sdcflows/utils/phasemanip.py | 127 +++++++++ 4 files changed, 192 insertions(+), 458 deletions(-) create mode 100644 sdcflows/utils/__init__.py create mode 100644 sdcflows/utils/phasemanip.py diff --git a/docs/api.rst b/docs/api.rst index dbe8a0e76a..89fc44eefe 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -8,5 +8,6 @@ Information on specific functions, classes, and methods. api/sdcflows.fieldmaps api/sdcflows.interfaces + api/sdcflows.utils api/sdcflows.viz api/sdcflows.workflows \ No newline at end of file diff --git a/sdcflows/interfaces/fmap.py b/sdcflows/interfaces/fmap.py index 9f2d1cb7a7..f9e804c89b 100644 --- a/sdcflows/interfaces/fmap.py +++ b/sdcflows/interfaces/fmap.py @@ -13,214 +13,88 @@ """ -import numpy as np -import nibabel as nb from nipype import logging -from nipype.utils.filemanip import fname_presuffix from nipype.interfaces.base import ( - BaseInterfaceInputSpec, TraitedSpec, File, isdefined, traits, - SimpleInterface) + BaseInterfaceInputSpec, + TraitedSpec, + File, + traits, + SimpleInterface, +) -LOGGER = logging.getLogger('nipype.interface') +LOGGER = logging.getLogger("nipype.interface") -class _SubtractPhasesInputSpec(BaseInterfaceInputSpec): - in_phases = traits.List(File(exists=True), min=1, max=2, - desc='input phase maps') - in_meta = traits.List(traits.Dict(), min=1, max=2, - desc='metadata corresponding to the inputs') - - -class _SubtractPhasesOutputSpec(TraitedSpec): - phase_diff = File(exists=True, desc='phase difference map') - metadata = traits.Dict(desc='output metadata') - - -class SubtractPhases(SimpleInterface): - """Calculate a phase difference map.""" - - input_spec = _SubtractPhasesInputSpec - output_spec = _SubtractPhasesOutputSpec - - def _run_interface(self, runtime): - if len(self.inputs.in_phases) != len(self.inputs.in_meta): - raise ValueError( - 'Length of input phase-difference maps and metadata files ' - 'should match.') - - if len(self.inputs.in_phases) == 1: - self._results['phase_diff'] = self.inputs.in_phases[0] - self._results['metadata'] = self.inputs.in_meta[0] - return runtime - - self._results['phase_diff'], self._results['metadata'] = \ - _subtract_phases(self.inputs.in_phases, - self.inputs.in_meta, - newpath=runtime.cwd) - - return runtime - - -class _FieldEnhanceInputSpec(BaseInterfaceInputSpec): - in_file = File(exists=True, mandatory=True, desc='input fieldmap') - in_mask = File(exists=True, desc='brain mask') - in_magnitude = File(exists=True, desc='input magnitude') - unwrap = traits.Bool(False, usedefault=True, desc='run phase unwrap') - despike = traits.Bool(True, usedefault=True, desc='run despike filter') - bspline_smooth = traits.Bool(True, usedefault=True, desc='run 3D bspline smoother') - mask_erode = traits.Int(1, usedefault=True, desc='mask erosion iterations') - despike_threshold = traits.Float(0.2, usedefault=True, desc='mask erosion iterations') - num_threads = traits.Int(1, usedefault=True, nohash=True, desc='number of jobs') +class _PhaseMap2radsInputSpec(BaseInterfaceInputSpec): + in_file = File(exists=True, mandatory=True, desc="input (wrapped) phase map") -class _FieldEnhanceOutputSpec(TraitedSpec): - out_file = File(desc='the output fieldmap') - out_unwrapped = File(desc='unwrapped fieldmap') +class _PhaseMap2radsOutputSpec(TraitedSpec): + out_file = File(desc="the phase map in the range 0 - 6.28") -class FieldEnhance(SimpleInterface): - """Massage the input fieldmap (masking, despiking, etc.).""" +class PhaseMap2rads(SimpleInterface): + """Convert a phase map given in a.u. (e.g., 0-4096) to radians.""" - input_spec = _FieldEnhanceInputSpec - output_spec = _FieldEnhanceOutputSpec + input_spec = _PhaseMap2radsInputSpec + output_spec = _PhaseMap2radsOutputSpec def _run_interface(self, runtime): - from scipy import ndimage as sim - - fmap_nii = nb.load(self.inputs.in_file) - data = np.squeeze(fmap_nii.get_fdata(dtype='float32')) - - # Despike / denoise (no-mask) - if self.inputs.despike: - data = _despike2d(data, self.inputs.despike_threshold) - - mask = None - if isdefined(self.inputs.in_mask): - masknii = nb.load(self.inputs.in_mask) - mask = np.asanyarray(masknii.dataobj).astype('uint8') - - # Dilate mask - if self.inputs.mask_erode > 0: - struc = sim.iterate_structure(sim.generate_binary_structure(3, 2), 1) - mask = sim.binary_erosion( - mask, struc, - iterations=self.inputs.mask_erode - ).astype(np.uint8) # pylint: disable=no-member - - self._results['out_file'] = fname_presuffix( - self.inputs.in_file, suffix='_enh', newpath=runtime.cwd) - datanii = nb.Nifti1Image(data, fmap_nii.affine, fmap_nii.header) - - if self.inputs.unwrap: - data = _unwrap(data, self.inputs.in_magnitude, mask) - self._results['out_unwrapped'] = fname_presuffix( - self.inputs.in_file, suffix='_unwrap', newpath=runtime.cwd) - nb.Nifti1Image(data, fmap_nii.affine, fmap_nii.header).to_filename( - self._results['out_unwrapped']) - - if not self.inputs.bspline_smooth: - datanii.to_filename(self._results['out_file']) - return runtime - else: - from ..utils import bspline as fbsp - from statsmodels.robust.scale import mad - - # Fit BSplines (coarse) - bspobj = fbsp.BSplineFieldmap(datanii, weights=mask, - njobs=self.inputs.num_threads) - bspobj.fit() - smoothed1 = bspobj.get_smoothed() - - # Manipulate the difference map - diffmap = data - smoothed1.get_fdata(dtype='float32') - sderror = mad(diffmap[mask > 0]) - LOGGER.info('SD of error after B-Spline fitting is %f', sderror) - errormask = np.zeros_like(diffmap) - errormask[np.abs(diffmap) > (10 * sderror)] = 1 - errormask *= mask - - nslices = 0 - try: - errorslice = np.squeeze(np.argwhere(errormask.sum(0).sum(0) > 0)) - nslices = errorslice[-1] - errorslice[0] - except IndexError: # mask is empty, do not refine - pass - - if nslices > 1: - diffmapmsk = mask[..., errorslice[0]:errorslice[-1]] - diffmapnii = nb.Nifti1Image( - diffmap[..., errorslice[0]:errorslice[-1]] * diffmapmsk, - datanii.affine, datanii.header) - - bspobj2 = fbsp.BSplineFieldmap(diffmapnii, knots_zooms=[24., 24., 4.], - njobs=self.inputs.num_threads) - bspobj2.fit() - smoothed2 = bspobj2.get_smoothed().get_fdata(dtype='float32') - - final = smoothed1.get_fdata(dtype='float32').copy() - final[..., errorslice[0]:errorslice[-1]] += smoothed2 - else: - final = smoothed1.get_fdata(dtype='float32') - - nb.Nifti1Image(final, datanii.affine, datanii.header).to_filename( - self._results['out_file']) + from ..utils.phasemanip import au2rads + self._results["out_file"] = au2rads(self.inputs.in_file, newpath=runtime.cwd) return runtime -class _FieldToRadSInputSpec(BaseInterfaceInputSpec): - in_file = File(exists=True, mandatory=True, desc='input fieldmap') - fmap_range = traits.Float(desc='range of input field map') +class _SubtractPhasesInputSpec(BaseInterfaceInputSpec): + in_phases = traits.List(File(exists=True), min=1, max=2, desc="input phase maps") + in_meta = traits.List( + traits.Dict(), min=1, max=2, desc="metadata corresponding to the inputs" + ) -class _FieldToRadSOutputSpec(TraitedSpec): - out_file = File(desc='the output fieldmap') - fmap_range = traits.Float(desc='range of input field map') +class _SubtractPhasesOutputSpec(TraitedSpec): + phase_diff = File(exists=True, desc="phase difference map") + metadata = traits.Dict(desc="output metadata") -class FieldToRadS(SimpleInterface): - """Convert from arbitrary units to rad/s.""" +class SubtractPhases(SimpleInterface): + """Calculate a phase difference map.""" - input_spec = _FieldToRadSInputSpec - output_spec = _FieldToRadSOutputSpec + input_spec = _SubtractPhasesInputSpec + output_spec = _SubtractPhasesOutputSpec def _run_interface(self, runtime): - fmap_range = None - if isdefined(self.inputs.fmap_range): - fmap_range = self.inputs.fmap_range - self._results['out_file'], self._results['fmap_range'] = _torads( - self.inputs.in_file, fmap_range, newpath=runtime.cwd) - return runtime - - -class _FieldToHzInputSpec(BaseInterfaceInputSpec): - in_file = File(exists=True, mandatory=True, desc='input fieldmap') - range_hz = traits.Float(mandatory=True, desc='range of input field map') - - -class _FieldToHzOutputSpec(TraitedSpec): - out_file = File(desc='the output fieldmap') + if len(self.inputs.in_phases) != len(self.inputs.in_meta): + raise ValueError( + "Length of input phase-difference maps and metadata files " + "should match." + ) + if len(self.inputs.in_phases) == 1: + self._results["phase_diff"] = self.inputs.in_phases[0] + self._results["metadata"] = self.inputs.in_meta[0] + return runtime -class FieldToHz(SimpleInterface): - """Convert from arbitrary units to Hz.""" + from ..utils.phasemanip import subtract_phases as _subtract_phases - input_spec = _FieldToHzInputSpec - output_spec = _FieldToHzOutputSpec + # Discard in_meta traits with copy(), so that pop() works. + self._results["phase_diff"], self._results["metadata"] = _subtract_phases( + self.inputs.in_phases, + (self.inputs.in_meta[0].copy(), self.inputs.in_meta[1].copy()), + newpath=runtime.cwd, + ) - def _run_interface(self, runtime): - self._results['out_file'] = _tohz( - self.inputs.in_file, self.inputs.range_hz, newpath=runtime.cwd) return runtime class _Phasediff2FieldmapInputSpec(BaseInterfaceInputSpec): - in_file = File(exists=True, mandatory=True, desc='input fieldmap') - metadata = traits.Dict(mandatory=True, desc='BIDS metadata dictionary') + in_file = File(exists=True, mandatory=True, desc="input fieldmap") + metadata = traits.Dict(mandatory=True, desc="BIDS metadata dictionary") class _Phasediff2FieldmapOutputSpec(TraitedSpec): - out_file = File(desc='the output fieldmap') + out_file = File(desc="the output fieldmap") class Phasediff2Fieldmap(SimpleInterface): @@ -239,242 +113,26 @@ class Phasediff2Fieldmap(SimpleInterface): output_spec = _Phasediff2FieldmapOutputSpec def _run_interface(self, runtime): - self._results['out_file'] = phdiff2fmap( - self.inputs.in_file, - _delta_te(self.inputs.metadata), - newpath=runtime.cwd) - return runtime - - -class _PhaseMap2radsInputSpec(BaseInterfaceInputSpec): - in_file = File(exists=True, mandatory=True, desc='input (wrapped) phase map') - - -class _PhaseMap2radsOutputSpec(TraitedSpec): - out_file = File(desc='the phase map in the range 0 - 6.28') - - -class PhaseMap2rads(SimpleInterface): - """Convert a phase map in a.u. to radians.""" - - input_spec = _PhaseMap2radsInputSpec - output_spec = _PhaseMap2radsOutputSpec - - def _run_interface(self, runtime): - self._results['out_file'] = au2rads( - self.inputs.in_file, - newpath=runtime.cwd) - return runtime - - -class _FUGUEvsm2ANTSwarpInputSpec(BaseInterfaceInputSpec): - in_file = File(exists=True, mandatory=True, - desc='input displacements field map') - pe_dir = traits.Enum('i', 'i-', 'j', 'j-', 'k', 'k-', - desc='phase-encoding axis') - - -class _FUGUEvsm2ANTSwarpOutputSpec(TraitedSpec): - out_file = File(desc='the output warp field') - fieldmap = File(desc='field map in mm') - - -class FUGUEvsm2ANTSwarp(SimpleInterface): - """Convert a voxel-shift-map to ants warp.""" - - _dtype = ' 1e-6 and - (abs(thisval - patch_med) / patch_range) > thres): - data[i, j, k] = patch_med - return data - - -def _unwrap(fmap_data, mag_file, mask=None): - from math import pi - from nipype.interfaces.fsl import PRELUDE - magnii = nb.load(mag_file) - - if mask is None: - mask = np.ones_like(fmap_data, dtype=np.uint8) - - fmapmax = max(abs(fmap_data[mask > 0].min()), fmap_data[mask > 0].max()) - fmap_data *= pi / fmapmax - - nb.Nifti1Image(fmap_data, magnii.affine).to_filename('fmap_rad.nii.gz') - nb.Nifti1Image(mask, magnii.affine).to_filename('fmap_mask.nii.gz') - nb.Nifti1Image(magnii.get_fdata(dtype='float32'), - magnii.affine).to_filename('fmap_mag.nii.gz') - - # Run prelude - res = PRELUDE(phase_file='fmap_rad.nii.gz', - magnitude_file='fmap_mag.nii.gz', - mask_file='fmap_mask.nii.gz').run() - - unwrapped = nb.load( - res.outputs.unwrapped_phase_file).get_fdata(dtype='float32') * (fmapmax / pi) - return unwrapped - - -def _torads(in_file, fmap_range=None, newpath=None): - """ - Convert a field map to rad/s units. - - If fmap_range is None, the range of the fieldmap - will be automatically calculated. - - Use fmap_range=0.5 to convert from Hz to rad/s - - """ - from math import pi - import nibabel as nb - from nipype.utils.filemanip import fname_presuffix - - out_file = fname_presuffix(in_file, suffix='_rad', newpath=newpath) - fmapnii = nb.load(in_file) - fmapdata = fmapnii.get_fdata(dtype='float32') - - if fmap_range is None: - fmap_range = max(abs(fmapdata.min()), fmapdata.max()) - fmapdata = fmapdata * (pi / fmap_range) - out_img = nb.Nifti1Image(fmapdata, fmapnii.affine, fmapnii.header) - out_img.set_data_dtype('float32') - out_img.to_filename(out_file) - return out_file, fmap_range - - -def _tohz(in_file, range_hz, newpath=None): - """Convert a field map to Hz units.""" - from math import pi - import nibabel as nb - from nipype.utils.filemanip import fname_presuffix - - out_file = fname_presuffix(in_file, suffix='_hz', newpath=newpath) - fmapnii = nb.load(in_file) - fmapdata = fmapnii.get_fdata(dtype='float32') - fmapdata = fmapdata * (range_hz / pi) - out_img = nb.Nifti1Image(fmapdata, fmapnii.affine, fmapnii.header) - out_img.set_data_dtype('float32') - out_img.to_filename(out_file) - return out_file - - -def phdiff2fmap(in_file, delta_te, newpath=None): - r""" - Convert the input phase-difference map into a fieldmap in Hz. - - Uses eq. (1) of [Hutton2002]_: - - .. math:: - - \Delta B_0 (\text{T}^{-1}) = \frac{\Delta \Theta}{2\pi\gamma \Delta\text{TE}} - - - In this case, we do not take into account the gyromagnetic ratio of the - proton (:math:`\gamma`), since it will be applied inside TOPUP: - - .. math:: - - \Delta B_0 (\text{Hz}) = \frac{\Delta \Theta}{2\pi \Delta\text{TE}} - - References - ---------- - .. [Hutton2002] Hutton et al., Image Distortion Correction in fMRI: A Quantitative - Evaluation, NeuroImage 16(1):217-240, 2002. doi:`10.1006/nimg.2001.1054 - `_. - - - """ - import math - import numpy as np - import nibabel as nb - from nipype.utils.filemanip import fname_presuffix - # GYROMAG_RATIO_H_PROTON_MHZ = 42.576 - - out_file = fname_presuffix(in_file, suffix='_fmap', newpath=newpath) - image = nb.load(in_file) - data = (image.get_fdata(dtype='float32') / (2. * math.pi * delta_te)) - nii = nb.Nifti1Image(data, image.affine, image.header) - nii.set_data_dtype(np.float32) - nii.to_filename(out_file) - return out_file - - def _delta_te(in_values, te1=None, te2=None): r"""Read :math:`\Delta_\text{TE}` from BIDS metadata dict.""" if isinstance(in_values, float): te2 = in_values - te1 = 0. + te1 = 0.0 if isinstance(in_values, dict): - te1 = in_values.get('EchoTime1') - te2 = in_values.get('EchoTime2') + te1 = in_values.get("EchoTime1") + te2 = in_values.get("EchoTime2") if not all((te1, te2)): - te2 = in_values.get('EchoTimeDifference') + te2 = in_values.get("EchoTimeDifference") te1 = 0 if isinstance(in_values, list): @@ -486,69 +144,17 @@ def _delta_te(in_values, te1=None, te2=None): # For convienience if both are missing we should give one error about them if te1 is None and te2 is None: - raise RuntimeError('EchoTime1 and EchoTime2 metadata fields not found. ' - 'Please consult the BIDS specification.') + raise RuntimeError( + "EchoTime1 and EchoTime2 metadata fields not found. " + "Please consult the BIDS specification." + ) if te1 is None: raise RuntimeError( - 'EchoTime1 metadata field not found. Please consult the BIDS specification.') + "EchoTime1 metadata field not found. Please consult the BIDS specification." + ) if te2 is None: raise RuntimeError( - 'EchoTime2 metadata field not found. Please consult the BIDS specification.') + "EchoTime2 metadata field not found. Please consult the BIDS specification." + ) return abs(float(te2) - float(te1)) - - -def au2rads(in_file, newpath=None): - """Convert the input phase difference map in arbitrary units (a.u.) to rads.""" - im = nb.load(in_file) - data = im.get_fdata(caching='unchanged') # Read as float64 for safety - hdr = im.header.copy() - - # Rescale to [0, 2*pi] - data = (data - data.min()) * (2 * np.pi / (data.max() - data.min())) - - # Round to float32 and clip - data = np.clip(np.float32(data), 0.0, 2 * np.pi) - - hdr.set_data_dtype(np.float32) - hdr.set_xyzt_units('mm') - out_file = fname_presuffix(in_file, suffix='_rads', newpath=newpath) - nb.Nifti1Image(data, None, hdr).to_filename(out_file) - return out_file - - -def _subtract_phases(in_phases, in_meta, newpath=None): - # Discard traits with copy(), so that pop() works. - in_meta = (in_meta[0].copy(), in_meta[1].copy()) - echo_times = tuple([m.pop('EchoTime', None) for m in in_meta]) - if not all(echo_times): - raise ValueError( - 'One or more missing EchoTime metadata parameter ' - 'associated to one or more phase map(s).') - - if echo_times[0] > echo_times[1]: - in_phases = (in_phases[1], in_phases[0]) - in_meta = (in_meta[1], in_meta[0]) - echo_times = (echo_times[1], echo_times[0]) - - in_phases_nii = [nb.load(ph) for ph in in_phases] - sub_data = in_phases_nii[1].get_fdata(dtype='float32') - \ - in_phases_nii[0].get_fdata(dtype='float32') - - # wrap negative radians back to [0, 2pi] - sub_data[sub_data < 0] += 2 * np.pi - sub_data = np.clip(sub_data, 0.0, 2 * np.pi) - - new_meta = in_meta[1].copy() - new_meta.update(in_meta[0]) - new_meta['EchoTime1'] = echo_times[0] - new_meta['EchoTime2'] = echo_times[1] - - hdr = in_phases_nii[0].header.copy() - hdr.set_data_dtype(np.float32) - hdr.set_xyzt_units('mm') - nii = nb.Nifti1Image(sub_data, in_phases_nii[0].affine, hdr) - out_phdiff = fname_presuffix(in_phases[0], suffix='_phdiff', - newpath=newpath) - nii.to_filename(out_phdiff) - return out_phdiff, new_meta diff --git a/sdcflows/utils/__init__.py b/sdcflows/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sdcflows/utils/phasemanip.py b/sdcflows/utils/phasemanip.py new file mode 100644 index 0000000000..96c93fd439 --- /dev/null +++ b/sdcflows/utils/phasemanip.py @@ -0,0 +1,127 @@ +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# vi: set ft=python sts=4 ts=4 sw=4 et: +"""Utilities to manipulate phase and phase difference maps.""" + + +def au2rads(in_file, newpath=None): + """Convert the input phase map in arbitrary units (a.u.) to rads.""" + import numpy as np + import nibabel as nb + from nipype.utils.filemanip import fname_presuffix + + im = nb.load(in_file) + data = im.get_fdata(caching="unchanged") # Read as float64 for safety + hdr = im.header.copy() + + # Rescale to [0, 2*pi] + data = (data - data.min()) * (2 * np.pi / (data.max() - data.min())) + + # Round to float32 and clip + data = np.clip(np.float32(data), 0.0, 2 * np.pi) + + hdr.set_data_dtype(np.float32) + hdr.set_xyzt_units("mm") + out_file = fname_presuffix(in_file, suffix="_rads", newpath=newpath) + nb.Nifti1Image(data, None, hdr).to_filename(out_file) + return out_file + + +def subtract_phases(in_phases, in_meta, newpath=None): + """Calculate the phase-difference map, given two input phase maps.""" + import numpy as np + import nibabel as nb + from nipype.utils.filemanip import fname_presuffix + + echo_times = tuple([m.pop("EchoTime", None) for m in in_meta]) + if not all(echo_times): + raise ValueError( + "One or more missing EchoTime metadata parameter " + "associated to one or more phase map(s)." + ) + + if echo_times[0] > echo_times[1]: + in_phases = (in_phases[1], in_phases[0]) + in_meta = (in_meta[1], in_meta[0]) + echo_times = (echo_times[1], echo_times[0]) + + in_phases_nii = [nb.load(ph) for ph in in_phases] + sub_data = ( + in_phases_nii[1].get_fdata(dtype="float32") + - in_phases_nii[0].get_fdata(dtype="float32") + ) + + # wrap negative radians back to [0, 2pi] + sub_data[sub_data < 0] += 2 * np.pi + sub_data = np.clip(sub_data, 0.0, 2 * np.pi) + + new_meta = in_meta[1].copy() + new_meta.update(in_meta[0]) + new_meta["EchoTime1"] = echo_times[0] + new_meta["EchoTime2"] = echo_times[1] + + hdr = in_phases_nii[0].header.copy() + hdr.set_data_dtype(np.float32) + hdr.set_xyzt_units("mm") + nii = nb.Nifti1Image(sub_data, in_phases_nii[0].affine, hdr) + out_phdiff = fname_presuffix(in_phases[0], suffix="_phdiff", newpath=newpath) + nii.to_filename(out_phdiff) + return out_phdiff, new_meta + + +def phdiff2fmap(in_file, delta_te, newpath=None): + r""" + Convert the input phase-difference map into a fieldmap in Hz. + + Uses eq. (1) of [Hutton2002]_: + + .. math:: + + \Delta B_0 (i, j, k) = \frac{\Delta \Theta (i, j, k)}{2\pi\gamma \, \Delta\text{TE}} + + where :math:`\Delta B_0 (i, j, k)` is the *fieldmap* in Hz, + :math:`\Delta \Theta (i, j, k)` is the phase-difference map in rad, + :math:`\gamma` is the gyromagnetic ratio of the H proton, + and :math:`\Delta\text{TE}` is the elapsed time between the two GRE echoes. + + + We can obtain a voxel displacement map following eq. (2) of the same paper: + + .. math:: + + d_\text{PE} (i, j, k) = \gamma \, \Delta B_0 (i, j, k) \, T_\text{ro} + + where :math:`T_\text{ro}` is the readout time of one slice of the EPI dataset + we want to correct for distortions, and + :math:`\Delta_\text{PE} (i, j, k)` is the *voxel-shift map* (VSM) along the *PE* + direction. + + Replacing (1) into (2), and eliminating the scaling effect of :math:`T_\text{ro}`, + we obtain the *voxel-shift-velocity map* (voxels/ms) which can be then used to + recover the actual displacement field of the target EPI dataset. + + .. math:: + + v(i, j, k) = \frac{\Delta \Theta (i, j, k)}{2\pi \, \Delta\text{TE}} + + References + ---------- + .. [Hutton2002] Hutton et al., Image Distortion Correction in fMRI: A Quantitative + Evaluation, NeuroImage 16(1):217-240, 2002. doi:`10.1006/nimg.2001.1054 + `_. + + + """ + import math + import numpy as np + import nibabel as nb + from nipype.utils.filemanip import fname_presuffix + + # GYROMAG_RATIO_H_PROTON_MHZ = 42.576 + + out_file = fname_presuffix(in_file, suffix="_fmap", newpath=newpath) + image = nb.load(in_file) + data = image.get_fdata(dtype="float32") / (2.0 * math.pi * delta_te) + nii = nb.Nifti1Image(data, image.affine, image.header) + nii.set_data_dtype(np.float32) + nii.to_filename(out_file) + return out_file From 0dca11e867720671e9bae226344aea214060679b Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 19 Nov 2020 15:39:21 +0100 Subject: [PATCH 04/68] maint: extensive cleanup of ``fmap`` interfaces --- sdcflows/interfaces/fmap.py | 24 +- sdcflows/utils/phasemanip.py | 45 +--- sdcflows/workflows/fit/fieldmap.py | 339 +++++++++++++++++------------ 3 files changed, 203 insertions(+), 205 deletions(-) diff --git a/sdcflows/interfaces/fmap.py b/sdcflows/interfaces/fmap.py index f9e804c89b..f01c55c971 100644 --- a/sdcflows/interfaces/fmap.py +++ b/sdcflows/interfaces/fmap.py @@ -1,17 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -""" -Interfaces to deal with the various types of fieldmap sources. - - .. testsetup:: - - >>> tmpdir = getfixture('tmpdir') - >>> tmp = tmpdir.chdir() # changing to a temporary directory - >>> nb.Nifti1Image(np.zeros((90, 90, 60)), None, None).to_filename( - ... tmpdir.join('epi.nii.gz').strpath) - - -""" +"""Interfaces to deal with the various types of fieldmap sources.""" from nipype import logging from nipype.interfaces.base import ( @@ -98,16 +87,7 @@ class _Phasediff2FieldmapOutputSpec(TraitedSpec): class Phasediff2Fieldmap(SimpleInterface): - """ - Convert a phase difference map into a fieldmap in Hz. - - This interface is equivalent to running the following steps: - #. Convert from rad to rad/s - (``niflow.nipype1.workflows.dmri.fsl.utils.rads2radsec``) - #. FUGUE execution: fsl.FUGUE(save_fmap=True) - #. Conversion from rad/s to Hz (divide by 2pi, ``rsec2hz``). - - """ + """Convert a phase difference map into a fieldmap in Hz.""" input_spec = _Phasediff2FieldmapInputSpec output_spec = _Phasediff2FieldmapOutputSpec diff --git a/sdcflows/utils/phasemanip.py b/sdcflows/utils/phasemanip.py index 96c93fd439..b0391d677c 100644 --- a/sdcflows/utils/phasemanip.py +++ b/sdcflows/utils/phasemanip.py @@ -69,55 +69,12 @@ def subtract_phases(in_phases, in_meta, newpath=None): def phdiff2fmap(in_file, delta_te, newpath=None): - r""" - Convert the input phase-difference map into a fieldmap in Hz. - - Uses eq. (1) of [Hutton2002]_: - - .. math:: - - \Delta B_0 (i, j, k) = \frac{\Delta \Theta (i, j, k)}{2\pi\gamma \, \Delta\text{TE}} - - where :math:`\Delta B_0 (i, j, k)` is the *fieldmap* in Hz, - :math:`\Delta \Theta (i, j, k)` is the phase-difference map in rad, - :math:`\gamma` is the gyromagnetic ratio of the H proton, - and :math:`\Delta\text{TE}` is the elapsed time between the two GRE echoes. - - - We can obtain a voxel displacement map following eq. (2) of the same paper: - - .. math:: - - d_\text{PE} (i, j, k) = \gamma \, \Delta B_0 (i, j, k) \, T_\text{ro} - - where :math:`T_\text{ro}` is the readout time of one slice of the EPI dataset - we want to correct for distortions, and - :math:`\Delta_\text{PE} (i, j, k)` is the *voxel-shift map* (VSM) along the *PE* - direction. - - Replacing (1) into (2), and eliminating the scaling effect of :math:`T_\text{ro}`, - we obtain the *voxel-shift-velocity map* (voxels/ms) which can be then used to - recover the actual displacement field of the target EPI dataset. - - .. math:: - - v(i, j, k) = \frac{\Delta \Theta (i, j, k)}{2\pi \, \Delta\text{TE}} - - References - ---------- - .. [Hutton2002] Hutton et al., Image Distortion Correction in fMRI: A Quantitative - Evaluation, NeuroImage 16(1):217-240, 2002. doi:`10.1006/nimg.2001.1054 - `_. - - - """ + """Convert the input phase-difference map into a *fieldmap* in Hz.""" import math import numpy as np import nibabel as nb from nipype.utils.filemanip import fname_presuffix - # GYROMAG_RATIO_H_PROTON_MHZ = 42.576 - out_file = fname_presuffix(in_file, suffix="_fmap", newpath=newpath) image = nb.load(in_file) data = image.get_fdata(dtype="float32") / (2.0 * math.pi * delta_te) diff --git a/sdcflows/workflows/fit/fieldmap.py b/sdcflows/workflows/fit/fieldmap.py index 4f0eb2e9ad..f29f782192 100644 --- a/sdcflows/workflows/fit/fieldmap.py +++ b/sdcflows/workflows/fit/fieldmap.py @@ -1,15 +1,79 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -""" -Processing phase-difference and *directly measured* B0 maps. +r""" +Processing phase-difference and *directly measured* :math:`B_0` maps. + +The displacement suffered by every voxel along the phase-encoding (PE) direction +can be derived from eq. (2) of [Hutton2002]_: + +.. math:: + + \Delta_\text{PE} (i, j, k) = \gamma \cdot \Delta B_0 (i, j, k) \cdot T_\text{ro}, + +where :math:`T_\text{ro}` is the readout time of one slice of the EPI dataset +we want to correct for distortions, and :math:`\Delta_\text{PE} (i, j, k)` +is the *voxel-shift map* (VSM) along the *PE* direction. + +Calling the *fieldmap* in Hz :math:`V`, +with :math:`V(i,j,k) = \gamma \cdot \Delta B_0 (i, j, k)`, and introducing +the voxel zoom along the phase-encoding direction, :math:`s_\text{PE}`, +we obtain the nonzero component of the associated displacements field +:math:`\Delta D_\text{PE} (i, j, k)` that unwarps the target EPI dataset: + +.. math:: + + \Delta D_\text{PE} (i, j, k) = V(i, j, k) \cdot T_\text{ro} \cdot s_\text{PE}. + + +Theory +~~~~~~ +The derivation of a fieldmap in Hz (or, as called thereafter, *voxel-shift-velocity +map*) results from eq. (1) of [Hutton2002]_: + +.. math:: + + \Delta B_0 (i, j, k) = \frac{\Delta \Theta (i, j, k)}{2\pi \cdot \gamma \, \Delta\text{TE}} + +where :math:`\Delta B_0 (i, j, k)` is the *fieldmap variation* in T, +:math:`\Delta \Theta (i, j, k)` is the phase-difference map in rad, +:math:`\gamma` is the gyromagnetic ratio of the H proton in Hz/T +(:math:`\gamma = 42.576 \cdot 10^6 \, \text{Hz} \cdot \text{T}^\text{-1}`), +and :math:`\Delta\text{TE}` is the elapsed time between the two GRE echoes. + +We can obtain a voxel displacement map following eq. (2) of the same paper: + +.. math:: + + \Delta_\text{PE} (i, j, k) = \gamma \cdot \Delta B_0 (i, j, k) \cdot T_\text{ro} + +where :math:`T_\text{ro}` is the readout time of one slice of the EPI dataset +we want to correct for distortions, and +:math:`\Delta_\text{PE} (i, j, k)` is the *voxel-shift map* (VSM) along the *PE* +direction. .. _sdc_phasediff : Phase-difference B0 estimation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The inhomogeneity of the :math:`B_0` field inside the scanner at each voxel is -proportional to the phase drift (at that voxel) between two subsequent -:abbr:`GRE (gradient-recalled echo)` acquisitions. +The inhomogeneity of the :math:`B_0` field inside the scanner at each voxel +(and hence, the *fieldmap* in Hz, :math:`V(i,j,k)`) is proportional to the +phase drift :math:`\Delta \Theta (i,j,k) = \Theta_2(i,j,k) - \Theta_1(i,j,k)` +between two subsequent :abbr:`GRE (gradient-recalled echo)` acquisitions +(:math:`\Theta_1`, :math:`\Theta_2`), separated by a time +:math:`\Delta \text{TE}` (s): +Replacing (1) into (2), and eliminating the scaling effect of :math:`T_\text{ro}`, +we obtain a *voxel-shift-velocity map* (voxels/s, or just Hz) which can be then +used to recover the actual displacement field of the target EPI dataset. + +.. math:: + + V(i, j, k) = \frac{\Delta \Theta (i, j, k)}{2\pi \cdot \Delta\text{TE}}. + +This calculation if further complicated by the fact that :math:`\Theta_i` +(and therfore, :math:`\Delta \Theta`) are clipped (or *wrapped*) within +the range :math:`[0 \dotsb 2\pi )`. +It is necessary to find the integer number of offsets that make a region +continuously smooth with its neighbors (*phase-unwrapping*, [Jenkinson2003]_). This corresponds to `this section of the BIDS specification `__. @@ -21,14 +85,22 @@ Direct B0 mapping sequences ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The inhomogeneity of the :math:`B_0` field can directly be mapped with a -some MR schemes (such as :abbr:`SE (spiral echo)`). +Some MR schemes such as :abbr:`SEI (spiral-echo imaging)` can directly +reconstruct an estimate of *the fieldmap in Hz*, :math:`V(i,j,k)`. These *fieldmaps* are described with more detail `here `__. This corresponds to `this section of the BIDS specification `__. +References +---------- +.. [Hutton2002] Hutton et al., Image Distortion Correction in fMRI: A Quantitative + Evaluation, NeuroImage 16(1):217-240, 2002. doi:`10.1006/nimg.2001.1054 + `__. +.. [Jenkinson2003] Jenkinson, M. (2003) Fast, automated, N-dimensional phase-unwrapping + algorithm. MRM 49(1):193-197. doi:`10.1002/mrm.10354 + `__. """ @@ -226,86 +298,6 @@ def init_magnitude_wf(omp_nthreads, name="magnitude_wf"): return workflow -def init_fmap_postproc_wf( - omp_nthreads, median_kernel_size=5, name="fmap_postproc_wf" -): - """ - Postprocess a :math:`B_0` map estimated elsewhere. - - This workflow denoises (mostly via smoothing) a :math:`B_0` fieldmap. - - Workflow Graph - .. workflow :: - :graph2use: orig - :simple_form: yes - - from sdcflows.workflows.fit.fieldmap import init_fmap_postproc_wf - wf = init_fmap_postproc_wf(omp_nthreads=6) - - Parameters - ---------- - omp_nthreads : :obj:`int` - Maximum number of threads an individual process may use - median_kernel_size : :obj:`int` - Size of the kernel when smoothing is done with a median filter. - name : :obj:`str` - Name of workflow (default: ``fmap_postproc_wf``) - - Inputs - ------ - fmap : :obj:`os.PathLike` - Fully preprocessed :math:`B_0` field nonuniformity map (aka *fieldmap*). - fmap_ref : :obj:`os.PathLike` - A preprocessed magnitude/reference image for the fieldmap. - fmap_mask : :obj:`os.PathLike` - A brain binary mask corresponding to this fieldmap. - - Outputs - ------- - out_fmap : :obj:`os.PathLike` - Postprocessed fieldmap. - - """ - from nipype.interfaces.fsl import SpatialFilter - from niflow.nipype1.workflows.dmri.fsl.utils import cleanup_edge_pipeline - workflow = Workflow(name=name) - inputnode = pe.Node( - niu.IdentityInterface(fields=["fmap_mask", "fmap_ref", "fmap", "metadata"]), - name="inputnode", - ) - outputnode = pe.Node( - niu.IdentityInterface(fields=["out_fmap", "metadata"]), name="outputnode" - ) - recenter = pe.Node( - niu.Function(function=_recenter), - name="recenter", - run_without_submitting=True, - ) - denoise = pe.Node( - SpatialFilter( - operation="median", - kernel_shape="sphere", - kernel_size=median_kernel_size, - ), - name="denoise", - ) - demean = pe.Node(niu.Function(function=_demean), name="demean") - cleanup_wf = cleanup_edge_pipeline(name="cleanup_wf") - - # fmt: off - workflow.connect([ - (inputnode, cleanup_wf, [("fmap_mask", "inputnode.in_mask")]), - (inputnode, recenter, [(("fmap", _pop), "in_file")]), - (recenter, denoise, [("out", "in_file")]), - (denoise, demean, [("out_file", "in_file")]), - (demean, cleanup_wf, [("out", "inputnode.in_file")]), - (cleanup_wf, outputnode, [("outputnode.out_file", "out_fmap")]), - (inputnode, outputnode, [(("metadata", _pop), "metadata")]), - ]) - # fmt: on - return workflow - - def init_phdiff_wf(omp_nthreads, name="phdiff_wf"): r""" Generate a :math:`B_0` field from consecutive-phases and phase-difference maps. @@ -317,9 +309,6 @@ def init_phdiff_wf(omp_nthreads, name="phdiff_wf"): Besides phase2 - phase1 subtraction, the core of this particular workflow relies in the phase-unwrapping with FSL PRELUDE [Jenkinson2003]_. - Because phase (and phase-difference) maps are clipped in the range - :math:`[0 \dotsb 2\pi )`, it is necessary to find the integer number of offsets - that make a region continuously smooth with its neighbors (*phase-unwrapping*). FSL PRELUDE takes wrapped maps in the range 0 to 6.28, `as per the user guide `__. @@ -328,8 +317,6 @@ def init_phdiff_wf(omp_nthreads, name="phdiff_wf"): After some massaging and with the scaling of the echo separation factor :math:`\Delta \text{TE}`, the phase-difference maps are converted into an actual :math:`B_0` map in Hz units. - This implementation derives `originally from Nipype - `__. Workflow Graph .. workflow :: @@ -360,11 +347,6 @@ def init_phdiff_wf(omp_nthreads, name="phdiff_wf"): fieldmap : :obj:`os.PathLike` The estimated fieldmap in Hz. # TODO: write metadata "Units" - References - ---------- - .. [Jenkinson2003] Jenkinson, M. (2003) Fast, automated, N-dimensional phase-unwrapping - algorithm. MRM 49(1):193-197. doi:`10.1002/mrm.10354 <10.1002/mrm.10354>`__. - """ from nipype.interfaces.fsl import PRELUDE from ...interfaces.fmap import Phasediff2Fieldmap, PhaseMap2rads, SubtractPhases @@ -424,60 +406,139 @@ def _split(phase): return workflow -def _recenter(in_file): - """Recenter the phase-map distribution to the -pi..pi range.""" - from os import getcwd - import numpy as np - import nibabel as nb - from nipype.utils.filemanip import fname_presuffix - - nii = nb.load(in_file) - data = nii.get_fdata(dtype="float32") - msk = data != 0 - msk[data == 0] = False - data[msk] -= np.median(data[msk]) +def init_fmap_postproc_wf( + omp_nthreads, median_kernel_size=5, name="fmap_postproc_wf" +): + """ + Postprocess a :math:`B_0` map estimated elsewhere. - out_file = fname_presuffix(in_file, suffix="_recentered", newpath=getcwd()) - nb.Nifti1Image(data, nii.affine, nii.header).to_filename(out_file) - return out_file + This workflow denoises (mostly via smoothing) a :math:`B_0` fieldmap. + Workflow Graph + .. workflow :: + :graph2use: orig + :simple_form: yes -def _demean(in_file, in_mask=None, usemode=True): - """ - Subtract the median (since it is robuster than the mean) from a map. + from sdcflows.workflows.fit.fieldmap import init_fmap_postproc_wf + wf = init_fmap_postproc_wf(omp_nthreads=6) Parameters ---------- - usemode : :obj:`bool` - Use the mode instead of the median (should be even more robust - against outliers). + omp_nthreads : :obj:`int` + Maximum number of threads an individual process may use + median_kernel_size : :obj:`int` + Size of the kernel when smoothing is done with a median filter. + name : :obj:`str` + Name of workflow (default: ``fmap_postproc_wf``) + + Inputs + ------ + fmap : :obj:`os.PathLike` + Fully preprocessed :math:`B_0` field nonuniformity map (aka *fieldmap*). + fmap_ref : :obj:`os.PathLike` + A preprocessed magnitude/reference image for the fieldmap. + fmap_mask : :obj:`os.PathLike` + A brain binary mask corresponding to this fieldmap. + + Outputs + ------- + out_fmap : :obj:`os.PathLike` + Postprocessed fieldmap. """ - from os import getcwd - import numpy as np - import nibabel as nb - from nipype.utils.filemanip import fname_presuffix + from nipype.interfaces.fsl import SpatialFilter + from niflow.nipype1.workflows.dmri.fsl.utils import cleanup_edge_pipeline + workflow = Workflow(name=name) + inputnode = pe.Node( + niu.IdentityInterface(fields=["fmap_mask", "fmap_ref", "fmap", "metadata"]), + name="inputnode", + ) + outputnode = pe.Node( + niu.IdentityInterface(fields=["out_fmap", "metadata"]), name="outputnode" + ) - nii = nb.load(in_file) - data = nii.get_fdata(dtype="float32") + def _recenter(in_file): + """Recenter the phase-map distribution to the -pi..pi range.""" + from os import getcwd + import numpy as np + import nibabel as nb + from nipype.utils.filemanip import fname_presuffix - msk = np.ones_like(data, dtype=bool) - if in_mask is not None: - msk[nb.load(in_mask).get_fdata(dtype="float32") < 1e-4] = False + nii = nb.load(in_file) + data = nii.get_fdata(dtype="float32") + msk = data != 0 + msk[data == 0] = False + data[msk] -= np.median(data[msk]) - if usemode: - from scipy.stats import mode + out_file = fname_presuffix(in_file, suffix="_recentered", newpath=getcwd()) + nb.Nifti1Image(data, nii.affine, nii.header).to_filename(out_file) + return out_file - data[msk] -= mode(data[msk], axis=None)[0][0] - else: - data[msk] -= np.median(data[msk], axis=None) + recenter = pe.Node( + niu.Function(function=_recenter), + name="recenter", + run_without_submitting=True, + ) + denoise = pe.Node( + SpatialFilter( + operation="median", + kernel_shape="sphere", + kernel_size=median_kernel_size, + ), + name="denoise", + ) - out_file = fname_presuffix(in_file, suffix="_demean", newpath=getcwd()) - nb.Nifti1Image(data, nii.affine, nii.header).to_filename(out_file) - return out_file + def _demean(in_file, in_mask=None, usemode=True): + """ + Subtract the median (since it is robuster than the mean) from a map. + Parameters + ---------- + usemode : :obj:`bool` + Use the mode instead of the median (should be even more robust + against outliers). -def _pop(inlist): - if isinstance(inlist, (tuple, list)): - return inlist[0] - return inlist + """ + from os import getcwd + import numpy as np + import nibabel as nb + from nipype.utils.filemanip import fname_presuffix + + nii = nb.load(in_file) + data = nii.get_fdata(dtype="float32") + + msk = np.ones_like(data, dtype=bool) + if in_mask is not None: + msk[nb.load(in_mask).get_fdata(dtype="float32") < 1e-4] = False + + if usemode: + from scipy.stats import mode + + data[msk] -= mode(data[msk], axis=None)[0][0] + else: + data[msk] -= np.median(data[msk], axis=None) + + out_file = fname_presuffix(in_file, suffix="_demean", newpath=getcwd()) + nb.Nifti1Image(data, nii.affine, nii.header).to_filename(out_file) + return out_file + + demean = pe.Node(niu.Function(function=_demean), name="demean") + cleanup_wf = cleanup_edge_pipeline(name="cleanup_wf") + + def _pop(inlist): + if isinstance(inlist, (tuple, list)): + return inlist[0] + return inlist + + # fmt: off + workflow.connect([ + (inputnode, cleanup_wf, [("fmap_mask", "inputnode.in_mask")]), + (inputnode, recenter, [(("fmap", _pop), "in_file")]), + (recenter, denoise, [("out", "in_file")]), + (denoise, demean, [("out_file", "in_file")]), + (demean, cleanup_wf, [("out", "inputnode.in_file")]), + (cleanup_wf, outputnode, [("outputnode.out_file", "out_fmap")]), + (inputnode, outputnode, [(("metadata", _pop), "metadata")]), + ]) + # fmt: on + return workflow From cad2b79074213d033539b3013f1ec7133a730b34 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 19 Nov 2020 21:58:34 +0100 Subject: [PATCH 05/68] enh: continue refactoring --- sdcflows/data/fmap-any_registration.json | 3 +- .../data/fmap-any_registration_testing.json | 1 - sdcflows/interfaces/fmap.py | 23 +++ sdcflows/workflows/apply/registration.py | 120 +++++++++++++ sdcflows/workflows/outputs.py | 159 +++++++++++++++--- 5 files changed, 282 insertions(+), 24 deletions(-) create mode 100644 sdcflows/workflows/apply/registration.py diff --git a/sdcflows/data/fmap-any_registration.json b/sdcflows/data/fmap-any_registration.json index b97b002fc7..97d4ffc756 100644 --- a/sdcflows/data/fmap-any_registration.json +++ b/sdcflows/data/fmap-any_registration.json @@ -4,10 +4,9 @@ "winsorize_lower_quantile": 0.005, "winsorize_upper_quantile": 0.998, "collapse_output_transforms": true, - "write_composite_transform": true, "use_histogram_matching": [ true, true ], "use_estimate_learning_rate_once": [ true, true ], - "transforms": [ "Translation", "Affine" ], + "transforms": [ "Translation", "Rigid" ], "number_of_iterations": [ [ 500 ], [ 200 ] ], "transform_parameters": [ [ 0.05 ], [ 0.01 ] ], "convergence_threshold": [ 1e-07, 1e-08 ], diff --git a/sdcflows/data/fmap-any_registration_testing.json b/sdcflows/data/fmap-any_registration_testing.json index 207e997ed5..37aff850ec 100644 --- a/sdcflows/data/fmap-any_registration_testing.json +++ b/sdcflows/data/fmap-any_registration_testing.json @@ -4,7 +4,6 @@ "winsorize_lower_quantile": 0.005, "winsorize_upper_quantile": 0.998, "collapse_output_transforms": true, - "write_composite_transform": true, "use_histogram_matching": [ true, true ], "use_estimate_learning_rate_once": [ true, true ], "transforms": [ "Rigid", "Rigid" ], diff --git a/sdcflows/interfaces/fmap.py b/sdcflows/interfaces/fmap.py index f01c55c971..85a949efa6 100644 --- a/sdcflows/interfaces/fmap.py +++ b/sdcflows/interfaces/fmap.py @@ -9,6 +9,7 @@ File, traits, SimpleInterface, + InputMultiObject, ) LOGGER = logging.getLogger("nipype.interface") @@ -101,6 +102,28 @@ def _run_interface(self, runtime): return runtime +class _Coefficients2WarpInputSpec(BaseInterfaceInputSpec): + in_target = File(exist=True, mandatory=True, desc="input EPI data to be corrected") + in_coeff = InputMultiObject(File(exists=True), mandatory=True, + desc="input coefficients, after alignment to the EPI data") + ro_time = traits.Float(mandatory=True, desc="EPI readout time") + + +class _Coefficients2WarpOutputSpec(TraitedSpec): + out_warp = File(exists=True) + + +class Coefficients2Warp(SimpleInterface): + """Convert coefficients to a full displacements map.""" + + input_spec = _Coefficients2WarpInputSpec + output_spec = _Coefficients2WarpOutputSpec + + def _run_interface(self, runtime): + raise NotImplementedError + return runtime + + def _delta_te(in_values, te1=None, te2=None): r"""Read :math:`\Delta_\text{TE}` from BIDS metadata dict.""" if isinstance(in_values, float): diff --git a/sdcflows/workflows/apply/registration.py b/sdcflows/workflows/apply/registration.py new file mode 100644 index 0000000000..c75a3a3138 --- /dev/null +++ b/sdcflows/workflows/apply/registration.py @@ -0,0 +1,120 @@ +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# vi: set ft=python sts=4 ts=4 sw=4 et: +""" +Align the fieldmap reference map to the target EPI. + +The fieldmap reference map may be a magnitude image (or an EPI dataset, +in the case of PEPOLAR estimation). + +The target EPI is the distorted dataset (or a reference thereof). + +""" +from pkg_resources import resource_filename as pkgrf +from nipype.pipeline import engine as pe +from nipype.interfaces import utility as niu +from niworkflows.engine.workflows import LiterateWorkflow as Workflow + + +def init_coeff2epi_wf( + omp_nthreads, + debug=False, + name="fmap2field_wf", +): + """ + Move the field coefficients on to the target (distorted) EPI space. + + Workflow Graph + .. workflow:: + :graph2use: orig + :simple_form: yes + + from sdcflows.workflows.apply.registration import init_coeff2epi_wf + wf = init_coeff2epi_wf(omp_nthreads=2) + + Parameters + ---------- + omp_nthreads : int + Maximum number of threads an individual process may use. + debug : bool + Run fast configurations of registrations. + name : str + Unique name of this workflow. + + Inputs + ------ + target_ref + the target EPI reference image + target_mask + the reference image (skull-stripped) + fmap_ref + the reference (anatomical) image corresponding to ``fmap`` + fmap_mask + a brain mask corresponding to ``fmap`` + fmap_coeff + fieldmap coefficients + + Outputs + ------- + fmap_coeff + fieldmap coefficients + + """ + from niworkflows.interfaces.fixes import FixHeaderRegistration as Registration + workflow = Workflow(name=name) + workflow.__desc__ = """\ +The estimated *fieldmap* was then aligned with rigid-registration to the target +EPI (echo-planar imaging) reference run. +The field coefficients were mapped on to the reference EPI using the transform. +""" + inputnode = pe.Node( + niu.IdentityInterface( + fields=[ + "target_ref", + "target_mask", + "fmap_ref", + "fmap_mask", + "fmap_coeff", + ] + ), + name="inputnode", + ) + outputnode = pe.Node(niu.IdentityInterface(fields=["fmap_coeff"]), + name="outputnode") + + # Register the reference of the fieldmap to the reference + # of the target image (the one that shall be corrected) + ants_settings = pkgrf( + "sdcflows", f"data/fmap-any_registration{'_testing' * debug}.json" + ) + + coregister = pe.Node( + Registration( + from_file=ants_settings, + ), + name="coregister", + n_procs=omp_nthreads, + ) + + # Map the coefficients into the EPI space + map_coeff = pe.Node(niu.Function(function=_move_coeff), name="map_coeff") + + # fmt: off + workflow.connect([ + (inputnode, coregister, [ + ("target_ref", "fixed_image"), + ("fmap_ref", "moving_image"), + ("target_mask", "fixed_image_masks"), + ("fmap_mask", "moving_image_masks"), + ]), + (inputnode, map_coeff, [("fmap_coeff", "in_coeff")]), + (coregister, map_coeff, [("forward_transforms", "transform")]), + (map_coeff, outputnode, [("out", "fmap_coeff")]), + ]) + # fmt: on + + return workflow + + +def _move_coeff(in_coeff, transform): + """Read in a rigid transform from ANTs, and update the coefficients field affine.""" + raise NotImplementedError diff --git a/sdcflows/workflows/outputs.py b/sdcflows/workflows/outputs.py index d78386d533..8912d26ee2 100644 --- a/sdcflows/workflows/outputs.py +++ b/sdcflows/workflows/outputs.py @@ -6,7 +6,105 @@ from niworkflows.interfaces.bids import DerivativesDataSink -def init_sdc_unwarp_report_wf(name='sdc_unwarp_report_wf', forcedsyn=False): +def init_fmap_derivatives_wf( + *, + bids_root, + output_dir, + write_coeff=False, + name="fmap_derivatives_wf", +): + """ + Set up datasinks to store derivatives in the right location. + + Parameters + ---------- + bids_root : :obj:`str` + Root path of BIDS dataset + output_dir : :obj:`str` + Directory in which to save derivatives + name : :obj:`str` + Workflow name (default: ``"fmap_derivatives_wf"``) + + Inputs + ------ + source_file + A fieldmap file of the BIDS dataset that will serve for naming reference. + fieldmap + The preprocessed fieldmap, in its original space with Hz units. + fmap_coeff + Field coefficient(s) file(s) + fmap_ref + An anatomical reference (e.g., magnitude file) + + """ + workflow = pe.Workflow(name=name) + + inputnode = pe.Node( + niu.IdentityInterface( + fields=[ + "source_file", + "fieldmap", + "fmap_coeff", + "fmap_ref", + ]), + name="inputnode" + ) + + ds_reference = pe.Node( + DerivativesDataSink( + base_directory=output_dir, + desc="reference", + suffix="fieldmap", + compress=True), + name="ds_reference", + ) + + ds_fieldmap = pe.Node( + DerivativesDataSink( + base_directory=output_dir, + desc="preproc", + suffix="fieldmap", + compress=True), + name="ds_fieldmap", + ) + ds_fieldmap.inputs.Units = "Hz" + + # fmt:off + workflow.connect([ + (inputnode, ds_reference, [("source_file", "source_file"), + ("fmap_ref", "in_file")]), + (inputnode, ds_fieldmap, [("source_file", "source_file"), + ("fieldmap", "in_file")]), + ]) + # fmt:on + + if not write_coeff: + return workflow + + ds_coeff = pe.MapNode( + DerivativesDataSink( + base_directory=output_dir, + suffix="fieldmap", + compress=True), + name="ds_coeff", + iterfield=("in_file", "desc"), + ) + + gen_desc = pe.Node(niu.Function(function=_gendesc), name="gen_desc") + + # fmt:off + workflow.connect([ + (inputnode, ds_coeff, [("source_file", "source_file"), + ("fmap_coeff", "in_file")]), + (inputnode, gen_desc, [("fmap_coeff", "infiles")]), + (gen_desc, ds_coeff, [("out", "desc")]), + ]) + # fmt:on + + return workflow + + +def init_sdc_unwarp_report_wf(name="sdc_unwarp_report_wf", forcedsyn=False): """ Save a reportlet showing how SDC unwarping performed. @@ -49,34 +147,53 @@ def init_sdc_unwarp_report_wf(name='sdc_unwarp_report_wf', forcedsyn=False): workflow = pe.Workflow(name=name) - inputnode = pe.Node(niu.IdentityInterface( - fields=['in_pre', 'in_post', 'in_seg', 'in_xfm']), name='inputnode') + inputnode = pe.Node( + niu.IdentityInterface(fields=["in_pre", "in_post", "in_seg", "in_xfm"]), + name="inputnode", + ) - map_seg = pe.Node(ApplyTransforms( - dimension=3, float=True, interpolation='MultiLabel'), - name='map_seg', mem_gb=0.3) + map_seg = pe.Node( + ApplyTransforms(dimension=3, float=True, interpolation="MultiLabel"), + name="map_seg", + mem_gb=0.3, + ) - sel_wm = pe.Node(niu.Function(function=_dseg_label), name='sel_wm', - mem_gb=DEFAULT_MEMORY_MIN_GB) + sel_wm = pe.Node( + niu.Function(function=_dseg_label), name="sel_wm", mem_gb=DEFAULT_MEMORY_MIN_GB + ) sel_wm.inputs.label = 2 - bold_rpt = pe.Node(SimpleBeforeAfter(), name='bold_rpt', - mem_gb=0.1) + bold_rpt = pe.Node(SimpleBeforeAfter(), name="bold_rpt", mem_gb=0.1) ds_report_sdc = pe.Node( - DerivativesDataSink(desc=('sdc', 'forcedsyn')[forcedsyn], suffix='bold', - datatype='figures'), name='ds_report_sdc', - mem_gb=DEFAULT_MEMORY_MIN_GB, run_without_submitting=True + DerivativesDataSink( + desc=("sdc", "forcedsyn")[forcedsyn], suffix="bold", datatype="figures" + ), + name="ds_report_sdc", + mem_gb=DEFAULT_MEMORY_MIN_GB, + run_without_submitting=True, ) + # fmt: off workflow.connect([ - (inputnode, bold_rpt, [('in_post', 'after'), - ('in_pre', 'before')]), - (bold_rpt, ds_report_sdc, [('out_report', 'in_file')]), - (inputnode, map_seg, [('in_post', 'reference_image'), - ('in_seg', 'input_image'), - ('in_xfm', 'transforms')]), - (map_seg, sel_wm, [('output_image', 'in_seg')]), - (sel_wm, bold_rpt, [('out', 'wm_seg')]), + (inputnode, bold_rpt, [("in_post", "after"), + ("in_pre", "before")]), + (bold_rpt, ds_report_sdc, [("out_report", "in_file")]), + (inputnode, map_seg, [("in_post", "reference_image"), + ("in_seg", "input_image"), + ("in_xfm", "transforms")]), + (map_seg, sel_wm, [("output_image", "in_seg")]), + (sel_wm, bold_rpt, [("out", "wm_seg")]), ]) + # fmt: on return workflow + + +def _gendesc(infiles): + if isinstance(infiles, (str, bytes)): + infiles = [infiles] + + if len(infiles) == 1: + return "coeff" + + return [f"coeff{i}" for i, _ in enumerate(infiles)] From f4f20ec163f4dcaacef75ea4c515d4e1104cf39e Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 19 Nov 2020 22:19:54 +0100 Subject: [PATCH 06/68] rf: remove sdcflows.workflows.base --- sdcflows/workflows/base.py | 224 -------------------------- sdcflows/workflows/tests/__init__.py | 0 sdcflows/workflows/tests/test_base.py | 117 -------------- 3 files changed, 341 deletions(-) delete mode 100644 sdcflows/workflows/base.py delete mode 100644 sdcflows/workflows/tests/__init__.py delete mode 100644 sdcflows/workflows/tests/test_base.py diff --git a/sdcflows/workflows/base.py b/sdcflows/workflows/base.py deleted file mode 100644 index 6e2dd92656..0000000000 --- a/sdcflows/workflows/base.py +++ /dev/null @@ -1,224 +0,0 @@ -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -# vi: set ft=python sts=4 ts=4 sw=4 et: -"""SDC workflows coordination.""" -from nipype.pipeline import engine as pe -from nipype.interfaces import utility as niu -from nipype import logging - -from niworkflows.engine.workflows import LiterateWorkflow as Workflow - - -LOGGER = logging.getLogger("nipype.workflow") -FMAP_PRIORITY = { - "epi": 0, - "fieldmap": 1, - "phasediff": 2, - "syn": 3, -} - -DEFAULT_MEMORY_MIN_GB = 0.01 - - -def init_sdc_estimate_wf(bids_fmaps, omp_nthreads=1, debug=False): - """ - Build a :abbr:`SDC (susceptibility distortion correction)` workflow. - - This workflow implements the heuristics to choose an estimation - methodology for :abbr:`SDC (susceptibility distortion correction)`. - When no field map information is present within the BIDS inputs, - the EXPERIMENTAL "fieldmap-less SyN" can be performed, using - the ``--use-syn`` argument. When ``--force-syn`` is specified, - then the "fieldmap-less SyN" is always executed and reported - despite of other fieldmaps available with higher priority. - In the latter case (some sort of fieldmap(s) is available and - ``--force-syn`` is requested), then the :abbr:`SDC (susceptibility - distortion correction)` method applied is that with the - highest priority. - - Parameters - ---------- - bids_fmaps : list of pybids dicts - A list of dictionaries with the available fieldmaps - (and their metadata using the key ``'metadata'`` for the - case of :abbr:`PEPOLAR (Phase-Encoding POLARity)` fieldmaps). - omp_nthreads : int - Maximum number of threads an individual process may use - debug : bool - Enable debugging outputs - - Inputs - ------ - epi_file - A reference image calculated at a previous stage - epi_brain - Same as above, but brain-masked - epi_mask - Brain mask for the run - t1w_brain - T1w image, brain-masked, for the fieldmap-less SyN method - std2anat_xfm - Standard-to-T1w transform generated during spatial - normalization (only for the fieldmap-less SyN method). - - Outputs - ------- - epi_corrected - The EPI scan reference after unwarping. - epi_mask - The corresponding new mask after unwarping - epi_brain - Brain-extracted, unwarped EPI scan reference - out_warp - The deformation field to unwarp the susceptibility distortions - method : str - Short description of the estimation method that was run. - - """ - workflow = Workflow(name="sdc_estimate_wf" if bids_fmaps else "sdc_bypass_wf") - outputnode = pe.Node( - niu.IdentityInterface(fields=["fieldmap", "fmap_ref", "method"]), - name="outputnode", - ) - - # No fieldmaps - forward inputs to outputs - if not bids_fmaps: - workflow.__postdesc__ = """\ -Susceptibility distortion correction (SDC) was omitted. -""" - outputnode.inputs.method = "None" - outputnode.inputs.fieldmap = "identity" - # fmt: off - workflow.add_nodes([outputnode]) - # fmt: on - return workflow - - workflow.__postdesc__ = """\ -Based on the estimated susceptibility distortion, a corrected -EPI (echo-planar imaging) reference was calculated for a more -accurate co-registration with the anatomical reference. -""" - - inputnode = pe.Node( - niu.IdentityInterface( - fields=["epi_file", "epi_brain", "epi_mask", "t1w_brain", "std2anat_xfm"] - ), - name="inputnode", - ) - # PEPOLAR path - if "epi" in bids_fmaps: - from .fit.pepolar import init_3dQwarp_wf - - outputnode.inputs.method = "PEB/PEPOLAR (phase-encoding based / PE-POLARity)" - - in_data, metadata = zip(*bids_fmaps["epi"]) - estimate_wf = init_3dQwarp_wf(omp_nthreads=omp_nthreads) - estimate_wf.inputs.inputnode.in_data = in_data - estimate_wf.inputs.inputnode.metadata = metadata - - # FIELDMAP path - elif "fieldmap" in bids_fmaps or "phasediff" in bids_fmaps: - from .fit.fieldmap import init_fmap_wf - - if "fieldmap" in bids_fmaps: - fmap = bids_fmaps["fieldmap"][0] - outputnode.inputs.method = "FMB (fieldmap-based) - directly measured B0 map" - estimate_wf = init_fmap_wf(omp_nthreads=omp_nthreads, mode="fieldmap") - estimate_wf.inputs.inputnode.fieldmap = [m for m, _ in fmap["fieldmap"]] - else: - fmap = bids_fmaps["phasediff"][0] - outputnode.inputs.method = "FMB (fieldmap-based) - phase-difference map" - estimate_wf = init_fmap_wf(omp_nthreads=omp_nthreads) - estimate_wf.inputs.inputnode.fieldmap = fmap["phases"] - - # set magnitude files (common for the three flavors) - estimate_wf.inputs.inputnode.magnitude = [m for m, _ in fmap["magnitude"]] - - # FIELDMAP-less path - elif "syn" in bids_fmaps: - from .fit.syn import init_syn_sdc_wf - - outputnode.inputs.method = 'FLB ("fieldmap-less", SyN-based)' - - estimate_wf = init_syn_sdc_wf(omp_nthreads=omp_nthreads) - - # fmt: off - workflow.connect([ - (inputnode, estimate_wf, [ - ("epi_file", "inputnode.in_reference"), - ("epi_brain", "inputnode.in_reference_brain"), - ("t1w_brain", "inputnode.t1w_brain"), - ("std2anat_xfm", "inputnode.std2anat_xfm")]), - ]) - # fmt: on - else: - raise ValueError("Unsupported field mapping strategy.") - - # fmt: off - workflow.connect([ - (estimate_wf, outputnode, [ - ("outputnode.fmap", "fieldmap"), - ("outputnode.fmap_ref", "reference")]), - ]) - # fmt: on - return workflow - - -def fieldmap_wrangler(layout, target_image, use_syn=False, force_syn=False): - """Query the BIDSLayout for fieldmaps, and arrange them for the orchestration workflow.""" - from collections import defaultdict - - fmap_bids = layout.get_fieldmap(target_image, return_list=True) - fieldmaps = defaultdict(list) - for fmap in fmap_bids: - if fmap["suffix"] == "epi": - fieldmaps["epi"].append((fmap["epi"], layout.get_metadata(fmap["epi"]))) - - if fmap["suffix"] == "fieldmap": - fieldmaps["fieldmap"].append( - { - "magnitude": [ - (fmap["magnitude"], layout.get_metadata(fmap["magnitude"])) - ], - "fieldmap": [ - (fmap["fieldmap"], layout.get_metadata(fmap["fieldmap"])) - ], - } - ) - - if fmap["suffix"] == "phasediff": - fieldmaps["phasediff"].append( - { - "magnitude": [ - (fmap[k], layout.get_metadata(fmap[k])) - for k in sorted(fmap.keys()) - if k.startswith("magnitude") - ], - "phases": [ - (fmap["phasediff"], layout.get_metadata(fmap["phasediff"])) - ], - } - ) - - if fmap["suffix"] == "phase": - fieldmaps["phasediff"].append( - { - "magnitude": [ - (fmap[k], layout.get_metadata(fmap[k])) - for k in sorted(fmap.keys()) - if k.startswith("magnitude") - ], - "phases": [ - (fmap[k], layout.get_metadata(fmap[k])) - for k in sorted(fmap.keys()) - if k.startswith("phase") - ], - } - ) - - if fieldmaps and force_syn: - # syn: True -> Run SyN in addition to fieldmap-based SDC - fieldmaps["syn"] = True - elif not fieldmaps and (force_syn or use_syn): - # syn: False -> Run SyN as only SDC - fieldmaps["syn"] = False - return fieldmaps diff --git a/sdcflows/workflows/tests/__init__.py b/sdcflows/workflows/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/sdcflows/workflows/tests/test_base.py b/sdcflows/workflows/tests/test_base.py deleted file mode 100644 index a399bb4e9d..0000000000 --- a/sdcflows/workflows/tests/test_base.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Test base workflows.""" -import pytest - -from ..base import init_sdc_estimate_wf - -EPI_METADATA = { - "MultibandAccelerationFactor": 8, - "PhaseEncodingDirection": "i", - "RepetitionTime": 0.72, - "TaskName": "Resting-state fMRI", -} -EPI_FMAP_METADATA_1 = { - "BandwidthPerPixelPhaseEncode": 2290, - "EPIFactor": 90, - "EffectiveEchoSpacing": 0.00058, - "IntendedFor": [ - "func/sub-HCP101006_task-rest_dir-LR_bold.nii.gz", - "func/sub-HCP101006_task-rest_dir-LR_sbref.nii.gz", - ], - "MultibandAccelerationFactor": 1, - "PhaseEncodingDirection": "i-", -} -EPI_FMAP_METADATA_2 = EPI_FMAP_METADATA_1.copy() -EPI_FMAP_METADATA_2["PhaseEncodingDirection"] = "i" - -PHDIFF_METADATA = { - "EchoTime1": 0.00492, - "EchoTime2": 0.00738, -} -PHASE1_METADATA = { - "EchoTime": 0.00492, -} -PHASE2_METADATA = { - "EchoTime": 0.00738, -} - - -FMAP_DICT_ELEMENTS = { - "epi1": [ - ( - "sub-HCP101006/fmap/sub-HCP101006_dir-RL_epi.nii.gz", - EPI_FMAP_METADATA_1.copy(), - ) - ], - "epi2": [ - ( - "sub-HCP101006/fmap/sub-HCP101006_dir-RL_epi.nii.gz", - EPI_FMAP_METADATA_1.copy(), - ), - ( - "sub-HCP101006/fmap/sub-HCP101006_dir-LR_epi.nii.gz", - EPI_FMAP_METADATA_2.copy(), - ), - ], - "phdiff1": { - "magnitude": [ - ("sub-HCP101006/fmap/sub-HCP101006_magnitude1.nii.gz", {}), - ("sub-HCP101006/fmap/sub-HCP101006_magnitude2.nii.gz", {}), - ], - "phases": [ - ("sub-HCP101006/fmap/sub-HCP101006_phasediff.nii.gz", PHDIFF_METADATA) - ], - }, - "phdiff2": { - "magnitude": [ - ("sub-HCP101006/fmap/sub-HCP101006_magnitude1.nii.gz", {}), - ("sub-HCP101006/fmap/sub-HCP101006_magnitude2.nii.gz", {}), - ], - "phases": [ - ("sub-HCP101006/fmap/sub-HCP101006_phase1.nii.gz", PHASE1_METADATA.copy()), - ("sub-HCP101006/fmap/sub-HCP101006_phase2.nii.gz", PHASE2_METADATA.copy()), - ], - }, - "fmap1": { - "magnitude": [("sub-HCP101006/fmap/sub-HCP101006_magnitude.nii.gz", {})], - "fieldmap": [("sub-HCP101006/fmap/sub-HCP101006_fieldmap.nii.gz", {})], - }, - "syn": {"t1w": [("sub-HCP101006/fmap/sub-HCP101006_T1w.nii.gz", {})]}, -} - - -@pytest.mark.parametrize("method", ["skip", "phasediff", "pepolar", "fieldmap", "syn"]) -def test_base(method): - """Check the heuristics are correctly applied.""" - fieldmaps = { - "epi": FMAP_DICT_ELEMENTS["epi1"].copy(), - "fieldmap": [FMAP_DICT_ELEMENTS["fmap1"].copy()], - "phasediff": [FMAP_DICT_ELEMENTS["phdiff1"].copy()], - } - - if method == "skip": - wf = init_sdc_estimate_wf(bids_fmaps=None) - assert wf.inputs.outputnode.method == "None" - - with pytest.raises(ValueError): - wf = init_sdc_estimate_wf(bids_fmaps={"unsupported": None}) - elif method == "pepolar": - wf = init_sdc_estimate_wf(bids_fmaps=fieldmaps) - assert "PEPOLAR" in wf.inputs.outputnode.method - elif method == "fieldmap": - fieldmaps = { - "fieldmap": [FMAP_DICT_ELEMENTS["fmap1"].copy()], - "phasediff": [FMAP_DICT_ELEMENTS["phdiff1"].copy()], - } - wf = init_sdc_estimate_wf(bids_fmaps=fieldmaps) - assert "directly measured B0 map" in wf.inputs.outputnode.method - elif method == "phasediff": - fieldmaps = { - "phasediff": [FMAP_DICT_ELEMENTS["phdiff1"].copy()], - } - - wf = init_sdc_estimate_wf(bids_fmaps=fieldmaps) - assert "phase-difference" in wf.inputs.outputnode.method - elif method == "syn": - fmaps_onlysyn = {"syn": FMAP_DICT_ELEMENTS["syn"]} - wf = init_sdc_estimate_wf(bids_fmaps=fmaps_onlysyn) - assert "SyN" in wf.inputs.outputnode.method From 773d34b248b2303e6996022b3f63e46aa901f027 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Fri, 20 Nov 2020 07:41:46 +0100 Subject: [PATCH 07/68] enh: finalize writting outputs as a standalone workflow + tests Resolves: #26. --- sdcflows/workflows/fit/tests/test_pepolar.py | 54 +++--- sdcflows/workflows/fit/tests/test_phdiff.py | 63 +++---- sdcflows/workflows/outputs.py | 174 ++++++++++++++++--- 3 files changed, 202 insertions(+), 89 deletions(-) diff --git a/sdcflows/workflows/fit/tests/test_pepolar.py b/sdcflows/workflows/fit/tests/test_pepolar.py index b74887f552..22f8391b87 100644 --- a/sdcflows/workflows/fit/tests/test_pepolar.py +++ b/sdcflows/workflows/fit/tests/test_pepolar.py @@ -3,11 +3,10 @@ from pathlib import Path from json import loads import pytest -from niworkflows.interfaces.bids import DerivativesDataSink from niworkflows.interfaces.images import IntraModalMerge from nipype.pipeline import engine as pe -from ..pepolar import Workflow, init_topup_wf +from ..pepolar import init_topup_wf @pytest.mark.skipif(os.getenv("TRAVIS") == "true", reason="this is TravisCI") @@ -30,7 +29,7 @@ def test_topup_wf(tmpdir, datadir, workdir, outdir, epi_path): epi_path = [datadir / f for f in epi_path] in_data = [str(f.absolute()) for f in epi_path] - wf = Workflow( + wf = pe.Workflow( name=f"topup_{epi_path[0].name.replace('.nii.gz', '').replace('-', '_')}" ) @@ -38,9 +37,10 @@ def test_topup_wf(tmpdir, datadir, workdir, outdir, epi_path): merge.inputs.in_files = in_data topup_wf = init_topup_wf(omp_nthreads=2, debug=True) - topup_wf.inputs.inputnode.metadata = [ + metadata = [ loads(Path(str(f).replace(".nii.gz", ".json")).read_text()) for f in in_data ] + topup_wf.inputs.inputnode.metadata = metadata # fmt: off wf.connect([ @@ -50,37 +50,37 @@ def test_topup_wf(tmpdir, datadir, workdir, outdir, epi_path): if outdir: from nipype.interfaces.afni import Automask - from ....interfaces.reportlets import FieldmapReportlet + from ...outputs import init_fmap_derivatives_wf, init_fmap_reports_wf - pre_mask = pe.Node(Automask(dilate=1, outputtype="NIFTI_GZ"), - name="pre_mask") - merge_corrected = pe.Node(IntraModalMerge(hmc=False), name="merge_corrected") - - rep = pe.Node( - FieldmapReportlet(reference_label="EPI Reference"), "simple_report" + fmap_derivatives_wf = init_fmap_derivatives_wf( + output_dir=str(outdir), + write_coeff=True, + custom_entities={"est": "pepolar"}, + bids_fmap_id="pepolar_id", ) - rep.interface._always_run = True - ds_report = pe.Node( - DerivativesDataSink( - base_directory=str(outdir), - out_path_base="sdcflows", - datatype="figures", - suffix="fieldmap", - desc="pepolar", - dismiss_entities="fmap", - ), - name="ds_report", + fmap_derivatives_wf.inputs.inputnode.source_files = in_data + fmap_derivatives_wf.inputs.inputnode.fmap_meta = metadata + + fmap_reports_wf = init_fmap_reports_wf( + output_dir=str(outdir), fmap_type="pepolar", ) - ds_report.inputs.source_file = in_data[0] + fmap_reports_wf.inputs.inputnode.source_files = in_data + + pre_mask = pe.Node(Automask(dilate=1, outputtype="NIFTI_GZ"), name="pre_mask") + merge_corrected = pe.Node(IntraModalMerge(hmc=False), name="merge_corrected") # fmt: off wf.connect([ (topup_wf, merge_corrected, [("outputnode.fmap_ref", "in_files")]), - (merge_corrected, rep, [("out_avg", "reference")]), (merge_corrected, pre_mask, [("out_avg", "in_file")]), - (topup_wf, rep, [("outputnode.fmap", "fieldmap")]), - (pre_mask, rep, [("out_file", "mask")]), - (rep, ds_report, [("out_report", "in_file")]), + (merge_corrected, fmap_reports_wf, [("out_avg", "inputnode.fmap_ref")]), + (topup_wf, fmap_reports_wf, [("outputnode.fmap", "inputnode.fieldmap")]), + (pre_mask, fmap_reports_wf, [("out_file", "inputnode.fmap_mask")]), + (topup_wf, fmap_derivatives_wf, [ + ("outputnode.fmap", "inputnode.fieldmap"), + ("outputnode.fmap_ref", "inputnode.fmap_ref"), + ("outputnode.coefficients", "inputnode.fmap_coeff"), + ]), ]) # fmt: on diff --git a/sdcflows/workflows/fit/tests/test_phdiff.py b/sdcflows/workflows/fit/tests/test_phdiff.py index 530351b6aa..8895d9b10b 100644 --- a/sdcflows/workflows/fit/tests/test_phdiff.py +++ b/sdcflows/workflows/fit/tests/test_phdiff.py @@ -4,8 +4,6 @@ from json import loads import pytest -from niworkflows.interfaces.bids import DerivativesDataSink -from nipype.pipeline import engine as pe from ..fieldmap import init_fmap_wf, Workflow @@ -28,58 +26,47 @@ def test_phdiff(tmpdir, datadir, workdir, outdir, fmap_path): tmpdir.chdir() fmap_path = [datadir / f for f in fmap_path] + fieldmaps = [ + (str(f.absolute()), loads(Path(str(f).replace(".nii.gz", ".json")).read_text())) + for f in fmap_path + ] wf = Workflow( name=f"phdiff_{fmap_path[0].name.replace('.nii.gz', '').replace('-', '_')}" ) phdiff_wf = init_fmap_wf(omp_nthreads=2) + phdiff_wf.inputs.inputnode.fieldmap = fieldmaps phdiff_wf.inputs.inputnode.magnitude = [ - str(f.absolute()).replace("diff", "1").replace("phase", "magnitude") - for f in fmap_path - ] - phdiff_wf.inputs.inputnode.fieldmap = [ - (str(f.absolute()), loads(Path(str(f).replace(".nii.gz", ".json")).read_text())) - for f in fmap_path + f.replace("diff", "1").replace("phase", "magnitude") for f, _ in fieldmaps ] if outdir: - from ....interfaces.reportlets import FieldmapReportlet - - rep = pe.Node(FieldmapReportlet(reference_label="Magnitude"), "simple_report") - rep.interface._always_run = True + from ...outputs import init_fmap_derivatives_wf, init_fmap_reports_wf - ds_report = pe.Node( - DerivativesDataSink( - base_directory=str(outdir), - out_path_base="sdcflows", - datatype="figures", - suffix="fieldmap", - desc="phasediff", - dismiss_entities="fmap", - ), - name="ds_report", + fmap_derivatives_wf = init_fmap_derivatives_wf( + output_dir=str(outdir), + custom_entities={"est": "phasediff"}, + bids_fmap_id="phasediff_id", ) - ds_report.inputs.source_file = str(fmap_path[0]) + fmap_derivatives_wf.inputs.inputnode.source_files = [f for f, _ in fieldmaps] + fmap_derivatives_wf.inputs.inputnode.fmap_meta = [f for _, f in fieldmaps] - dsink_fmap = pe.Node( - DerivativesDataSink( - base_directory=str(outdir), - dismiss_entities="fmap", - desc="phasediff", - suffix="fieldmap", - ), - name="dsink_fmap", + fmap_reports_wf = init_fmap_reports_wf( + output_dir=str(outdir), + fmap_type="phasediff" if len(fieldmaps) == 1 else "phases", ) - dsink_fmap.interface.out_path_base = "sdcflows" - dsink_fmap.inputs.source_file = str(fmap_path[0]) + fmap_reports_wf.inputs.inputnode.source_files = [f for f, _ in fieldmaps] # fmt: off wf.connect([ - (phdiff_wf, rep, [("outputnode.fmap", "fieldmap"), - ("outputnode.fmap_ref", "reference"), - ("outputnode.fmap_mask", "mask")]), - (rep, ds_report, [("out_report", "in_file")]), - (phdiff_wf, dsink_fmap, [("outputnode.fmap", "in_file")]), + (phdiff_wf, fmap_reports_wf, [ + ("outputnode.fmap", "inputnode.fieldmap"), + ("outputnode.fmap_ref", "inputnode.fmap_ref"), + ("outputnode.fmap_mask", "inputnode.fmap_mask")]), + (phdiff_wf, fmap_derivatives_wf, [ + ("outputnode.fmap", "inputnode.fieldmap"), + ("outputnode.fmap_ref", "inputnode.fmap_ref"), + ]), ]) # fmt: on else: diff --git a/sdcflows/workflows/outputs.py b/sdcflows/workflows/outputs.py index 8912d26ee2..dee2ccb7f8 100644 --- a/sdcflows/workflows/outputs.py +++ b/sdcflows/workflows/outputs.py @@ -3,23 +3,107 @@ """Writing out outputs.""" from nipype.pipeline import engine as pe from nipype.interfaces import utility as niu -from niworkflows.interfaces.bids import DerivativesDataSink +from niworkflows.interfaces.bids import DerivativesDataSink as _DDS + + +class DerivativesDataSink(_DDS): + """Overload the ``out_path_base`` setting.""" + + out_path_base = "sdcflows" + + +del _DDS + + +def init_fmap_reports_wf( + *, output_dir, fmap_type, custom_entities=None, name="fmap_reports_wf", +): + """ + Set up a battery of datasinks to store reports in the right location. + + Parameters + ---------- + fmap_type : :obj:`str` + The fieldmap estimator type. + custom_entities : :obj:`dict` + Define extra entities that will be written out in filenames. + output_dir : :obj:`str` + Directory in which to save derivatives + name : :obj:`str` + Workflow name (default: ``"fmap_reports_wf"``) + + Inputs + ------ + source_files + One or more fieldmap file(s) of the BIDS dataset that will serve for naming reference. + fieldmap + The preprocessed fieldmap, in its original space with Hz units. + fmap_ref + An anatomical reference (e.g., magnitude file) + fmap_mask + A brain mask in the fieldmap's space. + + """ + from ..interfaces.reportlets import FieldmapReportlet + + custom_entities = custom_entities or {} + + workflow = pe.Workflow(name=name) + inputnode = pe.Node( + niu.IdentityInterface( + fields=["source_files", "fieldmap", "fmap_ref", "fmap_mask"] + ), + name="inputnode", + ) + + rep = pe.Node(FieldmapReportlet(), "simple_report") + rep.interface._always_run = True + + ds_fmap_report = pe.Node( + DerivativesDataSink( + base_directory=str(output_dir), + datatype="figures", + suffix="fieldmap", + desc=fmap_type, + dismiss_entities=("fmap",), + allowed_entities=tuple(custom_entities.keys()), + ), + name="ds_fmap_report", + ) + for k, v in custom_entities.items(): + setattr(ds_fmap_report.inputs, k, v) + + # fmt:off + workflow.connect([ + (inputnode, rep, [("fieldmap", "fieldmap"), + ("fmap_ref", "reference"), + ("fmap_mask", "mask")]), + (rep, ds_fmap_report, [("out_report", "in_file")]), + (inputnode, ds_fmap_report, [("source_files", "source_file")]), + + ]) + # fmt:on + + return workflow def init_fmap_derivatives_wf( *, - bids_root, output_dir, - write_coeff=False, + bids_fmap_id=None, + custom_entities=None, name="fmap_derivatives_wf", + write_coeff=False, ): """ Set up datasinks to store derivatives in the right location. Parameters ---------- - bids_root : :obj:`str` - Root path of BIDS dataset + bids_fmap_id : :obj:`str` + Sets the ``B0FieldIdentifier`` metadata into the outputs. + custom_entities : :obj:`dict` + Define extra entities that will be written out in filenames. output_dir : :obj:`str` Directory in which to save derivatives name : :obj:`str` @@ -27,8 +111,8 @@ def init_fmap_derivatives_wf( Inputs ------ - source_file - A fieldmap file of the BIDS dataset that will serve for naming reference. + source_files + One or more fieldmap file(s) of the BIDS dataset that will serve for naming reference. fieldmap The preprocessed fieldmap, in its original space with Hz units. fmap_coeff @@ -37,25 +121,24 @@ def init_fmap_derivatives_wf( An anatomical reference (e.g., magnitude file) """ - workflow = pe.Workflow(name=name) + custom_entities = custom_entities or {} + workflow = pe.Workflow(name=name) inputnode = pe.Node( niu.IdentityInterface( - fields=[ - "source_file", - "fieldmap", - "fmap_coeff", - "fmap_ref", - ]), - name="inputnode" + fields=["source_files", "fieldmap", "fmap_coeff", "fmap_ref", "fmap_meta"] + ), + name="inputnode", ) ds_reference = pe.Node( DerivativesDataSink( base_directory=output_dir, - desc="reference", + compress=True, suffix="fieldmap", - compress=True), + dismiss_entities=("fmap",), + allowed_entities=tuple(custom_entities.keys()), + ), name="ds_reference", ) @@ -64,17 +147,31 @@ def init_fmap_derivatives_wf( base_directory=output_dir, desc="preproc", suffix="fieldmap", - compress=True), + compress=True, + allowed_entities=tuple(custom_entities.keys()), + ), name="ds_fieldmap", ) ds_fieldmap.inputs.Units = "Hz" + if bids_fmap_id: + ds_fieldmap.inputs.B0FieldIdentifier = bids_fmap_id + + for k, v in custom_entities.items(): + setattr(ds_reference.inputs, k, v) + setattr(ds_fieldmap.inputs, k, v) # fmt:off workflow.connect([ - (inputnode, ds_reference, [("source_file", "source_file"), - ("fmap_ref", "in_file")]), - (inputnode, ds_fieldmap, [("source_file", "source_file"), - ("fieldmap", "in_file")]), + (inputnode, ds_reference, [("source_files", "source_file"), + ("fmap_ref", "in_file"), + (("source_files", _getsourcetype), "desc")]), + (inputnode, ds_fieldmap, [("source_files", "source_file"), + ("fieldmap", "in_file"), + ("source_files", "RawSources")]), + (ds_reference, ds_fieldmap, [ + (("out_file", _getname), "AnatomicalReference"), + ]), + (inputnode, ds_fieldmap, [(("fmap_meta", _selectintent), "IntendedFor")]), ]) # fmt:on @@ -85,19 +182,25 @@ def init_fmap_derivatives_wf( DerivativesDataSink( base_directory=output_dir, suffix="fieldmap", - compress=True), + compress=True, + allowed_entities=tuple(custom_entities.keys()), + ), name="ds_coeff", iterfield=("in_file", "desc"), ) gen_desc = pe.Node(niu.Function(function=_gendesc), name="gen_desc") + for k, v in custom_entities.items(): + setattr(ds_coeff.inputs, k, v) + # fmt:off workflow.connect([ - (inputnode, ds_coeff, [("source_file", "source_file"), + (inputnode, ds_coeff, [("source_files", "source_file"), ("fmap_coeff", "in_file")]), (inputnode, gen_desc, [("fmap_coeff", "infiles")]), (gen_desc, ds_coeff, [("out", "desc")]), + (ds_coeff, ds_fieldmap, [(("out_file", _getname), "AssociatedCoefficients")]), ]) # fmt:on @@ -197,3 +300,26 @@ def _gendesc(infiles): return "coeff" return [f"coeff{i}" for i, _ in enumerate(infiles)] + + +def _getname(infile): + from pathlib import Path + + if isinstance(infile, (list, tuple)): + return [Path(f).name for f in infile] + return Path(infile).name + + +def _getsourcetype(infiles): + from pathlib import Path + + fname = Path(infiles[0]).name + return "epi" if fname.endswith(("_epi.nii.gz", "_epi.nii")) else "magnitude" + + +def _selectintent(metadata): + from bids.utils import listify + + return sorted( + set([el for m in metadata for el in listify(m.get("IntendedFor", []))]) + ) From 944c9c3fd781c980cd2cbe2b1304c3e571a753ca Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Fri, 20 Nov 2020 15:03:01 +0100 Subject: [PATCH 08/68] enh: revise co-registration module & corresponding test --- sdcflows/data/fmap-any_registration.json | 28 +++--- .../data/fmap-any_registration_testing.json | 28 +++--- sdcflows/workflows/apply/registration.py | 51 ++++++----- .../apply/tests/test_registration.py | 89 +++++++++++++++++++ 4 files changed, 146 insertions(+), 50 deletions(-) create mode 100644 sdcflows/workflows/apply/tests/test_registration.py diff --git a/sdcflows/data/fmap-any_registration.json b/sdcflows/data/fmap-any_registration.json index 97d4ffc756..ef14cae47f 100644 --- a/sdcflows/data/fmap-any_registration.json +++ b/sdcflows/data/fmap-any_registration.json @@ -1,23 +1,23 @@ { - "dimension": 3, - "float": true, - "winsorize_lower_quantile": 0.005, - "winsorize_upper_quantile": 0.998, "collapse_output_transforms": true, - "use_histogram_matching": [ true, true ], - "use_estimate_learning_rate_once": [ true, true ], - "transforms": [ "Translation", "Rigid" ], - "number_of_iterations": [ [ 500 ], [ 200 ] ], - "transform_parameters": [ [ 0.05 ], [ 0.01 ] ], "convergence_threshold": [ 1e-07, 1e-08 ], "convergence_window_size": [ 200, 100 ], + "dimension": 3, + "float": true, + "interpolation": "LanczosWindowedSinc", "metric": [ "Mattes", "Mattes" ], + "metric_weight": [ 1.0, 1.0 ], + "number_of_iterations": [ [ 500 ], [ 200 ] ], + "radius_or_number_of_bins": [ 64, 64 ], "sampling_percentage": [ 0.5, 0.5 ], "sampling_strategy": [ "Random", "Random" ], - "smoothing_sigmas": [ [ 8.0 ], [ 2.0 ] ], - "sigma_units": [ "mm", "mm" ], - "metric_weight": [ 1.0, 1.0 ], "shrink_factors": [ [ 2 ], [ 1 ] ], - "radius_or_number_of_bins": [ 64, 64 ], - "interpolation": "LanczosWindowedSinc" + "sigma_units": [ "mm", "mm" ], + "smoothing_sigmas": [ [ 8.0 ], [ 2.0 ] ], + "transform_parameters": [ [ 0.05 ], [ 0.01 ] ], + "transforms": [ "Translation", "Rigid" ], + "use_estimate_learning_rate_once": [ true, true ], + "use_histogram_matching": [ true, true ], + "winsorize_lower_quantile": 0.005, + "winsorize_upper_quantile": 0.998 } \ No newline at end of file diff --git a/sdcflows/data/fmap-any_registration_testing.json b/sdcflows/data/fmap-any_registration_testing.json index 37aff850ec..129993b34e 100644 --- a/sdcflows/data/fmap-any_registration_testing.json +++ b/sdcflows/data/fmap-any_registration_testing.json @@ -1,23 +1,23 @@ { - "dimension": 3, - "float": true, - "winsorize_lower_quantile": 0.005, - "winsorize_upper_quantile": 0.998, "collapse_output_transforms": true, - "use_histogram_matching": [ true, true ], - "use_estimate_learning_rate_once": [ true, true ], - "transforms": [ "Rigid", "Rigid" ], - "number_of_iterations": [ [ 500 ], [ 100 ] ], - "transform_parameters": [ [ 1.0 ], [ 0.5 ] ], "convergence_threshold": [ 1e-07, 1e-08 ], "convergence_window_size": [ 20, 10 ], + "dimension": 3, + "float": true, + "interpolation": "LanczosWindowedSinc", "metric": [ "Mattes", "Mattes" ], + "metric_weight": [ 1.0, 1.0 ], + "number_of_iterations": [ [ 500 ], [ 100 ] ], + "radius_or_number_of_bins": [ 64, 64 ], "sampling_percentage": [ 1.0, 1.0 ], "sampling_strategy": [ "Regular", "Regular" ], - "smoothing_sigmas": [ [ 8.0 ], [ 2.0 ] ], - "sigma_units": [ "mm", "mm" ], - "metric_weight": [ 1.0, 1.0 ], "shrink_factors": [ [ 2 ], [ 1 ] ], - "radius_or_number_of_bins": [ 64, 64 ], - "interpolation": "LanczosWindowedSinc" + "sigma_units": [ "mm", "mm" ], + "smoothing_sigmas": [ [ 8.0 ], [ 2.0 ] ], + "transform_parameters": [ [ 1.0 ], [ 0.5 ] ], + "transforms": [ "Rigid", "Rigid" ], + "use_estimate_learning_rate_once": [ true, true ], + "use_histogram_matching": [ true, true ], + "winsorize_lower_quantile": 0.005, + "winsorize_upper_quantile": 0.998 } \ No newline at end of file diff --git a/sdcflows/workflows/apply/registration.py b/sdcflows/workflows/apply/registration.py index c75a3a3138..04d20aa8b7 100644 --- a/sdcflows/workflows/apply/registration.py +++ b/sdcflows/workflows/apply/registration.py @@ -16,9 +16,7 @@ def init_coeff2epi_wf( - omp_nthreads, - debug=False, - name="fmap2field_wf", + omp_nthreads, debug=False, write_coeff=False, name="fmap2field_wf", ): """ Move the field coefficients on to the target (distorted) EPI space. @@ -33,12 +31,14 @@ def init_coeff2epi_wf( Parameters ---------- - omp_nthreads : int + omp_nthreads : :obj:`int` Maximum number of threads an individual process may use. - debug : bool + debug : :obj:`bool` Run fast configurations of registrations. - name : str + name : :obj:`str` Unique name of this workflow. + write_coeff : :obj:`bool` + Map coefficients file Inputs ------ @@ -59,7 +59,9 @@ def init_coeff2epi_wf( fieldmap coefficients """ + from packaging.version import parse as parseversion, Version from niworkflows.interfaces.fixes import FixHeaderRegistration as Registration + workflow = Workflow(name=name) workflow.__desc__ = """\ The estimated *fieldmap* was then aligned with rigid-registration to the target @@ -68,18 +70,13 @@ def init_coeff2epi_wf( """ inputnode = pe.Node( niu.IdentityInterface( - fields=[ - "target_ref", - "target_mask", - "fmap_ref", - "fmap_mask", - "fmap_coeff", - ] + fields=["target_ref", "target_mask", "fmap_ref", "fmap_mask", "fmap_coeff"] ), name="inputnode", ) - outputnode = pe.Node(niu.IdentityInterface(fields=["fmap_coeff"]), - name="outputnode") + outputnode = pe.Node( + niu.IdentityInterface(fields=["fmap_ref", "fmap_coeff"]), name="outputnode" + ) # Register the reference of the fieldmap to the reference # of the target image (the one that shall be corrected) @@ -88,24 +85,34 @@ def init_coeff2epi_wf( ) coregister = pe.Node( - Registration( - from_file=ants_settings, - ), + Registration(from_file=ants_settings, output_warped_image=True,), name="coregister", n_procs=omp_nthreads, ) - # Map the coefficients into the EPI space - map_coeff = pe.Node(niu.Function(function=_move_coeff), name="map_coeff") + ver = coregister.interface.version or "2.2.0" + mask_trait_s = "s" if parseversion(ver) >= Version("2.2.0") else "" # fmt: off workflow.connect([ (inputnode, coregister, [ ("target_ref", "fixed_image"), ("fmap_ref", "moving_image"), - ("target_mask", "fixed_image_masks"), - ("fmap_mask", "moving_image_masks"), + ("target_mask", f"fixed_image_mask{mask_trait_s}"), + ("fmap_mask", f"moving_image_mask{mask_trait_s}"), ]), + (coregister, outputnode, [("warped_image", "fmap_ref")]), + ]) + # fmt: on + + if not write_coeff: + return workflow + + # Map the coefficients into the EPI space + map_coeff = pe.Node(niu.Function(function=_move_coeff), name="map_coeff") + + # fmt: off + workflow.connect([ (inputnode, map_coeff, [("fmap_coeff", "in_coeff")]), (coregister, map_coeff, [("forward_transforms", "transform")]), (map_coeff, outputnode, [("out", "fmap_coeff")]), diff --git a/sdcflows/workflows/apply/tests/test_registration.py b/sdcflows/workflows/apply/tests/test_registration.py new file mode 100644 index 0000000000..2777548297 --- /dev/null +++ b/sdcflows/workflows/apply/tests/test_registration.py @@ -0,0 +1,89 @@ +"""Test pepolar type of fieldmaps.""" +import os +import pytest +from nipype.pipeline import engine as pe + +from ...fit.fieldmap import init_magnitude_wf +from ..registration import init_coeff2epi_wf + + +@pytest.mark.skipif(os.getenv("TRAVIS") == "true", reason="this is TravisCI") +def test_registration_wf(tmpdir, datadir, workdir, outdir): + """Test fieldmap-to-target alignment workflow.""" + epi_ref_wf = init_magnitude_wf(2, name="epi_ref_wf") + epi_ref_wf.inputs.inputnode.magnitude = ( + datadir + / "testdata" + / "sub-HCP101006" + / "func" + / "sub-HCP101006_task-rest_dir-LR_sbref.nii.gz" + ) + + magnitude = ( + datadir + / "testdata" + / "sub-HCP101006" + / "fmap" + / "sub-HCP101006_magnitude1.nii.gz" + ) + fmap_ref_wf = init_magnitude_wf(2, name="fmap_ref_wf") + fmap_ref_wf.inputs.inputnode.magnitude = magnitude + + reg_wf = init_coeff2epi_wf(2, debug=True) + + workflow = pe.Workflow(name="test_registration_wf") + workflow.connect( + [ + ( + epi_ref_wf, + reg_wf, + [ + ("outputnode.fmap_ref", "inputnode.target_ref"), + ("outputnode.fmap_mask", "inputnode.target_mask"), + ], + ), + ( + fmap_ref_wf, + reg_wf, + [ + ("outputnode.fmap_ref", "inputnode.fmap_ref"), + ("outputnode.fmap_mask", "inputnode.fmap_mask"), + ], + ), + ] + ) + + if outdir: + from niworkflows.interfaces import SimpleBeforeAfter + from ...outputs import DerivativesDataSink + + report = pe.Node( + SimpleBeforeAfter(before_label="Target EPI", after_label="B0 Reference",), + name="report", + mem_gb=0.1, + ) + ds_report = pe.Node( + DerivativesDataSink( + base_directory=str(outdir), + suffix="fieldmap", + space="sbref", + datatype="figures", + dismiss_entities=("fmap",), + source_file=magnitude, + ), + name="ds_report", + run_without_submitting=True, + ) + + workflow.connect( + [ + (epi_ref_wf, report, [("outputnode.fmap_ref", "before")]), + (reg_wf, report, [("outputnode.fmap_ref", "after")]), + (report, ds_report, [("out_report", "in_file")]), + ] + ) + + if workdir: + workflow.base_dir = str(workdir) + + workflow.run() From 42c097839431df47ca3b8fd99bbbc9951646cc5b Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Fri, 20 Nov 2020 15:06:01 +0100 Subject: [PATCH 09/68] rf: remove unnecessary workflow init_sdc_unwarp_wf --- sdcflows/workflows/apply/base.py | 106 ------------------------------- 1 file changed, 106 deletions(-) delete mode 100644 sdcflows/workflows/apply/base.py diff --git a/sdcflows/workflows/apply/base.py b/sdcflows/workflows/apply/base.py deleted file mode 100644 index 9dcfd90654..0000000000 --- a/sdcflows/workflows/apply/base.py +++ /dev/null @@ -1,106 +0,0 @@ -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -# vi: set ft=python sts=4 ts=4 sw=4 et: -"""Apply the estimated fieldmap to perform susceptibility-derived distortion correction.""" -from nipype.pipeline import engine as pe -from nipype.interfaces import utility as niu -from niworkflows.engine.workflows import LiterateWorkflow as Workflow - - -def init_sdc_unwarp_wf(omp_nthreads, debug, name="sdc_unwarp_wf"): - """ - Apply the warping given by a displacements fieldmap. - - This workflow takes in a displacements field through which the - input reference can be corrected for susceptibility-derived distortion. - - It also calculates a new mask for the input dataset, after the distortions - have been accounted for. - - .. workflow :: - :graph2use: orig - :simple_form: yes - - from sdcflows.workflows.apply.base import init_sdc_unwarp_wf - wf = init_sdc_unwarp_wf(omp_nthreads=8, - debug=False) - - Parameters - ---------- - omp_nthreads : :obj:`int` - Maximum number of threads an individual process may use. - debug : :obj:`bool` - Run fast configurations of registrations. - name : :obj:`str` - Unique name of this workflow. - - Inputs - ------ - in_warp : :obj:`os.PathLike` - The :abbr:`DFM (displacements field map)` that corrects for - susceptibility-derived distortions estimated elsewhere. - in_reference : :obj:`os.PathLike` - the reference image to be unwarped. - in_reference_mask : :obj:`os.PathLike` - the reference image mask to be unwarped - - Outputs - ------- - out_reference : :obj:`str` - the ``in_reference`` after unwarping - out_mask : :obj:`str` - mask of the unwarped input file - out_warp : :obj:`str` - the ``in_warp`` field is forwarded for compatibility - - """ - from niworkflows.interfaces.registration import ANTSApplyTransformsRPT - workflow = Workflow(name=name) - inputnode = pe.Node( - niu.IdentityInterface(fields=["in_warp", "in_reference", "in_reference_mask"]), - name="inputnode", - ) - outputnode = pe.Node( - niu.IdentityInterface( - fields=["out_reference", "out_warp", "out_mask"] - ), - name="outputnode", - ) - - unwarp_reference = pe.Node( - ANTSApplyTransformsRPT( - dimension=3, - generate_report=False, - float=True, - interpolation="LanczosWindowedSinc", - ), - name="unwarp_reference", - ) - - unwarp_mask = pe.Node( - ANTSApplyTransformsRPT( - dimension=3, - generate_report=False, - float=True, - interpolation="NearestNeighbor", - ), - name="unwarp_mask", - ) - - # fmt:off - workflow.connect([ - (inputnode, unwarp_reference, [ - ("in_warp", "transforms"), - ("in_reference", "reference_image"), - ("in_reference", "input_image"), - ]), - (inputnode, unwarp_mask, [ - ("in_warp", "transforms"), - ("in_reference_mask", "reference_image"), - ("in_reference_mask", "input_image"), - ]), - (inputnode, outputnode, [("in_warp", "out_warp")]), - (unwarp_reference, outputnode, [("output_image", "out_reference")]), - (unwarp_mask, outputnode, [("output_image", "out_mask")]), - ]) - # fmt: on - return workflow From b0bb3c521244b04970ed53125c2b0042d288820a Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Fri, 20 Nov 2020 15:43:06 +0100 Subject: [PATCH 10/68] enh: create a test that just builds all estimation workflows --- sdcflows/workflows/fit/fieldmap.py | 24 +++++++++------------- sdcflows/workflows/fit/syn.py | 2 +- sdcflows/workflows/fit/tests/test_fit.py | 26 ++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 15 deletions(-) create mode 100644 sdcflows/workflows/fit/tests/test_fit.py diff --git a/sdcflows/workflows/fit/fieldmap.py b/sdcflows/workflows/fit/fieldmap.py index f29f782192..9d0b272288 100644 --- a/sdcflows/workflows/fit/fieldmap.py +++ b/sdcflows/workflows/fit/fieldmap.py @@ -109,7 +109,7 @@ from niworkflows.engine.workflows import LiterateWorkflow as Workflow -def init_fmap_wf(omp_nthreads, mode="phasediff", name="fmap_wf"): +def init_fmap_wf(omp_nthreads=1, mode="phasediff", name="fmap_wf"): """ Estimate the fieldmap based on a field-mapping MRI acquisition. @@ -206,12 +206,15 @@ def init_fmap_wf(omp_nthreads, mode="phasediff", name="fmap_wf"): else: from niworkflows.interfaces.nibabel import ApplyMask from niworkflows.interfaces.images import IntraModalMerge + workflow.__desc__ = """\ A *B0* nonuniformity map (or *fieldmap*) was directly measured with an MRI scheme designed with that purpose (e.g., a spiral pulse sequence). """ # Merge input fieldmap images - fmapmrg = pe.Node(IntraModalMerge(zero_based_avg=False, hmc=False), name="fmapmrg") + fmapmrg = pe.Node( + IntraModalMerge(zero_based_avg=False, hmc=False), name="fmapmrg" + ) applymsk = pe.Node(ApplyMask(), name="applymsk") # fmt: off workflow.connect([ @@ -360,9 +363,7 @@ def init_phdiff_wf(omp_nthreads, name="phdiff_wf"): niu.IdentityInterface(fields=["magnitude", "phase", "mask"]), name="inputnode" ) - outputnode = pe.Node( - niu.IdentityInterface(fields=["fieldmap"]), name="outputnode", - ) + outputnode = pe.Node(niu.IdentityInterface(fields=["fieldmap"]), name="outputnode",) def _split(phase): return phase @@ -406,9 +407,7 @@ def _split(phase): return workflow -def init_fmap_postproc_wf( - omp_nthreads, median_kernel_size=5, name="fmap_postproc_wf" -): +def init_fmap_postproc_wf(omp_nthreads, median_kernel_size=5, name="fmap_postproc_wf"): """ Postprocess a :math:`B_0` map estimated elsewhere. @@ -448,6 +447,7 @@ def init_fmap_postproc_wf( """ from nipype.interfaces.fsl import SpatialFilter from niflow.nipype1.workflows.dmri.fsl.utils import cleanup_edge_pipeline + workflow = Workflow(name=name) inputnode = pe.Node( niu.IdentityInterface(fields=["fmap_mask", "fmap_ref", "fmap", "metadata"]), @@ -475,15 +475,11 @@ def _recenter(in_file): return out_file recenter = pe.Node( - niu.Function(function=_recenter), - name="recenter", - run_without_submitting=True, + niu.Function(function=_recenter), name="recenter", run_without_submitting=True, ) denoise = pe.Node( SpatialFilter( - operation="median", - kernel_shape="sphere", - kernel_size=median_kernel_size, + operation="median", kernel_shape="sphere", kernel_size=median_kernel_size, ), name="denoise", ) diff --git a/sdcflows/workflows/fit/syn.py b/sdcflows/workflows/fit/syn.py index 9f55d680a6..cb1cafa63c 100644 --- a/sdcflows/workflows/fit/syn.py +++ b/sdcflows/workflows/fit/syn.py @@ -40,7 +40,7 @@ LOGGER = logging.getLogger("nipype.workflow") -def init_syn_sdc_wf(omp_nthreads, epi_pe=None, atlas_threshold=3, name="syn_sdc_wf"): +def init_syn_sdc_wf(omp_nthreads=1, epi_pe=None, atlas_threshold=3, name="syn_sdc_wf"): """ Build the *fieldmap-less* susceptibility-distortion estimation workflow. diff --git a/sdcflows/workflows/fit/tests/test_fit.py b/sdcflows/workflows/fit/tests/test_fit.py new file mode 100644 index 0000000000..a4df18756f --- /dev/null +++ b/sdcflows/workflows/fit/tests/test_fit.py @@ -0,0 +1,26 @@ +"""Test that workflows build.""" +import pytest +import sys + +from .. import fieldmap # noqa +from .. import pepolar # noqa +from .. import syn # noqa + + +@pytest.mark.parametrize( + "workflow,kwargs", + ( + ("sdcflows.workflows.fit.pepolar.init_topup_wf", {}), + ("sdcflows.workflows.fit.pepolar.init_3dQwarp_wf", {}), + ("sdcflows.workflows.fit.syn.init_syn_sdc_wf", {}), + ("sdcflows.workflows.fit.fieldmap.init_fmap_wf", {}), + ("sdcflows.workflows.fit.fieldmap.init_fmap_wf", {"mode": "fieldmap"}), + ("sdcflows.workflows.fit.fieldmap.init_fmap_postproc_wf", {"omp_nthreads": 1}), + ("sdcflows.workflows.fit.fieldmap.init_phdiff_wf", {"omp_nthreads": 1}), + ), +) +def test_build_1(workflow, kwargs): + """Make sure the workflow builds.""" + module = ".".join(workflow.split(".")[:-1]) + func = workflow.split(".")[-1] + getattr(sys.modules[module], func)(**kwargs) From 7bdefa5068944e163e84d43e2f6c1c35033036c6 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Fri, 20 Nov 2020 16:02:04 +0100 Subject: [PATCH 11/68] enh: remove init_sdc_unwarp_report_wf --- sdcflows/workflows/outputs.py | 85 ----------------------------------- 1 file changed, 85 deletions(-) diff --git a/sdcflows/workflows/outputs.py b/sdcflows/workflows/outputs.py index dee2ccb7f8..dd8d3afecc 100644 --- a/sdcflows/workflows/outputs.py +++ b/sdcflows/workflows/outputs.py @@ -207,91 +207,6 @@ def init_fmap_derivatives_wf( return workflow -def init_sdc_unwarp_report_wf(name="sdc_unwarp_report_wf", forcedsyn=False): - """ - Save a reportlet showing how SDC unwarping performed. - - This workflow generates and saves a reportlet showing the effect of fieldmap - unwarping a BOLD image. - - Workflow Graph - .. workflow:: - :graph2use: orig - :simple_form: yes - - from sdcflows.workflows.outputs import init_sdc_unwarp_report_wf - wf = init_sdc_unwarp_report_wf() - - Parameters - ---------- - name : str, optional - Workflow name (default: ``sdc_unwarp_report_wf``) - forcedsyn : bool, optional - Whether SyN-SDC was forced. - - Inputs - ------ - in_pre - Reference image, before unwarping - in_post - Reference image, after unwarping - in_seg - Segmentation of preprocessed structural image, including - gray-matter (GM), white-matter (WM) and cerebrospinal fluid (CSF) - in_xfm - Affine transform from T1 space to BOLD space (ITK format) - - """ - from niworkflows.interfaces import SimpleBeforeAfter - from niworkflows.interfaces.fixes import FixHeaderApplyTransforms as ApplyTransforms - from niworkflows.utils.images import dseg_label as _dseg_label - - DEFAULT_MEMORY_MIN_GB = 0.01 - - workflow = pe.Workflow(name=name) - - inputnode = pe.Node( - niu.IdentityInterface(fields=["in_pre", "in_post", "in_seg", "in_xfm"]), - name="inputnode", - ) - - map_seg = pe.Node( - ApplyTransforms(dimension=3, float=True, interpolation="MultiLabel"), - name="map_seg", - mem_gb=0.3, - ) - - sel_wm = pe.Node( - niu.Function(function=_dseg_label), name="sel_wm", mem_gb=DEFAULT_MEMORY_MIN_GB - ) - sel_wm.inputs.label = 2 - - bold_rpt = pe.Node(SimpleBeforeAfter(), name="bold_rpt", mem_gb=0.1) - ds_report_sdc = pe.Node( - DerivativesDataSink( - desc=("sdc", "forcedsyn")[forcedsyn], suffix="bold", datatype="figures" - ), - name="ds_report_sdc", - mem_gb=DEFAULT_MEMORY_MIN_GB, - run_without_submitting=True, - ) - - # fmt: off - workflow.connect([ - (inputnode, bold_rpt, [("in_post", "after"), - ("in_pre", "before")]), - (bold_rpt, ds_report_sdc, [("out_report", "in_file")]), - (inputnode, map_seg, [("in_post", "reference_image"), - ("in_seg", "input_image"), - ("in_xfm", "transforms")]), - (map_seg, sel_wm, [("output_image", "in_seg")]), - (sel_wm, bold_rpt, [("out", "wm_seg")]), - ]) - # fmt: on - - return workflow - - def _gendesc(infiles): if isinstance(infiles, (str, bytes)): infiles = [infiles] From e88d2ff0a17025306402578378d8f808be87c2a5 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Fri, 20 Nov 2020 16:11:13 +0100 Subject: [PATCH 12/68] doc: remove unnecessary docstring --- sdcflows/workflows/__init__.py | 37 ---------------------------------- 1 file changed, 37 deletions(-) diff --git a/sdcflows/workflows/__init__.py b/sdcflows/workflows/__init__.py index 80f9302559..e69de29bb2 100644 --- a/sdcflows/workflows/__init__.py +++ b/sdcflows/workflows/__init__.py @@ -1,37 +0,0 @@ -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -# vi: set ft=python sts=4 ts=4 sw=4 et: -""" -Fieldmap estimation and unwarping workflows. - -.. _sdc_base : - -Automatic selection of the appropriate susceptibility-distortion correction (SDC) method -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If the dataset metadata indicate that more than one field map acquisition is -``IntendedFor`` (see the `BIDS Specification -`__), -the following priority will be used: - - 1. :ref:`sdc_pepolar` (or **blip-up/blip-down**) - - 2. :ref:`sdc_direct_b0` - - 3. :ref:`sdc_phasediff` - - 4. :ref:`sdc_fieldmapless` - - -Table of behavior (fieldmap use-cases): - -=============== =========== ============= =============== -Fieldmaps found ``use_syn`` ``force_syn`` Action -=============== =========== ============= =============== -True * True Fieldmaps + SyN -True * False Fieldmaps -False * True SyN -False True False SyN -False False False HMC only -=============== =========== ============= =============== - -""" From 103a79f8f603f39119d2294b9c514cbeb826e4e5 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Fri, 20 Nov 2020 16:28:52 +0100 Subject: [PATCH 13/68] maint: run workflows in Linear mode so that pytest-cov keeps up --- sdcflows/workflows/apply/tests/test_registration.py | 2 +- sdcflows/workflows/fit/tests/test_pepolar.py | 2 +- sdcflows/workflows/fit/tests/test_phdiff.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdcflows/workflows/apply/tests/test_registration.py b/sdcflows/workflows/apply/tests/test_registration.py index 2777548297..59d7f64252 100644 --- a/sdcflows/workflows/apply/tests/test_registration.py +++ b/sdcflows/workflows/apply/tests/test_registration.py @@ -86,4 +86,4 @@ def test_registration_wf(tmpdir, datadir, workdir, outdir): if workdir: workflow.base_dir = str(workdir) - workflow.run() + workflow.run(plugin="Linear") diff --git a/sdcflows/workflows/fit/tests/test_pepolar.py b/sdcflows/workflows/fit/tests/test_pepolar.py index 22f8391b87..a223f20a73 100644 --- a/sdcflows/workflows/fit/tests/test_pepolar.py +++ b/sdcflows/workflows/fit/tests/test_pepolar.py @@ -87,4 +87,4 @@ def test_topup_wf(tmpdir, datadir, workdir, outdir, epi_path): if workdir: wf.base_dir = str(workdir) - wf.run() + wf.run(plugin="Linear") diff --git a/sdcflows/workflows/fit/tests/test_phdiff.py b/sdcflows/workflows/fit/tests/test_phdiff.py index 8895d9b10b..a8ed63e291 100644 --- a/sdcflows/workflows/fit/tests/test_phdiff.py +++ b/sdcflows/workflows/fit/tests/test_phdiff.py @@ -75,4 +75,4 @@ def test_phdiff(tmpdir, datadir, workdir, outdir, fmap_path): if workdir: wf.base_dir = str(workdir) - wf.run() + wf.run(plugin="Linear") From eaaaa97f3a0281b492c59f350b84291cb6ecb31a Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Fri, 20 Nov 2020 18:59:42 +0100 Subject: [PATCH 14/68] wip: drafting base workflow leveraging new objects --- sdcflows/conftest.py | 1 + sdcflows/workflows/base.py | 105 +++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 sdcflows/workflows/base.py diff --git a/sdcflows/conftest.py b/sdcflows/conftest.py index 99aa3ebebc..bffaaf275b 100644 --- a/sdcflows/conftest.py +++ b/sdcflows/conftest.py @@ -29,6 +29,7 @@ def add_np(doctest_namespace): doctest_namespace['nb'] = nibabel doctest_namespace['os'] = os doctest_namespace['Path'] = Path + doctest_namespace['layouts'] = layouts for key, val in list(layouts.items()): doctest_namespace[key] = Path(val.root) diff --git a/sdcflows/workflows/base.py b/sdcflows/workflows/base.py new file mode 100644 index 0000000000..540eb2e7b4 --- /dev/null +++ b/sdcflows/workflows/base.py @@ -0,0 +1,105 @@ +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# vi: set ft=python sts=4 ts=4 sw=4 et: +"""Estimate fieldmaps for :abbr:`SDC (susceptibility distortion correction)`.""" +from nipype.pipeline import engine as pe +from nipype.interfaces import utility as niu +from nipype import logging + +from niworkflows.engine.workflows import LiterateWorkflow as Workflow + + +LOGGER = logging.getLogger('nipype.workflow') +DEFAULT_MEMORY_MIN_GB = 0.01 + + +def init_fmap_preproc_wf( + *, + layout, + omp_nthreads, + output_dir, + subject, + debug=False, + name='fmap_preproc_wf', +): + """ + Stage the fieldmap data preprocessing steps of *SDCFlows*. + + Parameters + ---------- + layout : :obj:`bids.layout.BIDSLayout` + An initialized PyBIDS layout. + omp_nthreads : :obj:`int` + Maximum number of threads an individual process may use + output_dir : :obj:`str` + Directory in which to save derivatives + subject : :obj:`str` + Participant label for this single-subject workflow. + debug : :obj:`bool` + Enable debugging outputs + name : :obj:`str`, optional + Workflow name (default: ``"fmap_preproc_wf"``) + + Examples + -------- + >>> init_fmap_preproc_wf( + ... layout=layouts['ds001600'], + ... omp_nthreads=1, + ... output_dir="/tmp", + ... subject="1", + ... ) + [FieldmapEstimation(sources=<4 files>, method=), + FieldmapEstimation(sources=<4 files>, method=), + FieldmapEstimation(sources=<3 files>, method=), + FieldmapEstimation(sources=<2 files>, method=)] + + >>> init_fmap_preproc_wf( + ... layout=layouts['testdata'], + ... omp_nthreads=1, + ... output_dir="/tmp", + ... subject="HCP101006", + ... ) + [FieldmapEstimation(sources=<2 files>, method=), + FieldmapEstimation(sources=<2 files>, method=)] + + """ + from ..fieldmaps import FieldmapEstimation, FieldmapFile + + base_entities = { + "subject": subject, + "extension": [".nii", ".nii.gz"], + "space": None, # Ensure derivatives are not captured + } + + estimators = [] + + # Set up B0 fieldmap strategies: + for fmap in layout.get( + suffix=["fieldmap", "phasediff", "phase1"], **base_entities + ): + e = FieldmapEstimation( + FieldmapFile(fmap.path, metadata=fmap.get_metadata()) + ) + estimators.append(e) + + # A bunch of heuristics to select EPI fieldmaps + sessions = layout.get_sessions() or [None] + for session in sessions: + dirs = layout.get_directions( + suffix="epi", + session=session, + **base_entities, + ) + if len(dirs) > 1: + e = FieldmapEstimation([ + FieldmapFile(fmap.path, metadata=fmap.get_metadata()) + for fmap in layout.get(suffix="epi", session=session, + direction=dirs, **base_entities) + ]) + estimators.append(e) + + for e in estimators: + LOGGER.info( + f"{e.method}:: <{':'.join(s.path.name for s in e.sources)}>." + ) + + return estimators From b45218a84734233ba3c76a7fbbd68010624c5376 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Sun, 22 Nov 2020 11:07:05 +0100 Subject: [PATCH 15/68] enh: increase test coverage of utilities --- sdcflows/interfaces/fmap.py | 71 ++++++------------------- sdcflows/utils/phasemanip.py | 60 +++++++++++++++++++-- sdcflows/utils/tests/__init__.py | 0 sdcflows/utils/tests/test_phasemanip.py | 39 ++++++++++++++ sdcflows/workflows/base.py | 5 -- 5 files changed, 111 insertions(+), 64 deletions(-) create mode 100644 sdcflows/utils/tests/__init__.py create mode 100644 sdcflows/utils/tests/test_phasemanip.py diff --git a/sdcflows/interfaces/fmap.py b/sdcflows/interfaces/fmap.py index 85a949efa6..988d978089 100644 --- a/sdcflows/interfaces/fmap.py +++ b/sdcflows/interfaces/fmap.py @@ -9,7 +9,6 @@ File, traits, SimpleInterface, - InputMultiObject, ) LOGGER = logging.getLogger("nipype.interface") @@ -94,7 +93,7 @@ class Phasediff2Fieldmap(SimpleInterface): output_spec = _Phasediff2FieldmapOutputSpec def _run_interface(self, runtime): - from ..utils.phasemanip import phdiff2fmap + from ..utils.phasemanip import phdiff2fmap, delta_te as _delta_te self._results["out_file"] = phdiff2fmap( self.inputs.in_file, _delta_te(self.inputs.metadata), newpath=runtime.cwd @@ -102,62 +101,24 @@ def _run_interface(self, runtime): return runtime -class _Coefficients2WarpInputSpec(BaseInterfaceInputSpec): - in_target = File(exist=True, mandatory=True, desc="input EPI data to be corrected") - in_coeff = InputMultiObject(File(exists=True), mandatory=True, - desc="input coefficients, after alignment to the EPI data") - ro_time = traits.Float(mandatory=True, desc="EPI readout time") +# Code stub for #119 (B-Spline smoothing) +# class _Coefficients2WarpInputSpec(BaseInterfaceInputSpec): +# in_target = File(exist=True, mandatory=True, desc="input EPI data to be corrected") +# in_coeff = InputMultiObject(File(exists=True), mandatory=True, +# desc="input coefficients, after alignment to the EPI data") +# ro_time = traits.Float(mandatory=True, desc="EPI readout time") -class _Coefficients2WarpOutputSpec(TraitedSpec): - out_warp = File(exists=True) +# class _Coefficients2WarpOutputSpec(TraitedSpec): +# out_warp = File(exists=True) -class Coefficients2Warp(SimpleInterface): - """Convert coefficients to a full displacements map.""" +# class Coefficients2Warp(SimpleInterface): +# """Convert coefficients to a full displacements map.""" - input_spec = _Coefficients2WarpInputSpec - output_spec = _Coefficients2WarpOutputSpec +# input_spec = _Coefficients2WarpInputSpec +# output_spec = _Coefficients2WarpOutputSpec - def _run_interface(self, runtime): - raise NotImplementedError - return runtime - - -def _delta_te(in_values, te1=None, te2=None): - r"""Read :math:`\Delta_\text{TE}` from BIDS metadata dict.""" - if isinstance(in_values, float): - te2 = in_values - te1 = 0.0 - - if isinstance(in_values, dict): - te1 = in_values.get("EchoTime1") - te2 = in_values.get("EchoTime2") - - if not all((te1, te2)): - te2 = in_values.get("EchoTimeDifference") - te1 = 0 - - if isinstance(in_values, list): - te2, te1 = in_values - if isinstance(te1, list): - te1 = te1[1] - if isinstance(te2, list): - te2 = te2[1] - - # For convienience if both are missing we should give one error about them - if te1 is None and te2 is None: - raise RuntimeError( - "EchoTime1 and EchoTime2 metadata fields not found. " - "Please consult the BIDS specification." - ) - if te1 is None: - raise RuntimeError( - "EchoTime1 metadata field not found. Please consult the BIDS specification." - ) - if te2 is None: - raise RuntimeError( - "EchoTime2 metadata field not found. Please consult the BIDS specification." - ) - - return abs(float(te2) - float(te1)) +# def _run_interface(self, runtime): +# raise NotImplementedError +# return runtime diff --git a/sdcflows/utils/phasemanip.py b/sdcflows/utils/phasemanip.py index b0391d677c..088efcd9b1 100644 --- a/sdcflows/utils/phasemanip.py +++ b/sdcflows/utils/phasemanip.py @@ -21,7 +21,7 @@ def au2rads(in_file, newpath=None): hdr.set_data_dtype(np.float32) hdr.set_xyzt_units("mm") - out_file = fname_presuffix(in_file, suffix="_rads", newpath=newpath) + out_file = fname_presuffix(str(in_file), suffix="_rads", newpath=newpath) nb.Nifti1Image(data, None, hdr).to_filename(out_file) return out_file @@ -70,15 +70,67 @@ def subtract_phases(in_phases, in_meta, newpath=None): def phdiff2fmap(in_file, delta_te, newpath=None): """Convert the input phase-difference map into a *fieldmap* in Hz.""" - import math import numpy as np import nibabel as nb from nipype.utils.filemanip import fname_presuffix - out_file = fname_presuffix(in_file, suffix="_fmap", newpath=newpath) + out_file = fname_presuffix(str(in_file), suffix="_fmap", newpath=newpath) image = nb.load(in_file) - data = image.get_fdata(dtype="float32") / (2.0 * math.pi * delta_te) + data = image.get_fdata(dtype="float32") / (2.0 * np.pi * delta_te) nii = nb.Nifti1Image(data, image.affine, image.header) nii.set_data_dtype(np.float32) nii.to_filename(out_file) return out_file + + +def delta_te(in_values): + r""" + Read :math:`\Delta_\text{TE}` from BIDS metadata dict. + + Examples + -------- + >>> t = delta_te({"EchoTime1": 0.00522, "EchoTime2": 0.00768}) + >>> f"{t:.5f}" + '0.00246' + + >>> t = delta_te({'EchoTimeDifference': 0.00246}) + >>> f"{t:.5f}" + '0.00246' + + >>> delta_te({"EchoTime1": "a"}) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ValueError: + + >>> delta_te({"EchoTime2": "a"}) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ValueError: + + >>> delta_te({}) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ValueError: + + """ + te2 = in_values.get("EchoTime2") + te1 = in_values.get("EchoTime1") + + if te2 is None and te1 is None: + try: + te2 = float(in_values.get("EchoTimeDifference")) + return abs(te2) + except TypeError: + raise ValueError( + "Phase/phase-difference fieldmaps: no echo-times information." + ) + except ValueError: + raise ValueError( + f"Could not interpret metadata ." + ) + try: + te2 = float(te2 or "unknown") + te1 = float(te1 or "unknown") + except ValueError: + raise ValueError( + f"Could not interpret metadata ." + ) + + return abs(te2 - te1) diff --git a/sdcflows/utils/tests/__init__.py b/sdcflows/utils/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sdcflows/utils/tests/test_phasemanip.py b/sdcflows/utils/tests/test_phasemanip.py new file mode 100644 index 0000000000..376f1b87f8 --- /dev/null +++ b/sdcflows/utils/tests/test_phasemanip.py @@ -0,0 +1,39 @@ +"""Test phase manipulation routines.""" +import numpy as np +import nibabel as nb + +from ..phasemanip import au2rads, phdiff2fmap + + +def test_au2rads(tmp_path): + """Check the conversion.""" + data = np.random.randint(0, high=4096, size=(5, 5, 5)) + data[0, 0, 0] = 0 + data[-1, -1, -1] = 4096 + + nb.Nifti1Image( + data.astype("int16"), + np.eye(4) + ).to_filename(tmp_path / "testdata.nii.gz") + + out_file = au2rads(tmp_path / "testdata.nii.gz") + + assert np.allclose( + (data / 4096).astype("float32") * 2.0 * np.pi, + nb.load(out_file).get_fdata(dtype="float32") + ) + + +def test_phdiff2fmap(tmp_path): + """Check the conversion.""" + nb.Nifti1Image( + np.ones((5, 5, 5), dtype="float32") * 2.0 * np.pi * 2.46e-3, + np.eye(4) + ).to_filename(tmp_path / "testdata.nii.gz") + + out_file = phdiff2fmap(tmp_path / "testdata.nii.gz", 2.46e-3) + + assert np.allclose( + np.ones((5, 5, 5)), + nb.load(out_file).get_fdata(dtype="float32") + ) diff --git a/sdcflows/workflows/base.py b/sdcflows/workflows/base.py index 540eb2e7b4..4e61b58662 100644 --- a/sdcflows/workflows/base.py +++ b/sdcflows/workflows/base.py @@ -1,13 +1,8 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: """Estimate fieldmaps for :abbr:`SDC (susceptibility distortion correction)`.""" -from nipype.pipeline import engine as pe -from nipype.interfaces import utility as niu from nipype import logging -from niworkflows.engine.workflows import LiterateWorkflow as Workflow - - LOGGER = logging.getLogger('nipype.workflow') DEFAULT_MEMORY_MIN_GB = 0.01 From 74496c5ccb6bde0fd48e6abc84a1eee076c05749 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Fri, 13 Nov 2020 10:55:26 +0100 Subject: [PATCH 16/68] ENH: A 3D tensor B-Spline approximator and extrapolator This PR finally adds an implementation for B-Spline smoothing and extrapolation of fieldmaps. References: #71, #22. Resolves: #72. Resolves: #14. --- sdcflows/interfaces/bspline.py | 188 ++++++++++++++++++++ sdcflows/workflows/fit/fieldmap.py | 26 +-- sdcflows/workflows/fit/tests/test_phdiff.py | 2 +- setup.cfg | 1 + 4 files changed, 203 insertions(+), 14 deletions(-) create mode 100644 sdcflows/interfaces/bspline.py diff --git a/sdcflows/interfaces/bspline.py b/sdcflows/interfaces/bspline.py new file mode 100644 index 0000000000..bf2e344b27 --- /dev/null +++ b/sdcflows/interfaces/bspline.py @@ -0,0 +1,188 @@ +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# vi: set ft=python sts=4 ts=4 sw=4 et: +""" +B-Spline filtering. + + .. testsetup:: + + >>> tmpdir = getfixture('tmpdir') + >>> tmp = tmpdir.chdir() # changing to a temporary directory + >>> nb.Nifti1Image(np.zeros((90, 90, 60)), None, None).to_filename( + ... tmpdir.join('epi.nii.gz').strpath) + +""" +from pathlib import Path +import numpy as np +import nibabel as nb + +from nipype.utils.filemanip import fname_presuffix +from nipype.interfaces.base import ( + BaseInterfaceInputSpec, + TraitedSpec, + File, + traits, + SimpleInterface, + InputMultiObject, + OutputMultiObject, +) + + +DEFAULT_ZOOMS_MM = (40.0, 40.0, 20.0) # For human adults (mid-frequency), in mm +DEFAULT_LF_ZOOMS_MM = (100.0, 100.0, 40.0) # For human adults (low-frequency), in mm +DEFAULT_HF_ZOOMS_MM = (16.0, 16.0, 10.0) # For human adults (high-frequency), in mm + + +class _BSplineApproxInputSpec(BaseInterfaceInputSpec): + in_data = File(exists=True, mandatory=True, desc="path to a fieldmap") + in_mask = File(exists=True, mandatory=True, desc="path to a brain mask") + bs_spacing = InputMultiObject( + [DEFAULT_ZOOMS_MM], + traits.Tuple(traits.Float, traits.Float, traits.Float), + usedefault=True, + desc="spacing between B-Spline control points", + ) + ridge_alpha = traits.Float( + 1e-4, usedefault=True, desc="controls the regularization" + ) + + +class _BSplineApproxOutputSpec(TraitedSpec): + out_field = File(exists=True) + out_coeff = OutputMultiObject(File(exists=True)) + + +class BSplineApprox(SimpleInterface): + """ + Approximate the field to smooth it removing spikes and extrapolating beyond the brain mask. + + Examples + -------- + + """ + + input_spec = _BSplineApproxInputSpec + output_spec = _BSplineApproxOutputSpec + + def _run_interface(self, runtime): + from gridbspline.maths import cubic + from sklearn import linear_model as lm + + _vbspl = np.vectorize(cubic) + + # Load in the fieldmap + fmapnii = nb.load(self.inputs.in_data) + data = fmapnii.get_fdata() + mask = nb.load(self.inputs.in_mask).get_fdata() > 0 + bs_spacing = [np.array(sp, dtype="float32") for sp in self.inputs.bs_spacing] + + # Calculate B-Splines grid(s) + bs_levels = [] + for sp in bs_spacing: + bs_levels.append(bspline_grid(fmapnii, control_zooms_mm=sp)) + + # Calculate spatial location of voxels, and normalize per B-Spline grid + fmap_points = grid_coords(fmapnii) + sample_points = [] + for sp in bs_spacing: + sample_points.append((fmap_points / sp).astype("float32")) + + # Calculate the spatial location of control points + bs_x = [] + ncoeff = [] + for sp, level, points in zip(bs_spacing, bs_levels, sample_points): + ncoeff.append(level.dataobj.size) + control_points = grid_coords(level, control_zooms_mm=sp) + bs_x.append(control_points[:, np.newaxis, :] - points[np.newaxis, ...]) + + # Calculate the cubic spline weights per dimension and tensor-product + dist = np.vstack(bs_x) + dist_support = (np.abs(dist) < 2).all(axis=-1) + weights = _vbspl(dist[dist_support]).prod(axis=-1) + + # Compose the interpolation matrix + interp_mat = np.zeros(dist.shape[:2]) + interp_mat[dist_support] = weights + + # Fit the model + model = lm.Ridge(alpha=self.inputs.ridge_alpha, fit_intercept=False) + model.fit( + interp_mat[..., mask.reshape(-1)].T, # Regress only within brainmask + data[mask], + ) + + # Store outputs + out_name = str( + Path( + fname_presuffix( + self.inputs.in_data, suffix="_field", newpath=runtime.cwd + ) + ).absolute() + ) + hdr = fmapnii.header.copy() + hdr.set_data_dtype("float32") + nb.Nifti1Image( + (model.intercept_ + np.array(model.coef_) @ interp_mat) + .astype("float32") # Interpolation + .reshape(data.shape), + fmapnii.affine, + hdr, + ).to_filename(out_name) + self._results["out_field"] = out_name + + index = 0 + self._results["out_coeff"] = [] + for i, (n, bsl) in enumerate(zip(ncoeff, bs_levels)): + out_level = out_name.replace("_field.", f"_coeff{i:03}.") + nb.Nifti1Image( + np.array(model.coef_, dtype="float32")[index : index + n].reshape( + bsl.shape + ), + bsl.affine, + bsl.header, + ).to_filename(out_level) + index += n + self._results["out_coeff"].append(out_level) + return runtime + + +def bspline_grid(img, control_zooms_mm=DEFAULT_ZOOMS_MM): + """Calculate a Nifti1Image object encoding the location of control points.""" + if isinstance(img, (str, Path)): + img = nb.load(img) + + im_zooms = np.array(img.header.get_zooms()) + im_shape = np.array(img.shape[:3]) + + # Calculate the direction cosines of the target image + dir_cos = img.affine[:3, :3] / im_zooms + + # Initialize the affine of the B-Spline grid + bs_affine = np.diag(np.hstack((np.array(control_zooms_mm) @ dir_cos, 1))) + bs_zooms = nb.affines.voxel_sizes(bs_affine) + + # Calculate the shape of the B-Spline grid + im_extent = im_zooms * (im_shape - 1) + bs_shape = (im_extent // bs_zooms + 3).astype(int) + + # Center both images + im_center = img.affine @ np.hstack((0.5 * (im_shape - 1), 1)) + bs_center = bs_affine @ np.hstack((0.5 * (bs_shape - 1), 1)) + bs_affine[:3, 3] = im_center[:3] - bs_center[:3] + + return nb.Nifti1Image(np.zeros(bs_shape, dtype="float32"), bs_affine) + + +def grid_coords(img, control_zooms_mm=None, dtype="float32"): + """Create a linear space of physical coordinates.""" + if isinstance(img, (str, Path)): + img = nb.load(img) + + grid = np.array( + np.meshgrid(*[range(s) for s in img.shape[:3]]), dtype=dtype + ).reshape(3, -1) + coords = (img.affine @ np.vstack((grid, np.ones(grid.shape[-1])))).T[..., :3] + + if control_zooms_mm is not None: + coords /= np.array(control_zooms_mm) + + return coords.astype(dtype) diff --git a/sdcflows/workflows/fit/fieldmap.py b/sdcflows/workflows/fit/fieldmap.py index 9d0b272288..892097d89a 100644 --- a/sdcflows/workflows/fit/fieldmap.py +++ b/sdcflows/workflows/fit/fieldmap.py @@ -109,7 +109,7 @@ from niworkflows.engine.workflows import LiterateWorkflow as Workflow -def init_fmap_wf(omp_nthreads=1, mode="phasediff", name="fmap_wf"): +def init_fmap_wf(omp_nthreads=1, debug=False, mode="phasediff", name="fmap_wf"): """ Estimate the fieldmap based on a field-mapping MRI acquisition. @@ -156,6 +156,10 @@ def init_fmap_wf(omp_nthreads=1, mode="phasediff", name="fmap_wf"): pair. """ + from ...interfaces.bspline import ( + BSplineApprox, DEFAULT_LF_ZOOMS_MM, DEFAULT_HF_ZOOMS_MM + ) + workflow = Workflow(name=name) inputnode = pe.Node( @@ -167,19 +171,19 @@ def init_fmap_wf(omp_nthreads=1, mode="phasediff", name="fmap_wf"): ) magnitude_wf = init_magnitude_wf(omp_nthreads=omp_nthreads) - fmap_postproc_wf = init_fmap_postproc_wf(omp_nthreads=omp_nthreads) + bs_filter = pe.Node(BSplineApprox( + bs_spacing=[DEFAULT_LF_ZOOMS_MM] if debug else [DEFAULT_LF_ZOOMS_MM, DEFAULT_HF_ZOOMS_MM], + ), n_procs=omp_nthreads, name="bs_filter") # fmt: off workflow.connect([ (inputnode, magnitude_wf, [("magnitude", "inputnode.magnitude")]), - (magnitude_wf, fmap_postproc_wf, [ - ("outputnode.fmap_mask", "inputnode.fmap_mask"), - ("outputnode.fmap_ref", "inputnode.fmap_ref")]), + (magnitude_wf, bs_filter, [("outputnode.fmap_mask", "in_mask")]), (magnitude_wf, outputnode, [ ("outputnode.fmap_mask", "fmap_mask"), ("outputnode.fmap_ref", "fmap_ref"), ]), - (fmap_postproc_wf, outputnode, [("outputnode.out_fmap", "fmap")]), + (bs_filter, outputnode, [("out_field", "fmap")]), ]) # fmt: on @@ -198,13 +202,12 @@ def init_fmap_wf(omp_nthreads=1, mode="phasediff", name="fmap_wf"): ("outputnode.fmap_ref", "inputnode.magnitude"), ("outputnode.fmap_mask", "inputnode.mask"), ]), - (phdiff_wf, fmap_postproc_wf, [ - ("outputnode.fieldmap", "inputnode.fmap"), + (phdiff_wf, bs_filter, [ + ("outputnode.fieldmap", "in_data"), ]), ]) # fmt: on else: - from niworkflows.interfaces.nibabel import ApplyMask from niworkflows.interfaces.images import IntraModalMerge workflow.__desc__ = """\ @@ -215,13 +218,10 @@ def init_fmap_wf(omp_nthreads=1, mode="phasediff", name="fmap_wf"): fmapmrg = pe.Node( IntraModalMerge(zero_based_avg=False, hmc=False), name="fmapmrg" ) - applymsk = pe.Node(ApplyMask(), name="applymsk") # fmt: off workflow.connect([ (inputnode, fmapmrg, [("fieldmap", "in_files")]), - (fmapmrg, applymsk, [("out_avg", "in_file")]), - (magnitude_wf, applymsk, [("outputnode.fmap_mask", "in_mask")]), - (applymsk, fmap_postproc_wf, [("out_file", "inputnode.fmap")]), + (fmapmrg, bs_filter, [("out_avg", "in_data")]), ]) # fmt: on diff --git a/sdcflows/workflows/fit/tests/test_phdiff.py b/sdcflows/workflows/fit/tests/test_phdiff.py index a8ed63e291..8365de5f5f 100644 --- a/sdcflows/workflows/fit/tests/test_phdiff.py +++ b/sdcflows/workflows/fit/tests/test_phdiff.py @@ -34,7 +34,7 @@ def test_phdiff(tmpdir, datadir, workdir, outdir, fmap_path): wf = Workflow( name=f"phdiff_{fmap_path[0].name.replace('.nii.gz', '').replace('-', '_')}" ) - phdiff_wf = init_fmap_wf(omp_nthreads=2) + phdiff_wf = init_fmap_wf(omp_nthreads=2, debug=True) phdiff_wf.inputs.inputnode.fieldmap = fieldmaps phdiff_wf.inputs.inputnode.magnitude = [ f.replace("diff", "1").replace("phase", "magnitude") for f, _ in fieldmaps diff --git a/setup.cfg b/setup.cfg index aa421ca242..4e389ffaba 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,6 +26,7 @@ setup_requires = setuptools_scm >= 3.4 toml install_requires = + gridbspline nibabel >=3.0.1 niflow-nipype1-workflows ~= 0.0.1 nipype >=1.5.1,<2.0 From a6548b3e76a2b6926b11f3402db7f4e5da550ce4 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Sun, 15 Nov 2020 09:11:02 +0100 Subject: [PATCH 17/68] enh: alleviate memory utilization --- sdcflows/interfaces/bspline.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/sdcflows/interfaces/bspline.py b/sdcflows/interfaces/bspline.py index bf2e344b27..4a99e24bce 100644 --- a/sdcflows/interfaces/bspline.py +++ b/sdcflows/interfaces/bspline.py @@ -87,21 +87,24 @@ def _run_interface(self, runtime): sample_points.append((fmap_points / sp).astype("float32")) # Calculate the spatial location of control points - bs_x = [] + w_x = [] ncoeff = [] for sp, level, points in zip(bs_spacing, bs_levels, sample_points): ncoeff.append(level.dataobj.size) control_points = grid_coords(level, control_zooms_mm=sp) - bs_x.append(control_points[:, np.newaxis, :] - points[np.newaxis, ...]) + _w = _vbspl( + control_points[:, np.newaxis, :] - points[np.newaxis, ...] + ).prod(axis=-1, dtype="float32") + _w[_w < 1e-6] = 0.0 + w_x.append(_w) # Calculate the cubic spline weights per dimension and tensor-product - dist = np.vstack(bs_x) - dist_support = (np.abs(dist) < 2).all(axis=-1) - weights = _vbspl(dist[dist_support]).prod(axis=-1) + weights = np.vstack(w_x) + dist_support = weights > 0.0 # Compose the interpolation matrix - interp_mat = np.zeros(dist.shape[:2]) - interp_mat[dist_support] = weights + interp_mat = np.zeros((np.sum(ncoeff), data.size)) + interp_mat[dist_support] = weights[dist_support] # Fit the model model = lm.Ridge(alpha=self.inputs.ridge_alpha, fit_intercept=False) From ff71d6922baff5b4341b359de1e4cce6e818f5d8 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Sun, 15 Nov 2020 09:40:52 +0100 Subject: [PATCH 18/68] enh: second attempt to alleviate memory issues --- sdcflows/interfaces/bspline.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/sdcflows/interfaces/bspline.py b/sdcflows/interfaces/bspline.py index 4a99e24bce..a47b0aa133 100644 --- a/sdcflows/interfaces/bspline.py +++ b/sdcflows/interfaces/bspline.py @@ -72,6 +72,7 @@ def _run_interface(self, runtime): # Load in the fieldmap fmapnii = nb.load(self.inputs.in_data) data = fmapnii.get_fdata() + nsamples = data.size mask = nb.load(self.inputs.in_mask).get_fdata() > 0 bs_spacing = [np.array(sp, dtype="float32") for sp in self.inputs.bs_spacing] @@ -91,19 +92,23 @@ def _run_interface(self, runtime): ncoeff = [] for sp, level, points in zip(bs_spacing, bs_levels, sample_points): ncoeff.append(level.dataobj.size) - control_points = grid_coords(level, control_zooms_mm=sp) - _w = _vbspl( - control_points[:, np.newaxis, :] - points[np.newaxis, ...] - ).prod(axis=-1, dtype="float32") + d = ( + grid_coords(level, control_zooms_mm=sp)[:, np.newaxis, :] + - points[np.newaxis, ...] + ) + d_support = (np.abs(d) < 2).all(axis=-1) + _w = _vbspl(d[d_support]).prod(axis=-1, dtype="float32") _w[_w < 1e-6] = 0.0 - w_x.append(_w) + w = np.zeros((ncoeff[-1], nsamples), dtype="float32") + w[d_support] = _w + w_x.append(w) # Calculate the cubic spline weights per dimension and tensor-product weights = np.vstack(w_x) dist_support = weights > 0.0 # Compose the interpolation matrix - interp_mat = np.zeros((np.sum(ncoeff), data.size)) + interp_mat = np.zeros((np.sum(ncoeff), nsamples)) interp_mat[dist_support] = weights[dist_support] # Fit the model From f1d8f779a332955abd18c574f346bbe8d4060499 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Mon, 16 Nov 2020 08:47:33 +0100 Subject: [PATCH 19/68] enh: try a pure-numpy implementation --- sdcflows/interfaces/bspline.py | 31 +++++++++++++++++-------------- setup.cfg | 1 - 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/sdcflows/interfaces/bspline.py b/sdcflows/interfaces/bspline.py index a47b0aa133..1dbce54f46 100644 --- a/sdcflows/interfaces/bspline.py +++ b/sdcflows/interfaces/bspline.py @@ -64,11 +64,8 @@ class BSplineApprox(SimpleInterface): output_spec = _BSplineApproxOutputSpec def _run_interface(self, runtime): - from gridbspline.maths import cubic from sklearn import linear_model as lm - _vbspl = np.vectorize(cubic) - # Load in the fieldmap fmapnii = nb.load(self.inputs.in_data) data = fmapnii.get_fdata() @@ -88,23 +85,29 @@ def _run_interface(self, runtime): sample_points.append((fmap_points / sp).astype("float32")) # Calculate the spatial location of control points - w_x = [] + w_l = [] ncoeff = [] for sp, level, points in zip(bs_spacing, bs_levels, sample_points): ncoeff.append(level.dataobj.size) - d = ( - grid_coords(level, control_zooms_mm=sp)[:, np.newaxis, :] - - points[np.newaxis, ...] - ) - d_support = (np.abs(d) < 2).all(axis=-1) - _w = _vbspl(d[d_support]).prod(axis=-1, dtype="float32") + _w = np.ones((ncoeff[-1], nsamples), dtype="float32") + + _gc = grid_coords(level, control_zooms_mm=sp) + + for i in range(3): + d = np.abs((_gc[:, np.newaxis, i] - points[np.newaxis, :, i])[_w > 1e-6]) + _w[_w > 1e-6] *= np.piecewise( + d, + [d >= 2.0, d < 1.0, (d >= 1.0) & (d < 2)], + [0., + lambda d: (4. - 6. * d ** 2 + 3. * d ** 3) / 6., + lambda d: (2. - d) ** 3 / 6.] + ) + _w[_w < 1e-6] = 0.0 - w = np.zeros((ncoeff[-1], nsamples), dtype="float32") - w[d_support] = _w - w_x.append(w) + w_l.append(_w) # Calculate the cubic spline weights per dimension and tensor-product - weights = np.vstack(w_x) + weights = np.vstack(w_l) dist_support = weights > 0.0 # Compose the interpolation matrix diff --git a/setup.cfg b/setup.cfg index 4e389ffaba..aa421ca242 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,7 +26,6 @@ setup_requires = setuptools_scm >= 3.4 toml install_requires = - gridbspline nibabel >=3.0.1 niflow-nipype1-workflows ~= 0.0.1 nipype >=1.5.1,<2.0 From 4162dd9f7c9e8ac0bb39967af09f426c9de2a14b Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Mon, 16 Nov 2020 10:37:41 +0100 Subject: [PATCH 20/68] enh: do fit the intercept (but do not use it in interpolation) --- sdcflows/interfaces/bspline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdcflows/interfaces/bspline.py b/sdcflows/interfaces/bspline.py index 1dbce54f46..72211e8b93 100644 --- a/sdcflows/interfaces/bspline.py +++ b/sdcflows/interfaces/bspline.py @@ -115,7 +115,7 @@ def _run_interface(self, runtime): interp_mat[dist_support] = weights[dist_support] # Fit the model - model = lm.Ridge(alpha=self.inputs.ridge_alpha, fit_intercept=False) + model = lm.Ridge(alpha=self.inputs.ridge_alpha, fit_intercept=True) model.fit( interp_mat[..., mask.reshape(-1)].T, # Regress only within brainmask data[mask], @@ -132,7 +132,7 @@ def _run_interface(self, runtime): hdr = fmapnii.header.copy() hdr.set_data_dtype("float32") nb.Nifti1Image( - (model.intercept_ + np.array(model.coef_) @ interp_mat) + (np.array(model.coef_) @ interp_mat) .astype("float32") # Interpolation .reshape(data.shape), fmapnii.affine, From 450fe99e324174eadd7c4ba9374a828dcc5dcd95 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Mon, 16 Nov 2020 02:20:00 -0800 Subject: [PATCH 21/68] enh: recenter the distribution before fitting the B-Splines --- sdcflows/interfaces/bspline.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdcflows/interfaces/bspline.py b/sdcflows/interfaces/bspline.py index 72211e8b93..f606b9af00 100644 --- a/sdcflows/interfaces/bspline.py +++ b/sdcflows/interfaces/bspline.py @@ -72,7 +72,8 @@ def _run_interface(self, runtime): nsamples = data.size mask = nb.load(self.inputs.in_mask).get_fdata() > 0 bs_spacing = [np.array(sp, dtype="float32") for sp in self.inputs.bs_spacing] - + data -= np.median(data[mask]) + # Calculate B-Splines grid(s) bs_levels = [] for sp in bs_spacing: From 39145c2aa9d20af1026d8f4a5a8615eeadbcc930 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Mon, 16 Nov 2020 11:35:45 +0100 Subject: [PATCH 22/68] enh: allow recentering data before and after fitting --- sdcflows/interfaces/bspline.py | 46 ++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/sdcflows/interfaces/bspline.py b/sdcflows/interfaces/bspline.py index f606b9af00..57a985dc89 100644 --- a/sdcflows/interfaces/bspline.py +++ b/sdcflows/interfaces/bspline.py @@ -44,11 +44,16 @@ class _BSplineApproxInputSpec(BaseInterfaceInputSpec): ridge_alpha = traits.Float( 1e-4, usedefault=True, desc="controls the regularization" ) + recenter = traits.Enum( + "mode", "median", "mean", "no", usedefault=True, + desc="strategy to recenter the distribution of the input fieldmap" + ) class _BSplineApproxOutputSpec(TraitedSpec): out_field = File(exists=True) out_coeff = OutputMultiObject(File(exists=True)) + out_error = File(exists=True) class BSplineApprox(SimpleInterface): @@ -72,8 +77,16 @@ def _run_interface(self, runtime): nsamples = data.size mask = nb.load(self.inputs.in_mask).get_fdata() > 0 bs_spacing = [np.array(sp, dtype="float32") for sp in self.inputs.bs_spacing] - data -= np.median(data[mask]) - + + # Recenter the fieldmap + if self.inputs.recenter == "mode": + from scipy.stats import mode + data -= mode(data[mask], axis=None)[0][0] + elif self.inputs.recenter == "median": + data -= np.median(data[mask]) + elif self.inputs.recenter == "mean": + data -= np.mean(data[mask]) + # Calculate B-Splines grid(s) bs_levels = [] for sp in bs_spacing: @@ -122,6 +135,20 @@ def _run_interface(self, runtime): data[mask], ) + fit_data = ( + (np.array(model.coef_) @ interp_mat) # Interpolation + .astype("float32") + .reshape(data.shape) + ) + # Recenter the fieldmap + if self.inputs.recenter == "mode": + from scipy.stats import mode + fit_data -= mode(fit_data[mask], axis=None)[0][0] + elif self.inputs.recenter == "median": + fit_data -= np.median(fit_data[mask]) + elif self.inputs.recenter == "mean": + fit_data -= np.mean(fit_data[mask]) + # Store outputs out_name = str( Path( @@ -132,13 +159,7 @@ def _run_interface(self, runtime): ) hdr = fmapnii.header.copy() hdr.set_data_dtype("float32") - nb.Nifti1Image( - (np.array(model.coef_) @ interp_mat) - .astype("float32") # Interpolation - .reshape(data.shape), - fmapnii.affine, - hdr, - ).to_filename(out_name) + nb.Nifti1Image(fit_data, fmapnii.affine, hdr).to_filename(out_name) self._results["out_field"] = out_name index = 0 @@ -146,7 +167,7 @@ def _run_interface(self, runtime): for i, (n, bsl) in enumerate(zip(ncoeff, bs_levels)): out_level = out_name.replace("_field.", f"_coeff{i:03}.") nb.Nifti1Image( - np.array(model.coef_, dtype="float32")[index : index + n].reshape( + np.array(model.coef_, dtype="float32")[index:index + n].reshape( bsl.shape ), bsl.affine, @@ -154,6 +175,11 @@ def _run_interface(self, runtime): ).to_filename(out_level) index += n self._results["out_coeff"].append(out_level) + + # Write out fitting-error map + self._results["out_error"] = out_name.replace("_field.", "_error.") + nb.Nifti1Image(data - fit_data * mask, fmapnii.affine, fmapnii.header).to_filename( + self._results["out_error"]) return runtime From 852e1661f426c374d819cb1be143fc8d875b087d Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Mon, 16 Nov 2020 11:37:39 +0100 Subject: [PATCH 23/68] maint: cleanup after the old niflow is not necessary --- docs/requirements.txt | 1 - sdcflows/workflows/fit/fieldmap.py | 133 ----------------------- sdcflows/workflows/fit/tests/test_fit.py | 1 - setup.cfg | 1 - 4 files changed, 136 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 3e4809c4c3..d740f24fb1 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,6 @@ git+https://github.com/AleksandarPetrov/napoleon.git@0dc3f28a309ad602be5f44a9049785a1026451b3#egg=sphinxcontrib-napoleon git+https://github.com/rwblair/sphinxcontrib-versioning.git@39b40b0b84bf872fc398feff05344051bbce0f63#egg=sphinxcontrib-versioning nbsphinx -niflow-nipype1-workflows ~= 0.0.1 nipype>=1.3.1 git+https://github.com/nipreps/niworkflows.git@master packaging diff --git a/sdcflows/workflows/fit/fieldmap.py b/sdcflows/workflows/fit/fieldmap.py index 892097d89a..56ca208d22 100644 --- a/sdcflows/workflows/fit/fieldmap.py +++ b/sdcflows/workflows/fit/fieldmap.py @@ -405,136 +405,3 @@ def _split(phase): ]) # fmt: on return workflow - - -def init_fmap_postproc_wf(omp_nthreads, median_kernel_size=5, name="fmap_postproc_wf"): - """ - Postprocess a :math:`B_0` map estimated elsewhere. - - This workflow denoises (mostly via smoothing) a :math:`B_0` fieldmap. - - Workflow Graph - .. workflow :: - :graph2use: orig - :simple_form: yes - - from sdcflows.workflows.fit.fieldmap import init_fmap_postproc_wf - wf = init_fmap_postproc_wf(omp_nthreads=6) - - Parameters - ---------- - omp_nthreads : :obj:`int` - Maximum number of threads an individual process may use - median_kernel_size : :obj:`int` - Size of the kernel when smoothing is done with a median filter. - name : :obj:`str` - Name of workflow (default: ``fmap_postproc_wf``) - - Inputs - ------ - fmap : :obj:`os.PathLike` - Fully preprocessed :math:`B_0` field nonuniformity map (aka *fieldmap*). - fmap_ref : :obj:`os.PathLike` - A preprocessed magnitude/reference image for the fieldmap. - fmap_mask : :obj:`os.PathLike` - A brain binary mask corresponding to this fieldmap. - - Outputs - ------- - out_fmap : :obj:`os.PathLike` - Postprocessed fieldmap. - - """ - from nipype.interfaces.fsl import SpatialFilter - from niflow.nipype1.workflows.dmri.fsl.utils import cleanup_edge_pipeline - - workflow = Workflow(name=name) - inputnode = pe.Node( - niu.IdentityInterface(fields=["fmap_mask", "fmap_ref", "fmap", "metadata"]), - name="inputnode", - ) - outputnode = pe.Node( - niu.IdentityInterface(fields=["out_fmap", "metadata"]), name="outputnode" - ) - - def _recenter(in_file): - """Recenter the phase-map distribution to the -pi..pi range.""" - from os import getcwd - import numpy as np - import nibabel as nb - from nipype.utils.filemanip import fname_presuffix - - nii = nb.load(in_file) - data = nii.get_fdata(dtype="float32") - msk = data != 0 - msk[data == 0] = False - data[msk] -= np.median(data[msk]) - - out_file = fname_presuffix(in_file, suffix="_recentered", newpath=getcwd()) - nb.Nifti1Image(data, nii.affine, nii.header).to_filename(out_file) - return out_file - - recenter = pe.Node( - niu.Function(function=_recenter), name="recenter", run_without_submitting=True, - ) - denoise = pe.Node( - SpatialFilter( - operation="median", kernel_shape="sphere", kernel_size=median_kernel_size, - ), - name="denoise", - ) - - def _demean(in_file, in_mask=None, usemode=True): - """ - Subtract the median (since it is robuster than the mean) from a map. - - Parameters - ---------- - usemode : :obj:`bool` - Use the mode instead of the median (should be even more robust - against outliers). - - """ - from os import getcwd - import numpy as np - import nibabel as nb - from nipype.utils.filemanip import fname_presuffix - - nii = nb.load(in_file) - data = nii.get_fdata(dtype="float32") - - msk = np.ones_like(data, dtype=bool) - if in_mask is not None: - msk[nb.load(in_mask).get_fdata(dtype="float32") < 1e-4] = False - - if usemode: - from scipy.stats import mode - - data[msk] -= mode(data[msk], axis=None)[0][0] - else: - data[msk] -= np.median(data[msk], axis=None) - - out_file = fname_presuffix(in_file, suffix="_demean", newpath=getcwd()) - nb.Nifti1Image(data, nii.affine, nii.header).to_filename(out_file) - return out_file - - demean = pe.Node(niu.Function(function=_demean), name="demean") - cleanup_wf = cleanup_edge_pipeline(name="cleanup_wf") - - def _pop(inlist): - if isinstance(inlist, (tuple, list)): - return inlist[0] - return inlist - - # fmt: off - workflow.connect([ - (inputnode, cleanup_wf, [("fmap_mask", "inputnode.in_mask")]), - (inputnode, recenter, [(("fmap", _pop), "in_file")]), - (recenter, denoise, [("out", "in_file")]), - (denoise, demean, [("out_file", "in_file")]), - (demean, cleanup_wf, [("out", "inputnode.in_file")]), - (cleanup_wf, outputnode, [("outputnode.out_file", "out_fmap")]), - (inputnode, outputnode, [(("metadata", _pop), "metadata")]), - ]) - # fmt: on - return workflow diff --git a/sdcflows/workflows/fit/tests/test_fit.py b/sdcflows/workflows/fit/tests/test_fit.py index a4df18756f..c97b9ca096 100644 --- a/sdcflows/workflows/fit/tests/test_fit.py +++ b/sdcflows/workflows/fit/tests/test_fit.py @@ -15,7 +15,6 @@ ("sdcflows.workflows.fit.syn.init_syn_sdc_wf", {}), ("sdcflows.workflows.fit.fieldmap.init_fmap_wf", {}), ("sdcflows.workflows.fit.fieldmap.init_fmap_wf", {"mode": "fieldmap"}), - ("sdcflows.workflows.fit.fieldmap.init_fmap_postproc_wf", {"omp_nthreads": 1}), ("sdcflows.workflows.fit.fieldmap.init_phdiff_wf", {"omp_nthreads": 1}), ), ) diff --git a/setup.cfg b/setup.cfg index aa421ca242..b8a4ef6564 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,7 +27,6 @@ setup_requires = toml install_requires = nibabel >=3.0.1 - niflow-nipype1-workflows ~= 0.0.1 nipype >=1.5.1,<2.0 niworkflows ~= 1.3.0 numpy From 56587da8742ba4364a630110ad56e205bf89a3b2 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Mon, 16 Nov 2020 15:55:08 +0100 Subject: [PATCH 24/68] fix: do not recenter after fitting, do not fit an intercept Co-authored-by: Chris Markiewicz --- sdcflows/interfaces/bspline.py | 173 ++++++++++++++++------------- sdcflows/workflows/fit/fieldmap.py | 17 ++- 2 files changed, 109 insertions(+), 81 deletions(-) diff --git a/sdcflows/interfaces/bspline.py b/sdcflows/interfaces/bspline.py index 57a985dc89..0c1ac914e6 100644 --- a/sdcflows/interfaces/bspline.py +++ b/sdcflows/interfaces/bspline.py @@ -14,6 +14,7 @@ from pathlib import Path import numpy as np import nibabel as nb +from nibabel.affines import apply_affine from nipype.utils.filemanip import fname_presuffix from nipype.interfaces.base import ( @@ -45,8 +46,17 @@ class _BSplineApproxInputSpec(BaseInterfaceInputSpec): 1e-4, usedefault=True, desc="controls the regularization" ) recenter = traits.Enum( - "mode", "median", "mean", "no", usedefault=True, - desc="strategy to recenter the distribution of the input fieldmap" + "mode", + "median", + "mean", + "no", + usedefault=True, + desc="strategy to recenter the distribution of the input fieldmap", + ) + extrapolate = traits.Bool( + True, + usedefault=True, + desc="generate a field, extrapolated outside the brain mask", ) @@ -54,6 +64,7 @@ class _BSplineApproxOutputSpec(TraitedSpec): out_field = File(exists=True) out_coeff = OutputMultiObject(File(exists=True)) out_error = File(exists=True) + out_extrapolated = File() class BSplineApprox(SimpleInterface): @@ -73,93 +84,55 @@ def _run_interface(self, runtime): # Load in the fieldmap fmapnii = nb.load(self.inputs.in_data) - data = fmapnii.get_fdata() - nsamples = data.size + data = fmapnii.get_fdata(dtype="float32") + oriented_nii = canonical_orientation(fmapnii) + oriented_nii.to_filename("data.nii.gz") mask = nb.load(self.inputs.in_mask).get_fdata() > 0 bs_spacing = [np.array(sp, dtype="float32") for sp in self.inputs.bs_spacing] # Recenter the fieldmap if self.inputs.recenter == "mode": from scipy.stats import mode + data -= mode(data[mask], axis=None)[0][0] elif self.inputs.recenter == "median": data -= np.median(data[mask]) elif self.inputs.recenter == "mean": data -= np.mean(data[mask]) - # Calculate B-Splines grid(s) - bs_levels = [] - for sp in bs_spacing: - bs_levels.append(bspline_grid(fmapnii, control_zooms_mm=sp)) - # Calculate spatial location of voxels, and normalize per B-Spline grid - fmap_points = grid_coords(fmapnii) - sample_points = [] - for sp in bs_spacing: - sample_points.append((fmap_points / sp).astype("float32")) + mask_indices = np.argwhere(mask) + fmap_points = apply_affine( + oriented_nii.affine.astype("float32"), mask_indices + ) # Calculate the spatial location of control points + bs_levels = [] w_l = [] ncoeff = [] - for sp, level, points in zip(bs_spacing, bs_levels, sample_points): + for sp in bs_spacing: + level = bspline_grid(oriented_nii, control_zooms_mm=sp) + bs_levels.append(level) ncoeff.append(level.dataobj.size) - _w = np.ones((ncoeff[-1], nsamples), dtype="float32") - - _gc = grid_coords(level, control_zooms_mm=sp) - - for i in range(3): - d = np.abs((_gc[:, np.newaxis, i] - points[np.newaxis, :, i])[_w > 1e-6]) - _w[_w > 1e-6] *= np.piecewise( - d, - [d >= 2.0, d < 1.0, (d >= 1.0) & (d < 2)], - [0., - lambda d: (4. - 6. * d ** 2 + 3. * d ** 3) / 6., - lambda d: (2. - d) ** 3 / 6.] - ) - - _w[_w < 1e-6] = 0.0 - w_l.append(_w) - - # Calculate the cubic spline weights per dimension and tensor-product - weights = np.vstack(w_l) - dist_support = weights > 0.0 + w_l.append(bspline_weights(fmap_points, level)) # Compose the interpolation matrix - interp_mat = np.zeros((np.sum(ncoeff), nsamples)) - interp_mat[dist_support] = weights[dist_support] + regressors = np.vstack(w_l) # Fit the model - model = lm.Ridge(alpha=self.inputs.ridge_alpha, fit_intercept=True) - model.fit( - interp_mat[..., mask.reshape(-1)].T, # Regress only within brainmask - data[mask], - ) + model = lm.Ridge(alpha=self.inputs.ridge_alpha, fit_intercept=False) + model.fit(regressors.T, data[mask]) - fit_data = ( - (np.array(model.coef_) @ interp_mat) # Interpolation - .astype("float32") - .reshape(data.shape) - ) - # Recenter the fieldmap - if self.inputs.recenter == "mode": - from scipy.stats import mode - fit_data -= mode(fit_data[mask], axis=None)[0][0] - elif self.inputs.recenter == "median": - fit_data -= np.median(fit_data[mask]) - elif self.inputs.recenter == "mean": - fit_data -= np.mean(fit_data[mask]) + interp_data = np.zeros_like(data) + interp_data[mask] = np.array(model.coef_) @ regressors # Interpolation # Store outputs - out_name = str( - Path( - fname_presuffix( - self.inputs.in_data, suffix="_field", newpath=runtime.cwd - ) - ).absolute() + out_name = fname_presuffix( + self.inputs.in_data, suffix="_field", newpath=runtime.cwd ) hdr = fmapnii.header.copy() hdr.set_data_dtype("float32") - nb.Nifti1Image(fit_data, fmapnii.affine, hdr).to_filename(out_name) + nb.Nifti1Image(interp_data, fmapnii.affine, hdr).to_filename(out_name) self._results["out_field"] = out_name index = 0 @@ -178,11 +151,42 @@ def _run_interface(self, runtime): # Write out fitting-error map self._results["out_error"] = out_name.replace("_field.", "_error.") - nb.Nifti1Image(data - fit_data * mask, fmapnii.affine, fmapnii.header).to_filename( - self._results["out_error"]) + nb.Nifti1Image( + data * mask - interp_data, fmapnii.affine, fmapnii.header + ).to_filename(self._results["out_error"]) + + if not self.inputs.extrapolate: + return runtime + + bg_indices = np.argwhere(~mask) + bg_points = apply_affine( + oriented_nii.affine.astype("float32"), bg_indices + ) + + extrapolators = np.vstack( + [bspline_weights(bg_points, level) for level in bs_levels] + ) + interp_data[~mask] = np.array(model.coef_) @ extrapolators # Extrapolation + self._results["out_extrapolated"] = out_name.replace("_field.", "_extra.") + nb.Nifti1Image(interp_data, fmapnii.affine, hdr).to_filename( + self._results["out_extrapolated"] + ) return runtime +def canonical_orientation(img): + """Generate an alternative image aligned with the array axes.""" + if isinstance(img, (str, Path)): + img = nb.load(img) + + shape = np.array(img.shape[:3]) + affine = np.diag(np.hstack((img.header.get_zooms()[:3], 1))) + affine[:3, 3] -= affine[:3, :3] @ (0.5 * (shape - 1)) + nii = nb.Nifti1Image(img.dataobj, affine) + nii.header.set_xyzt_units(*img.header.get_xyzt_units()) + return nii + + def bspline_grid(img, control_zooms_mm=DEFAULT_ZOOMS_MM): """Calculate a Nifti1Image object encoding the location of control points.""" if isinstance(img, (str, Path)): @@ -195,7 +199,8 @@ def bspline_grid(img, control_zooms_mm=DEFAULT_ZOOMS_MM): dir_cos = img.affine[:3, :3] / im_zooms # Initialize the affine of the B-Spline grid - bs_affine = np.diag(np.hstack((np.array(control_zooms_mm) @ dir_cos, 1))) + bs_affine = np.eye(4) + bs_affine[:3, :3] = np.array(control_zooms_mm) * dir_cos bs_zooms = nb.affines.voxel_sizes(bs_affine) # Calculate the shape of the B-Spline grid @@ -210,17 +215,33 @@ def bspline_grid(img, control_zooms_mm=DEFAULT_ZOOMS_MM): return nb.Nifti1Image(np.zeros(bs_shape, dtype="float32"), bs_affine) -def grid_coords(img, control_zooms_mm=None, dtype="float32"): - """Create a linear space of physical coordinates.""" - if isinstance(img, (str, Path)): - img = nb.load(img) - - grid = np.array( - np.meshgrid(*[range(s) for s in img.shape[:3]]), dtype=dtype - ).reshape(3, -1) - coords = (img.affine @ np.vstack((grid, np.ones(grid.shape[-1])))).T[..., :3] +def bspline_weights(points, level): + """Calculate the tensor-product cubic B-Spline weights for a list of 3D points.""" + ctl_spacings = [float(sp) for sp in level.header.get_zooms()[:3]] + ncoeff = level.dataobj.size + ctl_points = apply_affine( + level.affine.astype("float32"), np.argwhere(np.isclose(level.dataobj, 0)) + ) - if control_zooms_mm is not None: - coords /= np.array(control_zooms_mm) + weights = np.ones((ncoeff, points.shape[0]), dtype="float32") + for i in range(3): + d = ( + np.abs( + (ctl_points[:, np.newaxis, i] - points[np.newaxis, :, i])[ + weights > 1e-6 + ] + ) + / ctl_spacings[i] + ) + weights[weights > 1e-6] *= np.piecewise( + d, + [d >= 2.0, d < 1.0, (d >= 1.0) & (d < 2)], + [ + 0.0, + lambda d: (4.0 - 6.0 * d ** 2 + 3.0 * d ** 3) / 6.0, + lambda d: (2.0 - d) ** 3 / 6.0, + ], + ) - return coords.astype(dtype) + weights[weights < 1e-6] = 0.0 + return weights diff --git a/sdcflows/workflows/fit/fieldmap.py b/sdcflows/workflows/fit/fieldmap.py index 56ca208d22..59d9574c22 100644 --- a/sdcflows/workflows/fit/fieldmap.py +++ b/sdcflows/workflows/fit/fieldmap.py @@ -157,7 +157,10 @@ def init_fmap_wf(omp_nthreads=1, debug=False, mode="phasediff", name="fmap_wf"): """ from ...interfaces.bspline import ( - BSplineApprox, DEFAULT_LF_ZOOMS_MM, DEFAULT_HF_ZOOMS_MM + BSplineApprox, + DEFAULT_LF_ZOOMS_MM, + DEFAULT_HF_ZOOMS_MM, + DEFAULT_ZOOMS_MM, ) workflow = Workflow(name=name) @@ -171,9 +174,12 @@ def init_fmap_wf(omp_nthreads=1, debug=False, mode="phasediff", name="fmap_wf"): ) magnitude_wf = init_magnitude_wf(omp_nthreads=omp_nthreads) - bs_filter = pe.Node(BSplineApprox( - bs_spacing=[DEFAULT_LF_ZOOMS_MM] if debug else [DEFAULT_LF_ZOOMS_MM, DEFAULT_HF_ZOOMS_MM], - ), n_procs=omp_nthreads, name="bs_filter") + bs_filter = pe.Node(BSplineApprox(), n_procs=omp_nthreads, name="bs_filter") + bs_filter.interface._always_run = debug + bs_filter.inputs.bs_spacing = ( + [DEFAULT_LF_ZOOMS_MM, DEFAULT_HF_ZOOMS_MM] if not debug else [DEFAULT_ZOOMS_MM] + ) + bs_filter.inputs.extrapolate = not debug # fmt: off workflow.connect([ @@ -183,7 +189,8 @@ def init_fmap_wf(omp_nthreads=1, debug=False, mode="phasediff", name="fmap_wf"): ("outputnode.fmap_mask", "fmap_mask"), ("outputnode.fmap_ref", "fmap_ref"), ]), - (bs_filter, outputnode, [("out_field", "fmap")]), + (bs_filter, outputnode, [ + ("out_extrapolated" if not debug else "out_field", "fmap")]), ]) # fmt: on From 9138bb22768d55f2b82e859c78b45fa11ef23b6e Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Mon, 23 Nov 2020 17:20:43 +0100 Subject: [PATCH 25/68] docs: add methodological references for the implementation of B-Splines Co-authored-by: Chris Markiewicz --- sdcflows/interfaces/bspline.py | 97 +++++++++++++++++++++++++++------- 1 file changed, 77 insertions(+), 20 deletions(-) diff --git a/sdcflows/interfaces/bspline.py b/sdcflows/interfaces/bspline.py index 0c1ac914e6..bd1f4206c1 100644 --- a/sdcflows/interfaces/bspline.py +++ b/sdcflows/interfaces/bspline.py @@ -1,16 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -""" -B-Spline filtering. - - .. testsetup:: - - >>> tmpdir = getfixture('tmpdir') - >>> tmp = tmpdir.chdir() # changing to a temporary directory - >>> nb.Nifti1Image(np.zeros((90, 90, 60)), None, None).to_filename( - ... tmpdir.join('epi.nii.gz').strpath) - -""" +"""Filtering of :math:`B_0` field mappings with B-Splines.""" from pathlib import Path import numpy as np import nibabel as nb @@ -68,11 +58,35 @@ class _BSplineApproxOutputSpec(TraitedSpec): class BSplineApprox(SimpleInterface): - """ - Approximate the field to smooth it removing spikes and extrapolating beyond the brain mask. + r""" + Approximate the :math:`B_0` field using tensor-product B-Splines. + + The approximation effectively smooths the data, removing spikes and other + sources of noise, as well as enables the extrapolation of the :math:`B_0` field + beyond the brain mask, which alleviates boundary effects in correction. - Examples + This interface resolves the optimization problem of obtaining the B-Spline coefficients + :math:`c(\mathbf{k})` that best approximate the data samples within the + brain mask :math:`f(\mathbf{s})`, following Eq. (17) -- in that case for 2D -- + of [Unser1999]_. + Here, and adapted to 3D: + + .. math:: + + f(\mathbf{s}) = + \sum_{k_1} \sum_{k_2} \sum_{k_3} c(\mathbf{k}) \Psi^3(\mathbf{k}, \mathbf{s}) + \label{eq:1}\tag{1}. + + References + ---------- + .. [Unser1999] M. Unser, "`Splines: A Perfect Fit for Signal and Image Processing + `__," IEEE Signal Processing + Magazine 16(6):22-38, 1999. + + See Also -------- + :py:func:`bspline_weights` - for Eq. :math:`\eqref{eq:2}` and the evaluation of + the tri-cubic B-Splines :math:`\Psi^3(\mathbf{k}, \mathbf{s})`. """ @@ -188,7 +202,7 @@ def canonical_orientation(img): def bspline_grid(img, control_zooms_mm=DEFAULT_ZOOMS_MM): - """Calculate a Nifti1Image object encoding the location of control points.""" + """Create a :obj:`~nibabel.nifti1.Nifti1Image` embedding the location of control points.""" if isinstance(img, (str, Path)): img = nb.load(img) @@ -215,12 +229,55 @@ def bspline_grid(img, control_zooms_mm=DEFAULT_ZOOMS_MM): return nb.Nifti1Image(np.zeros(bs_shape, dtype="float32"), bs_affine) -def bspline_weights(points, level): - """Calculate the tensor-product cubic B-Spline weights for a list of 3D points.""" - ctl_spacings = [float(sp) for sp in level.header.get_zooms()[:3]] - ncoeff = level.dataobj.size +def bspline_weights(points, ctrl_nii): + r""" + Calculate the tensor-product cubic B-Spline kernel weights for a list of 3D points. + + For each of the *N* input samples :math:`(s_1, s_2, s_3)` and *K* control + points or *knots* :math:`\mathbf{k} =(k_1, k_2, k_3)`, the tensor-product + cubic B-Spline kernel weights are calculated: + + .. math:: + + \Psi^3(\mathbf{k}, \mathbf{s}) = + \beta^3(s_1 - k_1) \cdot \beta^3(s_2 - k_2) \cdot \beta^3(s_3 - k_3) + \label{eq:2}\tag{2}, + + where each :math:`\beta^3` represents the cubic B-Spline for one dimension. + The 1D B-Spline kernel implementation uses :obj:`numpy.piecewise`, and is based on the + closed-form given by Eq. (6) of [Unser1999]_. + + By iterating over dimensions, the data samples that fall outside of the compact + support of the tensor-product kernel associated to each control point can be filtered + out and dismissed to lighten computation. + + Finally, the resulting weights matrix :math:`\Psi^3(\mathbf{k}, \mathbf{s})` + can be easily identified in Eq. :math:`\eqref{eq:1}` and used as the design matrix + for approximation of data. + + Parameters + ---------- + points : :obj:`numpy.ndarray`; :math:`N \times 3` + Array of 3D coordinates of samples from the data to be approximated, + in index (i,j,k) coordinates with respect to the control points grid. + ctrl_nii : :obj:`nibabel.spatialimages` + An spatial image object (typically, a :obj:`~nibabel.nifti1.Nifti1Image`) + embedding the location of the control points of the B-Spline grid. + The data array should contain a total of :math:`K` knots (control points). + + Returns + ------- + weights : :obj:`numpy.ndarray` (:math:`K \times N`) + A sparse matrix of interpolating weights :math:`\Psi^3(\mathbf{k}, \mathbf{s})` + for the *N* samples in ``points``, for each of the total *K* knots. + This sparse matrix can be directly used as design matrix for the fitting + step of approximation/extrapolation. + + """ + ctl_spacings = [float(sp) for sp in ctrl_nii.header.get_zooms()[:3]] + ncoeff = ctrl_nii.dataobj.size ctl_points = apply_affine( - level.affine.astype("float32"), np.argwhere(np.isclose(level.dataobj, 0)) + ctrl_nii.affine.astype("float32"), np.argwhere(np.isclose(ctrl_nii.dataobj, 0)) ) weights = np.ones((ncoeff, points.shape[0]), dtype="float32") From f66affdec1f1ce3ff6cf48e18f67a2f690170c62 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Mon, 23 Nov 2020 19:24:29 +0100 Subject: [PATCH 26/68] fix: undo the rotation to canonical --- sdcflows/interfaces/bspline.py | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/sdcflows/interfaces/bspline.py b/sdcflows/interfaces/bspline.py index bd1f4206c1..7f0e1cb1ad 100644 --- a/sdcflows/interfaces/bspline.py +++ b/sdcflows/interfaces/bspline.py @@ -99,8 +99,6 @@ def _run_interface(self, runtime): # Load in the fieldmap fmapnii = nb.load(self.inputs.in_data) data = fmapnii.get_fdata(dtype="float32") - oriented_nii = canonical_orientation(fmapnii) - oriented_nii.to_filename("data.nii.gz") mask = nb.load(self.inputs.in_mask).get_fdata() > 0 bs_spacing = [np.array(sp, dtype="float32") for sp in self.inputs.bs_spacing] @@ -117,7 +115,7 @@ def _run_interface(self, runtime): # Calculate spatial location of voxels, and normalize per B-Spline grid mask_indices = np.argwhere(mask) fmap_points = apply_affine( - oriented_nii.affine.astype("float32"), mask_indices + fmapnii.affine.astype("float32"), mask_indices ) # Calculate the spatial location of control points @@ -125,7 +123,7 @@ def _run_interface(self, runtime): w_l = [] ncoeff = [] for sp in bs_spacing: - level = bspline_grid(oriented_nii, control_zooms_mm=sp) + level = bspline_grid(fmapnii, control_zooms_mm=sp) bs_levels.append(level) ncoeff.append(level.dataobj.size) w_l.append(bspline_weights(fmap_points, level)) @@ -174,7 +172,7 @@ def _run_interface(self, runtime): bg_indices = np.argwhere(~mask) bg_points = apply_affine( - oriented_nii.affine.astype("float32"), bg_indices + fmapnii.affine.astype("float32"), bg_indices ) extrapolators = np.vstack( @@ -188,19 +186,6 @@ def _run_interface(self, runtime): return runtime -def canonical_orientation(img): - """Generate an alternative image aligned with the array axes.""" - if isinstance(img, (str, Path)): - img = nb.load(img) - - shape = np.array(img.shape[:3]) - affine = np.diag(np.hstack((img.header.get_zooms()[:3], 1))) - affine[:3, 3] -= affine[:3, :3] @ (0.5 * (shape - 1)) - nii = nb.Nifti1Image(img.dataobj, affine) - nii.header.set_xyzt_units(*img.header.get_xyzt_units()) - return nii - - def bspline_grid(img, control_zooms_mm=DEFAULT_ZOOMS_MM): """Create a :obj:`~nibabel.nifti1.Nifti1Image` embedding the location of control points.""" if isinstance(img, (str, Path)): @@ -222,8 +207,8 @@ def bspline_grid(img, control_zooms_mm=DEFAULT_ZOOMS_MM): bs_shape = (im_extent // bs_zooms + 3).astype(int) # Center both images - im_center = img.affine @ np.hstack((0.5 * (im_shape - 1), 1)) - bs_center = bs_affine @ np.hstack((0.5 * (bs_shape - 1), 1)) + im_center = apply_affine(img.affine, 0.5 * (im_shape - 1)) + bs_center = apply_affine(bs_affine, 0.5 * (bs_shape - 1)) bs_affine[:3, 3] = im_center[:3] - bs_center[:3] return nb.Nifti1Image(np.zeros(bs_shape, dtype="float32"), bs_affine) From 5d2fedb02450412b04201424440b8d30c61cd6c2 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Mon, 23 Nov 2020 23:23:01 +0100 Subject: [PATCH 27/68] wip: finalize handling of coefficients --- sdcflows/interfaces/bspline.py | 7 +- sdcflows/workflows/apply/registration.py | 32 +++++++- .../apply/tests/test_registration.py | 82 ++++++++++++------- sdcflows/workflows/fit/fieldmap.py | 7 +- sdcflows/workflows/fit/pepolar.py | 8 +- sdcflows/workflows/fit/tests/test_pepolar.py | 2 +- sdcflows/workflows/fit/tests/test_phdiff.py | 2 + 7 files changed, 98 insertions(+), 42 deletions(-) diff --git a/sdcflows/interfaces/bspline.py b/sdcflows/interfaces/bspline.py index 7f0e1cb1ad..f9c3a1d557 100644 --- a/sdcflows/interfaces/bspline.py +++ b/sdcflows/interfaces/bspline.py @@ -207,9 +207,10 @@ def bspline_grid(img, control_zooms_mm=DEFAULT_ZOOMS_MM): bs_shape = (im_extent // bs_zooms + 3).astype(int) # Center both images - im_center = apply_affine(img.affine, 0.5 * (im_shape - 1)) - bs_center = apply_affine(bs_affine, 0.5 * (bs_shape - 1)) - bs_affine[:3, 3] = im_center[:3] - bs_center[:3] + bs_affine[:3, 3] = ( + apply_affine(img.affine, 0.5 * (im_shape - 1)) + - apply_affine(bs_affine, 0.5 * (bs_shape - 1)) + ) return nb.Nifti1Image(np.zeros(bs_shape, dtype="float32"), bs_affine) diff --git a/sdcflows/workflows/apply/registration.py b/sdcflows/workflows/apply/registration.py index 04d20aa8b7..1c262d3758 100644 --- a/sdcflows/workflows/apply/registration.py +++ b/sdcflows/workflows/apply/registration.py @@ -113,8 +113,9 @@ def init_coeff2epi_wf( # fmt: off workflow.connect([ - (inputnode, map_coeff, [("fmap_coeff", "in_coeff")]), - (coregister, map_coeff, [("forward_transforms", "transform")]), + (inputnode, map_coeff, [("fmap_coeff", "in_coeff"), + ("target_ref", "ref_image")]), + (coregister, map_coeff, [("reverse_transforms", "transform")]), (map_coeff, outputnode, [("out", "fmap_coeff")]), ]) # fmt: on @@ -122,6 +123,29 @@ def init_coeff2epi_wf( return workflow -def _move_coeff(in_coeff, transform): +def _move_coeff(in_coeff, ref_image, transform): """Read in a rigid transform from ANTs, and update the coefficients field affine.""" - raise NotImplementedError + from pathlib import Path + import numpy as np + import nibabel as nb + import nitransforms as nt + + if isinstance(in_coeff, str): + in_coeff = [in_coeff] + + xfm = nt.io.itk.ITKLinearTransform.from_filename(transform[0]).to_ras() + + refaff = nb.load(ref_image).affine + refdir = refaff[:3, :3] / nb.affines.voxel_sizes(refaff) + + out = [] + for i, c in enumerate(in_coeff): + img = nb.load(c) + + out.append(str(Path(f"moved_coeff_{i:03d}.nii.gz").absolute())) + + newaff = np.eye(4) + newaff[:3, :3] = nb.affines.voxel_sizes(img.affine) * refdir + newaff[:3, 3] = nb.affines.apply_affine(xfm, img.affine[:3, 3].T) + img.__class__(img.dataobj, newaff, img.header).to_filename(out[-1]) + return out diff --git a/sdcflows/workflows/apply/tests/test_registration.py b/sdcflows/workflows/apply/tests/test_registration.py index 59d7f64252..729df6f3be 100644 --- a/sdcflows/workflows/apply/tests/test_registration.py +++ b/sdcflows/workflows/apply/tests/test_registration.py @@ -1,10 +1,14 @@ """Test pepolar type of fieldmaps.""" import os import pytest + +import numpy as np +import nibabel as nb from nipype.pipeline import engine as pe +from nipype.interfaces import utility as niu from ...fit.fieldmap import init_magnitude_wf -from ..registration import init_coeff2epi_wf +from ..registration import init_coeff2epi_wf, _move_coeff @pytest.mark.skipif(os.getenv("TRAVIS") == "true", reason="this is TravisCI") @@ -29,29 +33,25 @@ def test_registration_wf(tmpdir, datadir, workdir, outdir): fmap_ref_wf = init_magnitude_wf(2, name="fmap_ref_wf") fmap_ref_wf.inputs.inputnode.magnitude = magnitude - reg_wf = init_coeff2epi_wf(2, debug=True) + gen_coeff = pe.Node(niu.Function(function=_gen_coeff), name="gen_coeff") + + reg_wf = init_coeff2epi_wf(2, debug=True, write_coeff=True) workflow = pe.Workflow(name="test_registration_wf") - workflow.connect( - [ - ( - epi_ref_wf, - reg_wf, - [ - ("outputnode.fmap_ref", "inputnode.target_ref"), - ("outputnode.fmap_mask", "inputnode.target_mask"), - ], - ), - ( - fmap_ref_wf, - reg_wf, - [ - ("outputnode.fmap_ref", "inputnode.fmap_ref"), - ("outputnode.fmap_mask", "inputnode.fmap_mask"), - ], - ), - ] - ) + # fmt: off + workflow.connect([ + (epi_ref_wf, reg_wf, [ + ("outputnode.fmap_ref", "inputnode.target_ref"), + ("outputnode.fmap_mask", "inputnode.target_mask"), + ]), + (fmap_ref_wf, reg_wf, [ + ("outputnode.fmap_ref", "inputnode.fmap_ref"), + ("outputnode.fmap_mask", "inputnode.fmap_mask"), + ]), + (fmap_ref_wf, gen_coeff, [("outputnode.fmap_ref", "img")]), + (gen_coeff, reg_wf, [("out", "inputnode.fmap_coeff")]), + ]) + # fmt: on if outdir: from niworkflows.interfaces import SimpleBeforeAfter @@ -75,15 +75,39 @@ def test_registration_wf(tmpdir, datadir, workdir, outdir): run_without_submitting=True, ) - workflow.connect( - [ - (epi_ref_wf, report, [("outputnode.fmap_ref", "before")]), - (reg_wf, report, [("outputnode.fmap_ref", "after")]), - (report, ds_report, [("out_report", "in_file")]), - ] - ) + # fmt: off + workflow.connect([ + (epi_ref_wf, report, [("outputnode.fmap_ref", "before")]), + (reg_wf, report, [("outputnode.fmap_ref", "after")]), + (report, ds_report, [("out_report", "in_file")]), + ]) + # fmt: on if workdir: workflow.base_dir = str(workdir) workflow.run(plugin="Linear") + + +# def test_map_coeffs(tmpdir): +# from pathlib import Path +# tmpdir.chdir() +# outnii = nb.load(_move_coeff( +# str(Path(__file__).parent / "sample-coeff.nii.gz"), +# [str(Path(__file__).parent / "sample-rigid_xfm.mat")] +# )[0]) +# vs = nb.affines.voxel_sizes(outnii.affine) +# assert np.allclose(vs, (40., 40., 20.)) + +# dircos = outnii.affine[:3, :3] / vs +# assert np.allclose(dircos, np.eye(3)) + + + +def _gen_coeff(img): + from pathlib import Path + from sdcflows.interfaces.bspline import bspline_grid + + out_file = Path("coeff.nii.gz").absolute() + bspline_grid(img).to_filename(out_file) + return str(out_file) diff --git a/sdcflows/workflows/fit/fieldmap.py b/sdcflows/workflows/fit/fieldmap.py index 59d9574c22..53a16a6c33 100644 --- a/sdcflows/workflows/fit/fieldmap.py +++ b/sdcflows/workflows/fit/fieldmap.py @@ -151,6 +151,8 @@ def init_fmap_wf(omp_nthreads=1, debug=False, mode="phasediff", name="fmap_wf"): Path to the estimated fieldmap. fmap_ref : :obj:`str` Path to a preprocessed magnitude image reference. + fmap_coeff : :obj:`str` or :obj:`list` of :obj:`str` + The path(s) of the B-Spline coefficients supporting the fieldmap. fmap_mask : :obj:`str` Path to a binary brain mask corresponding to the ``fmap`` and ``fmap_ref`` pair. @@ -169,7 +171,7 @@ def init_fmap_wf(omp_nthreads=1, debug=False, mode="phasediff", name="fmap_wf"): niu.IdentityInterface(fields=["magnitude", "fieldmap"]), name="inputnode" ) outputnode = pe.Node( - niu.IdentityInterface(fields=["fmap", "fmap_ref", "fmap_mask"]), + niu.IdentityInterface(fields=["fmap", "fmap_ref", "fmap_mask", "fmap_coeff"]), name="outputnode", ) @@ -190,7 +192,8 @@ def init_fmap_wf(omp_nthreads=1, debug=False, mode="phasediff", name="fmap_wf"): ("outputnode.fmap_ref", "fmap_ref"), ]), (bs_filter, outputnode, [ - ("out_extrapolated" if not debug else "out_field", "fmap")]), + ("out_extrapolated" if not debug else "out_field", "fmap"), + ("out_coeff", "fmap_coeff")]), ]) # fmt: on diff --git a/sdcflows/workflows/fit/pepolar.py b/sdcflows/workflows/fit/pepolar.py index 10948dd381..c02f1e4ecd 100644 --- a/sdcflows/workflows/fit/pepolar.py +++ b/sdcflows/workflows/fit/pepolar.py @@ -57,6 +57,8 @@ def init_topup_wf(omp_nthreads=1, debug=False, name="pepolar_estimate_wf"): The path of the estimated fieldmap. fmap_ref : :obj:`str` The path of an unwarped conversion of files in ``in_data``. + fmap_coeff : :obj:`str` or :obj:`list` of :obj:`str` + The path(s) of the B-Spline coefficients supporting the fieldmap. """ from nipype.interfaces.fsl.epi import TOPUP @@ -74,9 +76,9 @@ def init_topup_wf(omp_nthreads=1, debug=False, name="pepolar_estimate_wf"): outputnode = pe.Node( niu.IdentityInterface( fields=[ - "fmap_ref", "fmap", - "coefficients", + "fmap_ref", + "fmap_coeff", "jacobians", "xfms", "out_warps", @@ -110,7 +112,7 @@ def init_topup_wf(omp_nthreads=1, debug=False, name="pepolar_estimate_wf"): (concat_blips, topup, [("out_file", "in_file")]), (topup, outputnode, [("out_corrected", "fmap_ref"), ("out_field", "fmap"), - ("out_fieldcoef", "coefficients"), + ("out_fieldcoef", "fmap_coeff"), ("out_jacs", "jacobians"), ("out_mats", "xfms"), ("out_warps", "out_warps")]), diff --git a/sdcflows/workflows/fit/tests/test_pepolar.py b/sdcflows/workflows/fit/tests/test_pepolar.py index a223f20a73..a0447f4860 100644 --- a/sdcflows/workflows/fit/tests/test_pepolar.py +++ b/sdcflows/workflows/fit/tests/test_pepolar.py @@ -79,7 +79,7 @@ def test_topup_wf(tmpdir, datadir, workdir, outdir, epi_path): (topup_wf, fmap_derivatives_wf, [ ("outputnode.fmap", "inputnode.fieldmap"), ("outputnode.fmap_ref", "inputnode.fmap_ref"), - ("outputnode.coefficients", "inputnode.fmap_coeff"), + ("outputnode.fmap_coeff", "inputnode.fmap_coeff"), ]), ]) # fmt: on diff --git a/sdcflows/workflows/fit/tests/test_phdiff.py b/sdcflows/workflows/fit/tests/test_phdiff.py index 8365de5f5f..8a15738f6d 100644 --- a/sdcflows/workflows/fit/tests/test_phdiff.py +++ b/sdcflows/workflows/fit/tests/test_phdiff.py @@ -45,6 +45,7 @@ def test_phdiff(tmpdir, datadir, workdir, outdir, fmap_path): fmap_derivatives_wf = init_fmap_derivatives_wf( output_dir=str(outdir), + write_coeff=True, custom_entities={"est": "phasediff"}, bids_fmap_id="phasediff_id", ) @@ -66,6 +67,7 @@ def test_phdiff(tmpdir, datadir, workdir, outdir, fmap_path): (phdiff_wf, fmap_derivatives_wf, [ ("outputnode.fmap", "inputnode.fieldmap"), ("outputnode.fmap_ref", "inputnode.fmap_ref"), + ("outputnode.fmap_coeff", "inputnode.fmap_coeff"), ]), ]) # fmt: on From de78f96e3f8343389cf8bd67a26bf99a09443dfd Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Tue, 24 Nov 2020 10:49:19 +0100 Subject: [PATCH 28/68] enh: improve calculation of distances to bspline knots --- sdcflows/interfaces/bspline.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdcflows/interfaces/bspline.py b/sdcflows/interfaces/bspline.py index f9c3a1d557..ad22e8ee82 100644 --- a/sdcflows/interfaces/bspline.py +++ b/sdcflows/interfaces/bspline.py @@ -260,21 +260,21 @@ def bspline_weights(points, ctrl_nii): step of approximation/extrapolation. """ - ctl_spacings = [float(sp) for sp in ctrl_nii.header.get_zooms()[:3]] ncoeff = ctrl_nii.dataobj.size + knots = np.argwhere(np.isclose(ctrl_nii.dataobj, 0)).astype("float32") ctl_points = apply_affine( - ctrl_nii.affine.astype("float32"), np.argwhere(np.isclose(ctrl_nii.dataobj, 0)) + np.linalg.inv(ctrl_nii.affine).astype("float32"), + points ) weights = np.ones((ncoeff, points.shape[0]), dtype="float32") for i in range(3): d = ( np.abs( - (ctl_points[:, np.newaxis, i] - points[np.newaxis, :, i])[ + (knots[:, np.newaxis, i] - ctl_points[np.newaxis, :, i])[ weights > 1e-6 ] ) - / ctl_spacings[i] ) weights[weights > 1e-6] *= np.piecewise( d, From 34f1822c748475f84968a7d730c708d4cf815ba9 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Tue, 24 Nov 2020 11:56:16 +0100 Subject: [PATCH 29/68] enh: add a B-Spline interpolator interface + tests --- sdcflows/interfaces/bspline.py | 87 +++++++++++++++++-- sdcflows/interfaces/fmap.py | 23 ----- sdcflows/interfaces/tests/__init__.py | 0 sdcflows/interfaces/tests/test_bspline.py | 56 ++++++++++++ .../apply/tests/test_registration.py | 4 +- sdcflows/workflows/fit/fieldmap.py | 12 ++- 6 files changed, 145 insertions(+), 37 deletions(-) create mode 100644 sdcflows/interfaces/tests/__init__.py create mode 100644 sdcflows/interfaces/tests/test_bspline.py diff --git a/sdcflows/interfaces/bspline.py b/sdcflows/interfaces/bspline.py index ad22e8ee82..88afe088a3 100644 --- a/sdcflows/interfaces/bspline.py +++ b/sdcflows/interfaces/bspline.py @@ -74,8 +74,8 @@ class BSplineApprox(SimpleInterface): .. math:: f(\mathbf{s}) = - \sum_{k_1} \sum_{k_2} \sum_{k_3} c(\mathbf{k}) \Psi^3(\mathbf{k}, \mathbf{s}) - \label{eq:1}\tag{1}. + \sum_{k_1} \sum_{k_2} \sum_{k_3} c(\mathbf{k}) \Psi^3(\mathbf{k}, \mathbf{s}). + \label{eq:1}\tag{1} References ---------- @@ -186,6 +186,79 @@ def _run_interface(self, runtime): return runtime +class _Coefficients2WarpInputSpec(BaseInterfaceInputSpec): + in_target = File(exist=True, mandatory=True, desc="input EPI data to be corrected") + in_coeff = InputMultiObject(File(exists=True), mandatory=True, + desc="input coefficients, after alignment to the EPI data") + ro_time = traits.Float(1.0, usedefault=True, desc="EPI readout time (s).") + pe_dir = traits.Enum("i", "i-", "j", "j-", "k", "k-", mandatory=True, + desc="the phase-encoding direction corresponding to in_target") + + +class _Coefficients2WarpOutputSpec(TraitedSpec): + out_field = File(exists=True) + out_warp = File(exists=True) + + +class Coefficients2Warp(SimpleInterface): + r""" + Convert a set of B-Spline coefficients to a full displacements map. + + Implements Eq. :math:`\eqref{eq:1}`, interpolating :math:`f(\mathbf{s})` + for all voxels in the target-image's extent. + When the readout time is known, the displacements field can be calculated + following Eq. `(2) `__. + + """ + + input_spec = _Coefficients2WarpInputSpec + output_spec = _Coefficients2WarpOutputSpec + + def _run_interface(self, runtime): + + # Calculate the physical coordinates of target grid + targetnii = nb.load(self.inputs.in_target) + allmask = np.ones_like(targetnii.dataobj, dtype=bool) + points = apply_affine( + targetnii.affine.astype("float32"), np.argwhere(allmask).astype("float32") + ) + + weights = [] + coeffs = [] + for cname in self.inputs.in_coeff: + cnii = nb.load(cname) + cdata = cnii.get_fdata(dtype="float32") + weights.append(bspline_weights(points, cnii)) + coeffs.append(cdata.reshape(-1)) + + data = np.zeros_like(targetnii.dataobj, dtype="float32") + data[allmask] = np.squeeze(np.vstack(coeffs).T) @ np.vstack(weights) + + hdr = targetnii.header.copy() + hdr.set_data_dtype("float32") + self._results["out_field"] = fname_presuffix( + self.inputs.in_target, suffix="_field", newpath=runtime.cwd + ) + nb.Nifti1Image(data, targetnii.affine, hdr).to_filename(self._results["out_field"]) + + # Generate warp field + phaseEncDim = {'i': 0, 'j': 1, 'k': 2}[self.inputs.pe_dir[0]] + phaseEncSign = [1.0, -1.0][len(self.inputs.pe_dir) != 2] + + data *= phaseEncSign * targetnii.header.get_zooms()[phaseEncDim] * self.inputs.ro_time + + self._results["out_warp"] = fname_presuffix( + self.inputs.in_target, suffix="_xfm", newpath=runtime.cwd + ) + # Compose a vector field + field = np.zeros(list(data.shape) + [3], dtype="float32") + field[..., phaseEncDim] = data + warpnii = nb.Nifti1Image(field, targetnii.affine, None) + warpnii.header.set_intent('vector', (), '') + warpnii.to_filename(self._results["out_warp"]) + return runtime + + def bspline_grid(img, control_zooms_mm=DEFAULT_ZOOMS_MM): """Create a :obj:`~nibabel.nifti1.Nifti1Image` embedding the location of control points.""" if isinstance(img, (str, Path)): @@ -226,8 +299,8 @@ def bspline_weights(points, ctrl_nii): .. math:: \Psi^3(\mathbf{k}, \mathbf{s}) = - \beta^3(s_1 - k_1) \cdot \beta^3(s_2 - k_2) \cdot \beta^3(s_3 - k_3) - \label{eq:2}\tag{2}, + \beta^3(s_1 - k_1) \cdot \beta^3(s_2 - k_2) \cdot \beta^3(s_3 - k_3), + \label{eq:2}\tag{2} where each :math:`\beta^3` represents the cubic B-Spline for one dimension. The 1D B-Spline kernel implementation uses :obj:`numpy.piecewise`, and is based on the @@ -260,8 +333,8 @@ def bspline_weights(points, ctrl_nii): step of approximation/extrapolation. """ - ncoeff = ctrl_nii.dataobj.size - knots = np.argwhere(np.isclose(ctrl_nii.dataobj, 0)).astype("float32") + ncoeff = np.prod(ctrl_nii.shape[:]) + knots = np.argwhere(np.ones_like(ctrl_nii.dataobj, dtype=bool)) ctl_points = apply_affine( np.linalg.inv(ctrl_nii.affine).astype("float32"), points @@ -271,7 +344,7 @@ def bspline_weights(points, ctrl_nii): for i in range(3): d = ( np.abs( - (knots[:, np.newaxis, i] - ctl_points[np.newaxis, :, i])[ + (knots[:, np.newaxis, i].astype("float32") - ctl_points[np.newaxis, :, i])[ weights > 1e-6 ] ) diff --git a/sdcflows/interfaces/fmap.py b/sdcflows/interfaces/fmap.py index 988d978089..3dc9b3b961 100644 --- a/sdcflows/interfaces/fmap.py +++ b/sdcflows/interfaces/fmap.py @@ -99,26 +99,3 @@ def _run_interface(self, runtime): self.inputs.in_file, _delta_te(self.inputs.metadata), newpath=runtime.cwd ) return runtime - - -# Code stub for #119 (B-Spline smoothing) -# class _Coefficients2WarpInputSpec(BaseInterfaceInputSpec): -# in_target = File(exist=True, mandatory=True, desc="input EPI data to be corrected") -# in_coeff = InputMultiObject(File(exists=True), mandatory=True, -# desc="input coefficients, after alignment to the EPI data") -# ro_time = traits.Float(mandatory=True, desc="EPI readout time") - - -# class _Coefficients2WarpOutputSpec(TraitedSpec): -# out_warp = File(exists=True) - - -# class Coefficients2Warp(SimpleInterface): -# """Convert coefficients to a full displacements map.""" - -# input_spec = _Coefficients2WarpInputSpec -# output_spec = _Coefficients2WarpOutputSpec - -# def _run_interface(self, runtime): -# raise NotImplementedError -# return runtime diff --git a/sdcflows/interfaces/tests/__init__.py b/sdcflows/interfaces/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sdcflows/interfaces/tests/test_bspline.py b/sdcflows/interfaces/tests/test_bspline.py new file mode 100644 index 0000000000..5ee4b0c262 --- /dev/null +++ b/sdcflows/interfaces/tests/test_bspline.py @@ -0,0 +1,56 @@ +"""Test B-Spline interfaces.""" +import os +import numpy as np +import nibabel as nb +import pytest + +from ..bspline import bspline_grid, Coefficients2Warp, BSplineApprox + + +@pytest.mark.parametrize("testnum", range(100)) +def test_bsplines(tmp_path, testnum): + """Test idempotency of B-Splines interpolation + approximation.""" + targetshape = (10, 12, 9) + + # Generate an oblique affine matrix for the target - it will be a common case. + targetaff = nb.affines.from_matvec( + nb.eulerangles.euler2mat(x=0.9, y=0.001, z=0.001) @ np.diag((2, 3, 4)), + ) + + # Intendedly mis-centered (exercise we may not have volume-centered NIfTIs) + targetaff[:3, 3] = nb.affines.apply_affine( + targetaff, 0.5 * (np.array(targetshape) - 3) + ) + + # Generate some target grid + targetnii = nb.Nifti1Image( + np.ones(targetshape), + targetaff, + None + ) + targetnii.to_filename(tmp_path / "target.nii.gz") + + # Generate random coefficients + gridnii = bspline_grid(targetnii, control_zooms_mm=(4, 6, 8)) + coeff = (np.random.random(size=gridnii.shape) - 0.5) * 500 + coeffnii = nb.Nifti1Image(coeff.astype("float32"), gridnii.affine, gridnii.header) + coeffnii.to_filename(tmp_path / "coeffs.nii.gz") + + os.chdir(tmp_path) + # Check that we can interpolate the coefficients on a target + test1 = Coefficients2Warp( + in_target=str(tmp_path / "target.nii.gz"), + in_coeff=str(tmp_path / "coeffs.nii.gz"), + pe_dir="j-", + ).run() + + # Approximate the interpolated target + test2 = BSplineApprox( + in_data=test1.outputs.out_field, + in_mask=str(tmp_path / "target.nii.gz"), + bs_spacing=[(4, 6, 8)], + recenter="no", + ).run() + + # Absolute error of the interpolated field is always below 2 Hz + assert np.all(np.abs(nb.load(test2.outputs.out_error).get_fdata()) < 2) diff --git a/sdcflows/workflows/apply/tests/test_registration.py b/sdcflows/workflows/apply/tests/test_registration.py index 729df6f3be..ed568daccd 100644 --- a/sdcflows/workflows/apply/tests/test_registration.py +++ b/sdcflows/workflows/apply/tests/test_registration.py @@ -2,13 +2,11 @@ import os import pytest -import numpy as np -import nibabel as nb from nipype.pipeline import engine as pe from nipype.interfaces import utility as niu from ...fit.fieldmap import init_magnitude_wf -from ..registration import init_coeff2epi_wf, _move_coeff +from ..registration import init_coeff2epi_wf @pytest.mark.skipif(os.getenv("TRAVIS") == "true", reason="this is TravisCI") diff --git a/sdcflows/workflows/fit/fieldmap.py b/sdcflows/workflows/fit/fieldmap.py index 53a16a6c33..0030169a21 100644 --- a/sdcflows/workflows/fit/fieldmap.py +++ b/sdcflows/workflows/fit/fieldmap.py @@ -4,11 +4,12 @@ Processing phase-difference and *directly measured* :math:`B_0` maps. The displacement suffered by every voxel along the phase-encoding (PE) direction -can be derived from eq. (2) of [Hutton2002]_: +can be derived from Eq. (2) of [Hutton2002]_: .. math:: \Delta_\text{PE} (i, j, k) = \gamma \cdot \Delta B_0 (i, j, k) \cdot T_\text{ro}, + \label{eq:fieldmap-1}\tag{1} where :math:`T_\text{ro}` is the readout time of one slice of the EPI dataset we want to correct for distortions, and :math:`\Delta_\text{PE} (i, j, k)` @@ -23,7 +24,7 @@ .. math:: \Delta D_\text{PE} (i, j, k) = V(i, j, k) \cdot T_\text{ro} \cdot s_\text{PE}. - + \label{eq:fieldmap-2}\tag{2} Theory ~~~~~~ @@ -32,7 +33,8 @@ .. math:: - \Delta B_0 (i, j, k) = \frac{\Delta \Theta (i, j, k)}{2\pi \cdot \gamma \, \Delta\text{TE}} + \Delta B_0 (i, j, k) = \frac{\Delta \Theta (i, j, k)}{2\pi \cdot \gamma \, \Delta\text{TE}}, + \label{eq:fieldmap-3}\tag{3} where :math:`\Delta B_0 (i, j, k)` is the *fieldmap variation* in T, :math:`\Delta \Theta (i, j, k)` is the phase-difference map in rad, @@ -44,7 +46,8 @@ .. math:: - \Delta_\text{PE} (i, j, k) = \gamma \cdot \Delta B_0 (i, j, k) \cdot T_\text{ro} + \Delta_\text{PE} (i, j, k) = \gamma \cdot \Delta B_0 (i, j, k) \cdot T_\text{ro}, + \label{eq:fieldmap-4}\tag{4} where :math:`T_\text{ro}` is the readout time of one slice of the EPI dataset we want to correct for distortions, and @@ -68,6 +71,7 @@ .. math:: V(i, j, k) = \frac{\Delta \Theta (i, j, k)}{2\pi \cdot \Delta\text{TE}}. + \label{eq:fieldmap-5}\tag{5} This calculation if further complicated by the fact that :math:`\Theta_i` (and therfore, :math:`\Delta \Theta`) are clipped (or *wrapped*) within From 7f1c27b6b2911eafcf5fd4721aa8f6e66d6673da Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Tue, 24 Nov 2020 23:19:26 +0100 Subject: [PATCH 30/68] enh: drafted a correction workflow + tests --- sdcflows/interfaces/bspline.py | 106 +++++++++------- sdcflows/interfaces/epi.py | 22 ++-- sdcflows/interfaces/reportlets.py | 5 +- sdcflows/interfaces/tests/test_bspline.py | 1 + sdcflows/workflows/apply/correction.py | 77 ++++++++++++ sdcflows/workflows/apply/registration.py | 39 +++--- .../workflows/apply/tests/fieldcoeff.nii.gz | Bin 0 -> 2100 bytes .../workflows/apply/tests/test_correct.py | 114 ++++++++++++++++++ .../apply/tests/test_registration.py | 7 +- 9 files changed, 300 insertions(+), 71 deletions(-) create mode 100644 sdcflows/workflows/apply/correction.py create mode 100644 sdcflows/workflows/apply/tests/fieldcoeff.nii.gz create mode 100644 sdcflows/workflows/apply/tests/test_correct.py diff --git a/sdcflows/interfaces/bspline.py b/sdcflows/interfaces/bspline.py index 88afe088a3..5c62a40f74 100644 --- a/sdcflows/interfaces/bspline.py +++ b/sdcflows/interfaces/bspline.py @@ -18,6 +18,7 @@ ) +LOW_MEM_BLOCK_SIZE = 1000 DEFAULT_ZOOMS_MM = (40.0, 40.0, 20.0) # For human adults (mid-frequency), in mm DEFAULT_LF_ZOOMS_MM = (100.0, 100.0, 40.0) # For human adults (low-frequency), in mm DEFAULT_HF_ZOOMS_MM = (16.0, 16.0, 10.0) # For human adults (high-frequency), in mm @@ -33,7 +34,7 @@ class _BSplineApproxInputSpec(BaseInterfaceInputSpec): desc="spacing between B-Spline control points", ) ridge_alpha = traits.Float( - 1e-4, usedefault=True, desc="controls the regularization" + 0.01, usedefault=True, desc="controls the regularization" ) recenter = traits.Enum( "mode", @@ -114,9 +115,7 @@ def _run_interface(self, runtime): # Calculate spatial location of voxels, and normalize per B-Spline grid mask_indices = np.argwhere(mask) - fmap_points = apply_affine( - fmapnii.affine.astype("float32"), mask_indices - ) + fmap_points = apply_affine(fmapnii.affine.astype("float32"), mask_indices) # Calculate the spatial location of control points bs_levels = [] @@ -171,9 +170,7 @@ def _run_interface(self, runtime): return runtime bg_indices = np.argwhere(~mask) - bg_points = apply_affine( - fmapnii.affine.astype("float32"), bg_indices - ) + bg_points = apply_affine(fmapnii.affine.astype("float32"), bg_indices) extrapolators = np.vstack( [bspline_weights(bg_points, level) for level in bs_levels] @@ -188,11 +185,23 @@ def _run_interface(self, runtime): class _Coefficients2WarpInputSpec(BaseInterfaceInputSpec): in_target = File(exist=True, mandatory=True, desc="input EPI data to be corrected") - in_coeff = InputMultiObject(File(exists=True), mandatory=True, - desc="input coefficients, after alignment to the EPI data") + in_coeff = InputMultiObject( + File(exists=True), + mandatory=True, + desc="input coefficients, after alignment to the EPI data", + ) ro_time = traits.Float(1.0, usedefault=True, desc="EPI readout time (s).") - pe_dir = traits.Enum("i", "i-", "j", "j-", "k", "k-", mandatory=True, - desc="the phase-encoding direction corresponding to in_target") + pe_dir = traits.Enum( + "i", + "i-", + "j", + "j-", + "k", + "k-", + mandatory=True, + desc="the phase-encoding direction corresponding to in_target", + ) + low_mem = traits.Bool(False, usedefault=True, desc="perform on low-mem fingerprint regime") class _Coefficients2WarpOutputSpec(TraitedSpec): @@ -215,46 +224,67 @@ class Coefficients2Warp(SimpleInterface): output_spec = _Coefficients2WarpOutputSpec def _run_interface(self, runtime): - # Calculate the physical coordinates of target grid targetnii = nb.load(self.inputs.in_target) - allmask = np.ones_like(targetnii.dataobj, dtype=bool) - points = apply_affine( - targetnii.affine.astype("float32"), np.argwhere(allmask).astype("float32") - ) + targetaff = targetnii.affine + allmask = np.ones_like(targetnii.dataobj, dtype="uint8") + voxels = np.argwhere(allmask == 1).astype("float32") + points = apply_affine(targetaff.astype("float32"), voxels) weights = [] coeffs = [] + blocksize = LOW_MEM_BLOCK_SIZE if self.inputs.low_mem else len(points) for cname in self.inputs.in_coeff: cnii = nb.load(cname) cdata = cnii.get_fdata(dtype="float32") - weights.append(bspline_weights(points, cnii)) coeffs.append(cdata.reshape(-1)) - data = np.zeros_like(targetnii.dataobj, dtype="float32") - data[allmask] = np.squeeze(np.vstack(coeffs).T) @ np.vstack(weights) + idx = 0 + block_w = [] + while True: + end = idx + blocksize + subsample = points[idx:end, ...] + if subsample.shape[0] == 0: + break + + idx = end + block_w.append(bspline_weights(subsample, cnii)) + + weights.append(np.hstack(block_w)) + + data = np.zeros(targetnii.shape, dtype="float32") + data[allmask == 1] = np.squeeze(np.vstack(coeffs).T) @ np.vstack(weights) hdr = targetnii.header.copy() hdr.set_data_dtype("float32") self._results["out_field"] = fname_presuffix( self.inputs.in_target, suffix="_field", newpath=runtime.cwd ) - nb.Nifti1Image(data, targetnii.affine, hdr).to_filename(self._results["out_field"]) + nb.Nifti1Image(data, targetnii.affine, hdr).to_filename( + self._results["out_field"] + ) # Generate warp field - phaseEncDim = {'i': 0, 'j': 1, 'k': 2}[self.inputs.pe_dir[0]] + phaseEncDim = "ijk".index(self.inputs.pe_dir[0]) phaseEncSign = [1.0, -1.0][len(self.inputs.pe_dir) != 2] - data *= phaseEncSign * targetnii.header.get_zooms()[phaseEncDim] * self.inputs.ro_time + data *= phaseEncSign * self.inputs.ro_time + fieldshape = tuple(list(data.shape[:3]) + [3]) self._results["out_warp"] = fname_presuffix( self.inputs.in_target, suffix="_xfm", newpath=runtime.cwd ) # Compose a vector field - field = np.zeros(list(data.shape) + [3], dtype="float32") - field[..., phaseEncDim] = data - warpnii = nb.Nifti1Image(field, targetnii.affine, None) - warpnii.header.set_intent('vector', (), '') + field = np.zeros((data.size, 3), dtype="float32") + field[..., phaseEncDim] = data.reshape(-1) + aff = targetnii.affine.copy() + aff[:3, 3] = 0.0 + field = nb.affines.apply_affine(aff, field).reshape(fieldshape) + warpnii = nb.Nifti1Image( + field[:, :, :, np.newaxis, :].astype("float32"), targetnii.affine, None + ) + warpnii.header.set_intent("vector", (), "") + warpnii.header.set_xyzt_units("mm") warpnii.to_filename(self._results["out_warp"]) return runtime @@ -280,9 +310,8 @@ def bspline_grid(img, control_zooms_mm=DEFAULT_ZOOMS_MM): bs_shape = (im_extent // bs_zooms + 3).astype(int) # Center both images - bs_affine[:3, 3] = ( - apply_affine(img.affine, 0.5 * (im_shape - 1)) - - apply_affine(bs_affine, 0.5 * (bs_shape - 1)) + bs_affine[:3, 3] = apply_affine(img.affine, 0.5 * (im_shape - 1)) - apply_affine( + bs_affine, 0.5 * (bs_shape - 1) ) return nb.Nifti1Image(np.zeros(bs_shape, dtype="float32"), bs_affine) @@ -333,21 +362,16 @@ def bspline_weights(points, ctrl_nii): step of approximation/extrapolation. """ - ncoeff = np.prod(ctrl_nii.shape[:]) - knots = np.argwhere(np.ones_like(ctrl_nii.dataobj, dtype=bool)) - ctl_points = apply_affine( - np.linalg.inv(ctrl_nii.affine).astype("float32"), - points - ) + ncoeff = np.prod(ctrl_nii.shape[:3]) + knots = np.argwhere(np.ones(ctrl_nii.shape[:3], dtype="uint8") == 1) + ctl_points = apply_affine(np.linalg.inv(ctrl_nii.affine).astype("float32"), points) weights = np.ones((ncoeff, points.shape[0]), dtype="float32") for i in range(3): - d = ( - np.abs( - (knots[:, np.newaxis, i].astype("float32") - ctl_points[np.newaxis, :, i])[ - weights > 1e-6 - ] - ) + d = np.abs( + (knots[:, np.newaxis, i].astype("float32") - ctl_points[np.newaxis, :, i])[ + weights > 1e-6 + ] ) weights[weights > 1e-6] *= np.piecewise( d, diff --git a/sdcflows/interfaces/epi.py b/sdcflows/interfaces/epi.py index 5e4783be73..df025ad9fc 100644 --- a/sdcflows/interfaces/epi.py +++ b/sdcflows/interfaces/epi.py @@ -30,6 +30,7 @@ class _GetReadoutTimeInputSpec(BaseInterfaceInputSpec): class _GetReadoutTimeOutputSpec(TraitedSpec): readout_time = traits.Float + pe_direction = traits.Enum("i", "i-", "j", "j-", "k", "k-") class GetReadoutTime(SimpleInterface): @@ -43,6 +44,7 @@ def _run_interface(self, runtime): self.inputs.metadata, self.inputs.in_file if isdefined(self.inputs.in_file) else None, ) + self._results["pe_direction"] = self.inputs.metadata["PhaseEncodingDirection"] return runtime @@ -59,9 +61,9 @@ def get_trt(in_meta, in_file=None): use an ``'epi.nii.gz'`` file-stub which has 90 pixels in the j-axis encoding direction. - >>> meta = {'TotalReadoutTime': 0.02596} + >>> meta = {'TotalReadoutTime': 0.05251} >>> get_trt(meta) - 0.02596 + 0.05251 If the *effective echo spacing* :math:`t_\text{ees}` (``EffectiveEchoSpacing`` BIDS field) is provided, then the @@ -77,7 +79,7 @@ def get_trt(in_meta, in_file=None): ... 'PhaseEncodingDirection': 'j-', ... 'ParallelReductionFactorInPlane': 2} >>> get_trt(meta, in_file='epi.nii.gz') - 0.02596 + 0.05251 Some vendors, like Philips, store different parameter names: @@ -96,15 +98,21 @@ def get_trt(in_meta, in_file=None): if trt is not None: return trt - # All other cases require the parallel acc and npe (N vox in PE dir) - acc = float(in_meta.get("ParallelReductionFactorInPlane", 1.0)) + # npe = N voxels PE direction npe = nb.load(in_file).shape[_get_pe_index(in_meta)] - etl = npe // acc # Use case 2: TRT is defined ees = in_meta.get("EffectiveEchoSpacing", None) if ees is not None: - return ees * (etl - 1) + # Effective echo spacing means that acceleration factors have been accounted for. + return ees * (npe - 1) + + # All other cases require the parallel acc and npe (N vox in PE dir) + acc = float(in_meta.get("ParallelReductionFactorInPlane", 1.0)) + etl = npe // acc # effective train length + es = in_meta.get("EchoSpacing", None) + if es is not None: + return es * (etl - 1) # Use case 3 (philips scans) wfs = in_meta.get("WaterFatShift", None) diff --git a/sdcflows/interfaces/reportlets.py b/sdcflows/interfaces/reportlets.py index feadfe86b5..337ca83346 100644 --- a/sdcflows/interfaces/reportlets.py +++ b/sdcflows/interfaces/reportlets.py @@ -27,7 +27,7 @@ class _FieldmapReportletInputSpec(reporting.ReportCapableInputSpec): desc='a label name for the reference mosaic') moving_label = traits.Str('Fieldmap (Hz)', usedefault=True, desc='a label name for the reference mosaic') - # pe_dir = traits.Enum(*tuple("ijk"), desc="PE direction") + apply_mask = traits.Bool(False, usedefault=True, desc="zero values outside mask") class FieldmapReportlet(reporting.ReportCapableInterface): @@ -77,6 +77,9 @@ def _generate_report(self): fmapdata = fmapnii.get_fdata() vmax = max(abs(np.percentile(fmapdata[maskdata], 99.8)), abs(np.percentile(fmapdata[maskdata], 0.2))) + if self.inputs.apply_mask: + fmapdata[~maskdata] = 0 + fmapnii = nb.Nifti1Image(fmapdata, fmapnii.affine, fmapnii.header) fmap_overlay = [{ 'overlay': fmapnii, diff --git a/sdcflows/interfaces/tests/test_bspline.py b/sdcflows/interfaces/tests/test_bspline.py index 5ee4b0c262..62ed8a800f 100644 --- a/sdcflows/interfaces/tests/test_bspline.py +++ b/sdcflows/interfaces/tests/test_bspline.py @@ -50,6 +50,7 @@ def test_bsplines(tmp_path, testnum): in_mask=str(tmp_path / "target.nii.gz"), bs_spacing=[(4, 6, 8)], recenter="no", + ridge_alpha=1e-4, ).run() # Absolute error of the interpolated field is always below 2 Hz diff --git a/sdcflows/workflows/apply/correction.py b/sdcflows/workflows/apply/correction.py new file mode 100644 index 0000000000..d33841a5d5 --- /dev/null +++ b/sdcflows/workflows/apply/correction.py @@ -0,0 +1,77 @@ +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# vi: set ft=python sts=4 ts=4 sw=4 et: +"""Applying a fieldmap given its B-Spline coefficients in Hz.""" +from nipype.pipeline import engine as pe +from nipype.interfaces import utility as niu +from niworkflows.engine.workflows import LiterateWorkflow as Workflow + + +def init_unwarp_wf(omp_nthreads=1, debug=False, name="unwarp_wf"): + """ + Set up a workflow that unwarps the input :abbr:`EPI (echo-planar imaging)` dataset. + + Workflow Graph + .. workflow:: + :graph2use: orig + :simple_form: yes + + from sdcflows.workflows.apply.correction import init_unwarp_wf + wf = init_unwarp_wf(omp_nthreads=2) + + Parameters + ---------- + omp_nthreads : :obj:`int` + Maximum number of threads an individual process may use. + name : :obj:`str` + Unique name of this workflow. + + Inputs + ------ + distorted + the target EPI image. + metadata + dictionary of metadata corresponding to the target EPI image + fmap_coeff + fieldmap coefficients in distorted EPI space. + + Outputs + ------- + corrected + the target EPI reference image, after applying unwarping. + + """ + from niworkflows.interfaces.fixes import FixHeaderApplyTransforms as ApplyTransforms + from ...interfaces.epi import GetReadoutTime + from ...interfaces.bspline import Coefficients2Warp + + workflow = Workflow(name=name) + inputnode = pe.Node( + niu.IdentityInterface(fields=["distorted", "metadata", "fmap_coeff"]), + name="inputnode", + ) + outputnode = pe.Node( + niu.IdentityInterface(fields=["corrected", "fieldmap"]), name="outputnode" + ) + + rotime = pe.Node(GetReadoutTime(), name="rotime") + resample = pe.Node(Coefficients2Warp(low_mem=debug), name="resample") + unwarp = pe.Node( + ApplyTransforms(dimension=3, interpolation="BSpline"), name="unwarp" + ) + + # fmt:off + workflow.connect([ + (inputnode, rotime, [("distorted", "in_file"), + ("metadata", "metadata")]), + (inputnode, resample, [("distorted", "in_target"), + ("fmap_coeff", "in_coeff")]), + (rotime, resample, [("readout_time", "ro_time"), + ("pe_direction", "pe_dir")]), + (inputnode, unwarp, [("distorted", "reference_image"), + ("distorted", "input_image")]), + (resample, unwarp, [("out_warp", "transforms")]), + (resample, outputnode, [("out_field", "fieldmap")]), + (unwarp, outputnode, [("output_image", "corrected")]), + ]) + # fmt:on + return workflow diff --git a/sdcflows/workflows/apply/registration.py b/sdcflows/workflows/apply/registration.py index 1c262d3758..5475ba7f8c 100644 --- a/sdcflows/workflows/apply/registration.py +++ b/sdcflows/workflows/apply/registration.py @@ -56,7 +56,10 @@ def init_coeff2epi_wf( Outputs ------- fmap_coeff - fieldmap coefficients + fieldmap coefficients in the space of the target reference EPI + target_ref + the target reference EPI resampled into the fieldmap reference for + quality control purposes. """ from packaging.version import parse as parseversion, Version @@ -75,7 +78,7 @@ def init_coeff2epi_wf( name="inputnode", ) outputnode = pe.Node( - niu.IdentityInterface(fields=["fmap_ref", "fmap_coeff"]), name="outputnode" + niu.IdentityInterface(fields=["target_ref", "fmap_coeff"]), name="outputnode" ) # Register the reference of the fieldmap to the reference @@ -96,12 +99,12 @@ def init_coeff2epi_wf( # fmt: off workflow.connect([ (inputnode, coregister, [ - ("target_ref", "fixed_image"), - ("fmap_ref", "moving_image"), - ("target_mask", f"fixed_image_mask{mask_trait_s}"), - ("fmap_mask", f"moving_image_mask{mask_trait_s}"), + ("target_ref", "moving_image"), + ("fmap_ref", "fixed_image"), + ("target_mask", f"moving_image_mask{mask_trait_s}"), + ("fmap_mask", f"fixed_image_mask{mask_trait_s}"), ]), - (coregister, outputnode, [("warped_image", "fmap_ref")]), + (coregister, outputnode, [("warped_image", "target_ref")]), ]) # fmt: on @@ -114,8 +117,9 @@ def init_coeff2epi_wf( # fmt: off workflow.connect([ (inputnode, map_coeff, [("fmap_coeff", "in_coeff"), - ("target_ref", "ref_image")]), - (coregister, map_coeff, [("reverse_transforms", "transform")]), + ("fmap_ref", "fmap_ref"), + ("target_ref", "target_ref")]), + (coregister, map_coeff, [("forward_transforms", "transform")]), (map_coeff, outputnode, [("out", "fmap_coeff")]), ]) # fmt: on @@ -123,20 +127,20 @@ def init_coeff2epi_wf( return workflow -def _move_coeff(in_coeff, ref_image, transform): +def _move_coeff(in_coeff, target_ref, fmap_ref, transform): """Read in a rigid transform from ANTs, and update the coefficients field affine.""" from pathlib import Path - import numpy as np import nibabel as nb import nitransforms as nt if isinstance(in_coeff, str): in_coeff = [in_coeff] - xfm = nt.io.itk.ITKLinearTransform.from_filename(transform[0]).to_ras() - - refaff = nb.load(ref_image).affine - refdir = refaff[:3, :3] / nb.affines.voxel_sizes(refaff) + xfm = nt.linear.Affine( + nt.io.itk.ITKLinearTransform.from_filename(transform[0]).to_ras(), + reference=fmap_ref, + ) + xfm.apply(target_ref).to_filename("transformed.nii.gz") out = [] for i, c in enumerate(in_coeff): @@ -144,8 +148,7 @@ def _move_coeff(in_coeff, ref_image, transform): out.append(str(Path(f"moved_coeff_{i:03d}.nii.gz").absolute())) - newaff = np.eye(4) - newaff[:3, :3] = nb.affines.voxel_sizes(img.affine) * refdir - newaff[:3, 3] = nb.affines.apply_affine(xfm, img.affine[:3, 3].T) + newaff = xfm.matrix @ img.affine img.__class__(img.dataobj, newaff, img.header).to_filename(out[-1]) + return out diff --git a/sdcflows/workflows/apply/tests/fieldcoeff.nii.gz b/sdcflows/workflows/apply/tests/fieldcoeff.nii.gz new file mode 100644 index 0000000000000000000000000000000000000000..e6a0f969706d3671865e87c843c8c1f6499f0460 GIT binary patch literal 2100 zcmV-42+Q{$iwFo+*1TT=|8sR>El5L9F)%SOFg9OgWpiUKV{c_B@9>e2By1w5o zfnUdp$>ZMN*S$Hy5oFZ3UHL}tNQs?OpTG^T?? zvScXw+g_0GVu?mK<|*_js_LW@-Fvv4AyxAEwQsnUK_x_k*EHsvUoo-GO0vm0R^m}u zCwR;KFMX$*k0e^xgm{^$!1$O2a9{9KiG05q3x!PUS|uDUyaYZ|J5Fl5r?DpOI-GNU z1Y@vD04;o1K#j?XWCq^@l%>Rw^;@6A$^1+@<#I7RFPcXUJ*To)HieT$x4UfdKqhSw zzo2Q>tJ(Z>OE~ZI2Jpa@R8X_@1SpTw=GK^=C)dQeO!b;bq$3UDUU-?gm372$rsJ2Q zs1r_7xjxG){Ukbpb78=IUwE*+1kBM-9gEffj%trMD%s`2Jvf@e4Ok?x4jvf_A2kiz z!#j_cAPbQeZFR7OQG0KYdjsL9BfAUw``aPa%vkyhKO2e1u(Nju3CWlR)b@K5^}|I#ce)G-lGJ7Mju#NaVr8~>!akDUb%9CJOC?qP)l?g$ z(8aTrWe#^BYPgfkE$f_)EL&yVkZznpA9X4w^puTTqH_&bScTCSk50pXw=S*hBgW4s zjcsVWK9-xmL;TET!>BCz7#D1C5V_=f$^!geC}MtZ*c|*x?sHPfb&*FWkAl(e zx_3EAHMO`NGi#h+x{U_TIY*8q(cB_y}uhTOK>n^ zB#z4Hs3?{+!k6HFSv@(m-ydZSj~}ai z^I0BT&-R(9{KFbr(A-Pp{N{pSs$2Fm3A%iN?JD4;o7c#tXw~h*D)qe!* zAq$3LPwBe!Vdj%9HpFLkJ@{2{I_1fr=8z-(C?Rd0llkU#561!khDC?ciKq@))dn#Q)D3YmJ2qXeSt}S z8jJbOSs<)$ki6VrO=ic1{RDS*W0e_~Y2HOOdb{>4SeHZ< z^{U&XiSg|@3pw@>%?_>s*DGx4nSMx}?q88A4OYSyhX+I!qXW9Oyu$UN_t|tcW%BTJ zD)cE4Al)cKXnWNa3+gwqcRP}B>1zrytqPH+)iEe=X~Vy?PGlQi)eE;|w==!!Zs7Hy ze5gOg2k`D>(_L9=K)m=O?wfOtan3$Q96uB(;z{M7At-m{Lb^|u#};-k0UL&j*q|L} z6>msesU~x&VFJ{#w?nAhA4Wdyx-PfvE%x&_3ezMEcy1f=4q3oeu;t=HE-0BhTcwCdteJpagKafvqB!G;`qlV=qCGD zPJ5^xW*F)y;z)j^*j*cUv`j)0BVBat@?jJhGUm76z??3xepGD$qiP(Ho=FOf_MY@! et*@{-;o624GoNo^gm2gUr~Y4udl3*I3IG7!)DBAk literal 0 HcmV?d00001 diff --git a/sdcflows/workflows/apply/tests/test_correct.py b/sdcflows/workflows/apply/tests/test_correct.py new file mode 100644 index 0000000000..0365a292a7 --- /dev/null +++ b/sdcflows/workflows/apply/tests/test_correct.py @@ -0,0 +1,114 @@ +"""Test unwarp.""" +import os +from pathlib import Path +import pytest + +from nipype.pipeline import engine as pe + +from ...fit.fieldmap import init_magnitude_wf +from ..correction import init_unwarp_wf +from ..registration import init_coeff2epi_wf + + +@pytest.mark.skipif(os.getenv("TRAVIS") == "true", reason="this is TravisCI") +def test_unwarp_wf(tmpdir, datadir, workdir, outdir): + """Test the unwarping workflow.""" + distorted = ( + datadir + / "testdata" + / "sub-HCP101006" + / "func" + / "sub-HCP101006_task-rest_dir-LR_sbref.nii.gz" + ) + + magnitude = ( + datadir + / "testdata" + / "sub-HCP101006" + / "fmap" + / "sub-HCP101006_magnitude1.nii.gz" + ) + fmap_ref_wf = init_magnitude_wf(2, name="fmap_ref_wf") + fmap_ref_wf.inputs.inputnode.magnitude = magnitude + + epi_ref_wf = init_magnitude_wf(2, name="epi_ref_wf") + epi_ref_wf.inputs.inputnode.magnitude = distorted + + reg_wf = init_coeff2epi_wf(2, debug=True, write_coeff=True) + reg_wf.inputs.inputnode.fmap_coeff = [Path(__file__).parent / "fieldcoeff.nii.gz"] + + unwarp_wf = init_unwarp_wf(omp_nthreads=2, debug=True) + unwarp_wf.inputs.inputnode.metadata = { + "EffectiveEchoSpacing": 0.00058, + "PhaseEncodingDirection": "i-", + } + + workflow = pe.Workflow(name="test_unwarp_wf") + # fmt: off + workflow.connect([ + (epi_ref_wf, unwarp_wf, [("outputnode.fmap_ref", "inputnode.distorted")]), + (epi_ref_wf, reg_wf, [ + ("outputnode.fmap_ref", "inputnode.target_ref"), + ("outputnode.fmap_mask", "inputnode.target_mask"), + ]), + (fmap_ref_wf, reg_wf, [ + ("outputnode.fmap_ref", "inputnode.fmap_ref"), + ("outputnode.fmap_mask", "inputnode.fmap_mask"), + ]), + (reg_wf, unwarp_wf, [("outputnode.fmap_coeff", "inputnode.fmap_coeff")]), + ]) + # fmt:on + + if outdir: + from niworkflows.interfaces import SimpleBeforeAfter + from ...outputs import DerivativesDataSink + from ....interfaces.reportlets import FieldmapReportlet + + report = pe.Node( + SimpleBeforeAfter(before_label="Distorted", after_label="Corrected",), + name="report", + mem_gb=0.1, + ) + ds_report = pe.Node( + DerivativesDataSink( + base_directory=str(outdir), + suffix="bold", + desc="corrected", + datatype="figures", + dismiss_entities=("fmap",), + source_file=distorted, + ), + name="ds_report", + run_without_submitting=True, + ) + + rep = pe.Node(FieldmapReportlet(apply_mask=True), "simple_report") + rep.interface._always_run = True + + ds_fmap_report = pe.Node( + DerivativesDataSink( + base_directory=str(outdir), + datatype="figures", + suffix="bold", + desc="fieldmap", + dismiss_entities=("fmap",), + source_file=distorted, + ), + name="ds_fmap_report", + ) + + # fmt: off + workflow.connect([ + (epi_ref_wf, report, [("outputnode.fmap_ref", "before")]), + (unwarp_wf, report, [("outputnode.corrected", "after")]), + (report, ds_report, [("out_report", "in_file")]), + (epi_ref_wf, rep, [("outputnode.fmap_ref", "reference"), + ("outputnode.fmap_mask", "mask")]), + (unwarp_wf, rep, [("outputnode.fieldmap", "fieldmap")]), + (rep, ds_fmap_report, [("out_report", "in_file")]), + ]) + # fmt: on + + if workdir: + workflow.base_dir = str(workdir) + workflow.run(plugin="Linear") diff --git a/sdcflows/workflows/apply/tests/test_registration.py b/sdcflows/workflows/apply/tests/test_registration.py index ed568daccd..ffdc49acab 100644 --- a/sdcflows/workflows/apply/tests/test_registration.py +++ b/sdcflows/workflows/apply/tests/test_registration.py @@ -56,7 +56,7 @@ def test_registration_wf(tmpdir, datadir, workdir, outdir): from ...outputs import DerivativesDataSink report = pe.Node( - SimpleBeforeAfter(before_label="Target EPI", after_label="B0 Reference",), + SimpleBeforeAfter(after_label="Target EPI", before_label="B0 Reference",), name="report", mem_gb=0.1, ) @@ -75,8 +75,8 @@ def test_registration_wf(tmpdir, datadir, workdir, outdir): # fmt: off workflow.connect([ - (epi_ref_wf, report, [("outputnode.fmap_ref", "before")]), - (reg_wf, report, [("outputnode.fmap_ref", "after")]), + (fmap_ref_wf, report, [("outputnode.fmap_ref", "before")]), + (reg_wf, report, [("outputnode.target_ref", "after")]), (report, ds_report, [("out_report", "in_file")]), ]) # fmt: on @@ -101,7 +101,6 @@ def test_registration_wf(tmpdir, datadir, workdir, outdir): # assert np.allclose(dircos, np.eye(3)) - def _gen_coeff(img): from pathlib import Path from sdcflows.interfaces.bspline import bspline_grid From 2b8e76bd1921eb25042538f34b5340dbf306493b Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Wed, 25 Nov 2020 20:26:18 +0100 Subject: [PATCH 31/68] Apply suggestions from code review Co-authored-by: Mathias Goncalves --- sdcflows/interfaces/bspline.py | 18 +++++++++--------- sdcflows/interfaces/reportlets.py | 2 +- sdcflows/interfaces/tests/test_bspline.py | 3 ++- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/sdcflows/interfaces/bspline.py b/sdcflows/interfaces/bspline.py index 5c62a40f74..1e2fc10bf3 100644 --- a/sdcflows/interfaces/bspline.py +++ b/sdcflows/interfaces/bspline.py @@ -40,7 +40,7 @@ class _BSplineApproxInputSpec(BaseInterfaceInputSpec): "mode", "median", "mean", - "no", + False, usedefault=True, desc="strategy to recenter the distribution of the input fieldmap", ) @@ -143,14 +143,14 @@ def _run_interface(self, runtime): ) hdr = fmapnii.header.copy() hdr.set_data_dtype("float32") - nb.Nifti1Image(interp_data, fmapnii.affine, hdr).to_filename(out_name) + fmapnii.__class__(interp_data, fmapnii.affine, hdr).to_filename(out_name) self._results["out_field"] = out_name index = 0 self._results["out_coeff"] = [] for i, (n, bsl) in enumerate(zip(ncoeff, bs_levels)): out_level = out_name.replace("_field.", f"_coeff{i:03}.") - nb.Nifti1Image( + bsl.__class__( np.array(model.coef_, dtype="float32")[index:index + n].reshape( bsl.shape ), @@ -162,7 +162,7 @@ def _run_interface(self, runtime): # Write out fitting-error map self._results["out_error"] = out_name.replace("_field.", "_error.") - nb.Nifti1Image( + fmapnii.__class__( data * mask - interp_data, fmapnii.affine, fmapnii.header ).to_filename(self._results["out_error"]) @@ -177,7 +177,7 @@ def _run_interface(self, runtime): ) interp_data[~mask] = np.array(model.coef_) @ extrapolators # Extrapolation self._results["out_extrapolated"] = out_name.replace("_field.", "_extra.") - nb.Nifti1Image(interp_data, fmapnii.affine, hdr).to_filename( + fmapnii.__class__(interp_data, fmapnii.affine, hdr).to_filename( self._results["out_extrapolated"] ) return runtime @@ -190,7 +190,7 @@ class _Coefficients2WarpInputSpec(BaseInterfaceInputSpec): mandatory=True, desc="input coefficients, after alignment to the EPI data", ) - ro_time = traits.Float(1.0, usedefault=True, desc="EPI readout time (s).") + ro_time = traits.Float(mandatory=True, desc="EPI readout time (s).") pe_dir = traits.Enum( "i", "i-", @@ -260,7 +260,7 @@ def _run_interface(self, runtime): self._results["out_field"] = fname_presuffix( self.inputs.in_target, suffix="_field", newpath=runtime.cwd ) - nb.Nifti1Image(data, targetnii.affine, hdr).to_filename( + targetnii.__class__(data, targetnii.affine, hdr).to_filename( self._results["out_field"] ) @@ -280,7 +280,7 @@ def _run_interface(self, runtime): aff = targetnii.affine.copy() aff[:3, 3] = 0.0 field = nb.affines.apply_affine(aff, field).reshape(fieldshape) - warpnii = nb.Nifti1Image( + warpnii = targetnii.__class__( field[:, :, :, np.newaxis, :].astype("float32"), targetnii.affine, None ) warpnii.header.set_intent("vector", (), "") @@ -314,7 +314,7 @@ def bspline_grid(img, control_zooms_mm=DEFAULT_ZOOMS_MM): bs_affine, 0.5 * (bs_shape - 1) ) - return nb.Nifti1Image(np.zeros(bs_shape, dtype="float32"), bs_affine) + return img.__class__(np.zeros(bs_shape, dtype="float32"), bs_affine) def bspline_weights(points, ctrl_nii): diff --git a/sdcflows/interfaces/reportlets.py b/sdcflows/interfaces/reportlets.py index 337ca83346..4953744981 100644 --- a/sdcflows/interfaces/reportlets.py +++ b/sdcflows/interfaces/reportlets.py @@ -79,7 +79,7 @@ def _generate_report(self): abs(np.percentile(fmapdata[maskdata], 0.2))) if self.inputs.apply_mask: fmapdata[~maskdata] = 0 - fmapnii = nb.Nifti1Image(fmapdata, fmapnii.affine, fmapnii.header) + fmapnii = fmapnii.__class__(fmapdata, fmapnii.affine, fmapnii.header) fmap_overlay = [{ 'overlay': fmapnii, diff --git a/sdcflows/interfaces/tests/test_bspline.py b/sdcflows/interfaces/tests/test_bspline.py index 62ed8a800f..46de43b1bd 100644 --- a/sdcflows/interfaces/tests/test_bspline.py +++ b/sdcflows/interfaces/tests/test_bspline.py @@ -42,6 +42,7 @@ def test_bsplines(tmp_path, testnum): in_target=str(tmp_path / "target.nii.gz"), in_coeff=str(tmp_path / "coeffs.nii.gz"), pe_dir="j-", + ro_time=1.0, ).run() # Approximate the interpolated target @@ -49,7 +50,7 @@ def test_bsplines(tmp_path, testnum): in_data=test1.outputs.out_field, in_mask=str(tmp_path / "target.nii.gz"), bs_spacing=[(4, 6, 8)], - recenter="no", + recenter=False, ridge_alpha=1e-4, ).run() From 528ca12726abb4b85cccd6a7f1071d03fd9da5c5 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 26 Nov 2020 15:10:06 +0100 Subject: [PATCH 32/68] FIX: Convert SEI fieldmaps given in rad/s into Hz Adds a lightweight node (`run_without_submitting=True`) wrapping an interface that just checks the units of the input image and divides by 2pi when units are rad/s. Unfortunatelly, we don't currently have data to test these fieldmaps in some integration/smoke test. Resolves: #124. --- sdcflows/interfaces/fmap.py | 32 +++++++++++++++++++++++++- sdcflows/interfaces/tests/test_fmap.py | 24 +++++++++++++++++++ sdcflows/workflows/fit/fieldmap.py | 14 ++++++++--- 3 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 sdcflows/interfaces/tests/test_fmap.py diff --git a/sdcflows/interfaces/fmap.py b/sdcflows/interfaces/fmap.py index 3dc9b3b961..a011aaec5d 100644 --- a/sdcflows/interfaces/fmap.py +++ b/sdcflows/interfaces/fmap.py @@ -1,7 +1,9 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: """Interfaces to deal with the various types of fieldmap sources.""" - +import numpy as np +import nibabel as nb +from nipype.utils.filemanip import fname_presuffix from nipype import logging from nipype.interfaces.base import ( BaseInterfaceInputSpec, @@ -99,3 +101,31 @@ def _run_interface(self, runtime): self.inputs.in_file, _delta_te(self.inputs.metadata), newpath=runtime.cwd ) return runtime + + +class _CheckB0UnitsInputSpec(BaseInterfaceInputSpec): + in_file = File(exists=True, mandatory=True, desc="input fieldmap") + units = traits.Enum("Hz", "rad/s", mandatory=True, desc="fieldmap units") + + +class _CheckB0UnitsOutputSpec(TraitedSpec): + out_file = File(exists=True, desc="output fieldmap in Hz") + + +class CheckB0Units(SimpleInterface): + """Ensure the input fieldmap is given in Hz.""" + + input_spec = _CheckB0UnitsInputSpec + output_spec = _CheckB0UnitsOutputSpec + + def _run_interface(self, runtime): + if self.inputs.units == "Hz": + self._results["out_file"] = self.inputs.in_file + return runtime + + self._results["out_file"] = fname_presuffix( + self.inputs.in_file, suffix="_Hz", newpath=runtime.cwd) + img = nb.load(self.inputs.in_file) + data = np.asanyarray(img.dataobj) / (2.0 * np.pi) + img.__class__(data, img.affine, img.header).to_filename(self._results["out_file"]) + return runtime diff --git a/sdcflows/interfaces/tests/test_fmap.py b/sdcflows/interfaces/tests/test_fmap.py new file mode 100644 index 0000000000..5919f5cbce --- /dev/null +++ b/sdcflows/interfaces/tests/test_fmap.py @@ -0,0 +1,24 @@ +"""Test fieldmap interfaces.""" +import numpy as np +import nibabel as nb +import pytest + +from ..fmap import CheckB0Units + + +@pytest.mark.parametrize("units", ("rad/s", "Hz")) +def test_units(tmpdir, units): + """Check the conversion of units.""" + tmpdir.chdir() + hz = np.ones((5, 5, 5), dtype="float32") * 100 + data = hz.copy() + + if units == "rad/s": + data *= 2.0 * np.pi + + nb.Nifti1Image(data, np.eye(4), None).to_filename("data.nii.gz") + out_data = nb.load( + CheckB0Units(units=units, in_file="data.nii.gz").run().outputs.out_file + ).get_fdata(dtype="float32") + + assert np.allclose(hz, out_data) diff --git a/sdcflows/workflows/fit/fieldmap.py b/sdcflows/workflows/fit/fieldmap.py index 0030169a21..7c99670e99 100644 --- a/sdcflows/workflows/fit/fieldmap.py +++ b/sdcflows/workflows/fit/fieldmap.py @@ -148,6 +148,9 @@ def init_fmap_wf(omp_nthreads=1, debug=False, mode="phasediff", name="fmap_wf"): Path to the corresponding magnitude image for anatomical reference. fieldmap : :obj:`str` Path to the fieldmap acquisition (``*_fieldmap.nii[.gz]`` of BIDS). + units : :obj:`str` + Units (`"Hz"` or `"rad/s"`) of the fieldmap (only direct :math:`B_0` + acquisitions with :abbr:`SEI (Spiral-Echo Imaging)` fieldmaps). Outputs ------- @@ -223,19 +226,24 @@ def init_fmap_wf(omp_nthreads=1, debug=False, mode="phasediff", name="fmap_wf"): # fmt: on else: from niworkflows.interfaces.images import IntraModalMerge + from ...interfaces.fmap import CheckB0Units workflow.__desc__ = """\ A *B0* nonuniformity map (or *fieldmap*) was directly measured with -an MRI scheme designed with that purpose (e.g., a spiral pulse sequence). +an MRI scheme designed with that purpose such as SEI (Spiral-Echo Imaging). """ - # Merge input fieldmap images + # Merge input fieldmap images (assumes all are given in the same units!) fmapmrg = pe.Node( IntraModalMerge(zero_based_avg=False, hmc=False), name="fmapmrg" ) + units = pe.Node(CheckB0Units(), name="units", run_without_submitting=True) + # fmt: off workflow.connect([ + (inputnode, units, [("units", "units")]), (inputnode, fmapmrg, [("fieldmap", "in_files")]), - (fmapmrg, bs_filter, [("out_avg", "in_data")]), + (fmapmrg, units, [("out_avg", "in_file")]), + (units, bs_filter, [("out_file", "in_data")]), ]) # fmt: on From e97750bebfe54a3bd86d59706ca3a45b09417f9c Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 26 Nov 2020 15:14:42 +0100 Subject: [PATCH 33/68] sty: black these files --- sdcflows/interfaces/fmap.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sdcflows/interfaces/fmap.py b/sdcflows/interfaces/fmap.py index a011aaec5d..4d8ec6529e 100644 --- a/sdcflows/interfaces/fmap.py +++ b/sdcflows/interfaces/fmap.py @@ -124,8 +124,11 @@ def _run_interface(self, runtime): return runtime self._results["out_file"] = fname_presuffix( - self.inputs.in_file, suffix="_Hz", newpath=runtime.cwd) + self.inputs.in_file, suffix="_Hz", newpath=runtime.cwd + ) img = nb.load(self.inputs.in_file) data = np.asanyarray(img.dataobj) / (2.0 * np.pi) - img.__class__(data, img.affine, img.header).to_filename(self._results["out_file"]) + img.__class__(data, img.affine, img.header).to_filename( + self._results["out_file"] + ) return runtime From e6766bc9910f7bc92554d73313cc9e22e92de686 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 26 Nov 2020 16:03:37 +0100 Subject: [PATCH 34/68] maint: add one minimal doctest for coverage --- sdcflows/utils/phasemanip.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sdcflows/utils/phasemanip.py b/sdcflows/utils/phasemanip.py index 088efcd9b1..27ee9d857b 100644 --- a/sdcflows/utils/phasemanip.py +++ b/sdcflows/utils/phasemanip.py @@ -108,6 +108,10 @@ def delta_te(in_values): >>> delta_te({}) # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ValueError: + + >>> delta_te({"EchoTimeDifference": "a"}) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ValueError: """ te2 = in_values.get("EchoTime2") From 020c970998c1604647faf4d37d085c054bc099ed Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 26 Nov 2020 16:36:49 +0100 Subject: [PATCH 35/68] maint: remove unused, unmaintained sdcflows.cli module --- docs/conf.py | 2 +- sdcflows/cli/__init__.py | 0 sdcflows/cli/run.py | 113 --------------------------------------- setup.cfg | 4 -- 4 files changed, 1 insertion(+), 118 deletions(-) delete mode 100644 sdcflows/cli/__init__.py delete mode 100644 sdcflows/cli/run.py diff --git a/docs/conf.py b/docs/conf.py index 99b2eae04a..79722857e0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -92,7 +92,7 @@ exclude_patterns = [ '_build', 'Thumbs.db', '.DS_Store', 'api/sdcflows.rst', - 'api/sdcflows.cli.rst', 'api/sdcflows.cli.*.rst'] +] # The name of the Pygments (syntax highlighting) style to use. pygments_style = None diff --git a/sdcflows/cli/__init__.py b/sdcflows/cli/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/sdcflows/cli/run.py b/sdcflows/cli/run.py deleted file mode 100644 index f6426e9b96..0000000000 --- a/sdcflows/cli/run.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env python3 -import logging -from pathlib import Path - -logging.addLevelName(25, 'IMPORTANT') # Add a new level between INFO and WARNING -logging.addLevelName(15, 'VERBOSE') # Add a new level between INFO and DEBUG -logger = logging.getLogger('sdcflows') - - -def get_parser(): - """Define the command line interface""" - from argparse import ArgumentParser - from argparse import RawTextHelpFormatter - from .. import __version__ as _vstr - - parser = ArgumentParser(description='SDC Workflows', - formatter_class=RawTextHelpFormatter) - - parser.add_argument( - 'bids_dir', action='store', type=Path, - help='the root folder of a BIDS dataset') - parser.add_argument('output_dir', action='store', type=Path, - help='the output path for the outcomes of preprocessing and visual ' - 'reports') - parser.add_argument('analysis_level', choices=['participant', 'group'], nargs='+', - help='processing stage to be run, "participant" means individual analysis ' - 'and "group" is second level analysis.') - # optional arguments - parser.add_argument('--version', action='version', version='v{}'.format(_vstr)) - - # Options that affect how pyBIDS is configured - g_bids = parser.add_argument_group('Options for filtering BIDS queries') - g_bids.add_argument('--participant-label', action='store', type=str, - nargs='*', dest='subject', help='process only particular subjects') - g_bids.add_argument('--task', action='store', type=str, nargs='*', - help='select a specific task to be processed') - g_bids.add_argument('--dir', action='store', type=str, nargs='*', - help='select a specific direction entity to be processed') - g_bids.add_argument('--acq', action='store', type=str, nargs='*', dest='acquisition', - help='select a specific acquisition entity to be processed') - g_bids.add_argument('--run', action='store', type=int, nargs='*', - help='select a specific run identifier to be processed') - g_bids.add_argument('--suffix', action='store', type=str, nargs='*', default='bold', - help='select a specific run identifier to be processed') - - g_perfm = parser.add_argument_group('Options to handle performance') - g_perfm.add_argument("-v", "--verbose", dest="verbose_count", action="count", default=0, - help="increases log verbosity for each occurence, debug level is -vvv") - g_perfm.add_argument('--ncpus', '--nprocs', action='store', type=int, - help='maximum number of threads across all processes') - g_perfm.add_argument('--nthreads', '--omp-nthreads', action='store', type=int, - help='maximum number of threads per-process') - - g_other = parser.add_argument_group('Other options') - g_other.add_argument('-w', '--work-dir', action='store', type=Path, - help='path where intermediate results should be stored') - - return parser - - -def main(): - """Entry point""" - from os import cpu_count - from multiprocessing import set_start_method - # from bids.layout import BIDSLayout - from nipype import logging as nlogging - set_start_method('forkserver') - - opts = get_parser().parse_args() - - # Retrieve logging level - log_level = int(max(25 - 5 * opts.verbose_count, logging.DEBUG)) - # Set logging - logger.setLevel(log_level) - nlogging.getLogger('nipype.workflow').setLevel(log_level) - nlogging.getLogger('nipype.interface').setLevel(log_level) - nlogging.getLogger('nipype.utils').setLevel(log_level) - - # Resource management options - plugin_settings = { - 'plugin': 'MultiProc', - 'plugin_args': { - 'n_procs': opts.ncpus, - '' - 'raise_insufficient': False, - 'maxtasksperchild': 1, - } - } - # Permit overriding plugin config with specific CLI options - if not opts.ncpus or opts.ncpus < 1: - plugin_settings['plugin_args']['n_procs'] = cpu_count() - - nthreads = opts.nthreads - if not nthreads or nthreads < 1: - nthreads = cpu_count() - - # output_dir = opts.output_dir.resolve() - # bids_dir = opts.bids_dir or output_dir.parent - - # Get absolute path to BIDS directory - # bids_dir = opts.bids_dir.resolve() - # layout = BIDSLayout(str(bids_dir), validate=False, derivatives=str(output_dir)) - # query = {'suffix': opts.suffix, 'extension': ['.nii', '.nii.gz']} - - # for entity in ('subject', 'task', 'dir', 'acquisition', 'run'): - # arg = getattr(opts, entity, None) - # if arg is not None: - # query[entity] = arg - - -if __name__ == '__main__': - raise RuntimeError("sdcflows/cli/run.py should not be run directly;\n" - "Please `pip install` sdcflows and use the `sdcflows` command") diff --git a/setup.cfg b/setup.cfg index b8a4ef6564..6e48fcc405 100644 --- a/setup.cfg +++ b/setup.cfg @@ -70,10 +70,6 @@ sdcflows = data/flirtsch/*.cnf VERSION -[options.entry_points] -console_scripts = - sdcflows=sdcflows.cli.run:main - [versioneer] VCS = git style = pep440 From cb0ac847f76e967a7f6d5dff832df80b0e08ab2f Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 26 Nov 2020 16:42:12 +0100 Subject: [PATCH 36/68] sty: sphinx config file + cleanup --- docs/conf.py | 130 +++---- docs/sphinxext/docscrape.py | 570 ----------------------------- docs/sphinxext/docscrape_sphinx.py | 227 ------------ docs/sphinxext/github.py | 155 -------- docs/sphinxext/math_dollar.py | 63 ---- docs/sphinxext/numpydoc.py | 203 ---------- 6 files changed, 67 insertions(+), 1281 deletions(-) delete mode 100644 docs/sphinxext/docscrape.py delete mode 100644 docs/sphinxext/docscrape_sphinx.py delete mode 100644 docs/sphinxext/github.py delete mode 100644 docs/sphinxext/math_dollar.py delete mode 100644 docs/sphinxext/numpydoc.py diff --git a/docs/conf.py b/docs/conf.py index 79722857e0..7ce551cce5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,27 +1,24 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file does only contain a selection of the most common options. For a -# full list see the documentation: -# http://www.sphinx-doc.org/en/master/config +""" +Configuration file for the Sphinx documentation builder. -# -- Path setup -------------------------------------------------------------- +This file does only contain a selection of the most common options. For a +full list see the documentation: +http://www.sphinx-doc.org/en/master/config -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -import os -import sys +""" from packaging.version import Version - from sdcflows import __version__, __copyright__, __packagename__ -sys.path.append(os.path.abspath('sphinxext')) +# -- Path setup -------------------------------------------------------------- +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# sys.path.append(os.path.abspath("sphinxext")) # -- Project information ----------------------------------------------------- project = __packagename__ copyright = __copyright__ -author = 'The SDCflows Developers' +author = "The SDCflows Developers" # The short X.Y version version = Version(__version__).public @@ -31,30 +28,30 @@ # -- General configuration --------------------------------------------------- extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.doctest', - 'sphinx.ext.intersphinx', - 'sphinx.ext.coverage', - 'sphinx.ext.mathjax', - 'sphinx.ext.ifconfig', - 'sphinx.ext.viewcode', - 'sphinx.ext.githubpages', - 'nipype.sphinxext.plot_workflow', - 'sphinxcontrib.apidoc', - 'sphinxcontrib.napoleon' + "sphinx.ext.autodoc", + "sphinx.ext.doctest", + "sphinx.ext.intersphinx", + "sphinx.ext.coverage", + "sphinx.ext.mathjax", + "sphinx.ext.ifconfig", + "sphinx.ext.viewcode", + "sphinx.ext.githubpages", + "nipype.sphinxext.plot_workflow", + "sphinxcontrib.apidoc", + "sphinxcontrib.napoleon", ] autodoc_mock_imports = [ - 'matplotlib', - 'nilearn', - 'nipy', - 'nitime', - 'numpy', - 'pandas', - 'seaborn', - 'skimage', - 'svgutils', - 'transforms3d', + "matplotlib", + "nilearn", + "nipy", + "nitime", + "numpy", + "pandas", + "seaborn", + "skimage", + "svgutils", + "transforms3d", ] # Accept custom section names to be parsed for numpy-style docstrings @@ -63,21 +60,21 @@ # https://github.com/sphinx-contrib/napoleon/pull/10 is merged. napoleon_use_param = False napoleon_custom_sections = [ - ('Inputs', 'Parameters'), - ('Outputs', 'Parameters'), + ("Inputs", "Parameters"), + ("Outputs", "Parameters"), ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The master toctree document. -master_doc = 'index' +master_doc = "index" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -90,8 +87,10 @@ # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. exclude_patterns = [ - '_build', 'Thumbs.db', '.DS_Store', - 'api/sdcflows.rst', + "_build", + "Thumbs.db", + ".DS_Store", + "api/sdcflows.rst", ] # The name of the Pygments (syntax highlighting) style to use. @@ -103,7 +102,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -114,7 +113,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -130,7 +129,7 @@ # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. -htmlhelp_basename = 'sdcflowsdoc' +htmlhelp_basename = "sdcflowsdoc" # -- Options for LaTeX output ------------------------------------------------ @@ -139,15 +138,12 @@ # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -157,8 +153,13 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'smriprep.tex', 'sMRIPrep Documentation', - 'The sMRIPrep Developers', 'manual'), + ( + master_doc, + "smriprep.tex", + "sMRIPrep Documentation", + "The sMRIPrep Developers", + "manual", + ), ] @@ -166,10 +167,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'smriprep', 'sMRIPrep Documentation', - [author], 1) -] +man_pages = [(master_doc, "smriprep", "sMRIPrep Documentation", [author], 1)] # -- Options for Texinfo output ---------------------------------------------- @@ -178,9 +176,15 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'smriprep', 'sMRIPrep Documentation', - author, 'sMRIPrep', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "smriprep", + "sMRIPrep Documentation", + author, + "sMRIPrep", + "One line description of project.", + "Miscellaneous", + ), ] @@ -199,16 +203,16 @@ # epub_uid = '' # A list of files that should not be packed into the epub file. -epub_exclude_files = ['search.html'] +epub_exclude_files = ["search.html"] # -- Extension configuration ------------------------------------------------- -apidoc_module_dir = '../sdcflows' -apidoc_output_dir = 'api' -apidoc_excluded_paths = ['conftest.py', '*/tests/*', 'tests/*', 'data/*'] +apidoc_module_dir = "../sdcflows" +apidoc_output_dir = "api" +apidoc_excluded_paths = ["conftest.py", "*/tests/*", "tests/*", "data/*"] apidoc_separate_modules = True -apidoc_extra_args = ['--module-first', '-d 1', '-T'] +apidoc_extra_args = ["--module-first", "-d 1", "-T"] # -- Options for intersphinx extension --------------------------------------- diff --git a/docs/sphinxext/docscrape.py b/docs/sphinxext/docscrape.py deleted file mode 100644 index 470badd335..0000000000 --- a/docs/sphinxext/docscrape.py +++ /dev/null @@ -1,570 +0,0 @@ -"""Extract reference documentation from the NumPy source tree. - -""" -from __future__ import division, absolute_import, print_function - -import inspect -import textwrap -import re -import pydoc -from warnings import warn -import collections -import sys - - -class Reader(object): - """A line-based string reader. - - """ - def __init__(self, data): - """ - Parameters - ---------- - data : str - String with lines separated by '\n'. - - """ - if isinstance(data, list): - self._str = data - else: - self._str = data.split('\n') # store string as list of lines - - self.reset() - - def __getitem__(self, n): - return self._str[n] - - def reset(self): - self._l = 0 # current line nr - - def read(self): - if not self.eof(): - out = self[self._l] - self._l += 1 - return out - else: - return '' - - def seek_next_non_empty_line(self): - for l in self[self._l:]: - if l.strip(): - break - else: - self._l += 1 - - def eof(self): - return self._l >= len(self._str) - - def read_to_condition(self, condition_func): - start = self._l - for line in self[start:]: - if condition_func(line): - return self[start:self._l] - self._l += 1 - if self.eof(): - return self[start:self._l+1] - return [] - - def read_to_next_empty_line(self): - self.seek_next_non_empty_line() - - def is_empty(line): - return not line.strip() - - return self.read_to_condition(is_empty) - - def read_to_next_unindented_line(self): - def is_unindented(line): - return (line.strip() and (len(line.lstrip()) == len(line))) - return self.read_to_condition(is_unindented) - - def peek(self, n=0): - if self._l + n < len(self._str): - return self[self._l + n] - else: - return '' - - def is_empty(self): - return not ''.join(self._str).strip() - - -class NumpyDocString(collections.Mapping): - def __init__(self, docstring, config={}): - docstring = textwrap.dedent(docstring).split('\n') - - self._doc = Reader(docstring) - self._parsed_data = { - 'Signature': '', - 'Summary': [''], - 'Extended Summary': [], - 'Parameters': [], - 'Returns': [], - 'Yields': [], - 'Raises': [], - 'Warns': [], - 'Other Parameters': [], - 'Attributes': [], - 'Methods': [], - 'See Also': [], - 'Notes': [], - 'Warnings': [], - 'References': '', - 'Examples': '', - 'index': {} - } - - self._parse() - - def __getitem__(self, key): - return self._parsed_data[key] - - def __setitem__(self, key, val): - if key not in self._parsed_data: - warn("Unknown section %s" % key) - else: - self._parsed_data[key] = val - - def __iter__(self): - return iter(self._parsed_data) - - def __len__(self): - return len(self._parsed_data) - - def _is_at_section(self): - self._doc.seek_next_non_empty_line() - - if self._doc.eof(): - return False - - l1 = self._doc.peek().strip() # e.g. Parameters - - if l1.startswith('.. index::'): - return True - - l2 = self._doc.peek(1).strip() # ---------- or ========== - return l2.startswith('-'*len(l1)) or l2.startswith('='*len(l1)) - - def _strip(self, doc): - i = 0 - j = 0 - for i, line in enumerate(doc): - if line.strip(): - break - - for j, line in enumerate(doc[::-1]): - if line.strip(): - break - - return doc[i:len(doc)-j] - - def _read_to_next_section(self): - section = self._doc.read_to_next_empty_line() - - while not self._is_at_section() and not self._doc.eof(): - if not self._doc.peek(-1).strip(): # previous line was empty - section += [''] - - section += self._doc.read_to_next_empty_line() - - return section - - def _read_sections(self): - while not self._doc.eof(): - data = self._read_to_next_section() - name = data[0].strip() - - if name.startswith('..'): # index section - yield name, data[1:] - elif len(data) < 2: - yield StopIteration - else: - yield name, self._strip(data[2:]) - - def _parse_param_list(self, content): - r = Reader(content) - params = [] - while not r.eof(): - header = r.read().strip() - if ' : ' in header: - arg_name, arg_type = header.split(' : ')[:2] - else: - arg_name, arg_type = header, '' - - desc = r.read_to_next_unindented_line() - desc = dedent_lines(desc) - - params.append((arg_name, arg_type, desc)) - - return params - - _name_rgx = re.compile(r"^\s*(:(?P\w+):`(?P[a-zA-Z0-9_.-]+)`|" - r" (?P[a-zA-Z0-9_.-]+))\s*", re.X) - - def _parse_see_also(self, content): - """ - func_name : Descriptive text - continued text - another_func_name : Descriptive text - func_name1, func_name2, :meth:`func_name`, func_name3 - - """ - items = [] - - def parse_item_name(text): - """Match ':role:`name`' or 'name'""" - m = self._name_rgx.match(text) - if m: - g = m.groups() - if g[1] is None: - return g[3], None - else: - return g[2], g[1] - raise ValueError("%s is not a item name" % text) - - def push_item(name, rest): - if not name: - return - name, role = parse_item_name(name) - items.append((name, list(rest), role)) - del rest[:] - - current_func = None - rest = [] - - for line in content: - if not line.strip(): - continue - - m = self._name_rgx.match(line) - if m and line[m.end():].strip().startswith(':'): - push_item(current_func, rest) - current_func, line = line[:m.end()], line[m.end():] - rest = [line.split(':', 1)[1].strip()] - if not rest[0]: - rest = [] - elif not line.startswith(' '): - push_item(current_func, rest) - current_func = None - if ',' in line: - for func in line.split(','): - if func.strip(): - push_item(func, []) - elif line.strip(): - current_func = line - elif current_func is not None: - rest.append(line.strip()) - push_item(current_func, rest) - return items - - def _parse_index(self, section, content): - """ - .. index: default - :refguide: something, else, and more - - """ - def strip_each_in(lst): - return [s.strip() for s in lst] - - out = {} - section = section.split('::') - if len(section) > 1: - out['default'] = strip_each_in(section[1].split(','))[0] - for line in content: - line = line.split(':') - if len(line) > 2: - out[line[1]] = strip_each_in(line[2].split(',')) - return out - - def _parse_summary(self): - """Grab signature (if given) and summary""" - if self._is_at_section(): - return - - # If several signatures present, take the last one - while True: - summary = self._doc.read_to_next_empty_line() - summary_str = " ".join([s.strip() for s in summary]).strip() - if re.compile('^([\w., ]+=)?\s*[\w\.]+\(.*\)$').match(summary_str): - self['Signature'] = summary_str - if not self._is_at_section(): - continue - break - - if summary is not None: - self['Summary'] = summary - - if not self._is_at_section(): - self['Extended Summary'] = self._read_to_next_section() - - def _parse(self): - self._doc.reset() - self._parse_summary() - - sections = list(self._read_sections()) - section_names = set([section for section, content in sections]) - - has_returns = 'Returns' in section_names - has_yields = 'Yields' in section_names - # We could do more tests, but we are not. Arbitrarily. - if has_returns and has_yields: - msg = 'Docstring contains both a Returns and Yields section.' - raise ValueError(msg) - - for (section, content) in sections: - if not section.startswith('..'): - section = (s.capitalize() for s in section.split(' ')) - section = ' '.join(section) - if section in ('Parameters', 'Returns', 'Yields', 'Raises', - 'Warns', 'Other Parameters', 'Attributes', - 'Methods'): - self[section] = self._parse_param_list(content) - elif section.startswith('.. index::'): - self['index'] = self._parse_index(section, content) - elif section == 'See Also': - self['See Also'] = self._parse_see_also(content) - else: - self[section] = content - - # string conversion routines - - def _str_header(self, name, symbol='-'): - return [name, len(name)*symbol] - - def _str_indent(self, doc, indent=4): - out = [] - for line in doc: - out += [' '*indent + line] - return out - - def _str_signature(self): - if self['Signature']: - return [self['Signature'].replace('*', '\*')] + [''] - else: - return [''] - - def _str_summary(self): - if self['Summary']: - return self['Summary'] + [''] - else: - return [] - - def _str_extended_summary(self): - if self['Extended Summary']: - return self['Extended Summary'] + [''] - else: - return [] - - def _str_param_list(self, name): - out = [] - if self[name]: - out += self._str_header(name) - for param, param_type, desc in self[name]: - if param_type: - out += ['%s : %s' % (param, param_type)] - else: - out += [param] - out += self._str_indent(desc) - out += [''] - return out - - def _str_section(self, name): - out = [] - if self[name]: - out += self._str_header(name) - out += self[name] - out += [''] - return out - - def _str_see_also(self, func_role): - if not self['See Also']: - return [] - out = [] - out += self._str_header("See Also") - last_had_desc = True - for func, desc, role in self['See Also']: - if role: - link = ':%s:`%s`' % (role, func) - elif func_role: - link = ':%s:`%s`' % (func_role, func) - else: - link = "`%s`_" % func - if desc or last_had_desc: - out += [''] - out += [link] - else: - out[-1] += ", %s" % link - if desc: - out += self._str_indent([' '.join(desc)]) - last_had_desc = True - else: - last_had_desc = False - out += [''] - return out - - def _str_index(self): - idx = self['index'] - out = [] - out += ['.. index:: %s' % idx.get('default', '')] - for section, references in idx.items(): - if section == 'default': - continue - out += [' :%s: %s' % (section, ', '.join(references))] - return out - - def __str__(self, func_role=''): - out = [] - out += self._str_signature() - out += self._str_summary() - out += self._str_extended_summary() - for param_list in ('Parameters', 'Returns', 'Yields', - 'Other Parameters', 'Raises', 'Warns'): - out += self._str_param_list(param_list) - out += self._str_section('Warnings') - out += self._str_see_also(func_role) - for s in ('Notes', 'References', 'Examples'): - out += self._str_section(s) - for param_list in ('Attributes', 'Methods'): - out += self._str_param_list(param_list) - out += self._str_index() - return '\n'.join(out) - - -def indent(str, indent=4): - indent_str = ' '*indent - if str is None: - return indent_str - lines = str.split('\n') - return '\n'.join(indent_str + l for l in lines) - - -def dedent_lines(lines): - """Deindent a list of lines maximally""" - return textwrap.dedent("\n".join(lines)).split("\n") - - -def header(text, style='-'): - return text + '\n' + style*len(text) + '\n' - - -class FunctionDoc(NumpyDocString): - def __init__(self, func, role='func', doc=None, config={}): - self._f = func - self._role = role # e.g. "func" or "meth" - - if doc is None: - if func is None: - raise ValueError("No function or docstring given") - doc = inspect.getdoc(func) or '' - NumpyDocString.__init__(self, doc) - - if not self['Signature'] and func is not None: - func, func_name = self.get_func() - try: - # try to read signature - if sys.version_info[0] >= 3: - argspec = inspect.getfullargspec(func) - else: - argspec = inspect.getargspec(func) - argspec = inspect.formatargspec(*argspec) - argspec = argspec.replace('*', '\*') - signature = '%s%s' % (func_name, argspec) - except TypeError as e: - signature = '%s()' % func_name - self['Signature'] = signature - - def get_func(self): - func_name = getattr(self._f, '__name__', self.__class__.__name__) - if inspect.isclass(self._f): - func = getattr(self._f, '__call__', self._f.__init__) - else: - func = self._f - return func, func_name - - def __str__(self): - out = '' - - func, func_name = self.get_func() - signature = self['Signature'].replace('*', '\*') - - roles = {'func': 'function', - 'meth': 'method'} - - if self._role: - if self._role not in roles: - print("Warning: invalid role %s" % self._role) - out += '.. %s:: %s\n \n\n' % (roles.get(self._role, ''), - func_name) - - out += super(FunctionDoc, self).__str__(func_role=self._role) - return out - - -class ClassDoc(NumpyDocString): - - extra_public_methods = ['__call__'] - - def __init__(self, cls, doc=None, modulename='', func_doc=FunctionDoc, - config={}): - if not inspect.isclass(cls) and cls is not None: - raise ValueError("Expected a class or None, but got %r" % cls) - self._cls = cls - - self.show_inherited_members = config.get( - 'show_inherited_class_members', True) - - if modulename and not modulename.endswith('.'): - modulename += '.' - self._mod = modulename - - if doc is None: - if cls is None: - raise ValueError("No class or documentation string given") - doc = pydoc.getdoc(cls) - - NumpyDocString.__init__(self, doc) - - if config.get('show_class_members', True): - def splitlines_x(s): - if not s: - return [] - else: - return s.splitlines() - - for field, items in [('Methods', self.methods), - ('Attributes', self.properties)]: - if not self[field]: - doc_list = [] - for name in sorted(items): - try: - doc_item = pydoc.getdoc(getattr(self._cls, name)) - doc_list.append((name, '', splitlines_x(doc_item))) - except AttributeError: - pass # method doesn't exist - self[field] = doc_list - - @property - def methods(self): - if self._cls is None: - return [] - return [name for name, func in inspect.getmembers(self._cls) - if ((not name.startswith('_') - or name in self.extra_public_methods) - and isinstance(func, collections.Callable) - and self._is_show_member(name))] - - @property - def properties(self): - if self._cls is None: - return [] - return [name for name, func in inspect.getmembers(self._cls) - if (not name.startswith('_') and - (func is None or isinstance(func, property) or - inspect.isgetsetdescriptor(func)) - and self._is_show_member(name))] - - def _is_show_member(self, name): - if self.show_inherited_members: - return True # show all class members - if name not in self._cls.__dict__: - return False # class member is inherited, we do not show it - return True diff --git a/docs/sphinxext/docscrape_sphinx.py b/docs/sphinxext/docscrape_sphinx.py deleted file mode 100644 index e44e770ef8..0000000000 --- a/docs/sphinxext/docscrape_sphinx.py +++ /dev/null @@ -1,227 +0,0 @@ -import re, inspect, textwrap, pydoc -import sphinx -from docscrape import NumpyDocString, FunctionDoc, ClassDoc - -class SphinxDocString(NumpyDocString): - def __init__(self, docstring, config={}): - self.use_plots = config.get('use_plots', False) - NumpyDocString.__init__(self, docstring, config=config) - - # string conversion routines - def _str_header(self, name, symbol='`'): - return ['.. rubric:: ' + name, ''] - - def _str_field_list(self, name): - return [':' + name + ':'] - - def _str_indent(self, doc, indent=4): - out = [] - for line in doc: - out += [' '*indent + line] - return out - - def _str_signature(self): - return [''] - if self['Signature']: - return ['``%s``' % self['Signature']] + [''] - else: - return [''] - - def _str_summary(self): - return self['Summary'] + [''] - - def _str_extended_summary(self): - return self['Extended Summary'] + [''] - - def _str_param_list(self, name): - out = [] - if self[name]: - out += self._str_field_list(name) - out += [''] - for param,param_type,desc in self[name]: - out += self._str_indent(['**%s** : %s' % (param.strip(), - param_type)]) - out += [''] - out += self._str_indent(desc,8) - out += [''] - return out - - @property - def _obj(self): - if hasattr(self, '_cls'): - return self._cls - elif hasattr(self, '_f'): - return self._f - return None - - def _str_member_list(self, name): - """ - Generate a member listing, autosummary:: table where possible, - and a table where not. - - """ - out = [] - if self[name]: - out += ['.. rubric:: %s' % name, ''] - prefix = getattr(self, '_name', '') - - if prefix: - prefix = '~%s.' % prefix - - autosum = [] - others = [] - for param, param_type, desc in self[name]: - param = param.strip() - if not self._obj or hasattr(self._obj, param): - autosum += [" %s%s" % (prefix, param)] - else: - others.append((param, param_type, desc)) - - if autosum: - out += ['.. autosummary::', ' :toctree:', ''] - out += autosum - - if others: - maxlen_0 = max([len(x[0]) for x in others]) - maxlen_1 = max([len(x[1]) for x in others]) - hdr = "="*maxlen_0 + " " + "="*maxlen_1 + " " + "="*10 - fmt = '%%%ds %%%ds ' % (maxlen_0, maxlen_1) - n_indent = maxlen_0 + maxlen_1 + 4 - out += [hdr] - for param, param_type, desc in others: - out += [fmt % (param.strip(), param_type)] - out += self._str_indent(desc, n_indent) - out += [hdr] - out += [''] - return out - - def _str_section(self, name): - out = [] - if self[name]: - out += self._str_header(name) - out += [''] - content = textwrap.dedent("\n".join(self[name])).split("\n") - out += content - out += [''] - return out - - def _str_see_also(self, func_role): - out = [] - if self['See Also']: - see_also = super(SphinxDocString, self)._str_see_also(func_role) - out = ['.. seealso::', ''] - out += self._str_indent(see_also[2:]) - return out - - def _str_warnings(self): - out = [] - if self['Warnings']: - out = ['.. warning::', ''] - out += self._str_indent(self['Warnings']) - return out - - def _str_index(self): - idx = self['index'] - out = [] - if len(idx) == 0: - return out - - out += ['.. index:: %s' % idx.get('default','')] - for section, references in idx.iteritems(): - if section == 'default': - continue - elif section == 'refguide': - out += [' single: %s' % (', '.join(references))] - else: - out += [' %s: %s' % (section, ','.join(references))] - return out - - def _str_references(self): - out = [] - if self['References']: - out += self._str_header('References') - if isinstance(self['References'], str): - self['References'] = [self['References']] - out.extend(self['References']) - out += [''] - # Latex collects all references to a separate bibliography, - # so we need to insert links to it - if sphinx.__version__ >= "0.6": - out += ['.. only:: latex',''] - else: - out += ['.. latexonly::',''] - items = [] - for line in self['References']: - m = re.match(r'.. \[([a-z0-9._-]+)\]', line, re.I) - if m: - items.append(m.group(1)) - out += [' ' + ", ".join(["[%s]_" % item for item in items]), ''] - return out - - def _str_examples(self): - examples_str = "\n".join(self['Examples']) - - if (self.use_plots and 'import matplotlib' in examples_str - and 'plot::' not in examples_str): - out = [] - out += self._str_header('Examples') - out += ['.. plot::', ''] - out += self._str_indent(self['Examples']) - out += [''] - return out - else: - return self._str_section('Examples') - - def __str__(self, indent=0, func_role="obj"): - out = [] - out += self._str_signature() - out += self._str_index() + [''] - out += self._str_summary() - out += self._str_extended_summary() - for param_list in ('Parameters', 'Returns', 'Other Parameters', - 'Raises', 'Warns'): - out += self._str_param_list(param_list) - out += self._str_warnings() - out += self._str_see_also(func_role) - out += self._str_section('Notes') - out += self._str_references() - out += self._str_examples() - for param_list in ('Attributes', 'Methods'): - out += self._str_member_list(param_list) - out = self._str_indent(out,indent) - return '\n'.join(out) - -class SphinxFunctionDoc(SphinxDocString, FunctionDoc): - def __init__(self, obj, doc=None, config={}): - self.use_plots = config.get('use_plots', False) - FunctionDoc.__init__(self, obj, doc=doc, config=config) - -class SphinxClassDoc(SphinxDocString, ClassDoc): - def __init__(self, obj, doc=None, func_doc=None, config={}): - self.use_plots = config.get('use_plots', False) - ClassDoc.__init__(self, obj, doc=doc, func_doc=None, config=config) - -class SphinxObjDoc(SphinxDocString): - def __init__(self, obj, doc=None, config={}): - self._f = obj - SphinxDocString.__init__(self, doc, config=config) - -def get_doc_object(obj, what=None, doc=None, config={}): - if what is None: - if inspect.isclass(obj): - what = 'class' - elif inspect.ismodule(obj): - what = 'module' - elif callable(obj): - what = 'function' - else: - what = 'object' - if what == 'class': - return SphinxClassDoc(obj, func_doc=SphinxFunctionDoc, doc=doc, - config=config) - elif what in ('function', 'method'): - return SphinxFunctionDoc(obj, doc=doc, config=config) - else: - if doc is None: - doc = pydoc.getdoc(obj) - return SphinxObjDoc(obj, doc, config=config) diff --git a/docs/sphinxext/github.py b/docs/sphinxext/github.py deleted file mode 100644 index 519e146d19..0000000000 --- a/docs/sphinxext/github.py +++ /dev/null @@ -1,155 +0,0 @@ -"""Define text roles for GitHub - -* ghissue - Issue -* ghpull - Pull Request -* ghuser - User - -Adapted from bitbucket example here: -https://bitbucket.org/birkenfeld/sphinx-contrib/src/tip/bitbucket/sphinxcontrib/bitbucket.py - -Authors -------- - -* Doug Hellmann -* Min RK -""" -# -# Original Copyright (c) 2010 Doug Hellmann. All rights reserved. -# - -from docutils import nodes, utils -from docutils.parsers.rst.roles import set_classes - -def make_link_node(rawtext, app, type, slug, options): - """Create a link to a github resource. - - :param rawtext: Text being replaced with link node. - :param app: Sphinx application context - :param type: Link type (issues, changeset, etc.) - :param slug: ID of the thing to link to - :param options: Options dictionary passed to role func. - """ - - try: - base = app.config.github_project_url - if not base: - raise AttributeError - if not base.endswith('/'): - base += '/' - except AttributeError as err: - raise ValueError('github_project_url configuration value is not set (%s)' % str(err)) - - ref = base + type + '/' + slug + '/' - set_classes(options) - prefix = "#" - if type == 'pull': - prefix = "PR " + prefix - node = nodes.reference(rawtext, prefix + utils.unescape(slug), refuri=ref, - **options) - return node - -def ghissue_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - """Link to a GitHub issue. - - Returns 2 part tuple containing list of nodes to insert into the - document and a list of system messages. Both are allowed to be - empty. - - :param name: The role name used in the document. - :param rawtext: The entire markup snippet, with role. - :param text: The text marked with the role. - :param lineno: The line number where rawtext appears in the input. - :param inliner: The inliner instance that called us. - :param options: Directive options for customization. - :param content: The directive content for customization. - """ - - try: - issue_num = int(text) - if issue_num <= 0: - raise ValueError - except ValueError: - msg = inliner.reporter.error( - 'GitHub issue number must be a number greater than or equal to 1; ' - '"%s" is invalid.' % text, line=lineno) - prb = inliner.problematic(rawtext, rawtext, msg) - return [prb], [msg] - app = inliner.document.settings.env.app - #app.info('issue %r' % text) - if 'pull' in name.lower(): - category = 'pull' - elif 'issue' in name.lower(): - category = 'issues' - else: - msg = inliner.reporter.error( - 'GitHub roles include "ghpull" and "ghissue", ' - '"%s" is invalid.' % name, line=lineno) - prb = inliner.problematic(rawtext, rawtext, msg) - return [prb], [msg] - node = make_link_node(rawtext, app, category, str(issue_num), options) - return [node], [] - -def ghuser_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - """Link to a GitHub user. - - Returns 2 part tuple containing list of nodes to insert into the - document and a list of system messages. Both are allowed to be - empty. - - :param name: The role name used in the document. - :param rawtext: The entire markup snippet, with role. - :param text: The text marked with the role. - :param lineno: The line number where rawtext appears in the input. - :param inliner: The inliner instance that called us. - :param options: Directive options for customization. - :param content: The directive content for customization. - """ - app = inliner.document.settings.env.app - #app.info('user link %r' % text) - ref = 'https://www.github.com/' + text - node = nodes.reference(rawtext, text, refuri=ref, **options) - return [node], [] - -def ghcommit_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - """Link to a GitHub commit. - - Returns 2 part tuple containing list of nodes to insert into the - document and a list of system messages. Both are allowed to be - empty. - - :param name: The role name used in the document. - :param rawtext: The entire markup snippet, with role. - :param text: The text marked with the role. - :param lineno: The line number where rawtext appears in the input. - :param inliner: The inliner instance that called us. - :param options: Directive options for customization. - :param content: The directive content for customization. - """ - app = inliner.document.settings.env.app - #app.info('user link %r' % text) - try: - base = app.config.github_project_url - if not base: - raise AttributeError - if not base.endswith('/'): - base += '/' - except AttributeError as err: - raise ValueError('github_project_url configuration value is not set (%s)' % str(err)) - - ref = base + text - node = nodes.reference(rawtext, text[:6], refuri=ref, **options) - return [node], [] - - -def setup(app): - """Install the plugin. - - :param app: Sphinx application context. - """ - app.info('Initializing GitHub plugin') - app.add_role('ghissue', ghissue_role) - app.add_role('ghpull', ghissue_role) - app.add_role('ghuser', ghuser_role) - app.add_role('ghcommit', ghcommit_role) - app.add_config_value('github_project_url', None, 'env') - return diff --git a/docs/sphinxext/math_dollar.py b/docs/sphinxext/math_dollar.py deleted file mode 100644 index ad415deb90..0000000000 --- a/docs/sphinxext/math_dollar.py +++ /dev/null @@ -1,63 +0,0 @@ -import re - -def dollars_to_math(source): - r""" - Replace dollar signs with backticks. - - More precisely, do a regular expression search. Replace a plain - dollar sign ($) by a backtick (`). Replace an escaped dollar sign - (\$) by a dollar sign ($). Don't change a dollar sign preceded or - followed by a backtick (`$ or $`), because of strings like - "``$HOME``". Don't make any changes on lines starting with - spaces, because those are indented and hence part of a block of - code or examples. - - This also doesn't replaces dollar signs enclosed in curly braces, - to avoid nested math environments, such as :: - - $f(n) = 0 \text{ if $n$ is prime}$ - - Thus the above line would get changed to - - `f(n) = 0 \text{ if $n$ is prime}` - """ - s = "\n".join(source) - if s.find("$") == -1: - return - # This searches for "$blah$" inside a pair of curly braces -- - # don't change these, since they're probably coming from a nested - # math environment. So for each match, we replace it with a temporary - # string, and later on we substitute the original back. - global _data - _data = {} - def repl(matchobj): - global _data - s = matchobj.group(0) - t = "___XXX_REPL_%d___" % len(_data) - _data[t] = s - return t - s = re.sub(r"({[^{}$]*\$[^{}$]*\$[^{}]*})", repl, s) - # matches $...$ - dollars = re.compile(r"(?= 3: - sixu = lambda s: s -else: - sixu = lambda s: unicode(s, 'unicode_escape') - - -def mangle_docstrings(app, what, name, obj, options, lines, - reference_offset=[0]): - - cfg = {'use_plots': app.config.numpydoc_use_plots, - 'show_class_members': app.config.numpydoc_show_class_members, - 'show_inherited_class_members': - app.config.numpydoc_show_inherited_class_members, - 'class_members_toctree': app.config.numpydoc_class_members_toctree} - - u_NL = sixu('\n') - if what == 'module': - # Strip top title - pattern = '^\\s*[#*=]{4,}\\n[a-z0-9 -]+\\n[#*=]{4,}\\s*' - title_re = re.compile(sixu(pattern), re.I | re.S) - lines[:] = title_re.sub(sixu(''), u_NL.join(lines)).split(u_NL) - else: - doc = get_doc_object(obj, what, u_NL.join(lines), config=cfg) - if sys.version_info[0] >= 3: - doc = str(doc) - else: - doc = unicode(doc) - lines[:] = doc.split(u_NL) - - if (app.config.numpydoc_edit_link and hasattr(obj, '__name__') and - obj.__name__): - if hasattr(obj, '__module__'): - v = dict(full_name=sixu("%s.%s") % (obj.__module__, obj.__name__)) - else: - v = dict(full_name=obj.__name__) - lines += [sixu(''), sixu('.. htmlonly::'), sixu('')] - lines += [sixu(' %s') % x for x in - (app.config.numpydoc_edit_link % v).split("\n")] - - # replace reference numbers so that there are no duplicates - references = [] - for line in lines: - line = line.strip() - m = re.match(sixu('^.. \\[([a-z0-9_.-])\\]'), line, re.I) - if m: - references.append(m.group(1)) - - # start renaming from the longest string, to avoid overwriting parts - references.sort(key=lambda x: -len(x)) - if references: - for i, line in enumerate(lines): - for r in references: - if re.match(sixu('^\\d+$'), r): - new_r = sixu("R%d") % (reference_offset[0] + int(r)) - else: - new_r = sixu("%s%d") % (r, reference_offset[0]) - lines[i] = lines[i].replace(sixu('[%s]_') % r, - sixu('[%s]_') % new_r) - lines[i] = lines[i].replace(sixu('.. [%s]') % r, - sixu('.. [%s]') % new_r) - - reference_offset[0] += len(references) - - -def mangle_signature(app, what, name, obj, options, sig, retann): - # Do not try to inspect classes that don't define `__init__` - if (inspect.isclass(obj) and - (not hasattr(obj, '__init__') or - 'initializes x; see ' in pydoc.getdoc(obj.__init__))): - return '', '' - - if not (isinstance(obj, collections.Callable) or - hasattr(obj, '__argspec_is_invalid_')): - return - - if not hasattr(obj, '__doc__'): - return - - doc = SphinxDocString(pydoc.getdoc(obj)) - if doc['Signature']: - sig = re.sub(sixu("^[^(]*"), sixu(""), doc['Signature']) - return sig, sixu('') - - -def setup(app, get_doc_object_=get_doc_object): - if not hasattr(app, 'add_config_value'): - return # probably called by nose, better bail out - - global get_doc_object - get_doc_object = get_doc_object_ - - app.connect('autodoc-process-docstring', mangle_docstrings) - app.connect('autodoc-process-signature', mangle_signature) - app.add_config_value('numpydoc_edit_link', None, False) - app.add_config_value('numpydoc_use_plots', None, False) - app.add_config_value('numpydoc_show_class_members', True, True) - app.add_config_value('numpydoc_show_inherited_class_members', True, True) - app.add_config_value('numpydoc_class_members_toctree', True, True) - - # Extra mangling domains - app.add_domain(NumpyPythonDomain) - app.add_domain(NumpyCDomain) - -# ------------------------------------------------------------------------------ -# Docstring-mangling domains -# ------------------------------------------------------------------------------ - -from docutils.statemachine import ViewList -from sphinx.domains.c import CDomain -from sphinx.domains.python import PythonDomain - - -class ManglingDomainBase(object): - directive_mangling_map = {} - - def __init__(self, *a, **kw): - super(ManglingDomainBase, self).__init__(*a, **kw) - self.wrap_mangling_directives() - - def wrap_mangling_directives(self): - for name, objtype in list(self.directive_mangling_map.items()): - self.directives[name] = wrap_mangling_directive( - self.directives[name], objtype) - - -class NumpyPythonDomain(ManglingDomainBase, PythonDomain): - name = 'np' - directive_mangling_map = { - 'function': 'function', - 'class': 'class', - 'exception': 'class', - 'method': 'function', - 'classmethod': 'function', - 'staticmethod': 'function', - 'attribute': 'attribute', - } - indices = [] - - -class NumpyCDomain(ManglingDomainBase, CDomain): - name = 'np-c' - directive_mangling_map = { - 'function': 'function', - 'member': 'attribute', - 'macro': 'function', - 'type': 'class', - 'var': 'object', - } - - -def wrap_mangling_directive(base_directive, objtype): - class directive(base_directive): - def run(self): - env = self.state.document.settings.env - - name = None - if self.arguments: - m = re.match(r'^(.*\s+)?(.*?)(\(.*)?', self.arguments[0]) - name = m.group(2).strip() - - if not name: - name = self.arguments[0] - - lines = list(self.content) - mangle_docstrings(env.app, objtype, name, None, None, lines) - self.content = ViewList(lines, self.content.parent) - - return base_directive.run(self) - - return directive From ed0bb5dd8d73e94647061740764592f2c215a066 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 26 Nov 2020 16:49:34 +0100 Subject: [PATCH 37/68] DOC: Enable NiPype's sphinx-extension to better render Interfaces --- docs/conf.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 7ce551cce5..cb840c0e89 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -36,9 +36,9 @@ "sphinx.ext.ifconfig", "sphinx.ext.viewcode", "sphinx.ext.githubpages", - "nipype.sphinxext.plot_workflow", "sphinxcontrib.apidoc", - "sphinxcontrib.napoleon", + "nipype.sphinxext.plot_workflow", + "nipype.sphinxext.apidoc", ] autodoc_mock_imports = [ @@ -62,6 +62,9 @@ napoleon_custom_sections = [ ("Inputs", "Parameters"), ("Outputs", "Parameters"), + ("Attributes", "Parameters"), + ("Mandatory Inputs", "Parameters"), + ("Optional Inputs", "Parameters"), ] # Add any paths that contain templates here, relative to this directory. From 0843ffb6cb15d2ac5f5b655ec9b9bf23e3c88616 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 26 Nov 2020 18:28:55 +0100 Subject: [PATCH 38/68] docs: minimal revisions to the documentation --- sdcflows/interfaces/bspline.py | 3 +- sdcflows/workflows/fit/fieldmap.py | 98 +++++++++++++----------------- 2 files changed, 45 insertions(+), 56 deletions(-) diff --git a/sdcflows/interfaces/bspline.py b/sdcflows/interfaces/bspline.py index 1e2fc10bf3..6e72b3088a 100644 --- a/sdcflows/interfaces/bspline.py +++ b/sdcflows/interfaces/bspline.py @@ -216,7 +216,8 @@ class Coefficients2Warp(SimpleInterface): Implements Eq. :math:`\eqref{eq:1}`, interpolating :math:`f(\mathbf{s})` for all voxels in the target-image's extent. When the readout time is known, the displacements field can be calculated - following Eq. `(2) `__. + following `Eq. (2) in the fieldmap fitting section + `__. """ diff --git a/sdcflows/workflows/fit/fieldmap.py b/sdcflows/workflows/fit/fieldmap.py index 7c99670e99..afc7ba1c3e 100644 --- a/sdcflows/workflows/fit/fieldmap.py +++ b/sdcflows/workflows/fit/fieldmap.py @@ -3,6 +3,8 @@ r""" Processing phase-difference and *directly measured* :math:`B_0` maps. +Theory +~~~~~~ The displacement suffered by every voxel along the phase-encoding (PE) direction can be derived from Eq. (2) of [Hutton2002]_: @@ -11,13 +13,18 @@ \Delta_\text{PE} (i, j, k) = \gamma \cdot \Delta B_0 (i, j, k) \cdot T_\text{ro}, \label{eq:fieldmap-1}\tag{1} -where :math:`T_\text{ro}` is the readout time of one slice of the EPI dataset -we want to correct for distortions, and :math:`\Delta_\text{PE} (i, j, k)` -is the *voxel-shift map* (VSM) along the *PE* direction. - -Calling the *fieldmap* in Hz :math:`V`, -with :math:`V(i,j,k) = \gamma \cdot \Delta B_0 (i, j, k)`, and introducing -the voxel zoom along the phase-encoding direction, :math:`s_\text{PE}`, +where +:math:`\Delta_\text{PE} (i, j, k)` is the *voxel-shift map* (VSM) along the *PE* direction, +:math:`\gamma` is the gyromagnetic ratio of the H proton in Hz/T +(:math:`\gamma = 42.576 \cdot 10^6 \, \text{Hz} \cdot \text{T}^\text{-1}`), +:math:`\Delta B_0 (i, j, k)` is the *fieldmap variation* in T (Tesla), and +:math:`T_\text{ro}` is the readout time of one slice of the EPI dataset +we want to correct for distortions. + +Let :math:`V` represent the «*fieldmap in Hz*» (or equivalently, +«*voxel-shift-velocity map*» as Hz are equivalent to voxels/s), with +:math:`V(i,j,k) = \gamma \cdot \Delta B_0 (i, j, k)`, then, introducing +the voxel zoom along the phase-encoding direction, :math:`s_\text{PE}`, we obtain the nonzero component of the associated displacements field :math:`\Delta D_\text{PE} (i, j, k)` that unwarps the target EPI dataset: @@ -26,76 +33,57 @@ \Delta D_\text{PE} (i, j, k) = V(i, j, k) \cdot T_\text{ro} \cdot s_\text{PE}. \label{eq:fieldmap-2}\tag{2} -Theory -~~~~~~ -The derivation of a fieldmap in Hz (or, as called thereafter, *voxel-shift-velocity -map*) results from eq. (1) of [Hutton2002]_: -.. math:: +.. _sdc_direct_b0 : - \Delta B_0 (i, j, k) = \frac{\Delta \Theta (i, j, k)}{2\pi \cdot \gamma \, \Delta\text{TE}}, - \label{eq:fieldmap-3}\tag{3} +Direct B0 mapping sequences +~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Some MR schemes such as :abbr:`SEI (spiral-echo imaging)` can directly +reconstruct an estimate of *the fieldmap in Hz*, :math:`V(i,j,k)`. +These *fieldmaps* are described with more detail `here +`__. -where :math:`\Delta B_0 (i, j, k)` is the *fieldmap variation* in T, -:math:`\Delta \Theta (i, j, k)` is the phase-difference map in rad, -:math:`\gamma` is the gyromagnetic ratio of the H proton in Hz/T -(:math:`\gamma = 42.576 \cdot 10^6 \, \text{Hz} \cdot \text{T}^\text{-1}`), -and :math:`\Delta\text{TE}` is the elapsed time between the two GRE echoes. +This corresponds to `this section of the BIDS specification +`__. -We can obtain a voxel displacement map following eq. (2) of the same paper: +.. _sdc_phasediff : -.. math:: +Phase-difference B0 estimation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The fieldmap variation in T, :math:`\Delta B_0 (i, j, k)`, that is necessary to obtain +:math:`\Delta_\text{PE} (i, j, k)` in Eq. :math:`\eqref{eq:fieldmap-1}` can be +calculated from two subsequient :abbr:`GRE (Gradient-Recalled Echo)` echoes, +via eq. (1) of [Hutton2002]_: - \Delta_\text{PE} (i, j, k) = \gamma \cdot \Delta B_0 (i, j, k) \cdot T_\text{ro}, - \label{eq:fieldmap-4}\tag{4} +.. math:: -where :math:`T_\text{ro}` is the readout time of one slice of the EPI dataset -we want to correct for distortions, and -:math:`\Delta_\text{PE} (i, j, k)` is the *voxel-shift map* (VSM) along the *PE* -direction. + \Delta B_0 (i, j, k) = \frac{\Delta \Theta (i, j, k)}{2\pi \cdot \gamma \, \Delta\text{TE}}, + \label{eq:fieldmap-3}\tag{3} -.. _sdc_phasediff : +where +:math:`\Delta \Theta (i, j, k)` is the phase-difference map in radians, +and :math:`\Delta\text{TE}` is the elapsed time between the two GRE echoes. -Phase-difference B0 estimation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The inhomogeneity of the :math:`B_0` field inside the scanner at each voxel -(and hence, the *fieldmap* in Hz, :math:`V(i,j,k)`) is proportional to the -phase drift :math:`\Delta \Theta (i,j,k) = \Theta_2(i,j,k) - \Theta_1(i,j,k)` -between two subsequent :abbr:`GRE (gradient-recalled echo)` acquisitions -(:math:`\Theta_1`, :math:`\Theta_2`), separated by a time -:math:`\Delta \text{TE}` (s): -Replacing (1) into (2), and eliminating the scaling effect of :math:`T_\text{ro}`, -we obtain a *voxel-shift-velocity map* (voxels/s, or just Hz) which can be then -used to recover the actual displacement field of the target EPI dataset. +For simplicity, the «*voxel-shift-velocity map*» :math:`V(i,j,k)`, which we +can introduce in Eq. :math:`\eqref{eq:fieldmap-2}` to directly obtain +the displacements field, can be obtained as: .. math:: V(i, j, k) = \frac{\Delta \Theta (i, j, k)}{2\pi \cdot \Delta\text{TE}}. - \label{eq:fieldmap-5}\tag{5} + \label{eq:fieldmap-4}\tag{4} -This calculation if further complicated by the fact that :math:`\Theta_i` +This calculation is further complicated by the fact that :math:`\Theta_i` (and therfore, :math:`\Delta \Theta`) are clipped (or *wrapped*) within the range :math:`[0 \dotsb 2\pi )`. It is necessary to find the integer number of offsets that make a region continuously smooth with its neighbors (*phase-unwrapping*, [Jenkinson2003]_). This corresponds to `this section of the BIDS specification -`__. +`__. Some scanners produce one ``phasediff`` map, where the drift between the two echos has already been calulated (see `the corresponding section of BIDS -`__). - -.. _sdc_direct_b0 : - -Direct B0 mapping sequences -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Some MR schemes such as :abbr:`SEI (spiral-echo imaging)` can directly -reconstruct an estimate of *the fieldmap in Hz*, :math:`V(i,j,k)`. -These *fieldmaps* are described with more detail `here -`__. - -This corresponds to `this section of the BIDS specification -`__. +`__). References ---------- From 948cfcaaf2e44a0825590a22f10ae809b8f27b62 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 26 Nov 2020 22:31:35 +0100 Subject: [PATCH 39/68] fix: bug on Flatten input name + test --- sdcflows/interfaces/tests/test_utils.py | 33 +++++++++++++++++++++++++ sdcflows/interfaces/utils.py | 22 +++-------------- 2 files changed, 37 insertions(+), 18 deletions(-) create mode 100644 sdcflows/interfaces/tests/test_utils.py diff --git a/sdcflows/interfaces/tests/test_utils.py b/sdcflows/interfaces/tests/test_utils.py new file mode 100644 index 0000000000..231827cba7 --- /dev/null +++ b/sdcflows/interfaces/tests/test_utils.py @@ -0,0 +1,33 @@ +"""Test utilites.""" +import numpy as np +import nibabel as nb + +from ..utils import Flatten + + +def test_Flatten(tmpdir): + """Test the flattening interface.""" + tmpdir.chdir() + shape = (5, 5, 5) + nb.Nifti1Image( + np.zeros(shape), np.eye(4), None + ).to_filename("file1.nii.gz") + nb.Nifti1Image( + np.zeros((*shape, 6)), np.eye(4), None + ).to_filename("file2.nii.gz") + nb.Nifti1Image( + np.zeros((*shape, 2)), np.eye(4), None + ).to_filename("file3.nii.gz") + + out = Flatten( + in_data=["file1.nii.gz", "file2.nii.gz", "file3.nii.gz"], + in_meta=[{"a": 1}, {"b": 2}, {"c": 3}], + max_trs=3, + ).run() + + assert len(out.outputs.out_list) == 6 + + out_meta = out.outputs.out_meta + assert out_meta[0] == {"a": 1} + assert out_meta[1] == out_meta[2] == out_meta[3] == {"b": 2} + assert out_meta[4] == out_meta[5] == {"c": 3} diff --git a/sdcflows/interfaces/utils.py b/sdcflows/interfaces/utils.py index 5689bf4c2f..f1d1127aca 100644 --- a/sdcflows/interfaces/utils.py +++ b/sdcflows/interfaces/utils.py @@ -1,18 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -""" -Utilities. - - .. testsetup:: - - >>> tmpdir = getfixture('tmpdir') - >>> tmp = tmpdir.chdir() # changing to a temporary directory - >>> nb.Nifti1Image(np.zeros((90, 90, 60)), None, None).to_filename( - ... tmpdir.join('epi.nii.gz').strpath) - -""" - -from nipype import logging +"""Utilities.""" from nipype.interfaces.base import ( BaseInterfaceInputSpec, TraitedSpec, @@ -23,8 +11,6 @@ OutputMultiObject, ) -LOGGER = logging.getLogger("nipype.interface") - class _FlattenInputSpec(BaseInterfaceInputSpec): in_data = InputMultiObject( @@ -52,7 +38,7 @@ class Flatten(SimpleInterface): def _run_interface(self, runtime): self._results["out_list"] = _flatten( - zip(self.inputs.inlist, self.inputs.in_meta), + zip(self.inputs.in_data, self.inputs.in_meta), max_trs=self.inputs.max_trs, out_dir=runtime.cwd, ) @@ -72,9 +58,9 @@ def _flatten(inlist, max_trs=50, out_dir=None): ------ inlist : :obj:`list` of :obj:`tuple` List of pairs (filepath, metadata) - max_trs : int + max_trs : :obj:`int` Index of frame after which all volumes will be discarded - from the input EPI images. + from the input images. """ from pathlib import Path From d60ba664ea8b5dc685445fcbca631e05256d3d29 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 26 Nov 2020 22:53:23 +0100 Subject: [PATCH 40/68] tst: make sure the subtraction interface is always run --- sdcflows/workflows/fit/fieldmap.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/sdcflows/workflows/fit/fieldmap.py b/sdcflows/workflows/fit/fieldmap.py index afc7ba1c3e..3cb9478602 100644 --- a/sdcflows/workflows/fit/fieldmap.py +++ b/sdcflows/workflows/fit/fieldmap.py @@ -127,6 +127,8 @@ def init_fmap_wf(omp_nthreads=1, debug=False, mode="phasediff", name="fmap_wf"): ---------- omp_nthreads : :obj:`int` Maximum number of threads an individual process may use. + debug : :obj:`bool` + Run on debug mode name : :obj:`str` Unique name of this workflow. @@ -198,7 +200,7 @@ def init_fmap_wf(omp_nthreads=1, debug=False, mode="phasediff", name="fmap_wf"): phase-drift map(s) measure with two consecutive GRE (gradient-recall echo) acquisitions. """ - phdiff_wf = init_phdiff_wf(omp_nthreads) + phdiff_wf = init_phdiff_wf(omp_nthreads, debug=debug) # fmt: off workflow.connect([ @@ -260,7 +262,7 @@ def init_magnitude_wf(omp_nthreads, name="magnitude_wf"): omp_nthreads : :obj:`int` Maximum number of threads an individual process may use name : :obj:`str` - Name of workflow (default: ``prepare_magnitude_w``) + Name of workflow (default: ``magnitude_wf``) Inputs ------ @@ -311,7 +313,7 @@ def init_magnitude_wf(omp_nthreads, name="magnitude_wf"): return workflow -def init_phdiff_wf(omp_nthreads, name="phdiff_wf"): +def init_phdiff_wf(omp_nthreads, debug=False, name="phdiff_wf"): r""" Generate a :math:`B_0` field from consecutive-phases and phase-difference maps. @@ -343,6 +345,10 @@ def init_phdiff_wf(omp_nthreads, name="phdiff_wf"): ---------- omp_nthreads : :obj:`int` Maximum number of threads an individual process may use + debug : :obj:`bool` + Run on debug mode + name : :obj:`str` + Name of workflow (default: ``phdiff_wf``) Inputs ------ @@ -398,6 +404,7 @@ def _split(phase): calc_phdiff = pe.Node( SubtractPhases(), name="calc_phdiff", run_without_submitting=True ) + calc_phdiff.interface._always_run = debug compfmap = pe.Node(Phasediff2Fieldmap(), name="compfmap") # fmt: off From b13787f54932b6025a9dc5eb28be1ea3d540aa25 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 26 Nov 2020 23:25:57 +0100 Subject: [PATCH 41/68] fix: bug in assistive function + doctests --- sdcflows/workflows/outputs.py | 63 ++++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/sdcflows/workflows/outputs.py b/sdcflows/workflows/outputs.py index dd8d3afecc..d156843c29 100644 --- a/sdcflows/workflows/outputs.py +++ b/sdcflows/workflows/outputs.py @@ -208,6 +208,18 @@ def init_fmap_derivatives_wf( def _gendesc(infiles): + """ + Generate a desc entity value. + + Examples + -------- + >>> _gendesc("f") + 'coeff' + + >>> _gendesc(list("ab")) + ['coeff0', 'coeff1'] + + """ if isinstance(infiles, (str, bytes)): infiles = [infiles] @@ -218,6 +230,18 @@ def _gendesc(infiles): def _getname(infile): + """ + Get file names only. + + Examples + -------- + >>> _getname("drop/path/filename.txt") + 'filename.txt' + + >>> _getname(["drop/path/filename.txt", "other/path/filename2.txt"]) + ['filename.txt', 'filename2.txt'] + + """ from pathlib import Path if isinstance(infile, (list, tuple)): @@ -226,6 +250,18 @@ def _getname(infile): def _getsourcetype(infiles): + """ + Determine the type of fieldmap estimation strategy. + + Example + ------- + >>> _getsourcetype(["path/some_epi.nii.gz"]) + 'epi' + + >>> _getsourcetype(["path/some_notepi.nii.gz"]) + 'magnitude' + + """ from pathlib import Path fname = Path(infiles[0]).name @@ -233,8 +269,33 @@ def _getsourcetype(infiles): def _selectintent(metadata): + """ + Extract the IntendedFor metadata. + + Example + ------- + >>> _selectintent({}) + [] + + >>> _selectintent({"IntendedFor": "just/one/file.txt"}) + ['just/one/file.txt'] + + >>> _selectintent({"IntendedFor": ["file2.txt", "file1.txt"]}) + ['file1.txt', 'file2.txt'] + + >>> _selectintent([{"IntendedFor": "just/one/file.txt"}] * 2) + ['just/one/file.txt'] + + >>> _selectintent([ + ... {"IntendedFor": "just/one/file.txt"}, + ... {"IntendedFor": ["file2.txt", "file1.txt"]}, + ... ]) + ['file1.txt', 'file2.txt', 'just/one/file.txt'] + + """ from bids.utils import listify return sorted( - set([el for m in metadata for el in listify(m.get("IntendedFor", []))]) + set([el for m in listify(metadata) + for el in listify(m.get("IntendedFor", []))]) ) From 892f3facb64d53618e68cfd12f5ee87d7407ae33 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 26 Nov 2020 23:28:18 +0100 Subject: [PATCH 42/68] tst: make sure interface is always run in debug mode --- sdcflows/workflows/apply/registration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdcflows/workflows/apply/registration.py b/sdcflows/workflows/apply/registration.py index 5475ba7f8c..cb275d6466 100644 --- a/sdcflows/workflows/apply/registration.py +++ b/sdcflows/workflows/apply/registration.py @@ -113,6 +113,7 @@ def init_coeff2epi_wf( # Map the coefficients into the EPI space map_coeff = pe.Node(niu.Function(function=_move_coeff), name="map_coeff") + map_coeff.interface._always_run = debug # fmt: off workflow.connect([ From 4de43938c9d6f735aa447ede3cd97791c07d7836 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Fri, 27 Nov 2020 00:50:12 +0100 Subject: [PATCH 43/68] enh: upgrade from Function to fully-fledged interface --- sdcflows/interfaces/bspline.py | 53 +++++++++++++++++++++++- sdcflows/workflows/apply/registration.py | 39 ++++------------- 2 files changed, 61 insertions(+), 31 deletions(-) diff --git a/sdcflows/interfaces/bspline.py b/sdcflows/interfaces/bspline.py index 6e72b3088a..ddf046d6f1 100644 --- a/sdcflows/interfaces/bspline.py +++ b/sdcflows/interfaces/bspline.py @@ -201,7 +201,9 @@ class _Coefficients2WarpInputSpec(BaseInterfaceInputSpec): mandatory=True, desc="the phase-encoding direction corresponding to in_target", ) - low_mem = traits.Bool(False, usedefault=True, desc="perform on low-mem fingerprint regime") + low_mem = traits.Bool( + False, usedefault=True, desc="perform on low-mem fingerprint regime" + ) class _Coefficients2WarpOutputSpec(TraitedSpec): @@ -290,6 +292,31 @@ def _run_interface(self, runtime): return runtime +class _TransformCoefficientsInputSpec(BaseInterfaceInputSpec): + in_coeff = InputMultiObject( + File(exist=True), mandatory=True, desc="input coefficients file(s)" + ) + fmap_ref = File(exists=True, mandatory=True, desc="the fieldmap reference") + transform = File(exists=True, mandatory=True, desc="rigid-body transform file") + + +class _TransformCoefficientsOutputSpec(TraitedSpec): + out_coeff = OutputMultiObject(File(exists=True), desc="moved coefficients") + + +class TransformCoefficients(SimpleInterface): + """Project coefficients files to another space through a rigid-body transform.""" + + input_spec = _TransformCoefficientsInputSpec + output_spec = _TransformCoefficientsOutputSpec + + def _run_interface(self, runtime): + self._results["out_coeff"] = _move_coeff( + self.inputs.in_coeff, self.inputs.fmap_ref, self.inputs.transform, + ) + return runtime + + def bspline_grid(img, control_zooms_mm=DEFAULT_ZOOMS_MM): """Create a :obj:`~nibabel.nifti1.Nifti1Image` embedding the location of control points.""" if isinstance(img, (str, Path)): @@ -386,3 +413,27 @@ def bspline_weights(points, ctrl_nii): weights[weights < 1e-6] = 0.0 return weights + + +def _move_coeff(in_coeff, fmap_ref, transform): + """Read in a rigid transform from ANTs, and update the coefficients field affine.""" + from pathlib import Path + import nibabel as nb + import nitransforms as nt + + if isinstance(in_coeff, str): + in_coeff = [in_coeff] + + xfm = nt.linear.Affine( + nt.io.itk.ITKLinearTransform.from_filename(transform).to_ras(), + reference=fmap_ref, + ) + + out = [] + for i, c in enumerate(in_coeff): + out.append(str(Path(f"moved_coeff_{i:03d}.nii.gz").absolute())) + img = nb.load(c) + newaff = xfm.matrix @ img.affine + img.__class__(img.dataobj, newaff, img.header).to_filename(out[-1]) + + return out diff --git a/sdcflows/workflows/apply/registration.py b/sdcflows/workflows/apply/registration.py index cb275d6466..afeb3ba20f 100644 --- a/sdcflows/workflows/apply/registration.py +++ b/sdcflows/workflows/apply/registration.py @@ -64,6 +64,7 @@ def init_coeff2epi_wf( """ from packaging.version import parse as parseversion, Version from niworkflows.interfaces.fixes import FixHeaderRegistration as Registration + from ...interfaces.bspline import TransformCoefficients workflow = Workflow(name=name) workflow.__desc__ = """\ @@ -112,44 +113,22 @@ def init_coeff2epi_wf( return workflow # Map the coefficients into the EPI space - map_coeff = pe.Node(niu.Function(function=_move_coeff), name="map_coeff") + map_coeff = pe.Node(TransformCoefficients(), name="map_coeff") map_coeff.interface._always_run = debug # fmt: off workflow.connect([ (inputnode, map_coeff, [("fmap_coeff", "in_coeff"), - ("fmap_ref", "fmap_ref"), - ("target_ref", "target_ref")]), - (coregister, map_coeff, [("forward_transforms", "transform")]), - (map_coeff, outputnode, [("out", "fmap_coeff")]), + ("fmap_ref", "fmap_ref")]), + (coregister, map_coeff, [(("forward_transforms", _flatten), "transform")]), + (map_coeff, outputnode, [("out_coeff", "fmap_coeff")]), ]) # fmt: on return workflow -def _move_coeff(in_coeff, target_ref, fmap_ref, transform): - """Read in a rigid transform from ANTs, and update the coefficients field affine.""" - from pathlib import Path - import nibabel as nb - import nitransforms as nt - - if isinstance(in_coeff, str): - in_coeff = [in_coeff] - - xfm = nt.linear.Affine( - nt.io.itk.ITKLinearTransform.from_filename(transform[0]).to_ras(), - reference=fmap_ref, - ) - xfm.apply(target_ref).to_filename("transformed.nii.gz") - - out = [] - for i, c in enumerate(in_coeff): - img = nb.load(c) - - out.append(str(Path(f"moved_coeff_{i:03d}.nii.gz").absolute())) - - newaff = xfm.matrix @ img.affine - img.__class__(img.dataobj, newaff, img.header).to_filename(out[-1]) - - return out +def _flatten(inlist): + if isinstance(inlist, str): + return inlist + return inlist[0] From b9bc76a0d0087cdccdf66ef886d8aa9e560f299b Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 26 Nov 2020 16:10:19 +0100 Subject: [PATCH 44/68] enh: keep a registry of already-used identifiers (and auto-generate new) Resolves: #125. --- sdcflows/fieldmaps.py | 44 +++++++++++++++++++++++++ sdcflows/tests/test_fieldmaps.py | 56 +++++++++++++++++++++++++++++++- sdcflows/utils/phasemanip.py | 2 +- sdcflows/workflows/base.py | 16 ++++----- 4 files changed, 108 insertions(+), 10 deletions(-) diff --git a/sdcflows/fieldmaps.py b/sdcflows/fieldmaps.py index c2974e1d55..c8dc4b9498 100644 --- a/sdcflows/fieldmaps.py +++ b/sdcflows/fieldmaps.py @@ -9,6 +9,9 @@ from niworkflows.utils.bids import relative_to_root +_unique_ids = set() + + class MetadataError(ValueError): """A better name for a specific value error.""" @@ -59,6 +62,26 @@ def _type_setter(obj, attribute, value): return value +def _id_setter(obj, attribute, value): + """Ensure uniqueness of B0FieldIdentifier metadata.""" + if obj.bids_id: + if obj.bids_id != value: + raise ValueError("Unique identifier is already set") + _unique_ids.add(value) + return value + + if value is True: + value = f"auto_{len([el for el in _unique_ids if el.startswith('auto_')])}" + elif not value: + raise ValueError("Invalid unique identifier") + + if value in _unique_ids: + raise ValueError("Unique identifier has been previously registered.") + + _unique_ids.add(value) + return value + + @attr.s(slots=True) class FieldmapFile: """ @@ -253,6 +276,9 @@ class FieldmapEstimation: method = attr.ib(init=False, default=EstimatorType.UNKNOWN, on_setattr=_type_setter) """Flag indicating the estimator type inferred from the input sources.""" + bids_id = attr.ib(default=None, kw_only=True, type=str, on_setattr=_id_setter) + """The unique ``B0FieldIdentifier`` field of this fieldmap.""" + def __attrs_post_init__(self): """Determine the inteded fieldmap estimation type and check for data completeness.""" suffix_list = [f.suffix for f in self.sources] @@ -349,3 +375,21 @@ def __attrs_post_init__(self): # No method has been identified -> fail. if self.method == EstimatorType.UNKNOWN: raise ValueError("Insufficient sources to estimate a fieldmap.") + + if not self.bids_id: + bids_ids = set([ + f.metadata.get("B0FieldIdentifier") + for f in self.sources if f.metadata.get("B0FieldIdentifier") + ]) + if len(bids_ids) > 1: + raise ValueError( + f"Multiple ``B0FieldIdentifier`` set: <{', '.join(bids_ids)}>" + ) + elif not bids_ids: + self.bids_id = True + else: + self.bids_id = bids_ids.pop() + elif self.bids_id in _unique_ids: + raise ValueError("Unique identifier has been previously registered.") + else: + _unique_ids.add(self.bids_id) diff --git a/sdcflows/tests/test_fieldmaps.py b/sdcflows/tests/test_fieldmaps.py index 9311d402b4..0ef85e016c 100644 --- a/sdcflows/tests/test_fieldmaps.py +++ b/sdcflows/tests/test_fieldmaps.py @@ -54,9 +54,28 @@ def test_FieldmapEstimation(testdata_dir, inputfiles, method, nsources): """Test errors.""" sub_dir = testdata_dir / "sub-01" - fe = FieldmapEstimation([sub_dir / f for f in inputfiles]) + sources = [sub_dir / f for f in inputfiles] + fe = FieldmapEstimation(sources) assert fe.method == method assert len(fe.sources) == nsources + assert fe.bids_id is not None and fe.bids_id.startswith("auto_") + + # Attempt to change bids_id + with pytest.raises(ValueError): + fe.bids_id = "other" + + # Setting the same value should not raise + fe.bids_id = fe.bids_id + + # Ensure duplicate B0FieldIdentifier are not accepted + with pytest.raises(ValueError): + FieldmapEstimation(sources, bids_id=fe.bids_id) + + # B0FieldIdentifier can be generated manually + # Creating two FieldmapEstimation objects from the same sources SHOULD fail + # or be better handled in the future (see #129). + fe2 = FieldmapEstimation(sources, bids_id=f"no{fe.bids_id}") + assert fe2.bids_id and fe2.bids_id.startswith("noauto_") @pytest.mark.parametrize( @@ -74,3 +93,38 @@ def test_FieldmapEstimationError(testdata_dir, inputfiles, errortype): with pytest.raises(errortype): FieldmapEstimation([sub_dir / f for f in inputfiles]) + + +def test_FieldmapEstimationIdentifier(testdata_dir): + """Check some use cases of B0FieldIdentifier.""" + with pytest.raises(ValueError): + FieldmapEstimation([ + FieldmapFile(testdata_dir / "sub-01" / "fmap/sub-01_fieldmap.nii.gz", + metadata={"Units": "Hz", "B0FieldIdentifier": "fmap_0"}), + FieldmapFile(testdata_dir / "sub-01" / "fmap/sub-01_magnitude.nii.gz", + metadata={"Units": "Hz", "B0FieldIdentifier": "fmap_1"}) + ]) # Inconsistent identifiers + + fe = FieldmapEstimation([ + FieldmapFile(testdata_dir / "sub-01" / "fmap/sub-01_fieldmap.nii.gz", + metadata={"Units": "Hz", "B0FieldIdentifier": "fmap_0"}), + FieldmapFile(testdata_dir / "sub-01" / "fmap/sub-01_magnitude.nii.gz", + metadata={"Units": "Hz", "B0FieldIdentifier": "fmap_0"}) + ]) + assert fe.bids_id == "fmap_0" + + with pytest.raises(ValueError): + FieldmapEstimation([ + FieldmapFile(testdata_dir / "sub-01" / "fmap/sub-01_fieldmap.nii.gz", + metadata={"Units": "Hz", "B0FieldIdentifier": "fmap_0"}), + FieldmapFile(testdata_dir / "sub-01" / "fmap/sub-01_magnitude.nii.gz", + metadata={"Units": "Hz", "B0FieldIdentifier": "fmap_0"}) + ]) # Consistent, but already exists + + fe = FieldmapEstimation([ + FieldmapFile(testdata_dir / "sub-01" / "fmap/sub-01_fieldmap.nii.gz", + metadata={"Units": "Hz", "B0FieldIdentifier": "fmap_1"}), + FieldmapFile(testdata_dir / "sub-01" / "fmap/sub-01_magnitude.nii.gz", + metadata={"Units": "Hz", "B0FieldIdentifier": "fmap_1"}) + ]) + assert fe.bids_id == "fmap_1" diff --git a/sdcflows/utils/phasemanip.py b/sdcflows/utils/phasemanip.py index 27ee9d857b..4624b2fc39 100644 --- a/sdcflows/utils/phasemanip.py +++ b/sdcflows/utils/phasemanip.py @@ -108,7 +108,7 @@ def delta_te(in_values): >>> delta_te({}) # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ValueError: - + >>> delta_te({"EchoTimeDifference": "a"}) # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ValueError: diff --git a/sdcflows/workflows/base.py b/sdcflows/workflows/base.py index 4e61b58662..fe92e1c2e7 100644 --- a/sdcflows/workflows/base.py +++ b/sdcflows/workflows/base.py @@ -41,20 +41,20 @@ def init_fmap_preproc_wf( ... omp_nthreads=1, ... output_dir="/tmp", ... subject="1", - ... ) - [FieldmapEstimation(sources=<4 files>, method=), - FieldmapEstimation(sources=<4 files>, method=), - FieldmapEstimation(sources=<3 files>, method=), - FieldmapEstimation(sources=<2 files>, method=)] + ... ) # doctest: +ELLIPSIS + [FieldmapEstimation(sources=<4 files>, method=, bids_id='...'), + FieldmapEstimation(sources=<4 files>, method=, bids_id='...'), + FieldmapEstimation(sources=<3 files>, method=, bids_id='...'), + FieldmapEstimation(sources=<2 files>, method=, bids_id='...')] >>> init_fmap_preproc_wf( ... layout=layouts['testdata'], ... omp_nthreads=1, ... output_dir="/tmp", ... subject="HCP101006", - ... ) - [FieldmapEstimation(sources=<2 files>, method=), - FieldmapEstimation(sources=<2 files>, method=)] + ... ) # doctest: +ELLIPSIS + [FieldmapEstimation(sources=<2 files>, method=, bids_id='...'), + FieldmapEstimation(sources=<2 files>, method=, bids_id='...')] """ from ..fieldmaps import FieldmapEstimation, FieldmapFile From dfe93feb05839b9413f744b183d176526b9d1fcc Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Fri, 27 Nov 2020 14:09:34 +0100 Subject: [PATCH 45/68] ENH: Finalizing the API overhaul --- sdcflows/fieldmaps.py | 32 ++++++++++++++++++++++++++++++ sdcflows/tests/test_fieldmaps.py | 4 ++++ sdcflows/workflows/fit/fieldmap.py | 25 +++++++++++++++++------ 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/sdcflows/fieldmaps.py b/sdcflows/fieldmaps.py index c8dc4b9498..e8f17f4565 100644 --- a/sdcflows/fieldmaps.py +++ b/sdcflows/fieldmaps.py @@ -279,6 +279,9 @@ class FieldmapEstimation: bids_id = attr.ib(default=None, kw_only=True, type=str, on_setattr=_id_setter) """The unique ``B0FieldIdentifier`` field of this fieldmap.""" + _wf = attr.ib(init=False, default=None, repr=False) + """Internal pointer to a workflow.""" + def __attrs_post_init__(self): """Determine the inteded fieldmap estimation type and check for data completeness.""" suffix_list = [f.suffix for f in self.sources] @@ -393,3 +396,32 @@ def __attrs_post_init__(self): raise ValueError("Unique identifier has been previously registered.") else: _unique_ids.add(self.bids_id) + + def get_workflow(self, **kwargs): + """Build the estimation workflow corresponding to this instance.""" + if self._wf is not None: + return self._wf + + # Override workflow name + kwargs["name"] = f"wf_{self.bids_id}" + + if self.method in (EstimatorType.MAPPED, EstimatorType.PHASEDIFF): + from .workflows.fit.fieldmap import init_fmap_wf + + kwargs["mode"] = str(self.method).split(".")[-1].lower() + self._wf = init_fmap_wf(**kwargs) + self._wf.inputs.inputnode.magnitude = [ + str(f.path) for f in self.sources if f.suffix.startswith("magnitude") + ] + self._wf.inputs.inputnode.fieldmap = [ + (str(f.path), f.metadata) for f in self.sources + if f.suffix in ("fieldmap", "phasediff", "phase2", "phase1") + ] + elif self.method == EstimatorType.PEPOLAR: + from .workflows.fit.pepolar import init_topup_wf + self._wf = init_topup_wf(**kwargs) + elif self.method == EstimatorType.ANAT: + from .workflows.fit.syn import init_syn_sdc_wf + self._wf = init_syn_sdc_wf(**kwargs) + + return self._wf diff --git a/sdcflows/tests/test_fieldmaps.py b/sdcflows/tests/test_fieldmaps.py index 0ef85e016c..79c8e38f4b 100644 --- a/sdcflows/tests/test_fieldmaps.py +++ b/sdcflows/tests/test_fieldmaps.py @@ -77,6 +77,10 @@ def test_FieldmapEstimation(testdata_dir, inputfiles, method, nsources): fe2 = FieldmapEstimation(sources, bids_id=f"no{fe.bids_id}") assert fe2.bids_id and fe2.bids_id.startswith("noauto_") + # Exercise workflow creation + wf = fe.get_workflow() + wf == fe.get_workflow() + @pytest.mark.parametrize( "inputfiles,errortype", diff --git a/sdcflows/workflows/fit/fieldmap.py b/sdcflows/workflows/fit/fieldmap.py index 3cb9478602..0c8cbd8d3e 100644 --- a/sdcflows/workflows/fit/fieldmap.py +++ b/sdcflows/workflows/fit/fieldmap.py @@ -134,13 +134,10 @@ def init_fmap_wf(omp_nthreads=1, debug=False, mode="phasediff", name="fmap_wf"): Inputs ------ - magnitude : :obj:`str` + magnitude : :obj:`list` of :obj:`str` Path to the corresponding magnitude image for anatomical reference. - fieldmap : :obj:`str` + fieldmap : :obj:`list` of :obj:`tuple`(:obj:`str`, :obj:`dict`) Path to the fieldmap acquisition (``*_fieldmap.nii[.gz]`` of BIDS). - units : :obj:`str` - Units (`"Hz"` or `"rad/s"`) of the fieldmap (only direct :math:`B_0` - acquisitions with :abbr:`SEI (Spiral-Echo Imaging)` fieldmaps). Outputs ------- @@ -230,7 +227,7 @@ def init_fmap_wf(omp_nthreads=1, debug=False, mode="phasediff", name="fmap_wf"): # fmt: off workflow.connect([ - (inputnode, units, [("units", "units")]), + (inputnode, units, [(("fieldmap", _get_units), "units")]), (inputnode, fmapmrg, [("fieldmap", "in_files")]), (fmapmrg, units, [("out_avg", "in_file")]), (units, bs_filter, [("out_file", "in_data")]), @@ -422,3 +419,19 @@ def _split(phase): ]) # fmt: on return workflow + + +def _get_units(intuple): + """ + Extract Units from metadata. + + >>> _get_units([("fmap.nii.gz", {"Units": "rad/s"})]) + 'rad/s' + + >>> _get_units(("fmap.nii.gz", {"Units": "rad/s"})) + 'rad/s' + + """ + if isinstance(intuple, list): + intuple = intuple[0] + return intuple[1]["Units"] From 0fa398515f25bfa422a2a6604a23b88c8e51884d Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Fri, 27 Nov 2020 18:13:39 +0100 Subject: [PATCH 46/68] enh: refactor the fieldmapless-SyN method --- sdcflows/workflows/fit/syn.py | 294 ++++++++++++----------- sdcflows/workflows/fit/tests/test_fit.py | 8 +- 2 files changed, 154 insertions(+), 148 deletions(-) diff --git a/sdcflows/workflows/fit/syn.py b/sdcflows/workflows/fit/syn.py index cb1cafa63c..d80d927e26 100644 --- a/sdcflows/workflows/fit/syn.py +++ b/sdcflows/workflows/fit/syn.py @@ -5,58 +5,67 @@ .. _sdc_fieldmapless : -Fieldmap-less estimation (experimental) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In the absence of direct measurements of fieldmap data, we provide an (experimental) -option to estimate the susceptibility distortion based on the ANTs symmetric -normalization (SyN) technique. -This feature may be enabled, using the ``--use-syn-sdc`` flag, and will only be -applied if fieldmaps are unavailable. - -During the evaluation phase, the ``--force-syn`` flag will cause this estimation to -be performed *in addition to* fieldmap-based estimation, to permit the direct -comparison of the results of each technique. -Note that, even if ``--force-syn`` is given, the functional outputs of FMRIPREP will -be corrected using the fieldmap-based estimates. - +Fieldmap-less approaches +~~~~~~~~~~~~~~~~~~~~~~~~ +Many studies acquired (especially with legacy MRI protocols) do not have any +information to estimate susceptibility-derived distortions. +In the absence of data with the specific purpose of estimating the :math:`B_0` +inhomogeneity map, researchers resort to nonlinear registration to an +«*anatomically correct*» map of the same individual (normally acquired with +:abbr:`T1w (T1-weighted)`, or :abbr:`T2w (T2-weighted)` sequences). +One of the most prominent proposals of this approach is found in [Studholme2000]_. + +*SDCFlows* includes an (experimental) procedure (see :py:func:`init_syn_sdc_wf` below), +based on nonlinear image registration with ANTs' symmetric normalization (SyN) technique. +This workflow takes a skull-stripped :abbr:`T1w (T1-weighted)` image and +a reference :abbr:`EPI (Echo-Planar Imaging)` image, and estimates a field of nonlinear +displacements that accounts for susceptibility-derived distortions. +To more accurately estimate the warping on typically distorted regions, this +implementation uses an average :math:`B_0` mapping described in [Treiber2016]_. +The implementation is a variation on those developed in [Huntenburg2014]_ and +[Wang2017]_. Feedback will be enthusiastically received. -""" -from pkg_resources import resource_filename +References +---------- +.. [Studholme2000] Studholme et al. (2000) Accurate alignment of functional EPI data to + anatomical MRI using a physics-based distortion model, + IEEE Trans Med Imag 19(11):1115-1127, 2000, doi: `10.1109/42.896788 + `__. +.. [Treiber2016] Treiber, J. M. et al. (2016) Characterization and Correction + of Geometric Distortions in 814 Diffusion Weighted Images, + PLoS ONE 11(3): e0152472. doi:`10.1371/journal.pone.0152472 + `_. +.. [Wang2017] Wang S, et al. (2017) Evaluation of Field Map and Nonlinear + Registration Methods for Correction of Susceptibility Artifacts + in Diffusion MRI. Front. Neuroinform. 11:17. + doi:`10.3389/fninf.2017.00017 + `_. +.. [Huntenburg2014] Huntenburg, J. M. (2014) `Evaluating Nonlinear + Coregistration of BOLD EPI and T1w Images + `__, + Berlin: Master Thesis, Freie Universität. -from nipype import logging +""" from nipype.pipeline import engine as pe -from nipype.interfaces import fsl, utility as niu -from nipype.interfaces.image import Rescale - +from nipype.interfaces import utility as niu from niworkflows.engine.workflows import LiterateWorkflow as Workflow -from niworkflows.interfaces.fixes import ( - FixHeaderApplyTransforms as ApplyTransforms, - FixHeaderRegistration as Registration, -) -from niworkflows.func.util import init_skullstrip_bold_wf DEFAULT_MEMORY_MIN_GB = 0.01 -LOGGER = logging.getLogger("nipype.workflow") -def init_syn_sdc_wf(omp_nthreads=1, epi_pe=None, atlas_threshold=3, name="syn_sdc_wf"): +def init_syn_sdc_wf( + *, atlas_threshold=3, debug=False, name="syn_sdc_wf", omp_nthreads=1, +): """ Build the *fieldmap-less* susceptibility-distortion estimation workflow. - This workflow takes a skull-stripped T1w image and reference BOLD image and - estimates a susceptibility distortion correction warp, using ANTs symmetric - normalization (SyN) and the average fieldmap atlas described in - [Treiber2016]_. - SyN deformation is restricted to the phase-encoding (PE) direction. If no PE direction is specified, anterior-posterior PE is assumed. SyN deformation is also restricted to regions that are expected to have a >3mm (approximately 1 voxel) warp, based on the fieldmap atlas. - This technique is a variation on those developed in [Huntenburg2014]_ and - [Wang2017]_. Workflow Graph .. workflow :: @@ -64,69 +73,70 @@ def init_syn_sdc_wf(omp_nthreads=1, epi_pe=None, atlas_threshold=3, name="syn_sd :simple_form: yes from sdcflows.workflows.fit.syn import init_syn_sdc_wf - wf = init_syn_sdc_wf( - epi_pe="j", - omp_nthreads=8) + wf = init_syn_sdc_wf(omp_nthreads=8) + + Parameters + ---------- + atlas_threshold : :obj:`float` + Exclude from the registration metric computation areas with average distortions + below this threshold (in mm). + debug : :obj:`bool` + Whether a fast (less accurate) configuration of the workflow should be applied. + name : :obj:`str` + Name for this workflow + omp_nthreads : :obj:`int` + Parallelize internal tasks across the number of CPUs given by this option. Inputs ------ - in_reference - reference image - in_reference_brain - skull-stripped reference image - t1w_brain - skull-stripped, bias-corrected structural image - std2anat_xfm + epi_ref : :obj:`tuple` (:obj:`str`, :obj:`dict`) + A tuple, where the first element is the path of the distorted EPI + reference map (e.g., an average of *b=0* volumes), and the second + element is a dictionary of associated metadata. + epi_mask : :obj:`str` + A path to a brain mask corresponding to ``epi_ref``. + anat_brain : :obj:`str` + A preprocessed, skull-stripped anatomical (T1w or T2w) image. + std2anat_xfm : :obj:`str` inverse registration transform of T1w image to MNI template + anat2bold_xfm : :obj:`str` + transform mapping coordinates from the EPI space to the anatomical + space (i.e., the transform to resample anatomical info into EPI space.) Outputs ------- - fmap - the corresponding :abbr:`DFM (displacements field map)` compatible with - ANTs - fmap_ref - the ``in_reference`` image after unwarping - - References - ---------- - .. [Treiber2016] Treiber, J. M. et al. (2016) Characterization and Correction - of Geometric Distortions in 814 Diffusion Weighted Images, - PLoS ONE 11(3): e0152472. doi:`10.1371/journal.pone.0152472 - `_. - .. [Wang2017] Wang S, et al. (2017) Evaluation of Field Map and Nonlinear - Registration Methods for Correction of Susceptibility Artifacts - in Diffusion MRI. Front. Neuroinform. 11:17. - doi:`10.3389/fninf.2017.00017 - `_. - .. [Huntenburg2014] Huntenburg, J. M. (2014) Evaluating Nonlinear - Coregistration of BOLD EPI and T1w Images. Berlin: Master - Thesis, Freie Universität. `PDF - `_. + fmap : :obj:`str` + The path of the estimated fieldmap. + fmap_ref : :obj:`str` + The path of an unwarped conversion of files in ``epi_ref``. + fmap_coeff : :obj:`str` or :obj:`list` of :obj:`str` + The path(s) of the B-Spline coefficients supporting the fieldmap. """ - if epi_pe is None or epi_pe[0] not in ["i", "j"]: - LOGGER.warning( - "Incorrect phase-encoding direction, assuming PA (posterior-to-anterior)." - ) - epi_pe = "j" + from pkg_resources import resource_filename as pkgrf + from nipype.interfaces.image import Rescale + from niworkflows.interfaces.fixes import ( + FixHeaderApplyTransforms as ApplyTransforms, + FixHeaderRegistration as Registration, + ) + from niworkflows.interfaces.nibabel import Binarize workflow = Workflow(name=name) - workflow.__desc__ = """\ + workflow.__desc__ = f"""\ A deformation field to correct for susceptibility distortions was estimated based on *fMRIPrep*'s *fieldmap-less* approach. -The deformation field is that resulting from co-registering the BOLD reference +The deformation field is that resulting from co-registering the EPI reference to the same-subject T1w-reference with its intensity inverted [@fieldmapless1; @fieldmapless2]. -Registration is performed with `antsRegistration` (ANTs {ants_ver}), and +Registration is performed with `antsRegistration` +(ANTs {Registration().version or "-- version unknown"}), and the process regularized by constraining deformation to be nonzero only along the phase-encoding direction, and modulated with an average fieldmap template [@fieldmapless3]. -""".format( - ants_ver=Registration().version or "" - ) +""" inputnode = pe.Node( niu.IdentityInterface( - ["in_reference", "in_reference_brain", "t1w_brain", "std2anat_xfm"] + ["epi_ref", "epi_mask", "anat_brain", "std2anat_xfm", "anat2bold_xfm"] ), name="inputnode", ) @@ -134,97 +144,93 @@ def init_syn_sdc_wf(omp_nthreads=1, epi_pe=None, atlas_threshold=3, name="syn_sd niu.IdentityInterface(["fmap", "fmap_ref", "fmap_mask"]), name="outputnode", ) - # Collect predefined data - # Atlas image and registration affine - atlas_img = resource_filename("sdcflows", "data/fmap_atlas.nii.gz") - # Registration specifications - affine_transform = resource_filename("sdcflows", "data/affine.json") - syn_transform = resource_filename("sdcflows", "data/susceptibility_syn.json") - invert_t1w = pe.Node(Rescale(invert=True), name="invert_t1w", mem_gb=0.3) - - ref_2_t1 = pe.Node( - Registration(from_file=affine_transform), name="ref_2_t1", n_procs=omp_nthreads - ) - t1_2_ref = pe.Node( - ApplyTransforms(invert_transform_flags=[True]), - name="t1_2_ref", - n_procs=omp_nthreads, + anat2epi = pe.Node( + ApplyTransforms(interpolation="BSpline"), name="anat2epi", n_procs=omp_nthreads ) - # 1) BOLD -> T1; 2) MNI -> T1; 3) ATLAS -> MNI + # Mapping & preparing prior knowledge + # Concatenate transform files: + # 1) anat -> EPI; 2) MNI -> anat; 3) ATLAS -> MNI transform_list = pe.Node( niu.Merge(3), name="transform_list", mem_gb=DEFAULT_MEMORY_MIN_GB ) - transform_list.inputs.in3 = resource_filename( + transform_list.inputs.in3 = pkgrf( "sdcflows", "data/fmap_atlas_2_MNI152NLin2009cAsym_affine.mat" ) - - # Inverting (1), then applying in reverse order: - # - # ATLAS -> MNI -> T1 -> BOLD - atlas_2_ref = pe.Node( - ApplyTransforms(invert_transform_flags=[True, False, False]), - name="atlas_2_ref", + prior2epi = pe.Node( + ApplyTransforms(input_image=pkgrf("sdcflows", "data/fmap_atlas.nii.gz")), + name="prior2epi", n_procs=omp_nthreads, mem_gb=0.3, ) - atlas_2_ref.inputs.input_image = atlas_img - - threshold_atlas = pe.Node( - fsl.maths.MathsCommand( - args="-thr {:.8g} -bin".format(atlas_threshold), output_datatype="char" - ), - name="threshold_atlas", - mem_gb=0.3, - ) + atlas_msk = pe.Node(Binarize(thresh_low=atlas_threshold), name="atlas_msk") - fixed_image_masks = pe.Node( - niu.Merge(2), name="fixed_image_masks", mem_gb=DEFAULT_MEMORY_MIN_GB - ) - fixed_image_masks.inputs.in1 = "NULL" - - restrict = [[int(epi_pe[0] == "i"), int(epi_pe[0] == "j"), 0]] * 2 + # SyN Registration Core syn = pe.Node( - Registration(from_file=syn_transform, restrict_deformation=restrict), + Registration(from_file=pkgrf("sdcflows", "data/susceptibility_syn.json")), name="syn", n_procs=omp_nthreads, ) - unwarp_ref = pe.Node( - ApplyTransforms(dimension=3, float=True, interpolation="LanczosWindowedSinc"), - name="unwarp_ref", - ) - - skullstrip_bold_wf = init_skullstrip_bold_wf() + unwarp_ref = pe.Node(ApplyTransforms(interpolation="BSpline"), name="unwarp_ref",) # fmt: off workflow.connect([ - (inputnode, invert_t1w, [("t1w_brain", "in_file"), - ("in_reference", "ref_file")]), - (inputnode, ref_2_t1, [("in_reference_brain", "moving_image")]), - (invert_t1w, ref_2_t1, [("out_file", "fixed_image")]), - (inputnode, t1_2_ref, [("in_reference", "reference_image")]), - (invert_t1w, t1_2_ref, [("out_file", "input_image")]), - (ref_2_t1, t1_2_ref, [("forward_transforms", "transforms")]), - (ref_2_t1, transform_list, [("forward_transforms", "in1")]), - (inputnode, transform_list, [ - ("std2anat_xfm", "in2")]), - (inputnode, atlas_2_ref, [("in_reference", "reference_image")]), - (transform_list, atlas_2_ref, [("out", "transforms")]), - (atlas_2_ref, threshold_atlas, [("output_image", "in_file")]), - (threshold_atlas, fixed_image_masks, [("out_file", "in2")]), - (inputnode, syn, [("in_reference_brain", "moving_image")]), - (t1_2_ref, syn, [("output_image", "fixed_image")]), - (fixed_image_masks, syn, [("out", "fixed_image_masks")]), + (inputnode, transform_list, [("anat2bold_xfm", "in1"), + ("std2anat_xfm", "in2")]), + (inputnode, invert_t1w, [("anat_brain", "in_file"), + (("epi_ref", _pop), "ref_file")]), + (inputnode, anat2epi, [(("epi_ref", _pop), "reference_image")]), + (inputnode, syn, [(("epi_ref", _pop), "moving_image"), + ("epi_mask", "moving_image_masks"), + (("epi_ref", _warp_dir), "restrict_deformation")]), + (inputnode, prior2epi, [(("epi_ref", _pop), "reference_image")]), + (invert_t1w, anat2epi, [("out_file", "input_image")]), + (transform_list, prior2epi, [("out", "transforms")]), + (prior2epi, atlas_msk, [("output_image", "in_file")]), + (anat2epi, syn, [("output_image", "fixed_image")]), + (atlas_msk, syn, [(("out_mask", _fixed_masks_arg), "fixed_image_masks")]), (syn, outputnode, [("forward_transforms", "fmap")]), (syn, unwarp_ref, [("forward_transforms", "transforms")]), - (inputnode, unwarp_ref, [("in_reference", "reference_image"), - ("in_reference", "input_image")]), - (unwarp_ref, skullstrip_bold_wf, [ - ("output_image", "inputnode.in_file")]), + (inputnode, unwarp_ref, [(("epi_ref", _pop), "reference_image"), + (("epi_ref", _pop), "input_image")]), (unwarp_ref, outputnode, [("output_image", "fmap_ref")]), ]) # fmt: on return workflow + + +def _warp_dir(intuple): + """ + Extract the ``restrict_deformation`` argument from metadata. + + Example + ------- + >>> _warp_dir(("epi.nii.gz", {"PhaseEncodingDirection": "i-"})) + [[1, 0, 0], [1, 0, 0]] + + >>> _warp_dir(("epi.nii.gz", {"PhaseEncodingDirection": "j-"})) + [[0, 1, 0], [0, 1, 0]] + + """ + pe = intuple[1]["PhaseEncodingDirection"][0] + return 2 * [[int(pe == ax) for ax in "ijk"]] + + +def _fixed_masks_arg(mask): + """ + Prepare the ``fixed_image_masks`` argument of SyN. + + Example + ------- + >>> _fixed_masks_arg("atlas_mask.nii.gz") + ['NULL', 'atlas_mask.nii.gz'] + + """ + return ["NULL", mask] + + +def _pop(inlist): + return inlist[0] diff --git a/sdcflows/workflows/fit/tests/test_fit.py b/sdcflows/workflows/fit/tests/test_fit.py index c97b9ca096..f7561cce9c 100644 --- a/sdcflows/workflows/fit/tests/test_fit.py +++ b/sdcflows/workflows/fit/tests/test_fit.py @@ -10,12 +10,12 @@ @pytest.mark.parametrize( "workflow,kwargs", ( - ("sdcflows.workflows.fit.pepolar.init_topup_wf", {}), - ("sdcflows.workflows.fit.pepolar.init_3dQwarp_wf", {}), - ("sdcflows.workflows.fit.syn.init_syn_sdc_wf", {}), + ("sdcflows.workflows.fit.fieldmap.init_fmap_wf", {"mode": "mapped"}), ("sdcflows.workflows.fit.fieldmap.init_fmap_wf", {}), - ("sdcflows.workflows.fit.fieldmap.init_fmap_wf", {"mode": "fieldmap"}), ("sdcflows.workflows.fit.fieldmap.init_phdiff_wf", {"omp_nthreads": 1}), + ("sdcflows.workflows.fit.pepolar.init_3dQwarp_wf", {}), + ("sdcflows.workflows.fit.pepolar.init_topup_wf", {}), + ("sdcflows.workflows.fit.syn.init_syn_sdc_wf", {"omp_nthreads": 1}), ), ) def test_build_1(workflow, kwargs): From 10e77f2facbd3038447d81fb137f8b5a7ceb5a72 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Fri, 27 Nov 2020 21:14:04 +0100 Subject: [PATCH 47/68] tst: keep expanding coverage, add tiny doctests --- sdcflows/utils/misc.py | 37 +++++++ sdcflows/workflows/apply/registration.py | 9 +- sdcflows/workflows/fit/pepolar.py | 117 +++++++++++++---------- sdcflows/workflows/fit/syn.py | 5 +- 4 files changed, 108 insertions(+), 60 deletions(-) create mode 100644 sdcflows/utils/misc.py diff --git a/sdcflows/utils/misc.py b/sdcflows/utils/misc.py new file mode 100644 index 0000000000..0fc9a675da --- /dev/null +++ b/sdcflows/utils/misc.py @@ -0,0 +1,37 @@ +"""Basic miscelaneous utilities.""" + + +def front(inlist): + """ + Pop from a list or tuple, otherwise return untouched. + + Examples + -------- + >>> front([1, 0]) + 1 + + >>> front("/path/somewhere") + '/path/somewhere' + + """ + if isinstance(inlist, (list, tuple)): + return inlist[0] + return inlist + + +def last(inlist): + """ + Return the last element from a list or tuple, otherwise return untouched. + + Examples + -------- + >>> last([1, 0]) + 0 + + >>> last("/path/somewhere") + '/path/somewhere' + + """ + if isinstance(inlist, (list, tuple)): + return inlist[-1] + return inlist diff --git a/sdcflows/workflows/apply/registration.py b/sdcflows/workflows/apply/registration.py index afeb3ba20f..9be1fd3133 100644 --- a/sdcflows/workflows/apply/registration.py +++ b/sdcflows/workflows/apply/registration.py @@ -65,6 +65,7 @@ def init_coeff2epi_wf( from packaging.version import parse as parseversion, Version from niworkflows.interfaces.fixes import FixHeaderRegistration as Registration from ...interfaces.bspline import TransformCoefficients + from ...utils.misc import front as _pop workflow = Workflow(name=name) workflow.__desc__ = """\ @@ -120,15 +121,9 @@ def init_coeff2epi_wf( workflow.connect([ (inputnode, map_coeff, [("fmap_coeff", "in_coeff"), ("fmap_ref", "fmap_ref")]), - (coregister, map_coeff, [(("forward_transforms", _flatten), "transform")]), + (coregister, map_coeff, [(("forward_transforms", _pop), "transform")]), (map_coeff, outputnode, [("out_coeff", "fmap_coeff")]), ]) # fmt: on return workflow - - -def _flatten(inlist): - if isinstance(inlist, str): - return inlist - return inlist[0] diff --git a/sdcflows/workflows/fit/pepolar.py b/sdcflows/workflows/fit/pepolar.py index c02f1e4ecd..bfeb3b74b1 100644 --- a/sdcflows/workflows/fit/pepolar.py +++ b/sdcflows/workflows/fit/pepolar.py @@ -75,14 +75,7 @@ def init_topup_wf(omp_nthreads=1, debug=False, name="pepolar_estimate_wf"): ) outputnode = pe.Node( niu.IdentityInterface( - fields=[ - "fmap", - "fmap_ref", - "fmap_coeff", - "jacobians", - "xfms", - "out_warps", - ] + fields=["fmap", "fmap_ref", "fmap_coeff", "jacobians", "xfms", "out_warps"] ), name="outputnode", ) @@ -166,6 +159,7 @@ def init_3dQwarp_wf(omp_nthreads=1, name="pepolar_estimate_wf"): ) from niworkflows.interfaces.freesurfer import StructuralReference from niworkflows.func.util import init_enhance_and_skullstrip_bold_wf + from ...utils.misc import front as _front, last as _last from ...interfaces.utils import Flatten workflow = Workflow(name=name) @@ -173,33 +167,42 @@ def init_3dQwarp_wf(omp_nthreads=1, name="pepolar_estimate_wf"): with `3dQwarp` @afni (AFNI {''.join(['%02d' % v for v in afni.Info().version() or []])}). """ - inputnode = pe.Node(niu.IdentityInterface(fields=["in_data", "metadata"]), - name="inputnode") + inputnode = pe.Node( + niu.IdentityInterface(fields=["in_data", "metadata"]), name="inputnode" + ) outputnode = pe.Node( niu.IdentityInterface(fields=["fmap", "fmap_ref"]), name="outputnode" ) flatten = pe.Node(Flatten(), name="flatten") - sort_pe = pe.Node(niu.Function( - function=_sorted_pe, output_names=["sorted", "qwarp_args"]), - name="sort_pe", run_without_submitting=True) - - merge_pes = pe.MapNode(StructuralReference( - auto_detect_sensitivity=True, - initial_timepoint=1, - fixed_timepoint=True, # Align to first image - intensity_scaling=True, - # 7-DOF (rigid + intensity) - no_iteration=True, - subsample_threshold=200, - out_file='template.nii.gz'), - name='merge_pes', + sort_pe = pe.Node( + niu.Function(function=_sorted_pe, output_names=["sorted", "qwarp_args"]), + name="sort_pe", + run_without_submitting=True, + ) + + merge_pes = pe.MapNode( + StructuralReference( + auto_detect_sensitivity=True, + initial_timepoint=1, + fixed_timepoint=True, # Align to first image + intensity_scaling=True, + # 7-DOF (rigid + intensity) + no_iteration=True, + subsample_threshold=200, + out_file="template.nii.gz", + ), + name="merge_pes", iterfield=["in_files"], ) - pe0_wf = init_enhance_and_skullstrip_bold_wf(omp_nthreads=omp_nthreads, name="pe0_wf") - pe1_wf = init_enhance_and_skullstrip_bold_wf(omp_nthreads=omp_nthreads, name="pe1_wf") + pe0_wf = init_enhance_and_skullstrip_bold_wf( + omp_nthreads=omp_nthreads, name="pe0_wf" + ) + pe1_wf = init_enhance_and_skullstrip_bold_wf( + omp_nthreads=omp_nthreads, name="pe1_wf" + ) align_pes = pe.Node( Registration( @@ -228,11 +231,7 @@ def init_3dQwarp_wf(omp_nthreads=1, name="pepolar_estimate_wf"): cphdr_warp = pe.Node(CopyHeader(), name="cphdr_warp", mem_gb=0.01) unwarp_reference = pe.Node( - ApplyTransforms( - dimension=3, - float=True, - interpolation="LanczosWindowedSinc", - ), + ApplyTransforms(dimension=3, float=True, interpolation="LanczosWindowedSinc",), name="unwarp_reference", ) @@ -278,7 +277,15 @@ def _fix_hdr(in_file, newpath=None): def _pe2fsl(metadata): - """Convert ijk notation to xyz.""" + """ + Convert ijk notation to xyz. + + Example + ------- + >>> _pe2fsl([{"PhaseEncodingDirection": "j-"}, {"PhaseEncodingDirection": "i"}]) + ['y-', 'x'] + + """ return [ m["PhaseEncodingDirection"] .replace("i", "x") @@ -289,6 +296,29 @@ def _pe2fsl(metadata): def _sorted_pe(inlist): + """ + Generate suitable inputs to ``3dQwarp``. + + Example + ------- + >>> paths, args = _sorted_pe([ + ... ("dir-AP_epi.nii.gz", {"PhaseEncodingDirection": "j-"}), + ... ("dir-AP_bold.nii.gz", {"PhaseEncodingDirection": "j-"}), + ... ("dir-PA_epi.nii.gz", {"PhaseEncodingDirection": "j"}), + ... ("dir-PA_bold.nii.gz", {"PhaseEncodingDirection": "j"}), + ... ("dir-AP_sbref.nii.gz", {"PhaseEncodingDirection": "j-"}), + ... ("dir-PA_sbref.nii.gz", {"PhaseEncodingDirection": "j"}), + ... ]) + >>> paths[0] + ['dir-AP_epi.nii.gz', 'dir-AP_bold.nii.gz', 'dir-AP_sbref.nii.gz'] + + >>> paths[1] + ['dir-PA_epi.nii.gz', 'dir-PA_bold.nii.gz', 'dir-PA_sbref.nii.gz'] + + >>> args + '-noXdis -noZdis' + + """ out_ref = [inlist[0][0]] out_opp = [] @@ -302,20 +332,9 @@ def _sorted_pe(inlist): else: raise ValueError("Cannot handle orthogonal PE encodings.") - return [out_ref, out_opp], { - "i": "-noYdis -noZdis", - "j": "-noXdis -noZdis", - "k": "-noXdis -noYdis", - }[ref_pe[0]] - - -def _front(inlist): - if isinstance(inlist, (list, tuple)): - return inlist[0] - return inlist - - -def _last(inlist): - if isinstance(inlist, (list, tuple)): - return inlist[-1] - return inlist + return ( + [out_ref, out_opp], + {"i": "-noYdis -noZdis", "j": "-noXdis -noZdis", "k": "-noXdis -noYdis"}[ + ref_pe[0] + ], + ) diff --git a/sdcflows/workflows/fit/syn.py b/sdcflows/workflows/fit/syn.py index d80d927e26..652f20b074 100644 --- a/sdcflows/workflows/fit/syn.py +++ b/sdcflows/workflows/fit/syn.py @@ -120,6 +120,7 @@ def init_syn_sdc_wf( FixHeaderRegistration as Registration, ) from niworkflows.interfaces.nibabel import Binarize + from ...utils.misc import front as _pop workflow = Workflow(name=name) workflow.__desc__ = f"""\ @@ -230,7 +231,3 @@ def _fixed_masks_arg(mask): """ return ["NULL", mask] - - -def _pop(inlist): - return inlist[0] From a8b514a8fb03956b04c6a7cca50379c3d22443c6 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Sat, 28 Nov 2020 02:12:53 +0100 Subject: [PATCH 48/68] fix: revise total-readout-time, cleanup ees This commit makes a deep revisiting of our total readout time implementation. Most of the relevant details about this commit are included within the documentation. With thanks to @neurolabusc because this wouldn't have been possible without his thorough work, to @parekhpravesh for the thread he opened at Neurostars, to @Gilles86 for making us aware of these issues on the *fMRIPrep* repo, and to @treanus for reporting #5. Resolves: #5. --- sdcflows/fieldmaps.py | 27 ++-- sdcflows/interfaces/epi.py | 185 +--------------------- sdcflows/utils/epimanip.py | 202 +++++++++++++++++++++++++ sdcflows/workflows/apply/correction.py | 1 + 4 files changed, 223 insertions(+), 192 deletions(-) create mode 100644 sdcflows/utils/epimanip.py diff --git a/sdcflows/fieldmaps.py b/sdcflows/fieldmaps.py index e8f17f4565..6711f2f111 100644 --- a/sdcflows/fieldmaps.py +++ b/sdcflows/fieldmaps.py @@ -228,11 +228,12 @@ def __attrs_post_init__(self): raise MetadataError( f"Missing 'PhaseEncodingDirection' for <{self.path}>." ) - if not ( - set(("TotalReadoutTime", "EffectiveEchoSpacing")).intersection( - self.metadata.keys() - ) - ): + + from .utils.epimanip import get_trt + + try: + get_trt(self.metadata, in_file=self.path) + except ValueError: raise MetadataError( f"Missing readout timing information for <{self.path}>." ) @@ -380,10 +381,13 @@ def __attrs_post_init__(self): raise ValueError("Insufficient sources to estimate a fieldmap.") if not self.bids_id: - bids_ids = set([ - f.metadata.get("B0FieldIdentifier") - for f in self.sources if f.metadata.get("B0FieldIdentifier") - ]) + bids_ids = set( + [ + f.metadata.get("B0FieldIdentifier") + for f in self.sources + if f.metadata.get("B0FieldIdentifier") + ] + ) if len(bids_ids) > 1: raise ValueError( f"Multiple ``B0FieldIdentifier`` set: <{', '.join(bids_ids)}>" @@ -414,14 +418,17 @@ def get_workflow(self, **kwargs): str(f.path) for f in self.sources if f.suffix.startswith("magnitude") ] self._wf.inputs.inputnode.fieldmap = [ - (str(f.path), f.metadata) for f in self.sources + (str(f.path), f.metadata) + for f in self.sources if f.suffix in ("fieldmap", "phasediff", "phase2", "phase1") ] elif self.method == EstimatorType.PEPOLAR: from .workflows.fit.pepolar import init_topup_wf + self._wf = init_topup_wf(**kwargs) elif self.method == EstimatorType.ANAT: from .workflows.fit.syn import init_syn_sdc_wf + self._wf = init_syn_sdc_wf(**kwargs) return self._wf diff --git a/sdcflows/interfaces/epi.py b/sdcflows/interfaces/epi.py index df025ad9fc..eaad8ea1e1 100644 --- a/sdcflows/interfaces/epi.py +++ b/sdcflows/interfaces/epi.py @@ -1,17 +1,4 @@ -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -# vi: set ft=python sts=4 ts=4 sw=4 et: -""" -Interfaces to deal with the various types of fieldmap sources. - - .. testsetup:: - - >>> tmpdir = getfixture('tmpdir') - >>> tmp = tmpdir.chdir() # changing to a temporary directory - >>> nb.Nifti1Image(np.zeros((90, 90, 60)), None, None).to_filename( - ... tmpdir.join('epi.nii.gz').strpath) - - -""" +"""Interfaces to deal with the various types of fieldmap sources.""" from nipype.interfaces.base import ( BaseInterfaceInputSpec, @@ -40,177 +27,11 @@ class GetReadoutTime(SimpleInterface): output_spec = _GetReadoutTimeOutputSpec def _run_interface(self, runtime): + from ..utils.epimanip import get_trt + self._results["readout_time"] = get_trt( self.inputs.metadata, self.inputs.in_file if isdefined(self.inputs.in_file) else None, ) self._results["pe_direction"] = self.inputs.metadata["PhaseEncodingDirection"] return runtime - - -def get_trt(in_meta, in_file=None): - r""" - Extract the *total readout time* :math:`t_\text{RO}` from BIDS. - - Calculate the *total readout time* for an input - :abbr:`EPI (echo-planar imaging)` scan. - - There are several procedures to calculate the total - readout time. The basic one is that a ``TotalReadoutTime`` - field is set in the JSON sidecar. The following examples - use an ``'epi.nii.gz'`` file-stub which has 90 pixels in the - j-axis encoding direction. - - >>> meta = {'TotalReadoutTime': 0.05251} - >>> get_trt(meta) - 0.05251 - - If the *effective echo spacing* :math:`t_\text{ees}` - (``EffectiveEchoSpacing`` BIDS field) is provided, then the - total readout time can be calculated reading the number - of voxels along the readout direction :math:`T_\text{ro}` - and the parallel acceleration factor of the EPI :math:`f_\text{acc}`. - - .. math :: - - T_\text{ro} = t_\text{ees} \, (N_\text{PE} / f_\text{acc} - 1) - - >>> meta = {'EffectiveEchoSpacing': 0.00059, - ... 'PhaseEncodingDirection': 'j-', - ... 'ParallelReductionFactorInPlane': 2} - >>> get_trt(meta, in_file='epi.nii.gz') - 0.05251 - - Some vendors, like Philips, store different parameter names: - - >>> meta = {'WaterFatShift': 8.129, - ... 'MagneticFieldStrength': 3, - ... 'PhaseEncodingDirection': 'j-', - ... 'ParallelReductionFactorInPlane': 2} - >>> get_trt(meta, in_file='epi.nii.gz') - 0.018721183563864822 - - """ - import nibabel as nb - - # Use case 1: TRT is defined - trt = in_meta.get("TotalReadoutTime", None) - if trt is not None: - return trt - - # npe = N voxels PE direction - npe = nb.load(in_file).shape[_get_pe_index(in_meta)] - - # Use case 2: TRT is defined - ees = in_meta.get("EffectiveEchoSpacing", None) - if ees is not None: - # Effective echo spacing means that acceleration factors have been accounted for. - return ees * (npe - 1) - - # All other cases require the parallel acc and npe (N vox in PE dir) - acc = float(in_meta.get("ParallelReductionFactorInPlane", 1.0)) - etl = npe // acc # effective train length - es = in_meta.get("EchoSpacing", None) - if es is not None: - return es * (etl - 1) - - # Use case 3 (philips scans) - wfs = in_meta.get("WaterFatShift", None) - if wfs is not None: - fstrength = in_meta["MagneticFieldStrength"] - wfd_ppm = 3.4 # water-fat diff in ppm - g_ratio_mhz_t = 42.57 # gyromagnetic ratio for proton (1H) in MHz/T - wfs_hz = fstrength * wfd_ppm * g_ratio_mhz_t - return wfs / wfs_hz - - raise ValueError("Unknown total-readout time specification") - - -def get_ees(in_meta, in_file=None): - r""" - Extract the *effective echo spacing* :math:`t_\text{ees}` from BIDS. - - Calculate the *effective echo spacing* :math:`t_\text{ees}` - for an input :abbr:`EPI (echo-planar imaging)` scan. - - - There are several procedures to calculate the effective - echo spacing. The basic one is that an ``EffectiveEchoSpacing`` - field is set in the JSON sidecar. The following examples - use an ``'epi.nii.gz'`` file-stub which has 90 pixels in the - j-axis encoding direction. - - >>> meta = {'EffectiveEchoSpacing': 0.00059, - ... 'PhaseEncodingDirection': 'j-'} - >>> get_ees(meta) - 0.00059 - - If the *total readout time* :math:`T_\text{ro}` (``TotalReadoutTime`` - BIDS field) is provided, then the effective echo spacing can be - calculated reading the number of voxels :math:`N_\text{PE}` along the - readout direction and the parallel acceleration - factor of the EPI - - .. math :: - - = T_\text{ro} \, (N_\text{PE} / f_\text{acc} - 1)^{-1} - - where :math:`N_y` is the number of pixels along the phase-encoding direction - :math:`y`, and :math:`f_\text{acc}` is the parallel imaging acceleration factor - (:abbr:`GRAPPA (GeneRalized Autocalibrating Partial Parallel Acquisition)`, - :abbr:`ARC (Autocalibrating Reconstruction for Cartesian imaging)`, etc.). - - >>> meta = {'TotalReadoutTime': 0.02596, - ... 'PhaseEncodingDirection': 'j-', - ... 'ParallelReductionFactorInPlane': 2} - >>> get_ees(meta, in_file='epi.nii.gz') - 0.00059 - - Some vendors, like Philips, store different parameter names (see - http://dbic.dartmouth.edu/pipermail/mrusers/attachments/20141112/eb1d20e6/attachment.pdf - ): - - >>> meta = {'WaterFatShift': 8.129, - ... 'MagneticFieldStrength': 3, - ... 'PhaseEncodingDirection': 'j-', - ... 'ParallelReductionFactorInPlane': 2} - >>> get_ees(meta, in_file='epi.nii.gz') - 0.00041602630141921826 - - """ - import nibabel as nb - from sdcflows.interfaces.epi import _get_pe_index - - # Use case 1: EES is defined - ees = in_meta.get("EffectiveEchoSpacing", None) - if ees is not None: - return ees - - # All other cases require the parallel acc and npe (N vox in PE dir) - acc = float(in_meta.get("ParallelReductionFactorInPlane", 1.0)) - npe = nb.load(in_file).shape[_get_pe_index(in_meta)] - etl = npe // acc - - # Use case 2: TRT is defined - trt = in_meta.get("TotalReadoutTime", None) - if trt is not None: - return trt / (etl - 1) - - # Use case 3 (philips scans) - wfs = in_meta.get("WaterFatShift", None) - if wfs is not None: - fstrength = in_meta["MagneticFieldStrength"] - wfd_ppm = 3.4 # water-fat diff in ppm - g_ratio_mhz_t = 42.57 # gyromagnetic ratio for proton (1H) in MHz/T - wfs_hz = fstrength * wfd_ppm * g_ratio_mhz_t - return wfs / (wfs_hz * etl) - - raise ValueError("Unknown effective echo-spacing specification") - - -def _get_pe_index(meta): - pe = meta["PhaseEncodingDirection"] - try: - return {"i": 0, "j": 1, "k": 2}[pe[0]] - except KeyError: - raise RuntimeError('"%s" is an invalid PE string' % pe) diff --git a/sdcflows/utils/epimanip.py b/sdcflows/utils/epimanip.py new file mode 100644 index 0000000000..25d9d983b0 --- /dev/null +++ b/sdcflows/utils/epimanip.py @@ -0,0 +1,202 @@ +""" +Manipulation of EPI data. + +.. testsetup:: + + >>> tmpdir = getfixture('tmpdir') + >>> tmp = tmpdir.chdir() # changing to a temporary directory + >>> nb.Nifti1Image(np.zeros((90, 90, 60)), None, None).to_filename( + ... tmpdir.join('epi.nii.gz').strpath) + +""" + + +def get_trt(in_meta, in_file=None): + r""" + Obtain the *total readout time* :math:`T_\text{ro}` from available metadata. + + BIDS provides two standard mechanisms to store the total readout time, + :math:`T_\text{ro}`, of :abbr:`EPI (echo-planar imaging)` scans. + The first option is that a ``TotalReadoutTime`` field is found + in the JSON sidecar: + + >>> meta = {'TotalReadoutTime': 0.05251} + >>> get_trt(meta) + 0.05251 + + Alternatively, the *effective echo spacing* :math:`t_\text{ees}` + (``EffectiveEchoSpacing`` BIDS field) may be provided. + Then, the total readout time :math:`T_\text{ro}` can be calculated + as follows: + + .. math :: + + T_\text{ro} = t_\text{ees} \cdot (N_\text{PE} - 1), + \label{eq:rotime-ees}\tag{1} + + where :math:`N_\text{PE}` is the number of pixels along the + :abbr:`PE (phase-encoding)` direction **on the reconstructed matrix**. + + >>> meta = {'EffectiveEchoSpacing': 0.00059, + ... 'PhaseEncodingDirection': 'j-'} + >>> f"{get_trt(meta, in_file='epi.nii.gz'):g}" + '0.05251' + + Using nonstandard metadata, there are further options. + If the *echo spacing* :math:`t_\text{es}` (do not confuse with the + *effective echo spacing*, :math:`t_\text{ees}`) is set and the + parallel acceleration factor + (:abbr:`GRAPPA (GeneRalized Auto-calibrating Partial Parallel Acquisition)`, + :abbr:`ARC (Auto-calibrating Reconstruction for Cartesian imaging)`, etc.) + of the EPI :math:`f_\text{acc}` is known, then it is possible to calculate + the readout time as: + + .. math :: + + T_\text{ro} = t_\text{es} \cdot + (\left\lfloor\frac{N_\text{PE}}{f_\text{acc}} \right\rfloor - 1). + + >>> meta = {'EchoSpacing': 0.00119341, + ... 'PhaseEncodingDirection': 'j-', + ... 'ParallelReductionFactorInPlane': 2} + >>> f"{get_trt(meta, in_file='epi.nii.gz'):g}" + '0.05251' + + .. caution:: + + Philips stores different parameter names, and there has been quite a bit + of reverse-engineering and discussion around how to get the total + readout-time right for the vendor. + + The implementation done here follows the findings of Dr. Rorden, + summarized in `this post + `__. + + It seems to be possible to calculate the **effective** echo spacing + (in seconds) as: + + .. math :: + + t_\text{ees} = \frac{f_\text{wfs}} + {B_0 \gamma \Delta_\text{w/f} \cdot (f_\text{EPI} + 1)}, + \label{eq:philips-ees}\tag{2} + + where :math:`f_\text{wfs}` is the water-fat-shift in pixels, + :math:`B_0` is the field strength in T, :math:`\gamma` is the + gyromagnetic ratio, :math:`\Delta_\text{w/f}` is the water/fat + difference in ppm and :math:`f_\text{EPI}` is Philip's «*EPI factor*,» + which accounts for in-plane acceleration with :abbr:`SENSE + (SENSitivity Encoding)`. + The problem with Philip's «*EPI factor*» is that it is absolutely necessary + to calculate the effective echo spacing, because the reported SENSE + acceleration factor does not allow to calculate the effective train + length from the reconstructed matrix size along the PE direction + (neither from the acquisition matrix size if it is strangely found + stored within the metadata). + For :math:`B_0 = 3.0` [T], then + :math:`B_0 \gamma \Delta_\text{w/f} \approx 434.215`, as + in `early discussions held on the FSL listserv + `__. + + As per Dr. Rorden, Eq. :math:`\eqref{eq:philips-ees}` is equivalent to + the following formulation: + + .. math :: + + t_\text{ees} = \frac{f_\text{wfs}} + {3.4 \cdot F_\text{img} \cdot (f_\text{EPI} + 1)}, + + where :math:`F_\text{img}` is the «*Imaging Frequency*» in MHz, + as reported by the Philips console. + This second formulation seems to be preferred for the better accuracy + of the Imaging Frequency field over the Magnetic field strength. + + Once the effective echo spacing is obtained, the total readout time + can then be calculated with Eq. :math:`\eqref{eq:rotime-ees}`. + + >>> meta = {'WaterFatShift': 9.2227266, + ... 'EPIFactor': 35, + ... 'ImagingFrequency': 127.7325, + ... 'PhaseEncodingDirection': 'j-'} + >>> f"{get_trt(meta, in_file='epi.nii.gz'):0.5f}" + '0.05251' + + >>> meta = {'WaterFatShift': 9.2227266, + ... 'EPIFactor': 35, + ... 'MagneticFieldStrength': 3, + ... 'PhaseEncodingDirection': 'j-'} + >>> f"{get_trt(meta, in_file='epi.nii.gz'):0.5f}" + '0.05251' + + If enough metadata is not available, raise an error: + + >>> get_trt({'PhaseEncodingDirection': 'j-'}, + ... in_file='epi.nii.gz') # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ValueError: + + .. admonition:: Thanks + + With thanks to Dr. Rorden for his thourough + `assessment `__ + and `validation `__ on the matter, + and to Pravesh Parekh for `his wonderful review on NeuroStars + `__. + + .. admonition:: See Also + + Some useful links regarding the calculation of the readout time for Philips: + + * `Brain Voyager documentation + `__ + -- Please note that I (OE) *believe* the statement about the effective echo-spacing + on Philips **is wrong**, as the EPI factor should account for the in-plane + acceleration. + * `Disappeared documentation of the Spinoza Center + `__. + * This `guide for preprocessing of EPI data `__. + + """ + import nibabel as nb + + # Use case 1: TRT is defined + if "TotalReadoutTime" in in_meta: + return in_meta.get("TotalReadoutTime") + + # npe = N voxels PE direction + pe_index = "ijk".index(in_meta["PhaseEncodingDirection"][0]) + npe = nb.load(in_file).shape[pe_index] + + # Use case 2: EES is defined + ees = in_meta.get("EffectiveEchoSpacing") + if ees: + # Effective echo spacing means that acceleration factors have been accounted for. + return ees * (npe - 1) + + try: + echospacing = in_meta["EchoSpacing"] + acc_factor = in_meta["ParallelReductionFactorInPlane"] + except KeyError: + pass + else: + # etl = effective train length + etl = npe // acc_factor + return echospacing * (etl - 1) + + # Use case 3 (Philips scans) + try: + wfs = in_meta["WaterFatShift"] + epifactor = in_meta["EPIFactor"] + except KeyError: + pass + else: + wfs_hz = ( + (in_meta.get("ImagingFrequency", 0) * 3.39941) + or (in_meta.get("MagneticFieldStrength", 0) * 144.7383333) + or None + ) + if wfs_hz: + ees = wfs / (wfs_hz * (epifactor + 1)) + return ees * (npe - 1) + + raise ValueError("Unknown total-readout time specification") diff --git a/sdcflows/workflows/apply/correction.py b/sdcflows/workflows/apply/correction.py index d33841a5d5..5abaf8aee5 100644 --- a/sdcflows/workflows/apply/correction.py +++ b/sdcflows/workflows/apply/correction.py @@ -54,6 +54,7 @@ def init_unwarp_wf(omp_nthreads=1, debug=False, name="unwarp_wf"): ) rotime = pe.Node(GetReadoutTime(), name="rotime") + rotime.interface._always_run = debug resample = pe.Node(Coefficients2Warp(low_mem=debug), name="resample") unwarp = pe.Node( ApplyTransforms(dimension=3, interpolation="BSpline"), name="unwarp" From fcc6772f395d3940dca9b35a24682810aa6bdec1 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Sat, 28 Nov 2020 17:01:55 +0100 Subject: [PATCH 49/68] docs: remove outdated to-do note --- sdcflows/workflows/fit/fieldmap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdcflows/workflows/fit/fieldmap.py b/sdcflows/workflows/fit/fieldmap.py index 0c8cbd8d3e..c5d2a6a2a2 100644 --- a/sdcflows/workflows/fit/fieldmap.py +++ b/sdcflows/workflows/fit/fieldmap.py @@ -361,7 +361,7 @@ def init_phdiff_wf(omp_nthreads, debug=False, name="phdiff_wf"): Outputs ------- fieldmap : :obj:`os.PathLike` - The estimated fieldmap in Hz. # TODO: write metadata "Units" + The estimated fieldmap in Hz. """ from nipype.interfaces.fsl import PRELUDE From f3d10dd62888c1196e1830d3158e3fdfc74fac9c Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Sat, 28 Nov 2020 17:15:46 +0100 Subject: [PATCH 50/68] enh: cover one more line of code --- sdcflows/workflows/fit/pepolar.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sdcflows/workflows/fit/pepolar.py b/sdcflows/workflows/fit/pepolar.py index bfeb3b74b1..4a58be5f78 100644 --- a/sdcflows/workflows/fit/pepolar.py +++ b/sdcflows/workflows/fit/pepolar.py @@ -318,6 +318,13 @@ def _sorted_pe(inlist): >>> args '-noXdis -noZdis' + >>> paths, args = _sorted_pe([ + ... ("dir-AP_epi.nii.gz", {"PhaseEncodingDirection": "j-"}), + ... ("dir-LR_epi.nii.gz", {"PhaseEncodingDirection": "i"}), + ... ]) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ValueError: + """ out_ref = [inlist[0][0]] out_opp = [] From 15b8c6300eed900d42339ac2da8a130d1bb105f7 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Sat, 28 Nov 2020 19:03:20 +0100 Subject: [PATCH 51/68] enh: add minimal unit test and upgrade from Function to fully-fledged interface --- sdcflows/interfaces/tests/test_utils.py | 22 +++++++++++++- sdcflows/interfaces/utils.py | 39 +++++++++++++++++++++++++ sdcflows/workflows/fit/pepolar.py | 23 +++------------ 3 files changed, 64 insertions(+), 20 deletions(-) diff --git a/sdcflows/interfaces/tests/test_utils.py b/sdcflows/interfaces/tests/test_utils.py index 231827cba7..e5715e8490 100644 --- a/sdcflows/interfaces/tests/test_utils.py +++ b/sdcflows/interfaces/tests/test_utils.py @@ -1,8 +1,9 @@ """Test utilites.""" +import pytest import numpy as np import nibabel as nb -from ..utils import Flatten +from ..utils import Flatten, ConvertWarp def test_Flatten(tmpdir): @@ -31,3 +32,22 @@ def test_Flatten(tmpdir): assert out_meta[0] == {"a": 1} assert out_meta[1] == out_meta[2] == out_meta[3] == {"b": 2} assert out_meta[4] == out_meta[5] == {"c": 3} + + +@pytest.mark.parametrize("shape", [ + (10, 10, 10, 1, 3), + (10, 10, 10, 3) +]) +def test_ConvertWarp(tmpdir, shape): + """Exercise the interface.""" + tmpdir.chdir() + + nb.Nifti1Image(np.zeros(shape, dtype="uint8"), + np.eye(4), None).to_filename("3dQwarp.nii.gz") + + out = ConvertWarp(in_file="3dQwarp.nii.gz").run() + + nii = nb.load(out.outputs.out_file) + assert nii.header.get_data_dtype() == np.float32 + assert nii.header.get_intent() == ("vector", (), "") + assert nii.shape == (10, 10, 10, 1, 3) diff --git a/sdcflows/interfaces/utils.py b/sdcflows/interfaces/utils.py index f1d1127aca..7d0200b6ed 100644 --- a/sdcflows/interfaces/utils.py +++ b/sdcflows/interfaces/utils.py @@ -50,6 +50,27 @@ def _run_interface(self, runtime): return runtime +class _ConvertWarpInputSpec(BaseInterfaceInputSpec): + in_file = File(exists=True, mandatory=True, desc="output of 3dQwarp") + + +class _ConvertWarpOutputSpec(TraitedSpec): + out_file = File(exists=True, desc="the warp converted into ANTs") + + +class ConvertWarp(SimpleInterface): + """Convert a displacements field from ``3dQwarp`` to ANTS-compatible.""" + + input_spec = _ConvertWarpInputSpec + output_spec = _ConvertWarpOutputSpec + + def _run_interface(self, runtime): + self._results["out_file"] = _qwarp2ants( + self.inputs.in_file, newpath=runtime.cwd + ) + return runtime + + def _flatten(inlist, max_trs=50, out_dir=None): """ Split the input EPIs and generate a flattened list with corresponding metadata. @@ -83,3 +104,21 @@ def _flatten(inlist, max_trs=50, out_dir=None): output.append((str(out_name), meta)) return output + + +def _qwarp2ants(in_file, newpath=None): + """Ensure the data type and intent of a warp is acceptable by ITK-based tools.""" + import numpy as np + import nibabel as nb + from nipype.utils.filemanip import fname_presuffix + + nii = nb.load(in_file) + hdr = nii.header.copy() + hdr.set_data_dtype(" Date: Sun, 29 Nov 2020 03:18:43 -0800 Subject: [PATCH 52/68] maint: run ``tools/update_requirements.py`` --- min-requirements.txt | 1 - requirements.txt | 1 - 2 files changed, 2 deletions(-) diff --git a/min-requirements.txt b/min-requirements.txt index 415823bb8d..a10d3d8865 100644 --- a/min-requirements.txt +++ b/min-requirements.txt @@ -1,6 +1,5 @@ # Auto-generated by tools/update_requirements.py nibabel==3.0.1 -niflow-nipype1-workflows==0.0.1 nipype==1.5.1 niworkflows==1.3.0 numpy diff --git a/requirements.txt b/requirements.txt index 7a16a46837..8012b3d17f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ # Auto-generated by tools/update_requirements.py nibabel>=3.0.1 -niflow-nipype1-workflows~=0.0.1 nipype<2.0,>=1.5.1 niworkflows~=1.3.0 numpy From 7765ec920e1531c6f48d1677d7232eaf3688d148 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Sun, 29 Nov 2020 03:35:17 -0800 Subject: [PATCH 53/68] rel(1.4.0rc0): Update CHANGES --- CHANGES.rst | 118 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 90 insertions(+), 28 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index d7aac50ecc..4fda05ae9d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,49 +1,108 @@ +1.4.0 (TBD) +=========== +The *SDCFlows* 1.4.x series are release with a comprehensive overhaul of the software's API. +This overhaul has the vision of converting *SDCFlows* into some sort of subordinate pipeline +to other *d/fMRIPrep*, inline with *sMRIPrep*'s approach. +The idea is to consider fieldmaps a first-citizen input, for which derivatives are generated +at the output (on the same vein of, and effectively implementing `#26 +`__). +A bids's-eye view of this new release follows: + +* Two new base objects (:py:class:`~sdcflows.fieldmaps.FieldmapFile` and + :py:class:`~sdcflows.fieldmaps.FieldmapEstimation`) for the validation + and representation of fieldmap estimation strategies. + Validation of metadata and checking the sufficiency of imaging files + and necessary parameters is now done with these two objects. + :py:class:`~sdcflows.fieldmaps.FieldmapEstimation` also generates the + appropriate estimation workflow for the input data. +* Moved estimation workflows under the :py:mod:`sdcflows.workflows.fit` module. +* New outputs submodule :py:mod:`sdcflows.workflows.outputs` that writes out reportlets and + derivatives, following suit with higher-level *NiPreps* (*s/f/dMRIPrep*). + The two workflows are exercised in the CircleCI tests, and the artifacts are generated + this way. + Derivatives are populated with relevant pieces of metadata (for instance, they forward + the ``IntendedFor`` fields). +* A new :py:func:`~sdcflows.workflows.base.init_fmap_preproc_wf`, leveraging + :py:class:`~sdcflows.fieldmaps.FieldmapEstimation` objects. +* Separated out a new utilities module :py:mod:`sdcflows.utils` for the manipulation of + phase information and :abbr:`EPI (echo-planar imaging)` data. +* New :py:mod:`sdcflows.workflows.apply.registration` module, which aligns the reference map + of the fieldmap of choice (e.g., a magnitude image) to the reference EPI + (e.g., an SBRef, a *b=0* DWI, or a *fMRIPrep*'s *BOLDRef*) with ANTs. + The workflow resamples the fieldmap reference into the reference EPI's space for + reporting/visualization objectives. +* New :py:mod:`sdcflows.interfaces.bspline` set of utilities for the filtering and + extrapolation of fieldmaps with B-Splines. + Accordingly, all workflows have been updated to correctly handle (and better use) B-Spline + coefficients. +* A new PEPOLAR implementation based on TOPUP (see + :py:func:`sdcflows.workflows.fit.pepolar.init_topup_wf`). +* Pushed the code coverage with tests, along with a deep code cleanup. + +Some of the most prominent pull-requests conducive to this release are: + +* FIX: Convert SEI fieldmaps given in rad/s into Hz (#127) +* FIX: Limit ``3dQwarp`` to maximum 4 CPUs for stability reasons (#128) +* ENH: Finalizing the API overhaul (#132) +* ENH: Keep a registry of already-used identifiers (and auto-generate new) (#130) +* ENH: New objects for better representation of fieldmap estimation (#114) +* ENH: Show FieldmapReportlet oriented aligned with cardinal axes (#120) +* ENH: New estimation API (featuring a TOPUP implementation!) (#115) +* DOC: Enable NiPype's sphinx-extension to better render Interfaces (#131) +* MAINT: Minimal unit test and refactor of pepolar workflow node (#133) +* MAINT: Collect code coverage from tests on Circle (#122) +* MAINT: Test minimum dependencies with TravisCI (#96) +* MAINT: Add FLIRT config files to prepare for TOPUP integration (#116) + +A complete list of issues addressed by the release is found `in the GitHub repo +`__. + +1.3.x series +============ + 1.3.3 (September 4, 2020) -========================= +------------------------- Bug-fix release in 1.3.x series. Allows niworkflows 1.2.x or 1.3.x, as no API-breaking changes in 1.3.0 affect SDCflows. 1.3.2 (August 14, 2020) -======================= +----------------------- Bug-fix release in 1.3.x series. * FIX: Replace NaNs in fieldmap atlas with zeros (#104) * ENH: Return out_warp == "identity" if no SDC is applied (#108) 1.3.1 (May 22, 2020) -==================== +-------------------- Bug-fix release adapting to use newly refacored DerivativesDataSink * ENH: Use new ``DerivativesDataSink`` from NiWorkflows 1.2.0 (#102) 1.3.0 (May 4, 2020) -=================== +------------------- Minor release enforcing BIDS-Derivatives labels on ``dseg`` file. * FIX: WM mask selection from dseg before generating report (#100) +Pre-1.3.x releases +================== + 1.2.2 (April 16, 2020) -====================== +---------------------- Bug-fix release to fix phase-difference masking bug in the 1.2.x series. * FIX: Do not reorient magnitude images (#98) -1.0.6 (April 15, 2020) -====================== -Bug-fix release. - -* FIX: Do not reorient magnitude images (#98) - 1.2.1 (April 01, 2020) -====================== +---------------------- A patch release to make *SDCFlows* more amicable to downstream software. * MAINT: Migrate from versioneer to setuptools_scm (#97) * MAINT: Flexibilize dependencies -- nipype, niworkflows, pybids (#95) 1.2.0 (February 15, 2020) -========================= +------------------------- A minor version release that changes phasediff caclulations to improve robustness. This release is preparation for *fMRIPrep* 20.0.0. @@ -51,36 +110,42 @@ This release is preparation for *fMRIPrep* 20.0.0. * MNT: Fix package tests (#90) * MNT: Fix circle deployment (#91) -1.0.5 (February 14, 2020) -========================= -Bug-fix release. - -* FIX: Center phase maps around central mode, avoiding FoV-related outliers (#89) - 1.1.0 (February 3, 2020) -======================== +------------------------ This is a nominal release that enables downstream tools to depend on both SDCFlows and niworkflows 1.1.x. Bug fixes needed for the 1.5.x series of fMRIPrep will be accepted into the 1.0.x series of SDCFlows. +1.0.6 (April 15, 2020) +---------------------- +Bug-fix release. + +* FIX: Do not reorient magnitude images (#98) + +1.0.5 (February 14, 2020) +------------------------- +Bug-fix release. + +* FIX: Center phase maps around central mode, avoiding FoV-related outliers (#89) + 1.0.4 (January 27, 2020) -========================= +------------------------ Bug-fix release. * FIX: Connect SyN outputs whenever SyN is run (#82) * MNT: Skim Docker image, optimize CircleCI workflow, and reuse cached results (#80) 1.0.3 (December 18, 2019) -========================= +------------------------- A hotfix release preventing downstream dependency collisions on fMRIPrep. * PIN: niworkflows-1.0.3 `449c2c2 `__ 1.0.2 (December 18, 2019) -========================= +------------------------- A hotfix release. * FIX: NiWorkflows' ``IntraModalMerge`` choked with images of shape (x, y, z, 1) (#79, `2e6faa0 @@ -91,14 +156,14 @@ A hotfix release. `__) 1.0.1 (December 04, 2019) -========================= +------------------------- A bugfix release. * FIX: Flexibly and cheaply select initial PEPOLAR volumes (#75) * ENH: Phase1/2 - subtract phases before unwrapping (#70) 1.0.0 (November 25, 2019) -========================= +------------------------- A first stable release after detaching these workflows off from *fMRIPrep*. With thanks to Matthew Cieslak and Azeez Adebimpe. @@ -126,9 +191,6 @@ With thanks to Matthew Cieslak and Azeez Adebimpe. * MAINT: Rename boldrefs to distortedrefs (#41) * MAINT: Use niflow-nipype1-workflows for old nipype.workflows imports (#39) -Pre-1.0.0 releases -================== - 0.1.4 (November 22, 2019) ------------------------- A maintenance release to pin niworkflows to version 1.0.0rc1. From e2ce19f240d60bea125cac53bcf228ea9dfa55d0 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Mon, 30 Nov 2020 00:59:42 +0100 Subject: [PATCH 54/68] MAINT: Migrate test datasets to a datalad+osf infrastructure Allows us to share test data easily across projects, through the new nipreps-data org. Of particular interest is ds001771 (originally used in *dMRIPrep* for testing), because it has some "fieldmap" images that were actually generated with TOPUP, so we will be able to compare. --- .circleci/config.yml | 35 +++++++++++-------- .../workflows/apply/tests/test_correct.py | 12 +++---- .../apply/tests/test_registration.py | 12 +++---- sdcflows/workflows/base.py | 4 +-- sdcflows/workflows/fit/tests/test_pepolar.py | 4 +-- sdcflows/workflows/fit/tests/test_phdiff.py | 2 +- 6 files changed, 38 insertions(+), 31 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fe28b8e21c..33a6cbeb72 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,9 +48,9 @@ jobs: - env-v2- - restore_cache: keys: - - data-v2-{{ .Branch }}- - - data-v2-master- - - data-v2- + - data-v3-{{ .Branch }}- + - data-v3-master- + - data-v3- - run: name: Setup git-annex @@ -67,7 +67,7 @@ jobs: name: Setup DataLad & TemplateFlow command: | python -m pip install --no-cache-dir -U pip - python -m pip install --no-cache-dir -U datalad + python -m pip install --no-cache-dir -U datalad datalad-osf python -m pip install --no-cache-dir -U templateflow python -c "from templateflow import api as tfapi; \ tfapi.get('MNI152NLin2009cAsym', resolution=2, desc='brain', suffix='mask'); \ @@ -89,16 +89,23 @@ jobs: datalad install https://github.com/OpenNeuroDatasets/ds001600.git datalad update --merge -d ds001600/ datalad get -r -d ds001600/ + - run: - name: Get testdata + name: Install HCP/sub-101006 command: | - if [[ ! -d /tmp/data/testdata ]]; then - wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q \ - -O testdata.zip "https://files.osf.io/v1/resources/9sy2a/providers/osfstorage/5d44b940bcd6d900198ed6be/?zip=" - unzip testdata.zip -d /tmp/data/testdata - fi + datalad install -r https://github.com/nipreps-data/HCP101006.git + datalad update -r --merge -d HCP101006/ + datalad get -r -d HCP101006 + + - run: + name: Install ds001771 + command: | + datalad install -r https://github.com/nipreps-data/ds001771.git + datalad update --merge -d ds001771/ + datalad get -r -d ds001771/ ds001771/sub-36/fmap/* + - save_cache: - key: data-v2-{{ .Branch }}-{{ .BuildNum }} + key: data-v3-{{ .Branch }}-{{ .BuildNum }} paths: - /tmp/data - /tmp/templateflow @@ -241,9 +248,9 @@ jobs: - freesurfer-v1- - restore_cache: keys: - - data-v2-{{ .Branch }}- - - data-v2-master- - - data-v2- + - data-v3-{{ .Branch }}- + - data-v3-master- + - data-v3- - restore_cache: keys: - workdir-v2-{{ .Branch }}- diff --git a/sdcflows/workflows/apply/tests/test_correct.py b/sdcflows/workflows/apply/tests/test_correct.py index 0365a292a7..3774eeefb5 100644 --- a/sdcflows/workflows/apply/tests/test_correct.py +++ b/sdcflows/workflows/apply/tests/test_correct.py @@ -15,18 +15,18 @@ def test_unwarp_wf(tmpdir, datadir, workdir, outdir): """Test the unwarping workflow.""" distorted = ( datadir - / "testdata" - / "sub-HCP101006" + / "HCP101006" + / "sub-101006" / "func" - / "sub-HCP101006_task-rest_dir-LR_sbref.nii.gz" + / "sub-101006_task-rest_dir-LR_sbref.nii.gz" ) magnitude = ( datadir - / "testdata" - / "sub-HCP101006" + / "HCP101006" + / "sub-101006" / "fmap" - / "sub-HCP101006_magnitude1.nii.gz" + / "sub-101006_magnitude1.nii.gz" ) fmap_ref_wf = init_magnitude_wf(2, name="fmap_ref_wf") fmap_ref_wf.inputs.inputnode.magnitude = magnitude diff --git a/sdcflows/workflows/apply/tests/test_registration.py b/sdcflows/workflows/apply/tests/test_registration.py index ffdc49acab..bca74d30f3 100644 --- a/sdcflows/workflows/apply/tests/test_registration.py +++ b/sdcflows/workflows/apply/tests/test_registration.py @@ -15,18 +15,18 @@ def test_registration_wf(tmpdir, datadir, workdir, outdir): epi_ref_wf = init_magnitude_wf(2, name="epi_ref_wf") epi_ref_wf.inputs.inputnode.magnitude = ( datadir - / "testdata" - / "sub-HCP101006" + / "HCP101006" + / "sub-101006" / "func" - / "sub-HCP101006_task-rest_dir-LR_sbref.nii.gz" + / "sub-101006_task-rest_dir-LR_sbref.nii.gz" ) magnitude = ( datadir - / "testdata" - / "sub-HCP101006" + / "HCP101006" + / "sub-101006" / "fmap" - / "sub-HCP101006_magnitude1.nii.gz" + / "sub-101006_magnitude1.nii.gz" ) fmap_ref_wf = init_magnitude_wf(2, name="fmap_ref_wf") fmap_ref_wf.inputs.inputnode.magnitude = magnitude diff --git a/sdcflows/workflows/base.py b/sdcflows/workflows/base.py index fe92e1c2e7..c7efeab21e 100644 --- a/sdcflows/workflows/base.py +++ b/sdcflows/workflows/base.py @@ -48,10 +48,10 @@ def init_fmap_preproc_wf( FieldmapEstimation(sources=<2 files>, method=, bids_id='...')] >>> init_fmap_preproc_wf( - ... layout=layouts['testdata'], + ... layout=layouts['HCP101006'], ... omp_nthreads=1, ... output_dir="/tmp", - ... subject="HCP101006", + ... subject="101006", ... ) # doctest: +ELLIPSIS [FieldmapEstimation(sources=<2 files>, method=, bids_id='...'), FieldmapEstimation(sources=<2 files>, method=, bids_id='...')] diff --git a/sdcflows/workflows/fit/tests/test_pepolar.py b/sdcflows/workflows/fit/tests/test_pepolar.py index a0447f4860..bd9967c278 100644 --- a/sdcflows/workflows/fit/tests/test_pepolar.py +++ b/sdcflows/workflows/fit/tests/test_pepolar.py @@ -19,8 +19,8 @@ # "ds001600/sub-1/fmap/sub-1_dir-PA_epi.nii.gz", # ), ( - "testdata/sub-HCP101006/fmap/sub-HCP101006_dir-LR_epi.nii.gz", - "testdata/sub-HCP101006/fmap/sub-HCP101006_dir-RL_epi.nii.gz", + "HCP101006/sub-101006/fmap/sub-101006_dir-LR_epi.nii.gz", + "HCP101006/sub-101006/fmap/sub-101006_dir-RL_epi.nii.gz", ), ], ) diff --git a/sdcflows/workflows/fit/tests/test_phdiff.py b/sdcflows/workflows/fit/tests/test_phdiff.py index 8a15738f6d..5d4ce9e90a 100644 --- a/sdcflows/workflows/fit/tests/test_phdiff.py +++ b/sdcflows/workflows/fit/tests/test_phdiff.py @@ -18,7 +18,7 @@ "ds001600/sub-1/fmap/sub-1_acq-v2_phase1.nii.gz", "ds001600/sub-1/fmap/sub-1_acq-v2_phase2.nii.gz", ), - ("testdata/sub-HCP101006/fmap/sub-HCP101006_phasediff.nii.gz",), + ("HCP101006/sub-101006/fmap/sub-101006_phasediff.nii.gz",), ], ) def test_phdiff(tmpdir, datadir, workdir, outdir, fmap_path): From b282a4b8bfccb6e4513fa5427348b07916c35318 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Mon, 30 Nov 2020 10:19:23 +0100 Subject: [PATCH 55/68] MAINT: Migrate DS001600 to nipreps-data too --- .circleci/config.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 33a6cbeb72..20396f9a9d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,9 +48,9 @@ jobs: - env-v2- - restore_cache: keys: - - data-v3-{{ .Branch }}- - - data-v3-master- - - data-v3- + - data-v4-{{ .Branch }}- + - data-v4-master- + - data-v4- - run: name: Setup git-annex @@ -86,9 +86,9 @@ jobs: - run: name: Install ds001600 command: | - datalad install https://github.com/OpenNeuroDatasets/ds001600.git - datalad update --merge -d ds001600/ - datalad get -r -d ds001600/ + datalad install -r https://github.com/nipreps-data/ds001600.git + datalad update -r --merge -d ds001600/ + datalad get -r -d ds001600/ ds001600/sub-1/ - run: name: Install HCP/sub-101006 @@ -105,7 +105,7 @@ jobs: datalad get -r -d ds001771/ ds001771/sub-36/fmap/* - save_cache: - key: data-v3-{{ .Branch }}-{{ .BuildNum }} + key: data-v4-{{ .Branch }}-{{ .BuildNum }} paths: - /tmp/data - /tmp/templateflow @@ -248,9 +248,9 @@ jobs: - freesurfer-v1- - restore_cache: keys: - - data-v3-{{ .Branch }}- - - data-v3-master- - - data-v3- + - data-v4-{{ .Branch }}- + - data-v4-master- + - data-v4- - restore_cache: keys: - workdir-v2-{{ .Branch }}- From 75c37558149b5e854da27a227048914de8a7ec60 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Mon, 30 Nov 2020 17:39:10 +0100 Subject: [PATCH 56/68] fix(gha): location of test data --- sdcflows/conftest.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sdcflows/conftest.py b/sdcflows/conftest.py index bffaaf275b..e4718ed5c4 100644 --- a/sdcflows/conftest.py +++ b/sdcflows/conftest.py @@ -17,10 +17,12 @@ def pytest_report_header(config): - msg = "Datasets found: %s" % ', '.join([v.root for v in layouts.values()]) - if test_output_dir is not None: - msg += '\nOutput folder: %s' % Path(test_output_dir).resolve() - return msg + return f"""\ +TEST_DATA_HOME={test_data_env} +-> Available datasets: {', '.join(layouts.keys())}. +TEST_OUTPUT_DIR={test_output_dir or ' (output files will be discarded)'}. +TEST_WORK_DIR={test_workdir or ' (intermediate files will be discarded)'}. +""" @pytest.fixture(autouse=True) From fefafbddb77b2beff1d03bcadc9fb38a3a0f1724 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Mon, 30 Nov 2020 20:33:06 +0100 Subject: [PATCH 57/68] ENH: Use ds001771 as another test dataset This dataset contains two `_fieldmap` files that are not raw data, but outputs from TOPUP instead. This is interesting to compare to the output with the PEPOLAR data included in the dataset. --- sdcflows/workflows/base.py | 10 ++++++++++ sdcflows/workflows/fit/fieldmap.py | 18 +++++++++++++++++- sdcflows/workflows/fit/tests/test_pepolar.py | 12 ++++++++---- sdcflows/workflows/fit/tests/test_phdiff.py | 13 ++++++++++--- 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/sdcflows/workflows/base.py b/sdcflows/workflows/base.py index c7efeab21e..167319d604 100644 --- a/sdcflows/workflows/base.py +++ b/sdcflows/workflows/base.py @@ -36,6 +36,16 @@ def init_fmap_preproc_wf( Examples -------- + >>> init_fmap_preproc_wf( + ... layout=layouts['ds001771'], + ... omp_nthreads=1, + ... output_dir="/tmp", + ... subject="36", + ... ) # doctest: +ELLIPSIS + [FieldmapEstimation(sources=<2 files>, method=, bids_id='...'), + FieldmapEstimation(sources=<2 files>, method=, bids_id='...'), + FieldmapEstimation(sources=<4 files>, method=, bids_id='...')] + >>> init_fmap_preproc_wf( ... layout=layouts['ds001600'], ... omp_nthreads=1, diff --git a/sdcflows/workflows/fit/fieldmap.py b/sdcflows/workflows/fit/fieldmap.py index c5d2a6a2a2..928afd5a6c 100644 --- a/sdcflows/workflows/fit/fieldmap.py +++ b/sdcflows/workflows/fit/fieldmap.py @@ -228,7 +228,7 @@ def init_fmap_wf(omp_nthreads=1, debug=False, mode="phasediff", name="fmap_wf"): # fmt: off workflow.connect([ (inputnode, units, [(("fieldmap", _get_units), "units")]), - (inputnode, fmapmrg, [("fieldmap", "in_files")]), + (inputnode, fmapmrg, [(("fieldmap", _get_file), "in_files")]), (fmapmrg, units, [("out_avg", "in_file")]), (units, bs_filter, [("out_file", "in_data")]), ]) @@ -421,6 +421,22 @@ def _split(phase): return workflow +def _get_file(intuple): + """ + Extract the filename from the inputnode. + + >>> _get_file([("fmap.nii.gz", {"Units": "rad/s"})]) + 'fmap.nii.gz' + + >>> _get_file(("fmap.nii.gz", {"Units": "rad/s"})) + 'fmap.nii.gz' + + """ + if isinstance(intuple, list): + intuple = intuple[0] + return intuple[0] + + def _get_units(intuple): """ Extract Units from metadata. diff --git a/sdcflows/workflows/fit/tests/test_pepolar.py b/sdcflows/workflows/fit/tests/test_pepolar.py index bd9967c278..863b57aa26 100644 --- a/sdcflows/workflows/fit/tests/test_pepolar.py +++ b/sdcflows/workflows/fit/tests/test_pepolar.py @@ -14,10 +14,14 @@ @pytest.mark.parametrize( "epi_path", [ - # ( - # "ds001600/sub-1/fmap/sub-1_dir-AP_epi.nii.gz", - # "ds001600/sub-1/fmap/sub-1_dir-PA_epi.nii.gz", - # ), + ( + "ds001771/sub-36/fmap/sub-36_acq-topup1_dir-01_epi.nii.gz", + "ds001771/sub-36/fmap/sub-36_acq-topup1_dir-02_epi.nii.gz", + ), + ( + "ds001771/sub-36/fmap/sub-36_acq-topup2_dir-01_epi.nii.gz", + "ds001771/sub-36/fmap/sub-36_acq-topup2_dir-02_epi.nii.gz", + ), ( "HCP101006/sub-101006/fmap/sub-101006_dir-LR_epi.nii.gz", "HCP101006/sub-101006/fmap/sub-101006_dir-RL_epi.nii.gz", diff --git a/sdcflows/workflows/fit/tests/test_phdiff.py b/sdcflows/workflows/fit/tests/test_phdiff.py index 5d4ce9e90a..eabe85f445 100644 --- a/sdcflows/workflows/fit/tests/test_phdiff.py +++ b/sdcflows/workflows/fit/tests/test_phdiff.py @@ -18,6 +18,7 @@ "ds001600/sub-1/fmap/sub-1_acq-v2_phase1.nii.gz", "ds001600/sub-1/fmap/sub-1_acq-v2_phase2.nii.gz", ), + ("ds001771/sub-36/fmap/sub-36_acq-topup1_fieldmap.nii.gz",), ("HCP101006/sub-101006/fmap/sub-101006_phasediff.nii.gz",), ], ) @@ -34,10 +35,16 @@ def test_phdiff(tmpdir, datadir, workdir, outdir, fmap_path): wf = Workflow( name=f"phdiff_{fmap_path[0].name.replace('.nii.gz', '').replace('-', '_')}" ) - phdiff_wf = init_fmap_wf(omp_nthreads=2, debug=True) + mode = "mapped" if "fieldmap" in fmap_path[0].name else "phasediff" + phdiff_wf = init_fmap_wf( + omp_nthreads=2, + debug=True, + mode=mode, + ) phdiff_wf.inputs.inputnode.fieldmap = fieldmaps phdiff_wf.inputs.inputnode.magnitude = [ - f.replace("diff", "1").replace("phase", "magnitude") for f, _ in fieldmaps + f.replace("diff", "1").replace("phase", "magnitude").replace("fieldmap", "magnitude") + for f, _ in fieldmaps ] if outdir: @@ -54,7 +61,7 @@ def test_phdiff(tmpdir, datadir, workdir, outdir, fmap_path): fmap_reports_wf = init_fmap_reports_wf( output_dir=str(outdir), - fmap_type="phasediff" if len(fieldmaps) == 1 else "phases", + fmap_type=mode if len(fieldmaps) == 1 else "phases", ) fmap_reports_wf.inputs.inputnode.source_files = [f for f, _ in fieldmaps] From faa91a0c822d8c048b808fae1dc651be2d826418 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Tue, 1 Dec 2020 10:54:35 +0100 Subject: [PATCH 58/68] FIX: Consider ``acq-`` and ``ce-`` as grouping entities auto-determining PEPOLAR fieldmaps Improvement to the heuristics that determine available fieldmaps. --- sdcflows/workflows/base.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/sdcflows/workflows/base.py b/sdcflows/workflows/base.py index 167319d604..a780dd68ca 100644 --- a/sdcflows/workflows/base.py +++ b/sdcflows/workflows/base.py @@ -1,6 +1,7 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: """Estimate fieldmaps for :abbr:`SDC (susceptibility distortion correction)`.""" +from itertools import product from nipype import logging LOGGER = logging.getLogger('nipype.workflow') @@ -44,7 +45,8 @@ def init_fmap_preproc_wf( ... ) # doctest: +ELLIPSIS [FieldmapEstimation(sources=<2 files>, method=, bids_id='...'), FieldmapEstimation(sources=<2 files>, method=, bids_id='...'), - FieldmapEstimation(sources=<4 files>, method=, bids_id='...')] + FieldmapEstimation(sources=<2 files>, method=, bids_id='...'), + FieldmapEstimation(sources=<2 files>, method=, bids_id='...')] >>> init_fmap_preproc_wf( ... layout=layouts['ds001600'], @@ -87,18 +89,23 @@ def init_fmap_preproc_wf( estimators.append(e) # A bunch of heuristics to select EPI fieldmaps - sessions = layout.get_sessions() or [None] - for session in sessions: - dirs = layout.get_directions( - suffix="epi", - session=session, - **base_entities, - ) + sessions = layout.get_sessions() or (None, ) + acqs = tuple(layout.get_acquisitions(suffix="epi") + [None]) + contrasts = tuple(layout.get_ceagents(suffix="epi") + [None]) + + for ses, acq, ce in product(sessions, acqs, contrasts): + entities = base_entities.copy() + entities.update({ + "suffix": "epi", + "session": ses, + "acquisition": acq, + "ceagent": ce, + }) + dirs = layout.get_directions(**entities) if len(dirs) > 1: e = FieldmapEstimation([ FieldmapFile(fmap.path, metadata=fmap.get_metadata()) - for fmap in layout.get(suffix="epi", session=session, - direction=dirs, **base_entities) + for fmap in layout.get(direction=dirs, **entities) ]) estimators.append(e) From 5d51b7871a481d703ced31d64bab5178f8e2fb97 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Tue, 1 Dec 2020 20:30:59 +0100 Subject: [PATCH 59/68] ENH: Disallow ``FieldmapEstimation`` objects with same sources This PR implements a bi-directional hash-map (dictionary) instead of a set to track fieldmap estimator IDs. Instead, the registry can be queried by passing in the fieldmap estimator ID, OR a tuple of path-sources (in which case it returns the ID). This way, a ``ValueError`` is raised when one tries to establish a fieldmap estimation using the same set of images. Resolves: #129. --- sdcflows/fieldmaps.py | 40 ++++----- sdcflows/tests/test_fieldmaps.py | 146 +++++++++++++++++++----------- sdcflows/utils/bimap.py | 148 +++++++++++++++++++++++++++++++ 3 files changed, 260 insertions(+), 74 deletions(-) create mode 100644 sdcflows/utils/bimap.py diff --git a/sdcflows/fieldmaps.py b/sdcflows/fieldmaps.py index 6711f2f111..e4bf5ac936 100644 --- a/sdcflows/fieldmaps.py +++ b/sdcflows/fieldmaps.py @@ -7,9 +7,10 @@ from bids.layout import BIDSFile, parse_file_entities from bids.utils import listify from niworkflows.utils.bids import relative_to_root +from .utils.bimap import bidict -_unique_ids = set() +_estimators = bidict() class MetadataError(ValueError): @@ -63,23 +64,9 @@ def _type_setter(obj, attribute, value): def _id_setter(obj, attribute, value): - """Ensure uniqueness of B0FieldIdentifier metadata.""" - if obj.bids_id: - if obj.bids_id != value: - raise ValueError("Unique identifier is already set") - _unique_ids.add(value) + if obj.bids_id and obj.bids_id == value: return value - - if value is True: - value = f"auto_{len([el for el in _unique_ids if el.startswith('auto_')])}" - elif not value: - raise ValueError("Invalid unique identifier") - - if value in _unique_ids: - raise ValueError("Unique identifier has been previously registered.") - - _unique_ids.add(value) - return value + raise ValueError("Cannot edit the bids_id") @attr.s(slots=True) @@ -380,7 +367,9 @@ def __attrs_post_init__(self): if self.method == EstimatorType.UNKNOWN: raise ValueError("Insufficient sources to estimate a fieldmap.") + # Register this estimation method if not self.bids_id: + # If not manually set, try to get it from BIDS metadata bids_ids = set( [ f.metadata.get("B0FieldIdentifier") @@ -392,14 +381,17 @@ def __attrs_post_init__(self): raise ValueError( f"Multiple ``B0FieldIdentifier`` set: <{', '.join(bids_ids)}>" ) - elif not bids_ids: - self.bids_id = True + elif bids_ids: + object.__setattr__(self, "bids_id", bids_ids.pop()) else: - self.bids_id = bids_ids.pop() - elif self.bids_id in _unique_ids: - raise ValueError("Unique identifier has been previously registered.") - else: - _unique_ids.add(self.bids_id) + object.__setattr__(self, "bids_id", _estimators.add(self.paths())) + return + + _estimators[self.bids_id] = self.paths() + + def paths(self): + """Return a tuple of paths (sorted).""" + return tuple(sorted(str(f.path) for f in self.sources)) def get_workflow(self, **kwargs): """Build the estimation workflow corresponding to this instance.""" diff --git a/sdcflows/tests/test_fieldmaps.py b/sdcflows/tests/test_fieldmaps.py index 79c8e38f4b..9ce58a141e 100644 --- a/sdcflows/tests/test_fieldmaps.py +++ b/sdcflows/tests/test_fieldmaps.py @@ -1,20 +1,22 @@ """test_fieldmaps.""" import pytest -from ..fieldmaps import FieldmapFile, FieldmapEstimation, EstimatorType +from ..utils.bimap import bidict +from .. import fieldmaps as fm def test_FieldmapFile(testdata_dir): """Test one existing file.""" - FieldmapFile(testdata_dir / "sub-01" / "anat" / "sub-01_T1w.nii.gz") + fm.FieldmapFile(testdata_dir / "sub-01" / "anat" / "sub-01_T1w.nii.gz") @pytest.mark.parametrize( - "inputfiles,method,nsources", + "inputfiles,method,nsources,raises", [ ( ("fmap/sub-01_fieldmap.nii.gz", "fmap/sub-01_magnitude.nii.gz"), - EstimatorType.MAPPED, + fm.EstimatorType.MAPPED, 2, + False, ), ( ( @@ -23,39 +25,52 @@ def test_FieldmapFile(testdata_dir): "fmap/sub-01_magnitude1.nii.gz", "fmap/sub-01_magnitude2.nii.gz", ), - EstimatorType.PHASEDIFF, + fm.EstimatorType.PHASEDIFF, 4, + False, ), ( ("fmap/sub-01_phase1.nii.gz", "fmap/sub-01_phase2.nii.gz"), - EstimatorType.PHASEDIFF, + fm.EstimatorType.PHASEDIFF, 4, + True, ), - (("fmap/sub-01_phase2.nii.gz",), EstimatorType.PHASEDIFF, 4), - (("fmap/sub-01_phase1.nii.gz",), EstimatorType.PHASEDIFF, 4), + (("fmap/sub-01_phase2.nii.gz",), fm.EstimatorType.PHASEDIFF, 4, True), + (("fmap/sub-01_phase1.nii.gz",), fm.EstimatorType.PHASEDIFF, 4, True), ( ("fmap/sub-01_dir-LR_epi.nii.gz", "fmap/sub-01_dir-RL_epi.nii.gz"), - EstimatorType.PEPOLAR, + fm.EstimatorType.PEPOLAR, 2, + False, ), ( ("fmap/sub-01_dir-LR_epi.nii.gz", "dwi/sub-01_dir-RL_sbref.nii.gz"), - EstimatorType.PEPOLAR, + fm.EstimatorType.PEPOLAR, 2, + False, ), ( ("anat/sub-01_T1w.nii.gz", "dwi/sub-01_dir-RL_sbref.nii.gz"), - EstimatorType.ANAT, + fm.EstimatorType.ANAT, 2, + False, ), ], ) -def test_FieldmapEstimation(testdata_dir, inputfiles, method, nsources): +def test_FieldmapEstimation( + monkeypatch, testdata_dir, inputfiles, method, nsources, raises +): """Test errors.""" sub_dir = testdata_dir / "sub-01" sources = [sub_dir / f for f in inputfiles] - fe = FieldmapEstimation(sources) + + if raises is True: + with pytest.raises(ValueError): + fm.FieldmapEstimation(sources) + monkeypatch.setattr(fm, "_estimators", bidict()) + + fe = fm.FieldmapEstimation(sources) assert fe.method == method assert len(fe.sources) == nsources assert fe.bids_id is not None and fe.bids_id.startswith("auto_") @@ -68,14 +83,12 @@ def test_FieldmapEstimation(testdata_dir, inputfiles, method, nsources): fe.bids_id = fe.bids_id # Ensure duplicate B0FieldIdentifier are not accepted - with pytest.raises(ValueError): - FieldmapEstimation(sources, bids_id=fe.bids_id) + with pytest.raises(KeyError): + fm.FieldmapEstimation(sources, bids_id=fe.bids_id) - # B0FieldIdentifier can be generated manually - # Creating two FieldmapEstimation objects from the same sources SHOULD fail - # or be better handled in the future (see #129). - fe2 = FieldmapEstimation(sources, bids_id=f"no{fe.bids_id}") - assert fe2.bids_id and fe2.bids_id.startswith("noauto_") + # Ensure we can't instantiate one more estimation with same sources + with pytest.raises(ValueError): + fm.FieldmapEstimation(sources, bids_id=f"my{fe.bids_id}") # Exercise workflow creation wf = fe.get_workflow() @@ -91,44 +104,77 @@ def test_FieldmapEstimation(testdata_dir, inputfiles, method, nsources): (("anat/sub-01_T1w.nii.gz", "fmap/sub-01_phase2.nii.gz"), TypeError), ], ) -def test_FieldmapEstimationError(testdata_dir, inputfiles, errortype): +def test_FieldmapEstimationError(monkeypatch, testdata_dir, inputfiles, errortype): """Test errors.""" sub_dir = testdata_dir / "sub-01" + monkeypatch.setattr(fm, "_estimators", bidict()) + with pytest.raises(errortype): - FieldmapEstimation([sub_dir / f for f in inputfiles]) + fm.FieldmapEstimation([sub_dir / f for f in inputfiles]) -def test_FieldmapEstimationIdentifier(testdata_dir): +def test_FieldmapEstimationIdentifier(monkeypatch, testdata_dir): """Check some use cases of B0FieldIdentifier.""" + + monkeypatch.setattr(fm, "_estimators", bidict()) + with pytest.raises(ValueError): - FieldmapEstimation([ - FieldmapFile(testdata_dir / "sub-01" / "fmap/sub-01_fieldmap.nii.gz", - metadata={"Units": "Hz", "B0FieldIdentifier": "fmap_0"}), - FieldmapFile(testdata_dir / "sub-01" / "fmap/sub-01_magnitude.nii.gz", - metadata={"Units": "Hz", "B0FieldIdentifier": "fmap_1"}) - ]) # Inconsistent identifiers - - fe = FieldmapEstimation([ - FieldmapFile(testdata_dir / "sub-01" / "fmap/sub-01_fieldmap.nii.gz", - metadata={"Units": "Hz", "B0FieldIdentifier": "fmap_0"}), - FieldmapFile(testdata_dir / "sub-01" / "fmap/sub-01_magnitude.nii.gz", - metadata={"Units": "Hz", "B0FieldIdentifier": "fmap_0"}) - ]) + fm.FieldmapEstimation( + [ + fm.FieldmapFile( + testdata_dir / "sub-01" / "fmap/sub-01_fieldmap.nii.gz", + metadata={"Units": "Hz", "B0FieldIdentifier": "fmap_0"}, + ), + fm.FieldmapFile( + testdata_dir / "sub-01" / "fmap/sub-01_magnitude.nii.gz", + metadata={"Units": "Hz", "B0FieldIdentifier": "fmap_1"}, + ), + ] + ) # Inconsistent identifiers + + fe = fm.FieldmapEstimation( + [ + fm.FieldmapFile( + testdata_dir / "sub-01" / "fmap/sub-01_fieldmap.nii.gz", + metadata={"Units": "Hz", "B0FieldIdentifier": "fmap_0"}, + ), + fm.FieldmapFile( + testdata_dir / "sub-01" / "fmap/sub-01_magnitude.nii.gz", + metadata={"Units": "Hz", "B0FieldIdentifier": "fmap_0"}, + ), + ] + ) assert fe.bids_id == "fmap_0" - with pytest.raises(ValueError): - FieldmapEstimation([ - FieldmapFile(testdata_dir / "sub-01" / "fmap/sub-01_fieldmap.nii.gz", - metadata={"Units": "Hz", "B0FieldIdentifier": "fmap_0"}), - FieldmapFile(testdata_dir / "sub-01" / "fmap/sub-01_magnitude.nii.gz", - metadata={"Units": "Hz", "B0FieldIdentifier": "fmap_0"}) - ]) # Consistent, but already exists - - fe = FieldmapEstimation([ - FieldmapFile(testdata_dir / "sub-01" / "fmap/sub-01_fieldmap.nii.gz", - metadata={"Units": "Hz", "B0FieldIdentifier": "fmap_1"}), - FieldmapFile(testdata_dir / "sub-01" / "fmap/sub-01_magnitude.nii.gz", - metadata={"Units": "Hz", "B0FieldIdentifier": "fmap_1"}) - ]) + with pytest.raises(KeyError): + fm.FieldmapEstimation( + [ + fm.FieldmapFile( + testdata_dir / "sub-01" / "fmap/sub-01_fieldmap.nii.gz", + metadata={"Units": "Hz", "B0FieldIdentifier": "fmap_0"}, + ), + fm.FieldmapFile( + testdata_dir / "sub-01" / "fmap/sub-01_magnitude.nii.gz", + metadata={"Units": "Hz", "B0FieldIdentifier": "fmap_0"}, + ), + ] + ) # Consistent, but already exists + + monkeypatch.setattr(fm, "_estimators", bidict()) + + fe = fm.FieldmapEstimation( + [ + fm.FieldmapFile( + testdata_dir / "sub-01" / "fmap/sub-01_fieldmap.nii.gz", + metadata={"Units": "Hz", "B0FieldIdentifier": "fmap_1"}, + ), + fm.FieldmapFile( + testdata_dir / "sub-01" / "fmap/sub-01_magnitude.nii.gz", + metadata={"Units": "Hz", "B0FieldIdentifier": "fmap_1"}, + ), + ] + ) assert fe.bids_id == "fmap_1" + + monkeypatch.setattr(fm, "_estimators", bidict()) diff --git a/sdcflows/utils/bimap.py b/sdcflows/utils/bimap.py new file mode 100644 index 0000000000..37e373ab47 --- /dev/null +++ b/sdcflows/utils/bimap.py @@ -0,0 +1,148 @@ +"""A bidirectional hashmap.""" +import re + +_autokey_pat = re.compile(r"^auto_(\d+)$") + + +class bidict(dict): + """ + A bidirectional hashmap. + + >>> d = bidict({"a": 1, "b": 2}, c=3) + >>> d["a"] + 1 + + >>> d[1] + 'a' + + >>> 2 in d + True + + >>> "b" in d + True + + >>> d["d"] = 4 + >>> del d["d"] + >>> d["d"] = 4 + >>> del d[4] + >>> d + {'a': 1, 'b': 2, 'c': 3} + + >>> d["d"] = "d" # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + TypeError: 'd' <> 'd' is a self-mapping + + >>> d["d"] # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + KeyError: 'd' + + >>> d["b"] = None # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + KeyError: 'b' is already in mapping + + >>> d[1] = None # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + KeyError: '1' is already a value in mapping + + >>> d["d"] = 1 # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ValueError: '1' is already in mapping + + >>> d["d"] = "a" # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ValueError: 'a' is already a key in mapping + + >>> d["unhashable val"] = [] # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + TypeError: value '[]' of unhashable type: 'list' + + >>> d.add("a new value") + 'auto_00000' + + >>> d["auto_00000"] + 'a new value' + + >>> d["auto_00001"] = "another value" + >>> d.add("a new value") # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ValueError: 'a new value' is already in mapping + + >>> d.add("third value") + 'auto_00002' + + >>> d == bidict(reversed(list(d.items()))) + True + + """ + + _inverse = None + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._inverse = {v: k for k, v in self.items()} + if len(self) != len(self._inverse): + raise TypeError( + "Bidirectional dictionary cannot contain repeated keys or values." + ) + + def __setitem__(self, key, value): + if key == value: + raise TypeError(f"'{key}' <> '{value}' is a self-mapping") + + try: + hash(value) + except TypeError as exc: + raise TypeError(f"value '{value}' of {exc}") + try: + hash(key) + except TypeError as exc: + raise TypeError(f"key '{key}' of {exc}") + + if self.__contains__(key): + raise KeyError( + f"'{key}' is already {'a value' * (key in self._inverse)} in mapping" + ) + if self.__contains__(value): + raise ValueError( + f"'{value}' is already {'a key' * (value not in self._inverse)} in mapping" + ) + + super().__setitem__(key, value) + self._inverse[value] = key + + def __delitem__(self, key): + if not self.__contains__(key): + raise KeyError(f"'{key}") + + if super().__contains__(key): + del self._inverse[super().__getitem__(key)] + super().__delitem__(key) + else: + super().__delitem__(self._inverse[key]) + del self._inverse[key] + + def __getitem__(self, key): + if key in self._inverse: + return self._inverse[key] + return super().__getitem__(key) + + def __contains__(self, key): + if super().__contains__(key): + return True + return key in self._inverse + + def add(self, value): + """Insert a new value in the bidict, generating an automatic key.""" + _used = set( + int(i.groups()[0]) + for i in [ + _autokey_pat.match(k) for k in self.keys() if k.startswith("auto_") + ] + if i is not None + ) + for i in range(len(_used) + 1): + if i not in _used: + newkey = f"auto_{i:05d}" + + self.__setitem__(newkey, value) + return newkey From a96bd6bd190824224e196463383cc4e5b4676f8a Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Tue, 1 Dec 2020 22:57:25 +0100 Subject: [PATCH 60/68] enh: increase code coverage --- sdcflows/utils/bimap.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/sdcflows/utils/bimap.py b/sdcflows/utils/bimap.py index 37e373ab47..85637a2fd7 100644 --- a/sdcflows/utils/bimap.py +++ b/sdcflows/utils/bimap.py @@ -56,6 +56,10 @@ class bidict(dict): Traceback (most recent call last): TypeError: value '[]' of unhashable type: 'list' + >>> d[list()] = 1 # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + TypeError: key '[]' of unhashable type: 'list' + >>> d.add("a new value") 'auto_00000' @@ -73,6 +77,14 @@ class bidict(dict): >>> d == bidict(reversed(list(d.items()))) True + >>> bidict({"a": 1, "b": 1}) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + TypeError: Bidirectional dictionary cannot contain repeated values + + >>> del d["e"] # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + KeyError: 'e' + """ _inverse = None @@ -82,7 +94,7 @@ def __init__(self, *args, **kwargs): self._inverse = {v: k for k, v in self.items()} if len(self) != len(self._inverse): raise TypeError( - "Bidirectional dictionary cannot contain repeated keys or values." + "Bidirectional dictionary cannot contain repeated values" ) def __setitem__(self, key, value): From 83e9e09b8c80e9a95ed7a1ea4169f784094ae44b Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Wed, 2 Dec 2020 00:55:43 +0100 Subject: [PATCH 61/68] enh: set a test basis --- sdcflows/conftest.py | 2 + sdcflows/fieldmaps.py | 2 +- .../fmap/sub-01_acq-single_dir-PA_epi.json | 10 ++++ .../fmap/sub-01_acq-single_dir-PA_epi.nii.gz | 0 sdcflows/tests/test_fieldmaps.py | 13 ++++- sdcflows/workflows/base.py | 54 +++++++++---------- 6 files changed, 51 insertions(+), 30 deletions(-) create mode 100644 sdcflows/tests/data/dsA/sub-01/fmap/sub-01_acq-single_dir-PA_epi.json create mode 100644 sdcflows/tests/data/dsA/sub-01/fmap/sub-01_acq-single_dir-PA_epi.nii.gz diff --git a/sdcflows/conftest.py b/sdcflows/conftest.py index e4718ed5c4..b68fe8215e 100644 --- a/sdcflows/conftest.py +++ b/sdcflows/conftest.py @@ -15,6 +15,8 @@ data_dir = Path(__file__).parent / "tests" / "data" / "dsA" +layouts["dsA"] = BIDSLayout(data_dir, validate=False, derivatives=False) + def pytest_report_header(config): return f"""\ diff --git a/sdcflows/fieldmaps.py b/sdcflows/fieldmaps.py index e4bf5ac936..66343ce262 100644 --- a/sdcflows/fieldmaps.py +++ b/sdcflows/fieldmaps.py @@ -390,7 +390,7 @@ def __attrs_post_init__(self): _estimators[self.bids_id] = self.paths() def paths(self): - """Return a tuple of paths (sorted).""" + """Return a tuple of paths that are sorted.""" return tuple(sorted(str(f.path) for f in self.sources)) def get_workflow(self, **kwargs): diff --git a/sdcflows/tests/data/dsA/sub-01/fmap/sub-01_acq-single_dir-PA_epi.json b/sdcflows/tests/data/dsA/sub-01/fmap/sub-01_acq-single_dir-PA_epi.json new file mode 100644 index 0000000000..92f857af8b --- /dev/null +++ b/sdcflows/tests/data/dsA/sub-01/fmap/sub-01_acq-single_dir-PA_epi.json @@ -0,0 +1,10 @@ +{ + "PhaseEncodingDirection": "j", + "TotalReadoutTime": 0.005, + "IntendedFor": [ + "dwi/sub-01_dir-AP_dwi.nii.gz", + "dwi/sub-01_dir-AP_sbref.nii.gz", + "func/sub-01_task-rest_bold.nii.gz", + "func/sub-01_task-rest_sbref.nii.gz" + ] +} \ No newline at end of file diff --git a/sdcflows/tests/data/dsA/sub-01/fmap/sub-01_acq-single_dir-PA_epi.nii.gz b/sdcflows/tests/data/dsA/sub-01/fmap/sub-01_acq-single_dir-PA_epi.nii.gz new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sdcflows/tests/test_fieldmaps.py b/sdcflows/tests/test_fieldmaps.py index 9ce58a141e..0b68035fa6 100644 --- a/sdcflows/tests/test_fieldmaps.py +++ b/sdcflows/tests/test_fieldmaps.py @@ -9,6 +9,7 @@ def test_FieldmapFile(testdata_dir): fm.FieldmapFile(testdata_dir / "sub-01" / "anat" / "sub-01_T1w.nii.gz") +@pytest.fixture(scope="module") @pytest.mark.parametrize( "inputfiles,method,nsources,raises", [ @@ -66,8 +67,12 @@ def test_FieldmapEstimation( sources = [sub_dir / f for f in inputfiles] if raises is True: + # Ensure that _estimators is still holding values from previous + # parameter set of this parametrized execution. with pytest.raises(ValueError): fm.FieldmapEstimation(sources) + + # Clean up so this parameter set can be tested. monkeypatch.setattr(fm, "_estimators", bidict()) fe = fm.FieldmapEstimation(sources) @@ -92,7 +97,11 @@ def test_FieldmapEstimation( # Exercise workflow creation wf = fe.get_workflow() - wf == fe.get_workflow() + + yield wf == fe.get_workflow() # Code after this yield will only run on cleanup + + # Clean-up: reset _estimators registry + monkeypatch.setattr(fm, "_estimators", bidict()) @pytest.mark.parametrize( @@ -113,6 +122,8 @@ def test_FieldmapEstimationError(monkeypatch, testdata_dir, inputfiles, errortyp with pytest.raises(errortype): fm.FieldmapEstimation([sub_dir / f for f in inputfiles]) + monkeypatch.setattr(fm, "_estimators", bidict()) + def test_FieldmapEstimationIdentifier(monkeypatch, testdata_dir): """Check some use cases of B0FieldIdentifier.""" diff --git a/sdcflows/workflows/base.py b/sdcflows/workflows/base.py index a780dd68ca..9d651dc28f 100644 --- a/sdcflows/workflows/base.py +++ b/sdcflows/workflows/base.py @@ -4,18 +4,12 @@ from itertools import product from nipype import logging -LOGGER = logging.getLogger('nipype.workflow') +LOGGER = logging.getLogger("nipype.workflow") DEFAULT_MEMORY_MIN_GB = 0.01 def init_fmap_preproc_wf( - *, - layout, - omp_nthreads, - output_dir, - subject, - debug=False, - name='fmap_preproc_wf', + *, layout, omp_nthreads, output_dir, subject, debug=False, name="fmap_preproc_wf", ): """ Stage the fieldmap data preprocessing steps of *SDCFlows*. @@ -68,6 +62,17 @@ def init_fmap_preproc_wf( [FieldmapEstimation(sources=<2 files>, method=, bids_id='...'), FieldmapEstimation(sources=<2 files>, method=, bids_id='...')] + >>> init_fmap_preproc_wf( + ... layout=layouts['dsA'], + ... omp_nthreads=1, + ... output_dir="/tmp", + ... subject="01", + ... ) # doctest: +ELLIPSIS + [FieldmapEstimation(sources=<2 files>, method=, bids_id='...'), + FieldmapEstimation(sources=<4 files>, method=, bids_id='...'), + FieldmapEstimation(sources=<3 files>, method=, bids_id='...'), + FieldmapEstimation(sources=<4 files>, method=, bids_id='...')] + """ from ..fieldmaps import FieldmapEstimation, FieldmapFile @@ -80,38 +85,31 @@ def init_fmap_preproc_wf( estimators = [] # Set up B0 fieldmap strategies: - for fmap in layout.get( - suffix=["fieldmap", "phasediff", "phase1"], **base_entities - ): - e = FieldmapEstimation( - FieldmapFile(fmap.path, metadata=fmap.get_metadata()) - ) + for fmap in layout.get(suffix=["fieldmap", "phasediff", "phase1"], **base_entities): + e = FieldmapEstimation(FieldmapFile(fmap.path, metadata=fmap.get_metadata())) estimators.append(e) # A bunch of heuristics to select EPI fieldmaps - sessions = layout.get_sessions() or (None, ) + sessions = layout.get_sessions() or (None,) acqs = tuple(layout.get_acquisitions(suffix="epi") + [None]) contrasts = tuple(layout.get_ceagents(suffix="epi") + [None]) for ses, acq, ce in product(sessions, acqs, contrasts): entities = base_entities.copy() - entities.update({ - "suffix": "epi", - "session": ses, - "acquisition": acq, - "ceagent": ce, - }) + entities.update( + {"suffix": "epi", "session": ses, "acquisition": acq, "ceagent": ce} + ) dirs = layout.get_directions(**entities) if len(dirs) > 1: - e = FieldmapEstimation([ - FieldmapFile(fmap.path, metadata=fmap.get_metadata()) - for fmap in layout.get(direction=dirs, **entities) - ]) + e = FieldmapEstimation( + [ + FieldmapFile(fmap.path, metadata=fmap.get_metadata()) + for fmap in layout.get(direction=dirs, **entities) + ] + ) estimators.append(e) for e in estimators: - LOGGER.info( - f"{e.method}:: <{':'.join(s.path.name for s in e.sources)}>." - ) + LOGGER.info(f"{e.method}:: <{':'.join(s.path.name for s in e.sources)}>.") return estimators From 486cc86a3989747338547a6031cf8139149c9d3a Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Wed, 2 Dec 2020 21:10:21 +0100 Subject: [PATCH 62/68] enh: added automated detection of fieldmaps searching the IntendedFor Resolves: #126. --- .circleci/config.yml | 7 ++++ .github/workflows/travis.yml | 5 +++ sdcflows/fieldmaps.py | 4 +-- sdcflows/tests/test_fieldmaps.py | 29 ++++++---------- sdcflows/utils/bimap.py | 59 ++++++++++++++++++++++++++++++-- sdcflows/workflows/base.py | 56 +++++++++++++++++++++++++++--- 6 files changed, 131 insertions(+), 29 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 20396f9a9d..fbac43e47a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -104,6 +104,13 @@ jobs: datalad update --merge -d ds001771/ datalad get -r -d ds001771/ ds001771/sub-36/fmap/* + - run: + name: Install ds000054 + command: | + datalad install -r https://github.com/nipreps-data/ds000054.git + datalad update --merge -d ds000054/ + datalad get -r -d ds000054/ ds000054/sub-100185/fmap/* + - save_cache: key: data-v4-{{ .Branch }}-{{ .BuildNum }} paths: diff --git a/.github/workflows/travis.yml b/.github/workflows/travis.yml index bbe7b2fc52..e3b9d002c6 100644 --- a/.github/workflows/travis.yml +++ b/.github/workflows/travis.yml @@ -89,6 +89,11 @@ jobs: datalad install -r https://github.com/nipreps-data/ds001771.git datalad update --merge -d ds001771/ datalad get -r -d ds001771/ ds001771/sub-36/fmap/* + + # ds000054 + datalad install -r https://github.com/nipreps-data/ds000054.git + datalad update --merge -d ds000054/ + datalad get -r -d ds000054/ ds000054/sub-100185/fmap/* - uses: actions/cache@v2 with: path: /var/lib/apt diff --git a/sdcflows/fieldmaps.py b/sdcflows/fieldmaps.py index 66343ce262..869ca14780 100644 --- a/sdcflows/fieldmaps.py +++ b/sdcflows/fieldmaps.py @@ -7,10 +7,10 @@ from bids.layout import BIDSFile, parse_file_entities from bids.utils import listify from niworkflows.utils.bids import relative_to_root -from .utils.bimap import bidict +from .utils.bimap import EstimatorRegistry -_estimators = bidict() +_estimators = EstimatorRegistry() class MetadataError(ValueError): diff --git a/sdcflows/tests/test_fieldmaps.py b/sdcflows/tests/test_fieldmaps.py index 0b68035fa6..9f043d7e46 100644 --- a/sdcflows/tests/test_fieldmaps.py +++ b/sdcflows/tests/test_fieldmaps.py @@ -1,6 +1,5 @@ """test_fieldmaps.""" import pytest -from ..utils.bimap import bidict from .. import fieldmaps as fm @@ -9,7 +8,6 @@ def test_FieldmapFile(testdata_dir): fm.FieldmapFile(testdata_dir / "sub-01" / "anat" / "sub-01_T1w.nii.gz") -@pytest.fixture(scope="module") @pytest.mark.parametrize( "inputfiles,method,nsources,raises", [ @@ -58,9 +56,7 @@ def test_FieldmapFile(testdata_dir): ), ], ) -def test_FieldmapEstimation( - monkeypatch, testdata_dir, inputfiles, method, nsources, raises -): +def test_FieldmapEstimation(testdata_dir, inputfiles, method, nsources, raises): """Test errors.""" sub_dir = testdata_dir / "sub-01" @@ -73,7 +69,7 @@ def test_FieldmapEstimation( fm.FieldmapEstimation(sources) # Clean up so this parameter set can be tested. - monkeypatch.setattr(fm, "_estimators", bidict()) + fm._estimators.clear() fe = fm.FieldmapEstimation(sources) assert fe.method == method @@ -97,11 +93,7 @@ def test_FieldmapEstimation( # Exercise workflow creation wf = fe.get_workflow() - - yield wf == fe.get_workflow() # Code after this yield will only run on cleanup - - # Clean-up: reset _estimators registry - monkeypatch.setattr(fm, "_estimators", bidict()) + wf == fe.get_workflow() @pytest.mark.parametrize( @@ -113,22 +105,21 @@ def test_FieldmapEstimation( (("anat/sub-01_T1w.nii.gz", "fmap/sub-01_phase2.nii.gz"), TypeError), ], ) -def test_FieldmapEstimationError(monkeypatch, testdata_dir, inputfiles, errortype): +def test_FieldmapEstimationError(testdata_dir, inputfiles, errortype): """Test errors.""" sub_dir = testdata_dir / "sub-01" - monkeypatch.setattr(fm, "_estimators", bidict()) + fm._estimators.clear() with pytest.raises(errortype): fm.FieldmapEstimation([sub_dir / f for f in inputfiles]) - monkeypatch.setattr(fm, "_estimators", bidict()) + fm._estimators.clear() -def test_FieldmapEstimationIdentifier(monkeypatch, testdata_dir): +def test_FieldmapEstimationIdentifier(testdata_dir): """Check some use cases of B0FieldIdentifier.""" - - monkeypatch.setattr(fm, "_estimators", bidict()) + fm._estimators.clear() with pytest.raises(ValueError): fm.FieldmapEstimation( @@ -172,7 +163,7 @@ def test_FieldmapEstimationIdentifier(monkeypatch, testdata_dir): ] ) # Consistent, but already exists - monkeypatch.setattr(fm, "_estimators", bidict()) + fm._estimators.clear() fe = fm.FieldmapEstimation( [ @@ -188,4 +179,4 @@ def test_FieldmapEstimationIdentifier(monkeypatch, testdata_dir): ) assert fe.bids_id == "fmap_1" - monkeypatch.setattr(fm, "_estimators", bidict()) + fm._estimators.clear() diff --git a/sdcflows/utils/bimap.py b/sdcflows/utils/bimap.py index 85637a2fd7..3044c76347 100644 --- a/sdcflows/utils/bimap.py +++ b/sdcflows/utils/bimap.py @@ -85,6 +85,16 @@ class bidict(dict): Traceback (most recent call last): KeyError: 'e' + >>> list(d) + ['a', 'b', 'c', 'auto_00000', 'auto_00001', 'auto_00002'] + + >>> list(d.values()) + [1, 2, 3, 'a new value', 'another value', 'third value'] + + >>> d.clear() + >>> d + {} + """ _inverse = None @@ -93,9 +103,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._inverse = {v: k for k, v in self.items()} if len(self) != len(self._inverse): - raise TypeError( - "Bidirectional dictionary cannot contain repeated values" - ) + raise TypeError("Bidirectional dictionary cannot contain repeated values") def __setitem__(self, key, value): if key == value: @@ -158,3 +166,48 @@ def add(self, value): self.__setitem__(newkey, value) return newkey + + def clear(self): + """Empty of all key/value pairs.""" + self._inverse.clear() + super().clear() + + +class EstimatorRegistry(bidict): + """ + A specialized :py:class:`bidict` to track :py:class:`~sdcflows.fieldmaps.FieldmapEstimation`. + + Examples + -------- + >>> estimators = EstimatorRegistry() + >>> _ = estimators.add(("file3.txt", "file4.txt")) + >>> estimators.sources + ['file3.txt', 'file4.txt'] + + >>> _ = estimators.add(("file1.txt", "file2.txt")) + >>> estimators.sources + ['file1.txt', 'file2.txt', 'file3.txt', 'file4.txt'] + + >>> _ = estimators.add(("file3.txt", "file2.txt")) + >>> estimators.sources + ['file1.txt', 'file2.txt', 'file3.txt', 'file4.txt'] + + >>> estimators.get_key("file3.txt") + ('auto_00000', 'auto_00002') + + >>> estimators.get_key("file5.txt") + () + + """ + + @property + def sources(self): + """Return a flattened list of fieldmap sources.""" + return sorted(set([el for group in self.values() for el in group])) + + def get_key(self, value): + """Get the key(s) containing a particular value.""" + if value not in self.sources: + return tuple() + + return tuple(sorted(k for k, v in self.items() if value in v)) diff --git a/sdcflows/workflows/base.py b/sdcflows/workflows/base.py index 9d651dc28f..f5b4a8563b 100644 --- a/sdcflows/workflows/base.py +++ b/sdcflows/workflows/base.py @@ -2,6 +2,7 @@ # vi: set ft=python sts=4 ts=4 sw=4 et: """Estimate fieldmaps for :abbr:`SDC (susceptibility distortion correction)`.""" from itertools import product +from pathlib import Path from nipype import logging LOGGER = logging.getLogger("nipype.workflow") @@ -31,6 +32,14 @@ def init_fmap_preproc_wf( Examples -------- + >>> init_fmap_preproc_wf( + ... layout=layouts['ds000054'], + ... omp_nthreads=1, + ... output_dir="/tmp", + ... subject="100185", + ... ) # doctest: +ELLIPSIS + [FieldmapEstimation(sources=<3 files>, method=, bids_id='...')] + >>> init_fmap_preproc_wf( ... layout=layouts['ds001771'], ... omp_nthreads=1, @@ -71,10 +80,12 @@ def init_fmap_preproc_wf( [FieldmapEstimation(sources=<2 files>, method=, bids_id='...'), FieldmapEstimation(sources=<4 files>, method=, bids_id='...'), FieldmapEstimation(sources=<3 files>, method=, bids_id='...'), - FieldmapEstimation(sources=<4 files>, method=, bids_id='...')] + FieldmapEstimation(sources=<4 files>, method=, bids_id='...'), + FieldmapEstimation(sources=<2 files>, method=, bids_id='...')] """ - from ..fieldmaps import FieldmapEstimation, FieldmapFile + from .. import fieldmaps as fm + from bids.layout import Query base_entities = { "subject": subject, @@ -86,7 +97,9 @@ def init_fmap_preproc_wf( # Set up B0 fieldmap strategies: for fmap in layout.get(suffix=["fieldmap", "phasediff", "phase1"], **base_entities): - e = FieldmapEstimation(FieldmapFile(fmap.path, metadata=fmap.get_metadata())) + e = fm.FieldmapEstimation( + fm.FieldmapFile(fmap.path, metadata=fmap.get_metadata()) + ) estimators.append(e) # A bunch of heuristics to select EPI fieldmaps @@ -101,14 +114,47 @@ def init_fmap_preproc_wf( ) dirs = layout.get_directions(**entities) if len(dirs) > 1: - e = FieldmapEstimation( + e = fm.FieldmapEstimation( [ - FieldmapFile(fmap.path, metadata=fmap.get_metadata()) + fm.FieldmapFile(fmap.path, metadata=fmap.get_metadata()) for fmap in layout.get(direction=dirs, **entities) ] ) estimators.append(e) + # At this point, only single-PE _epi files WITH ``IntendedFor`` can be automatically processed + # (this will be easier with bids-standard/bids-specification#622 in). + try: + has_intended = layout.get(suffix="epi", IntendedFor=Query.ANY, **base_entities) + except ValueError: + has_intended = tuple() + + for epi_fmap in has_intended: + if epi_fmap.path in fm._estimators.sources: + continue # skip EPI images already considered above + + subject_root = Path(epi_fmap.path.rpartition("/sub-")[0]).parent + targets = [epi_fmap] + [ + layout.get_file(str(subject_root / intent)) + for intent in epi_fmap.get_metadata()["IntendedFor"] + ] + + epi_sources = [] + for fmap in targets: + try: + epi_sources.append( + fm.FieldmapFile(fmap.path, metadata=fmap.get_metadata()) + ) + except fm.MetadataError: + pass + + try: + estimators.append(fm.FieldmapEstimation(epi_sources)) + except (ValueError, TypeError) as exc: + LOGGER.warning( + f"FieldmapEstimation strategy failed for <{epi_fmap.path}>. Reason: {exc}." + ) + for e in estimators: LOGGER.info(f"{e.method}:: <{':'.join(s.path.name for s in e.sources)}>.") From 4548332fd1f369b7f6c6a183cffb76c1ab1f7bdc Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 3 Dec 2020 15:24:07 +0100 Subject: [PATCH 63/68] ENH: Putting the new API together on a base workflow Finalizes the new API, showing how the new SDCFlows works. --- sdcflows/fieldmaps.py | 2 +- sdcflows/utils/wrangler.py | 142 ++++++++++++++++ sdcflows/workflows/base.py | 230 ++++++++++++-------------- sdcflows/workflows/fit/fieldmap.py | 1 - sdcflows/workflows/tests/__init__.py | 0 sdcflows/workflows/tests/test_base.py | 34 ++++ setup.cfg | 1 + 7 files changed, 283 insertions(+), 127 deletions(-) create mode 100644 sdcflows/utils/wrangler.py create mode 100644 sdcflows/workflows/tests/__init__.py create mode 100644 sdcflows/workflows/tests/test_base.py diff --git a/sdcflows/fieldmaps.py b/sdcflows/fieldmaps.py index 869ca14780..eb41c7ba6d 100644 --- a/sdcflows/fieldmaps.py +++ b/sdcflows/fieldmaps.py @@ -404,7 +404,7 @@ def get_workflow(self, **kwargs): if self.method in (EstimatorType.MAPPED, EstimatorType.PHASEDIFF): from .workflows.fit.fieldmap import init_fmap_wf - kwargs["mode"] = str(self.method).split(".")[-1].lower() + kwargs["mode"] = str(self.method).rpartition(".")[-1].lower() self._wf = init_fmap_wf(**kwargs) self._wf.inputs.inputnode.magnitude = [ str(f.path) for f in self.sources if f.suffix.startswith("magnitude") diff --git a/sdcflows/utils/wrangler.py b/sdcflows/utils/wrangler.py new file mode 100644 index 0000000000..9478447c30 --- /dev/null +++ b/sdcflows/utils/wrangler.py @@ -0,0 +1,142 @@ +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# vi: set ft=python sts=4 ts=4 sw=4 et: +"""Find fieldmaps on the BIDS inputs for :abbr:`SDC (susceptibility distortion correction)`.""" +from itertools import product +from pathlib import Path + + +def find_estimators(layout, subject=None): + """ + Apply basic heuristics to automatically find available data for fieldmap estimation. + + Parameters + ---------- + layout : :obj:`bids.layout.BIDSLayout` + An initialized PyBIDS layout. + subject : :obj:`str` + Participant label for this single-subject workflow. + + Returns + ------- + estimators : :obj:`list` + The list of :py:class:`~sdcflows.fieldmaps.FieldmapEstimation` objects that have + successfully been built (meaning, all necessary inputs and corresponding metadata + are present in the given layout.) + + Examples + -------- + >>> find_estimators( + ... layout=layouts['ds000054'], + ... subject="100185", + ... ) # doctest: +ELLIPSIS + [FieldmapEstimation(sources=<3 files>, method=, bids_id='...')] + + >>> find_estimators( + ... layout=layouts['ds001771'], + ... subject="36", + ... ) # doctest: +ELLIPSIS + [FieldmapEstimation(sources=<2 files>, method=, bids_id='...'), + FieldmapEstimation(sources=<2 files>, method=, bids_id='...'), + FieldmapEstimation(sources=<2 files>, method=, bids_id='...'), + FieldmapEstimation(sources=<2 files>, method=, bids_id='...')] + + >>> find_estimators( + ... layout=layouts['ds001600'], + ... subject="1", + ... ) # doctest: +ELLIPSIS + [FieldmapEstimation(sources=<4 files>, method=, bids_id='...'), + FieldmapEstimation(sources=<4 files>, method=, bids_id='...'), + FieldmapEstimation(sources=<3 files>, method=, bids_id='...'), + FieldmapEstimation(sources=<2 files>, method=, bids_id='...')] + + >>> find_estimators( + ... layout=layouts['HCP101006'], + ... subject="101006", + ... ) # doctest: +ELLIPSIS + [FieldmapEstimation(sources=<2 files>, method=, bids_id='...'), + FieldmapEstimation(sources=<2 files>, method=, bids_id='...')] + + >>> find_estimators( + ... layout=layouts['dsA'], + ... subject="01", + ... ) # doctest: +ELLIPSIS + [FieldmapEstimation(sources=<2 files>, method=, bids_id='...'), + FieldmapEstimation(sources=<4 files>, method=, bids_id='...'), + FieldmapEstimation(sources=<3 files>, method=, bids_id='...'), + FieldmapEstimation(sources=<4 files>, method=, bids_id='...'), + FieldmapEstimation(sources=<2 files>, method=, bids_id='...')] + + """ + from .. import fieldmaps as fm + from bids.layout import Query + + if subject is None: + subject = Query.ANY + + base_entities = { + "subject": subject, + "extension": [".nii", ".nii.gz"], + "space": None, # Ensure derivatives are not captured + } + + estimators = [] + + # Set up B0 fieldmap strategies: + for fmap in layout.get(suffix=["fieldmap", "phasediff", "phase1"], **base_entities): + e = fm.FieldmapEstimation( + fm.FieldmapFile(fmap.path, metadata=fmap.get_metadata()) + ) + estimators.append(e) + + # A bunch of heuristics to select EPI fieldmaps + sessions = layout.get_sessions() or (None,) + acqs = tuple(layout.get_acquisitions(suffix="epi") + [None]) + contrasts = tuple(layout.get_ceagents(suffix="epi") + [None]) + + for ses, acq, ce in product(sessions, acqs, contrasts): + entities = base_entities.copy() + entities.update( + {"suffix": "epi", "session": ses, "acquisition": acq, "ceagent": ce} + ) + dirs = layout.get_directions(**entities) + if len(dirs) > 1: + e = fm.FieldmapEstimation( + [ + fm.FieldmapFile(fmap.path, metadata=fmap.get_metadata()) + for fmap in layout.get(direction=dirs, **entities) + ] + ) + estimators.append(e) + + # At this point, only single-PE _epi files WITH ``IntendedFor`` can be automatically processed + # (this will be easier with bids-standard/bids-specification#622 in). + try: + has_intended = layout.get(suffix="epi", IntendedFor=Query.ANY, **base_entities) + except ValueError: + has_intended = tuple() + + for epi_fmap in has_intended: + if epi_fmap.path in fm._estimators.sources: + continue # skip EPI images already considered above + + subject_root = Path(epi_fmap.path.rpartition("/sub-")[0]).parent + targets = [epi_fmap] + [ + layout.get_file(str(subject_root / intent)) + for intent in epi_fmap.get_metadata()["IntendedFor"] + ] + + epi_sources = [] + for fmap in targets: + try: + epi_sources.append( + fm.FieldmapFile(fmap.path, metadata=fmap.get_metadata()) + ) + except fm.MetadataError: + pass + + try: + estimators.append(fm.FieldmapEstimation(epi_sources)) + except (ValueError, TypeError): + pass + + return estimators diff --git a/sdcflows/workflows/base.py b/sdcflows/workflows/base.py index f5b4a8563b..a988a78862 100644 --- a/sdcflows/workflows/base.py +++ b/sdcflows/workflows/base.py @@ -1,24 +1,46 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: """Estimate fieldmaps for :abbr:`SDC (susceptibility distortion correction)`.""" -from itertools import product -from pathlib import Path from nipype import logging +from nipype.pipeline import engine as pe +from nipype.interfaces import utility as niu +from niworkflows.engine.workflows import LiterateWorkflow as Workflow +from ..fieldmaps import EstimatorType + LOGGER = logging.getLogger("nipype.workflow") DEFAULT_MEMORY_MIN_GB = 0.01 +INPUT_FIELDS = { + EstimatorType.MAPPED: ["magnitude", "fieldmap"], + EstimatorType.PHASEDIFF: ["magnitude", "fieldmap"], + EstimatorType.PEPOLAR: ["metadata", "in_data"], + EstimatorType.ANAT: [ + "epi_ref", + "epi_mask", + "anat_brain", + "std2anat_xfm", + "anat2bold_xfm", + ], +} + def init_fmap_preproc_wf( - *, layout, omp_nthreads, output_dir, subject, debug=False, name="fmap_preproc_wf", + *, + estimators, + omp_nthreads, + output_dir, + subject, + debug=False, + name="fmap_preproc_wf", ): """ - Stage the fieldmap data preprocessing steps of *SDCFlows*. + Create and combine estimator workflows. Parameters ---------- - layout : :obj:`bids.layout.BIDSLayout` - An initialized PyBIDS layout. + estimators : :obj:`list` of :py:class:`~sdcflows.fieldmaps.FieldmapEstimator` + A list of estimators. omp_nthreads : :obj:`int` Maximum number of threads an individual process may use output_dir : :obj:`str` @@ -30,132 +52,90 @@ def init_fmap_preproc_wf( name : :obj:`str`, optional Workflow name (default: ``"fmap_preproc_wf"``) - Examples - -------- - >>> init_fmap_preproc_wf( - ... layout=layouts['ds000054'], - ... omp_nthreads=1, - ... output_dir="/tmp", - ... subject="100185", - ... ) # doctest: +ELLIPSIS - [FieldmapEstimation(sources=<3 files>, method=, bids_id='...')] - - >>> init_fmap_preproc_wf( - ... layout=layouts['ds001771'], - ... omp_nthreads=1, - ... output_dir="/tmp", - ... subject="36", - ... ) # doctest: +ELLIPSIS - [FieldmapEstimation(sources=<2 files>, method=, bids_id='...'), - FieldmapEstimation(sources=<2 files>, method=, bids_id='...'), - FieldmapEstimation(sources=<2 files>, method=, bids_id='...'), - FieldmapEstimation(sources=<2 files>, method=, bids_id='...')] - - >>> init_fmap_preproc_wf( - ... layout=layouts['ds001600'], - ... omp_nthreads=1, - ... output_dir="/tmp", - ... subject="1", - ... ) # doctest: +ELLIPSIS - [FieldmapEstimation(sources=<4 files>, method=, bids_id='...'), - FieldmapEstimation(sources=<4 files>, method=, bids_id='...'), - FieldmapEstimation(sources=<3 files>, method=, bids_id='...'), - FieldmapEstimation(sources=<2 files>, method=, bids_id='...')] + Inputs + ------ + in_. : + The workflow generates inputs depending on the estimation strategy. - >>> init_fmap_preproc_wf( - ... layout=layouts['HCP101006'], - ... omp_nthreads=1, - ... output_dir="/tmp", - ... subject="101006", - ... ) # doctest: +ELLIPSIS - [FieldmapEstimation(sources=<2 files>, method=, bids_id='...'), - FieldmapEstimation(sources=<2 files>, method=, bids_id='...')] - - >>> init_fmap_preproc_wf( - ... layout=layouts['dsA'], - ... omp_nthreads=1, - ... output_dir="/tmp", - ... subject="01", - ... ) # doctest: +ELLIPSIS - [FieldmapEstimation(sources=<2 files>, method=, bids_id='...'), - FieldmapEstimation(sources=<4 files>, method=, bids_id='...'), - FieldmapEstimation(sources=<3 files>, method=, bids_id='...'), - FieldmapEstimation(sources=<4 files>, method=, bids_id='...'), - FieldmapEstimation(sources=<2 files>, method=, bids_id='...')] + Outputs + ------- + out_.fmap : + The preprocessed fieldmap. + out_.fmap_ref : + The preprocessed fieldmap reference. + out_.fmap_coeff : + The preprocessed fieldmap coefficients. """ - from .. import fieldmaps as fm - from bids.layout import Query - - base_entities = { - "subject": subject, - "extension": [".nii", ".nii.gz"], - "space": None, # Ensure derivatives are not captured - } - - estimators = [] - - # Set up B0 fieldmap strategies: - for fmap in layout.get(suffix=["fieldmap", "phasediff", "phase1"], **base_entities): - e = fm.FieldmapEstimation( - fm.FieldmapFile(fmap.path, metadata=fmap.get_metadata()) + from .outputs import init_fmap_derivatives_wf, init_fmap_reports_wf + + workflow = Workflow(name=name) + for estimator in estimators: + est_wf = estimator.get_workflow(omp_nthreads=omp_nthreads, debug=debug) + source_files = [str(f.path) for f in estimator.sources] + + outputnode = pe.Node( + niu.IdentityInterface( + fields=["fmap", "fmap_ref", "fmap_coeff", "fmap_mask"] + ), + name=f"out_{estimator.bids_id}", ) - estimators.append(e) - # A bunch of heuristics to select EPI fieldmaps - sessions = layout.get_sessions() or (None,) - acqs = tuple(layout.get_acquisitions(suffix="epi") + [None]) - contrasts = tuple(layout.get_ceagents(suffix="epi") + [None]) - - for ses, acq, ce in product(sessions, acqs, contrasts): - entities = base_entities.copy() - entities.update( - {"suffix": "epi", "session": ses, "acquisition": acq, "ceagent": ce} + fmap_derivatives_wf = init_fmap_derivatives_wf( + output_dir=str(output_dir), + write_coeff=True, + bids_fmap_id=estimator.bids_id.replace("_", ""), + name=f"fmap_derivatives_wf_{estimator.bids_id}", ) - dirs = layout.get_directions(**entities) - if len(dirs) > 1: - e = fm.FieldmapEstimation( - [ - fm.FieldmapFile(fmap.path, metadata=fmap.get_metadata()) - for fmap in layout.get(direction=dirs, **entities) - ] - ) - estimators.append(e) - - # At this point, only single-PE _epi files WITH ``IntendedFor`` can be automatically processed - # (this will be easier with bids-standard/bids-specification#622 in). - try: - has_intended = layout.get(suffix="epi", IntendedFor=Query.ANY, **base_entities) - except ValueError: - has_intended = tuple() - - for epi_fmap in has_intended: - if epi_fmap.path in fm._estimators.sources: - continue # skip EPI images already considered above - - subject_root = Path(epi_fmap.path.rpartition("/sub-")[0]).parent - targets = [epi_fmap] + [ - layout.get_file(str(subject_root / intent)) - for intent in epi_fmap.get_metadata()["IntendedFor"] + fmap_derivatives_wf.inputs.inputnode.source_files = source_files + fmap_derivatives_wf.inputs.inputnode.fmap_meta = [ + f.metadata for f in estimator.sources ] - epi_sources = [] - for fmap in targets: - try: - epi_sources.append( - fm.FieldmapFile(fmap.path, metadata=fmap.get_metadata()) - ) - except fm.MetadataError: - pass + fmap_reports_wf = init_fmap_reports_wf( + output_dir=str(output_dir), + fmap_type=str(estimator.method).rpartition(".")[-1].lower(), + name=f"fmap_reports_wf_{estimator.bids_id}", + ) + fmap_reports_wf.inputs.inputnode.source_files = source_files - try: - estimators.append(fm.FieldmapEstimation(epi_sources)) - except (ValueError, TypeError) as exc: - LOGGER.warning( - f"FieldmapEstimation strategy failed for <{epi_fmap.path}>. Reason: {exc}." + if estimator.method not in (EstimatorType.MAPPED, EstimatorType.PHASEDIFF): + fields = INPUT_FIELDS[estimator.method] + inputnode = pe.Node( + niu.IdentityInterface(fields=fields), name=f"in_{estimator.bids_id}", ) - - for e in estimators: - LOGGER.info(f"{e.method}:: <{':'.join(s.path.name for s in e.sources)}>.") - - return estimators + # fmt:off + workflow.connect([ + (inputnode, est_wf, [(f, f"inputnode.{f}") for f in fields]) + ]) + # fmt:on + else: + # PEPOLAR and ANAT do not produce masks + # fmt:off + workflow.connect([ + (est_wf, fmap_reports_wf, [ + ("outputnode.fmap_mask", "inputnode.fmap_mask"), + ]), + (est_wf, outputnode, [("outputnode.fmap_mask", "fmap_mask")]), + ]) + # fmt:on + + # fmt:off + workflow.connect([ + (est_wf, fmap_derivatives_wf, [ + ("outputnode.fmap", "inputnode.fieldmap"), + ("outputnode.fmap_ref", "inputnode.fmap_ref"), + ("outputnode.fmap_coeff", "inputnode.fmap_coeff"), + ]), + (est_wf, fmap_reports_wf, [ + ("outputnode.fmap", "inputnode.fieldmap"), + ("outputnode.fmap_ref", "inputnode.fmap_ref"), + ]), + (est_wf, outputnode, [ + ("outputnode.fmap", "fmap"), + ("outputnode.fmap_ref", "fmap_ref"), + ("outputnode.fmap_coeff", "fmap_coeff"), + ]), + ]) + # fmt:on + return workflow diff --git a/sdcflows/workflows/fit/fieldmap.py b/sdcflows/workflows/fit/fieldmap.py index 928afd5a6c..ff0e26e49c 100644 --- a/sdcflows/workflows/fit/fieldmap.py +++ b/sdcflows/workflows/fit/fieldmap.py @@ -160,7 +160,6 @@ def init_fmap_wf(omp_nthreads=1, debug=False, mode="phasediff", name="fmap_wf"): ) workflow = Workflow(name=name) - inputnode = pe.Node( niu.IdentityInterface(fields=["magnitude", "fieldmap"]), name="inputnode" ) diff --git a/sdcflows/workflows/tests/__init__.py b/sdcflows/workflows/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sdcflows/workflows/tests/test_base.py b/sdcflows/workflows/tests/test_base.py new file mode 100644 index 0000000000..fe50e98698 --- /dev/null +++ b/sdcflows/workflows/tests/test_base.py @@ -0,0 +1,34 @@ +"""Test the base workflow.""" +import pytest +from ... import fieldmaps as fm +from ...utils.wrangler import find_estimators +from ..base import init_fmap_preproc_wf + + +@pytest.mark.parametrize( + "dataset,subject", [("ds000054", "100185"), ("HCP101006", "101006")] +) +def test_fmap_wf(tmpdir, workdir, outdir, bids_layouts, dataset, subject): + """Test the encompassing of the wrangler and the workflow creator.""" + fm._estimators.clear() + estimators = find_estimators(bids_layouts[dataset], subject=subject) + wf = init_fmap_preproc_wf( + estimators=estimators, + omp_nthreads=2, + output_dir=str(outdir), + subject=subject, + debug=True, + ) + + for estimator in estimators: + if estimator.method != fm.EstimatorType.PEPOLAR: + continue + + inputnode = wf.get_node(f"in_{estimator.bids_id}") + inputnode.inputs.in_data = [str(f.path) for f in estimator.sources] + inputnode.inputs.metadata = [f.metadata for f in estimator.sources] + + if workdir: + wf.base_dir = str(workdir) + + wf.run(plugin="Linear") diff --git a/setup.cfg b/setup.cfg index 6e48fcc405..4e54959055 100644 --- a/setup.cfg +++ b/setup.cfg @@ -57,6 +57,7 @@ tests = pytest pytest-xdist >= 2.0 pytest-cov == 2.10.1 + pytest-env coverage all = %(doc)s From ca7fd50d81453391772b13ab312c3087dab1b75a Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Fri, 4 Dec 2020 10:18:56 +0100 Subject: [PATCH 64/68] enh: resolve some nuances of the pepolar workflow integration - BUGFIX: the ``IntraModalMerge`` interface of the fieldmap workflow lacked a ``to_ras=False`` setting. - ENH: the TOPUP workflow now uses ``Flatten`` (as the 3dQwarp did) to massage EPIs before processing (and allowing to remove one ``IntraModalMerge`` on unit tests. - ENH: the TOPUP workflow now generates a single-volume merged output after correction. - ENH: tests are using the ``outdir`` fixture in a better way. --- sdcflows/workflows/fit/fieldmap.py | 3 ++- sdcflows/workflows/fit/pepolar.py | 28 ++++++++++++++------ sdcflows/workflows/fit/tests/test_pepolar.py | 28 +++++++------------- sdcflows/workflows/fit/tests/test_phdiff.py | 20 +++++++------- sdcflows/workflows/outputs.py | 3 +-- sdcflows/workflows/tests/test_base.py | 7 +++++ 6 files changed, 49 insertions(+), 40 deletions(-) diff --git a/sdcflows/workflows/fit/fieldmap.py b/sdcflows/workflows/fit/fieldmap.py index ff0e26e49c..5f301454be 100644 --- a/sdcflows/workflows/fit/fieldmap.py +++ b/sdcflows/workflows/fit/fieldmap.py @@ -220,7 +220,8 @@ def init_fmap_wf(omp_nthreads=1, debug=False, mode="phasediff", name="fmap_wf"): """ # Merge input fieldmap images (assumes all are given in the same units!) fmapmrg = pe.Node( - IntraModalMerge(zero_based_avg=False, hmc=False), name="fmapmrg" + IntraModalMerge(zero_based_avg=False, hmc=False, to_ras=False), + name="fmapmrg", ) units = pe.Node(CheckB0Units(), name="units", run_without_submitting=True) diff --git a/sdcflows/workflows/fit/pepolar.py b/sdcflows/workflows/fit/pepolar.py index 949a9d7ce3..4b728f712b 100644 --- a/sdcflows/workflows/fit/pepolar.py +++ b/sdcflows/workflows/fit/pepolar.py @@ -63,7 +63,10 @@ def init_topup_wf(omp_nthreads=1, debug=False, name="pepolar_estimate_wf"): """ from nipype.interfaces.fsl.epi import TOPUP from niworkflows.interfaces.nibabel import MergeSeries - from sdcflows.interfaces.epi import GetReadoutTime + from niworkflows.interfaces.images import IntraModalMerge + + from ...interfaces.epi import GetReadoutTime + from ...interfaces.utils import Flatten workflow = Workflow(name=name) workflow.__desc__ = f"""\ @@ -80,6 +83,7 @@ def init_topup_wf(omp_nthreads=1, debug=False, name="pepolar_estimate_wf"): name="outputnode", ) + flatten = pe.Node(Flatten(), name="flatten") concat_blips = pe.Node(MergeSeries(), name="concat_blips") readout_time = pe.MapNode( GetReadoutTime(), @@ -94,28 +98,34 @@ def init_topup_wf(omp_nthreads=1, debug=False, name="pepolar_estimate_wf"): ), name="topup", ) + merge_corrected = pe.Node( + IntraModalMerge(hmc=False, to_ras=False), name="merge_corrected" + ) # fmt: off workflow.connect([ - (inputnode, concat_blips, [("in_data", "in_files")]), - (inputnode, readout_time, [("in_data", "in_file"), - ("metadata", "metadata")]), - (inputnode, topup, [(("metadata", _pe2fsl), "encoding_direction")]), + (inputnode, flatten, [("in_data", "in_data"), + ("metadata", "in_meta")]), + (flatten, readout_time, [("out_data", "in_file"), + ("out_meta", "metadata")]), + (flatten, concat_blips, [("out_data", "in_files")]), + (flatten, topup, [(("out_meta", _pe2fsl), "encoding_direction")]), (readout_time, topup, [("readout_time", "readout_times")]), (concat_blips, topup, [("out_file", "in_file")]), - (topup, outputnode, [("out_corrected", "fmap_ref"), - ("out_field", "fmap"), + (topup, merge_corrected, [("out_corrected", "in_files")]), + (topup, outputnode, [("out_field", "fmap"), ("out_fieldcoef", "fmap_coeff"), ("out_jacs", "jacobians"), ("out_mats", "xfms"), ("out_warps", "out_warps")]), + (merge_corrected, outputnode, [("out_avg", "fmap_ref")]), ]) # fmt: on return workflow -def init_3dQwarp_wf(omp_nthreads=1, name="pepolar_estimate_wf"): +def init_3dQwarp_wf(omp_nthreads=1, debug=False, name="pepolar_estimate_wf"): """ Create the PEPOLAR field estimation workflow based on AFNI's ``3dQwarp``. @@ -133,6 +143,8 @@ def init_3dQwarp_wf(omp_nthreads=1, name="pepolar_estimate_wf"): Parameters ---------- + debug : :obj:`bool` + Whether a fast configuration of topup (less accurate) should be applied. name : :obj:`str` Name for this workflow omp_nthreads : :obj:`int` diff --git a/sdcflows/workflows/fit/tests/test_pepolar.py b/sdcflows/workflows/fit/tests/test_pepolar.py index 863b57aa26..8a27da2131 100644 --- a/sdcflows/workflows/fit/tests/test_pepolar.py +++ b/sdcflows/workflows/fit/tests/test_pepolar.py @@ -3,7 +3,6 @@ from pathlib import Path from json import loads import pytest -from niworkflows.interfaces.images import IntraModalMerge from nipype.pipeline import engine as pe from ..pepolar import init_topup_wf @@ -12,7 +11,7 @@ @pytest.mark.skipif(os.getenv("TRAVIS") == "true", reason="this is TravisCI") @pytest.mark.skipif(os.getenv("GITHUB_ACTIONS") == "true", reason="this is GH Actions") @pytest.mark.parametrize( - "epi_path", + "epi_file", [ ( "ds001771/sub-36/fmap/sub-36_acq-topup1_dir-01_epi.nii.gz", @@ -28,34 +27,27 @@ ), ], ) -def test_topup_wf(tmpdir, datadir, workdir, outdir, epi_path): +def test_topup_wf(tmpdir, datadir, workdir, outdir, epi_file): """Test preparation workflow.""" - epi_path = [datadir / f for f in epi_path] + epi_path = [datadir / f for f in epi_file] in_data = [str(f.absolute()) for f in epi_path] wf = pe.Workflow( name=f"topup_{epi_path[0].name.replace('.nii.gz', '').replace('-', '_')}" ) - merge = pe.MapNode(IntraModalMerge(hmc=False), name="merge", iterfield=["in_files"]) - merge.inputs.in_files = in_data - topup_wf = init_topup_wf(omp_nthreads=2, debug=True) metadata = [ loads(Path(str(f).replace(".nii.gz", ".json")).read_text()) for f in in_data ] + topup_wf.inputs.inputnode.in_data = in_data topup_wf.inputs.inputnode.metadata = metadata - # fmt: off - wf.connect([ - (merge, topup_wf, [("out_avg", "inputnode.in_data")]), - ]) - # fmt: on - if outdir: from nipype.interfaces.afni import Automask from ...outputs import init_fmap_derivatives_wf, init_fmap_reports_wf + outdir = outdir / "unittests" / epi_file[0].split("/")[0] fmap_derivatives_wf = init_fmap_derivatives_wf( output_dir=str(outdir), write_coeff=True, @@ -71,15 +63,13 @@ def test_topup_wf(tmpdir, datadir, workdir, outdir, epi_path): fmap_reports_wf.inputs.inputnode.source_files = in_data pre_mask = pe.Node(Automask(dilate=1, outputtype="NIFTI_GZ"), name="pre_mask") - merge_corrected = pe.Node(IntraModalMerge(hmc=False), name="merge_corrected") # fmt: off wf.connect([ - (topup_wf, merge_corrected, [("outputnode.fmap_ref", "in_files")]), - (merge_corrected, pre_mask, [("out_avg", "in_file")]), - (merge_corrected, fmap_reports_wf, [("out_avg", "inputnode.fmap_ref")]), - (topup_wf, fmap_reports_wf, [("outputnode.fmap", "inputnode.fieldmap")]), + (topup_wf, pre_mask, [("outputnode.fmap_ref", "in_file")]), (pre_mask, fmap_reports_wf, [("out_file", "inputnode.fmap_mask")]), + (topup_wf, fmap_reports_wf, [("outputnode.fmap", "inputnode.fieldmap"), + ("outputnode.fmap_ref", "inputnode.fmap_ref")]), (topup_wf, fmap_derivatives_wf, [ ("outputnode.fmap", "inputnode.fieldmap"), ("outputnode.fmap_ref", "inputnode.fmap_ref"), @@ -87,6 +77,8 @@ def test_topup_wf(tmpdir, datadir, workdir, outdir, epi_path): ]), ]) # fmt: on + else: + wf.add_nodes([topup_wf]) if workdir: wf.base_dir = str(workdir) diff --git a/sdcflows/workflows/fit/tests/test_phdiff.py b/sdcflows/workflows/fit/tests/test_phdiff.py index eabe85f445..443267adc1 100644 --- a/sdcflows/workflows/fit/tests/test_phdiff.py +++ b/sdcflows/workflows/fit/tests/test_phdiff.py @@ -11,7 +11,7 @@ @pytest.mark.skipif(os.getenv("TRAVIS") == "true", reason="this is TravisCI") @pytest.mark.skipif(os.getenv("GITHUB_ACTIONS") == "true", reason="this is GH Actions") @pytest.mark.parametrize( - "fmap_path", + "fmap_file", [ ("ds001600/sub-1/fmap/sub-1_acq-v4_phasediff.nii.gz",), ( @@ -22,11 +22,11 @@ ("HCP101006/sub-101006/fmap/sub-101006_phasediff.nii.gz",), ], ) -def test_phdiff(tmpdir, datadir, workdir, outdir, fmap_path): +def test_phdiff(tmpdir, datadir, workdir, outdir, fmap_file): """Test creation of the workflow.""" tmpdir.chdir() - fmap_path = [datadir / f for f in fmap_path] + fmap_path = [datadir / f for f in fmap_file] fieldmaps = [ (str(f.absolute()), loads(Path(str(f).replace(".nii.gz", ".json")).read_text())) for f in fmap_path @@ -36,20 +36,19 @@ def test_phdiff(tmpdir, datadir, workdir, outdir, fmap_path): name=f"phdiff_{fmap_path[0].name.replace('.nii.gz', '').replace('-', '_')}" ) mode = "mapped" if "fieldmap" in fmap_path[0].name else "phasediff" - phdiff_wf = init_fmap_wf( - omp_nthreads=2, - debug=True, - mode=mode, - ) + phdiff_wf = init_fmap_wf(omp_nthreads=2, debug=True, mode=mode,) phdiff_wf.inputs.inputnode.fieldmap = fieldmaps phdiff_wf.inputs.inputnode.magnitude = [ - f.replace("diff", "1").replace("phase", "magnitude").replace("fieldmap", "magnitude") + f.replace("diff", "1") + .replace("phase", "magnitude") + .replace("fieldmap", "magnitude") for f, _ in fieldmaps ] if outdir: from ...outputs import init_fmap_derivatives_wf, init_fmap_reports_wf + outdir = outdir / "unittests" / fmap_file[0].split("/")[0] fmap_derivatives_wf = init_fmap_derivatives_wf( output_dir=str(outdir), write_coeff=True, @@ -60,8 +59,7 @@ def test_phdiff(tmpdir, datadir, workdir, outdir, fmap_path): fmap_derivatives_wf.inputs.inputnode.fmap_meta = [f for _, f in fieldmaps] fmap_reports_wf = init_fmap_reports_wf( - output_dir=str(outdir), - fmap_type=mode if len(fieldmaps) == 1 else "phases", + output_dir=str(outdir), fmap_type=mode if len(fieldmaps) == 1 else "phases", ) fmap_reports_wf.inputs.inputnode.source_files = [f for f, _ in fieldmaps] diff --git a/sdcflows/workflows/outputs.py b/sdcflows/workflows/outputs.py index d156843c29..255002f526 100644 --- a/sdcflows/workflows/outputs.py +++ b/sdcflows/workflows/outputs.py @@ -296,6 +296,5 @@ def _selectintent(metadata): from bids.utils import listify return sorted( - set([el for m in listify(metadata) - for el in listify(m.get("IntendedFor", []))]) + set([el for m in listify(metadata) for el in listify(m.get("IntendedFor", []))]) ) diff --git a/sdcflows/workflows/tests/test_base.py b/sdcflows/workflows/tests/test_base.py index fe50e98698..8c3126775f 100644 --- a/sdcflows/workflows/tests/test_base.py +++ b/sdcflows/workflows/tests/test_base.py @@ -1,15 +1,19 @@ """Test the base workflow.""" +import os import pytest from ... import fieldmaps as fm from ...utils.wrangler import find_estimators from ..base import init_fmap_preproc_wf +@pytest.mark.skipif(os.getenv("TRAVIS") == "true", reason="this is TravisCI") +@pytest.mark.skipif(os.getenv("GITHUB_ACTIONS") == "true", reason="this is GH Actions") @pytest.mark.parametrize( "dataset,subject", [("ds000054", "100185"), ("HCP101006", "101006")] ) def test_fmap_wf(tmpdir, workdir, outdir, bids_layouts, dataset, subject): """Test the encompassing of the wrangler and the workflow creator.""" + outdir = outdir / "test_base" / dataset fm._estimators.clear() estimators = find_estimators(bids_layouts[dataset], subject=subject) wf = init_fmap_preproc_wf( @@ -20,6 +24,9 @@ def test_fmap_wf(tmpdir, workdir, outdir, bids_layouts, dataset, subject): debug=True, ) + # PEPOLAR and fieldmap-less solutions typically cannot work directly on the + # raw inputs. Normally, some ad-hoc massaging and pre-processing is required. + # For that reason, the inputs cannot be set implicitly by init_fmap_preproc_wf. for estimator in estimators: if estimator.method != fm.EstimatorType.PEPOLAR: continue From 1a50b2a85f18bb416dfcc800cf1d716736837bcd Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Fri, 4 Dec 2020 14:13:51 +0100 Subject: [PATCH 65/68] sty: black the whole project --- .circleci/version.py | 1 + docs/tools/apigen.py | 230 ++++++++++-------- docs/tools/buildmodref.py | 24 +- sdcflows/__init__.py | 1 + sdcflows/conftest.py | 25 +- sdcflows/interfaces/bspline.py | 6 +- sdcflows/interfaces/reportlets.py | 94 ++++--- sdcflows/interfaces/tests/test_bspline.py | 6 +- sdcflows/interfaces/tests/test_utils.py | 22 +- sdcflows/interfaces/utils.py | 4 +- sdcflows/utils/phasemanip.py | 9 +- sdcflows/utils/tests/test_phasemanip.py | 17 +- sdcflows/viz/utils.py | 54 ++-- .../workflows/apply/tests/test_correct.py | 6 +- .../apply/tests/test_registration.py | 6 +- setup.cfg | 1 + setup.py | 5 +- tools/cache_templateflow.py | 6 +- tools/update_requirements.py | 10 +- 19 files changed, 273 insertions(+), 254 deletions(-) diff --git a/.circleci/version.py b/.circleci/version.py index a65be4e30c..8bbac435c3 100644 --- a/.circleci/version.py +++ b/.circleci/version.py @@ -1,3 +1,4 @@ """Get sdcflows version.""" import sdcflows + print(sdcflows.__version__, end="", file=open("/tmp/.docker-version.txt", "w")) diff --git a/docs/tools/apigen.py b/docs/tools/apigen.py index 758671fa3b..716fd488f0 100644 --- a/docs/tools/apigen.py +++ b/docs/tools/apigen.py @@ -27,21 +27,23 @@ # suppress print statements (warnings for empty files) DEBUG = True + class ApiDocWriter(object): - ''' Class for automatic detection and parsing of API docs - to Sphinx-parsable reST format''' + """ Class for automatic detection and parsing of API docs + to Sphinx-parsable reST format""" # only separating first two levels - rst_section_levels = ['*', '=', '-', '~', '^'] - - def __init__(self, - package_name, - rst_extension='.txt', - package_skip_patterns=None, - module_skip_patterns=None, - other_defines = True - ): - ''' Initialize package for parsing + rst_section_levels = ["*", "=", "-", "~", "^"] + + def __init__( + self, + package_name, + rst_extension=".txt", + package_skip_patterns=None, + module_skip_patterns=None, + other_defines=True, + ): + """ Initialize package for parsing Parameters ---------- @@ -69,11 +71,11 @@ def __init__(self, other_defines : {True, False}, optional Whether to include classes and functions that are imported in a particular module but not defined there. - ''' + """ if package_skip_patterns is None: - package_skip_patterns = ['\\.tests$'] + package_skip_patterns = ["\\.tests$"] if module_skip_patterns is None: - module_skip_patterns = ['\\.setup$', '\\._'] + module_skip_patterns = ["\\.setup$", "\\._"] self.package_name = package_name self.rst_extension = rst_extension self.package_skip_patterns = package_skip_patterns @@ -84,7 +86,7 @@ def get_package_name(self): return self._package_name def set_package_name(self, package_name): - ''' Set package_name + """ Set package_name >>> docwriter = ApiDocWriter('sphinx') >>> import sphinx @@ -94,26 +96,27 @@ def set_package_name(self, package_name): >>> import docutils >>> docwriter.root_path == docutils.__path__[0] True - ''' + """ # It's also possible to imagine caching the module parsing here self._package_name = package_name root_module = self._import(package_name) self.root_path = root_module.__path__[-1] self.written_modules = None - package_name = property(get_package_name, set_package_name, None, - 'get/set package_name') + package_name = property( + get_package_name, set_package_name, None, "get/set package_name" + ) def _import(self, name): - ''' Import namespace package ''' + """ Import namespace package """ mod = __import__(name) - components = name.split('.') + components = name.split(".") for comp in components[1:]: mod = getattr(mod, comp) return mod def _get_object_name(self, line): - ''' Get second token in line + """ Get second token in line >>> docwriter = ApiDocWriter('sphinx') >>> docwriter._get_object_name(" def func(): ") 'func' @@ -121,14 +124,14 @@ def _get_object_name(self, line): 'Klass' >>> docwriter._get_object_name(" class Klass: ") 'Klass' - ''' - name = line.split()[1].split('(')[0].strip() + """ + name = line.split()[1].split("(")[0].strip() # in case we have classes which are not derived from object # ie. old style classes - return name.rstrip(':') + return name.rstrip(":") def _uri2path(self, uri): - ''' Convert uri to absolute filepath + """ Convert uri to absolute filepath Parameters ---------- @@ -154,38 +157,38 @@ def _uri2path(self, uri): True >>> docwriter._uri2path('sphinx.does_not_exist') - ''' + """ if uri == self.package_name: - return os.path.join(self.root_path, '__init__.py') - path = uri.replace(self.package_name + '.', '') - path = path.replace('.', os.path.sep) + return os.path.join(self.root_path, "__init__.py") + path = uri.replace(self.package_name + ".", "") + path = path.replace(".", os.path.sep) path = os.path.join(self.root_path, path) # XXX maybe check for extensions as well? - if os.path.exists(path + '.py'): # file - path += '.py' - elif os.path.exists(os.path.join(path, '__init__.py')): - path = os.path.join(path, '__init__.py') + if os.path.exists(path + ".py"): # file + path += ".py" + elif os.path.exists(os.path.join(path, "__init__.py")): + path = os.path.join(path, "__init__.py") else: return None return path def _path2uri(self, dirpath): - ''' Convert directory path to uri ''' - package_dir = self.package_name.replace('.', os.path.sep) + """ Convert directory path to uri """ + package_dir = self.package_name.replace(".", os.path.sep) relpath = dirpath.replace(self.root_path, package_dir) if relpath.startswith(os.path.sep): relpath = relpath[1:] - return relpath.replace(os.path.sep, '.') + return relpath.replace(os.path.sep, ".") def _parse_module(self, uri): - ''' Parse module defined in *uri* ''' + """ Parse module defined in *uri* """ filename = self._uri2path(uri) if filename is None: - print(filename, 'erk') + print(filename, "erk") # nothing that we could handle here. - return ([],[]) + return ([], []) - f = open(filename, 'rt') + f = open(filename, "rt") functions, classes = self._parse_lines(f) f.close() return functions, classes @@ -208,7 +211,7 @@ def _parse_module_with_import(self, uri): """ mod = __import__(uri, fromlist=[uri]) # find all public objects in the module. - obj_strs = [obj for obj in dir(mod) if not obj.startswith('_')] + obj_strs = [obj for obj in dir(mod) if not obj.startswith("_")] functions = [] classes = [] for obj_str in obj_strs: @@ -220,9 +223,11 @@ def _parse_module_with_import(self, uri): if not self.other_defines and not getmodule(obj) == mod: continue # figure out if obj is a function or class - if hasattr(obj, 'func_name') or \ - isinstance(obj, BuiltinFunctionType) or \ - isinstance(obj, FunctionType): + if ( + hasattr(obj, "func_name") + or isinstance(obj, BuiltinFunctionType) + or isinstance(obj, FunctionType) + ): functions.append(obj_str) else: try: @@ -234,19 +239,19 @@ def _parse_module_with_import(self, uri): return functions, classes def _parse_lines(self, linesource): - ''' Parse lines of text for functions and classes ''' + """ Parse lines of text for functions and classes """ functions = [] classes = [] for line in linesource: - if line.startswith('def ') and line.count('('): + if line.startswith("def ") and line.count("("): # exclude private stuff name = self._get_object_name(line) - if not name.startswith('_'): + if not name.startswith("_"): functions.append(name) - elif line.startswith('class '): + elif line.startswith("class "): # exclude private stuff name = self._get_object_name(line) - if not name.startswith('_'): + if not name.startswith("_"): classes.append(name) else: pass @@ -255,7 +260,7 @@ def _parse_lines(self, linesource): return functions, classes def generate_api_doc(self, uri): - '''Make autodoc documentation template string for a module + """Make autodoc documentation template string for a module Parameters ---------- @@ -268,57 +273,63 @@ def generate_api_doc(self, uri): Module name, table of contents. body : string Function and class docstrings. - ''' + """ # get the names of all classes and functions functions, classes = self._parse_module_with_import(uri) if not len(functions) and not len(classes) and DEBUG: - print('WARNING: Empty -', uri) # dbg + print("WARNING: Empty -", uri) # dbg # Make a shorter version of the uri that omits the package name for # titles - uri_short = re.sub(r'^%s\.' % self.package_name,'',uri) + uri_short = re.sub(r"^%s\." % self.package_name, "", uri) - head = '.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n' - body = '' + head = ".. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n" + body = "" # Set the chapter title to read 'module' for all modules except for the # main packages - if '.' in uri_short: - title = 'Module: :mod:`' + uri_short + '`' - head += title + '\n' + self.rst_section_levels[2] * len(title) + if "." in uri_short: + title = "Module: :mod:`" + uri_short + "`" + head += title + "\n" + self.rst_section_levels[2] * len(title) else: - title = ':mod:`' + uri_short + '`' - head += title + '\n' + self.rst_section_levels[1] * len(title) + title = ":mod:`" + uri_short + "`" + head += title + "\n" + self.rst_section_levels[1] * len(title) - head += '\n.. automodule:: ' + uri + '\n' - head += '\n.. currentmodule:: ' + uri + '\n' - body += '\n.. currentmodule:: ' + uri + '\n\n' + head += "\n.. automodule:: " + uri + "\n" + head += "\n.. currentmodule:: " + uri + "\n" + body += "\n.. currentmodule:: " + uri + "\n\n" for c in classes: - body += '\n:class:`' + c + '`\n' \ - + self.rst_section_levels[3] * \ - (len(c)+9) + '\n\n' - body += '\n.. autoclass:: ' + c + '\n' + body += ( + "\n:class:`" + + c + + "`\n" + + self.rst_section_levels[3] * (len(c) + 9) + + "\n\n" + ) + body += "\n.. autoclass:: " + c + "\n" # must NOT exclude from index to keep cross-refs working - body += ' :members:\n' \ - ' :undoc-members:\n' \ - ' :show-inheritance:\n' \ - '\n' \ - ' .. automethod:: __init__\n\n' - head += '.. autosummary::\n\n' + body += ( + " :members:\n" + " :undoc-members:\n" + " :show-inheritance:\n" + "\n" + " .. automethod:: __init__\n\n" + ) + head += ".. autosummary::\n\n" for f in classes + functions: - head += ' ' + f + '\n' - head += '\n' + head += " " + f + "\n" + head += "\n" for f in functions: # must NOT exclude from index to keep cross-refs working - body += f + '\n' - body += self.rst_section_levels[3] * len(f) + '\n' - body += '\n.. autofunction:: ' + f + '\n\n' + body += f + "\n" + body += self.rst_section_levels[3] * len(f) + "\n" + body += "\n.. autofunction:: " + f + "\n\n" return head, body def _survives_exclude(self, matchstr, match_type): - ''' Returns True if *matchstr* does not match patterns + """ Returns True if *matchstr* does not match patterns ``self.package_name`` removed from front of string if present @@ -337,14 +348,13 @@ def _survives_exclude(self, matchstr, match_type): >>> dw.module_skip_patterns.append('^\\.badmod$') >>> dw._survives_exclude('sphinx.badmod', 'module') False - ''' - if match_type == 'module': + """ + if match_type == "module": patterns = self.module_skip_patterns - elif match_type == 'package': + elif match_type == "package": patterns = self.package_skip_patterns else: - raise ValueError('Cannot interpret match type "%s"' - % match_type) + raise ValueError('Cannot interpret match type "%s"' % match_type) # Match to URI without package name L = len(self.package_name) if matchstr[:L] == self.package_name: @@ -360,7 +370,7 @@ def _survives_exclude(self, matchstr, match_type): return True def discover_modules(self): - ''' Return module sequence discovered from ``self.package_name`` + """ Return module sequence discovered from ``self.package_name`` Parameters @@ -382,38 +392,42 @@ def discover_modules(self): >>> 'sphinx.util' in dw.discover_modules() False >>> - ''' + """ modules = [self.package_name] # raw directory parsing for dirpath, dirnames, filenames in os.walk(self.root_path): # Check directory names for packages - root_uri = self._path2uri(os.path.join(self.root_path, - dirpath)) + root_uri = self._path2uri(os.path.join(self.root_path, dirpath)) # Normally, we'd only iterate over dirnames, but since # dipy does not import a whole bunch of modules we'll # include those here as well (the *.py filenames). - filenames = [f[:-3] for f in filenames if - f.endswith('.py') and not f.startswith('__init__')] + filenames = [ + f[:-3] + for f in filenames + if f.endswith(".py") and not f.startswith("__init__") + ] for filename in filenames: - package_uri = '/'.join((dirpath, filename)) + package_uri = "/".join((dirpath, filename)) for subpkg_name in dirnames + filenames: - package_uri = '.'.join((root_uri, subpkg_name)) + package_uri = ".".join((root_uri, subpkg_name)) package_path = self._uri2path(package_uri) - if (package_path and - self._survives_exclude(package_uri, 'package')): + if package_path and self._survives_exclude(package_uri, "package"): modules.append(package_uri) return sorted(modules) def write_modules_api(self, modules, outdir): # upper-level modules - main_module = modules[0].split('.')[0] - ulms = ['.'.join(m.split('.')[:2]) if m.count('.') >= 1 - else m.split('.')[0] for m in modules] + main_module = modules[0].split(".")[0] + ulms = [ + ".".join(m.split(".")[:2]) if m.count(".") >= 1 else m.split(".")[0] + for m in modules + ] from collections import OrderedDict + module_by_ulm = OrderedDict() for v, k in zip(modules, ulms): @@ -438,7 +452,7 @@ def write_modules_api(self, modules, outdir): out_module = ulm + self.rst_extension outfile = os.path.join(outdir, out_module) - fileobj = open(outfile, 'wt') + fileobj = open(outfile, "wt") fileobj.writelines(document_head + document_body) fileobj.close() @@ -467,9 +481,9 @@ def write_api_docs(self, outdir): os.mkdir(outdir) # compose list of modules modules = self.discover_modules() - self.write_modules_api(modules,outdir) + self.write_modules_api(modules, outdir) - def write_index(self, outdir, froot='gen', relative_to=None): + def write_index(self, outdir, froot="gen", relative_to=None): """Make a reST API index file from written files Parameters @@ -488,22 +502,22 @@ def write_index(self, outdir, froot='gen', relative_to=None): leave path as it is. """ if self.written_modules is None: - raise ValueError('No modules written') + raise ValueError("No modules written") # Get full filename path - path = os.path.join(outdir, froot+self.rst_extension) + path = os.path.join(outdir, froot + self.rst_extension) # Path written into index is relative to rootpath if relative_to is not None: - relpath = (outdir + os.path.sep).replace(relative_to + os.path.sep, '') + relpath = (outdir + os.path.sep).replace(relative_to + os.path.sep, "") else: relpath = outdir - idx = open(path,'wt') + idx = open(path, "wt") w = idx.write - w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n') + w(".. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n") title = "API Reference" w(title + "\n") w("=" * len(title) + "\n\n") - w('.. toctree::\n\n') + w(".. toctree::\n\n") for f in self.written_modules: - w(' %s\n' % os.path.join(relpath,f)) + w(" %s\n" % os.path.join(relpath, f)) idx.close() diff --git a/docs/tools/buildmodref.py b/docs/tools/buildmodref.py index e2a7b9f614..769c696bf1 100755 --- a/docs/tools/buildmodref.py +++ b/docs/tools/buildmodref.py @@ -13,10 +13,11 @@ # version comparison from distutils.version import LooseVersion as V -#***************************************************************************** +# ***************************************************************************** + def abort(error): - print('*WARNING* API documentation not generated: %s' % error) + print("*WARNING* API documentation not generated: %s" % error) exit() @@ -40,18 +41,19 @@ def writeapi(package, outdir, source_version, other_defines=True): if source_version != installed_version: abort("Installed version does not match source version") - docwriter = ApiDocWriter(package, rst_extension='.rst', - other_defines=other_defines) + docwriter = ApiDocWriter(package, rst_extension=".rst", other_defines=other_defines) - docwriter.package_skip_patterns += [r'\.%s$' % package, - r'.*test.*$', - r'\.version.*$'] + docwriter.package_skip_patterns += [ + r"\.%s$" % package, + r".*test.*$", + r"\.version.*$", + ] docwriter.write_api_docs(outdir) - docwriter.write_index(outdir, 'index', relative_to=outdir) - print('%d files written' % len(docwriter.written_modules)) + docwriter.write_index(outdir, "index", relative_to=outdir) + print("%d files written" % len(docwriter.written_modules)) -if __name__ == '__main__': +if __name__ == "__main__": package = sys.argv[1] outdir = sys.argv[2] try: @@ -59,6 +61,6 @@ def writeapi(package, outdir, source_version, other_defines=True): except IndexError: other_defines = True else: - other_defines = other_defines in ('True', 'true', '1') + other_defines = other_defines in ("True", "true", "1") writeapi(package, outdir, other_defines=other_defines) diff --git a/sdcflows/__init__.py b/sdcflows/__init__.py index 74fe6e44e9..2c42248b3f 100644 --- a/sdcflows/__init__.py +++ b/sdcflows/__init__.py @@ -5,6 +5,7 @@ from ._version import __version__ except ModuleNotFoundError: from pkg_resources import get_distribution, DistributionNotFound + try: __version__ = get_distribution(__packagename__).version except DistributionNotFound: diff --git a/sdcflows/conftest.py b/sdcflows/conftest.py index b68fe8215e..e837168818 100644 --- a/sdcflows/conftest.py +++ b/sdcflows/conftest.py @@ -6,12 +6,15 @@ import pytest from bids.layout import BIDSLayout -test_data_env = os.getenv('TEST_DATA_HOME', str(Path.home() / 'sdcflows-tests')) -test_output_dir = os.getenv('TEST_OUTPUT_DIR') -test_workdir = os.getenv('TEST_WORK_DIR') +test_data_env = os.getenv("TEST_DATA_HOME", str(Path.home() / "sdcflows-tests")) +test_output_dir = os.getenv("TEST_OUTPUT_DIR") +test_workdir = os.getenv("TEST_WORK_DIR") -layouts = {p.name: BIDSLayout(str(p), validate=False, derivatives=True) - for p in Path(test_data_env).glob('*') if p.is_dir()} +layouts = { + p.name: BIDSLayout(str(p), validate=False, derivatives=True) + for p in Path(test_data_env).glob("*") + if p.is_dir() +} data_dir = Path(__file__).parent / "tests" / "data" / "dsA" @@ -29,15 +32,15 @@ def pytest_report_header(config): @pytest.fixture(autouse=True) def add_np(doctest_namespace): - doctest_namespace['np'] = numpy - doctest_namespace['nb'] = nibabel - doctest_namespace['os'] = os - doctest_namespace['Path'] = Path - doctest_namespace['layouts'] = layouts + doctest_namespace["np"] = numpy + doctest_namespace["nb"] = nibabel + doctest_namespace["os"] = os + doctest_namespace["Path"] = Path + doctest_namespace["layouts"] = layouts for key, val in list(layouts.items()): doctest_namespace[key] = Path(val.root) - doctest_namespace['testdata_dir'] = data_dir + doctest_namespace["testdata_dir"] = data_dir @pytest.fixture diff --git a/sdcflows/interfaces/bspline.py b/sdcflows/interfaces/bspline.py index ddf046d6f1..0875986d72 100644 --- a/sdcflows/interfaces/bspline.py +++ b/sdcflows/interfaces/bspline.py @@ -19,9 +19,9 @@ LOW_MEM_BLOCK_SIZE = 1000 -DEFAULT_ZOOMS_MM = (40.0, 40.0, 20.0) # For human adults (mid-frequency), in mm +DEFAULT_ZOOMS_MM = (40.0, 40.0, 20.0) # For human adults (mid-frequency), in mm DEFAULT_LF_ZOOMS_MM = (100.0, 100.0, 40.0) # For human adults (low-frequency), in mm -DEFAULT_HF_ZOOMS_MM = (16.0, 16.0, 10.0) # For human adults (high-frequency), in mm +DEFAULT_HF_ZOOMS_MM = (16.0, 16.0, 10.0) # For human adults (high-frequency), in mm class _BSplineApproxInputSpec(BaseInterfaceInputSpec): @@ -151,7 +151,7 @@ def _run_interface(self, runtime): for i, (n, bsl) in enumerate(zip(ncoeff, bs_levels)): out_level = out_name.replace("_field.", f"_coeff{i:03}.") bsl.__class__( - np.array(model.coef_, dtype="float32")[index:index + n].reshape( + np.array(model.coef_, dtype="float32")[index : index + n].reshape( bsl.shape ), bsl.affine, diff --git a/sdcflows/interfaces/reportlets.py b/sdcflows/interfaces/reportlets.py index 4953744981..33b5207e9f 100644 --- a/sdcflows/interfaces/reportlets.py +++ b/sdcflows/interfaces/reportlets.py @@ -14,19 +14,23 @@ class _FieldmapReportletInputSpec(reporting.ReportCapableInputSpec): - reference = File(exists=True, mandatory=True, desc='input reference') - moving = File(exists=True, desc='input moving') - fieldmap = File(exists=True, mandatory=True, desc='input fieldmap') - max_alpha = traits.Float(0.7, usedefault=True, desc='maximum alpha channel') - mask = File(exists=True, desc='brain mask') - out_report = File('report.svg', usedefault=True, - desc='filename for the visual report') - show = traits.Enum(1, 0, 'both', usedefault=True, - desc='where the fieldmap should be shown') - reference_label = traits.Str('Reference', usedefault=True, - desc='a label name for the reference mosaic') - moving_label = traits.Str('Fieldmap (Hz)', usedefault=True, - desc='a label name for the reference mosaic') + reference = File(exists=True, mandatory=True, desc="input reference") + moving = File(exists=True, desc="input moving") + fieldmap = File(exists=True, mandatory=True, desc="input fieldmap") + max_alpha = traits.Float(0.7, usedefault=True, desc="maximum alpha channel") + mask = File(exists=True, desc="brain mask") + out_report = File( + "report.svg", usedefault=True, desc="filename for the visual report" + ) + show = traits.Enum( + 1, 0, "both", usedefault=True, desc="where the fieldmap should be shown" + ) + reference_label = traits.Str( + "Reference", usedefault=True, desc="a label name for the reference mosaic" + ) + moving_label = traits.Str( + "Fieldmap (Hz)", usedefault=True, desc="a label name for the reference mosaic" + ) apply_mask = traits.Bool(False, usedefault=True, desc="zero values outside mask") @@ -39,7 +43,7 @@ class FieldmapReportlet(reporting.ReportCapableInterface): def __init__(self, **kwargs): """Instantiate FieldmapReportlet.""" - self._n_cuts = kwargs.pop('n_cuts', self._n_cuts) + self._n_cuts = kwargs.pop("n_cuts", self._n_cuts) super(FieldmapReportlet, self).__init__(generate_report=True, **kwargs) def _run_interface(self, runtime): @@ -47,7 +51,7 @@ def _run_interface(self, runtime): def _generate_report(self): """Generate a reportlet.""" - NIWORKFLOWS_LOG.info('Generating visual report') + NIWORKFLOWS_LOG.info("Generating visual report") movnii = load_img(self.inputs.reference) canonical_r = rotation2canonical(movnii) @@ -75,39 +79,49 @@ def _generate_report(self): maskdata = mask_nii.get_fdata() > 0 cuts = cuts_from_bbox(contour_nii or mask_nii, cuts=self._n_cuts) fmapdata = fmapnii.get_fdata() - vmax = max(abs(np.percentile(fmapdata[maskdata], 99.8)), - abs(np.percentile(fmapdata[maskdata], 0.2))) + vmax = max( + abs(np.percentile(fmapdata[maskdata], 99.8)), + abs(np.percentile(fmapdata[maskdata], 0.2)), + ) if self.inputs.apply_mask: fmapdata[~maskdata] = 0 fmapnii = fmapnii.__class__(fmapdata, fmapnii.affine, fmapnii.header) - fmap_overlay = [{ - 'overlay': fmapnii, - 'overlay_params': { - 'cmap': coolwarm_transparent(max_alpha=self.inputs.max_alpha), - 'vmax': vmax, - 'vmin': -vmax, + fmap_overlay = [ + { + "overlay": fmapnii, + "overlay_params": { + "cmap": coolwarm_transparent(max_alpha=self.inputs.max_alpha), + "vmax": vmax, + "vmin": -vmax, + }, } - }] * 2 + ] * 2 - if self.inputs.show != 'both': + if self.inputs.show != "both": fmap_overlay[not self.inputs.show] = {} # Call composer compose_view( - plot_registration(movnii, 'moving-image', - estimate_brightness=True, - cuts=cuts, - label=self.inputs.moving_label, - contour=contour_nii, - compress=False, - **fmap_overlay[1]), - plot_registration(refnii, 'fixed-image', - estimate_brightness=True, - cuts=cuts, - label=self.inputs.reference_label, - contour=contour_nii, - compress=False, - **fmap_overlay[0]), - out_file=self._out_report + plot_registration( + movnii, + "moving-image", + estimate_brightness=True, + cuts=cuts, + label=self.inputs.moving_label, + contour=contour_nii, + compress=False, + **fmap_overlay[1] + ), + plot_registration( + refnii, + "fixed-image", + estimate_brightness=True, + cuts=cuts, + label=self.inputs.reference_label, + contour=contour_nii, + compress=False, + **fmap_overlay[0] + ), + out_file=self._out_report, ) diff --git a/sdcflows/interfaces/tests/test_bspline.py b/sdcflows/interfaces/tests/test_bspline.py index 46de43b1bd..e683758d49 100644 --- a/sdcflows/interfaces/tests/test_bspline.py +++ b/sdcflows/interfaces/tests/test_bspline.py @@ -23,11 +23,7 @@ def test_bsplines(tmp_path, testnum): ) # Generate some target grid - targetnii = nb.Nifti1Image( - np.ones(targetshape), - targetaff, - None - ) + targetnii = nb.Nifti1Image(np.ones(targetshape), targetaff, None) targetnii.to_filename(tmp_path / "target.nii.gz") # Generate random coefficients diff --git a/sdcflows/interfaces/tests/test_utils.py b/sdcflows/interfaces/tests/test_utils.py index e5715e8490..651b3d4233 100644 --- a/sdcflows/interfaces/tests/test_utils.py +++ b/sdcflows/interfaces/tests/test_utils.py @@ -10,15 +10,9 @@ def test_Flatten(tmpdir): """Test the flattening interface.""" tmpdir.chdir() shape = (5, 5, 5) - nb.Nifti1Image( - np.zeros(shape), np.eye(4), None - ).to_filename("file1.nii.gz") - nb.Nifti1Image( - np.zeros((*shape, 6)), np.eye(4), None - ).to_filename("file2.nii.gz") - nb.Nifti1Image( - np.zeros((*shape, 2)), np.eye(4), None - ).to_filename("file3.nii.gz") + nb.Nifti1Image(np.zeros(shape), np.eye(4), None).to_filename("file1.nii.gz") + nb.Nifti1Image(np.zeros((*shape, 6)), np.eye(4), None).to_filename("file2.nii.gz") + nb.Nifti1Image(np.zeros((*shape, 2)), np.eye(4), None).to_filename("file3.nii.gz") out = Flatten( in_data=["file1.nii.gz", "file2.nii.gz", "file3.nii.gz"], @@ -34,16 +28,14 @@ def test_Flatten(tmpdir): assert out_meta[4] == out_meta[5] == {"c": 3} -@pytest.mark.parametrize("shape", [ - (10, 10, 10, 1, 3), - (10, 10, 10, 3) -]) +@pytest.mark.parametrize("shape", [(10, 10, 10, 1, 3), (10, 10, 10, 3)]) def test_ConvertWarp(tmpdir, shape): """Exercise the interface.""" tmpdir.chdir() - nb.Nifti1Image(np.zeros(shape, dtype="uint8"), - np.eye(4), None).to_filename("3dQwarp.nii.gz") + nb.Nifti1Image(np.zeros(shape, dtype="uint8"), np.eye(4), None).to_filename( + "3dQwarp.nii.gz" + ) out = ConvertWarp(in_file="3dQwarp.nii.gz").run() diff --git a/sdcflows/interfaces/utils.py b/sdcflows/interfaces/utils.py index 7d0200b6ed..d8ed2c28d0 100644 --- a/sdcflows/interfaces/utils.py +++ b/sdcflows/interfaces/utils.py @@ -118,7 +118,5 @@ def _qwarp2ants(in_file, newpath=None): hdr.set_intent("vector", (), "") out_file = fname_presuffix(in_file, "_warpfield", newpath=newpath) data = np.squeeze(nii.get_fdata(dtype="float32"))[..., np.newaxis, :] - nb.Nifti1Image(data, nii.affine, hdr).to_filename( - out_file - ) + nb.Nifti1Image(data, nii.affine, hdr).to_filename(out_file) return out_file diff --git a/sdcflows/utils/phasemanip.py b/sdcflows/utils/phasemanip.py index 4624b2fc39..fa71464b31 100644 --- a/sdcflows/utils/phasemanip.py +++ b/sdcflows/utils/phasemanip.py @@ -45,9 +45,8 @@ def subtract_phases(in_phases, in_meta, newpath=None): echo_times = (echo_times[1], echo_times[0]) in_phases_nii = [nb.load(ph) for ph in in_phases] - sub_data = ( - in_phases_nii[1].get_fdata(dtype="float32") - - in_phases_nii[0].get_fdata(dtype="float32") + sub_data = in_phases_nii[1].get_fdata(dtype="float32") - in_phases_nii[0].get_fdata( + dtype="float32" ) # wrap negative radians back to [0, 2pi] @@ -133,8 +132,6 @@ def delta_te(in_values): te2 = float(te2 or "unknown") te1 = float(te1 or "unknown") except ValueError: - raise ValueError( - f"Could not interpret metadata ." - ) + raise ValueError(f"Could not interpret metadata .") return abs(te2 - te1) diff --git a/sdcflows/utils/tests/test_phasemanip.py b/sdcflows/utils/tests/test_phasemanip.py index 376f1b87f8..1c070e855d 100644 --- a/sdcflows/utils/tests/test_phasemanip.py +++ b/sdcflows/utils/tests/test_phasemanip.py @@ -11,29 +11,24 @@ def test_au2rads(tmp_path): data[0, 0, 0] = 0 data[-1, -1, -1] = 4096 - nb.Nifti1Image( - data.astype("int16"), - np.eye(4) - ).to_filename(tmp_path / "testdata.nii.gz") + nb.Nifti1Image(data.astype("int16"), np.eye(4)).to_filename( + tmp_path / "testdata.nii.gz" + ) out_file = au2rads(tmp_path / "testdata.nii.gz") assert np.allclose( (data / 4096).astype("float32") * 2.0 * np.pi, - nb.load(out_file).get_fdata(dtype="float32") + nb.load(out_file).get_fdata(dtype="float32"), ) def test_phdiff2fmap(tmp_path): """Check the conversion.""" nb.Nifti1Image( - np.ones((5, 5, 5), dtype="float32") * 2.0 * np.pi * 2.46e-3, - np.eye(4) + np.ones((5, 5, 5), dtype="float32") * 2.0 * np.pi * 2.46e-3, np.eye(4) ).to_filename(tmp_path / "testdata.nii.gz") out_file = phdiff2fmap(tmp_path / "testdata.nii.gz", 2.46e-3) - assert np.allclose( - np.ones((5, 5, 5)), - nb.load(out_file).get_fdata(dtype="float32") - ) + assert np.allclose(np.ones((5, 5, 5)), nb.load(out_file).get_fdata(dtype="float32")) diff --git a/sdcflows/viz/utils.py b/sdcflows/viz/utils.py index 102df79395..6783e50c99 100644 --- a/sdcflows/viz/utils.py +++ b/sdcflows/viz/utils.py @@ -3,10 +3,19 @@ """Visualization tooling.""" -def plot_registration(anat_nii, div_id, plot_params=None, - order=('z', 'x', 'y'), cuts=None, - estimate_brightness=False, label=None, contour=None, - compress='auto', overlay=None, overlay_params=None): +def plot_registration( + anat_nii, + div_id, + plot_params=None, + order=("z", "x", "y"), + cuts=None, + estimate_brightness=False, + label=None, + contour=None, + compress="auto", + overlay=None, + overlay_params=None, +): """ Plot the foreground and background views. @@ -29,32 +38,31 @@ def plot_registration(anat_nii, div_id, plot_params=None, out_files = [] if estimate_brightness: plot_params = robust_set_limits( - anat_nii.get_fdata(dtype='float32').reshape(-1), - plot_params) + anat_nii.get_fdata(dtype="float32").reshape(-1), plot_params + ) # Plot each cut axis for i, mode in enumerate(list(order)): - plot_params['display_mode'] = mode - plot_params['cut_coords'] = cuts[mode] + plot_params["display_mode"] = mode + plot_params["cut_coords"] = cuts[mode] if i == 0: - plot_params['title'] = label + plot_params["title"] = label else: - plot_params['title'] = None + plot_params["title"] = None # Generate nilearn figure display = plot_anat(anat_nii, **plot_params) if overlay is not None: _overlay_params = { - 'vmin': overlay.get_fdata(dtype='float32').min(), - 'vmax': overlay.get_fdata(dtype='float32').max(), - 'cmap': plt.cm.gray, - 'interpolation': 'nearest', + "vmin": overlay.get_fdata(dtype="float32").min(), + "vmax": overlay.get_fdata(dtype="float32").max(), + "cmap": plt.cm.gray, + "interpolation": "nearest", } _overlay_params.update(overlay_params) display.add_overlay(overlay, **_overlay_params) if contour is not None: - display.add_contours(contour, colors='g', levels=[0.5], - linewidths=0.5) + display.add_contours(contour, colors="g", levels=[0.5], linewidths=0.5) svg = extract_svg(display, compress=compress) display.close() @@ -62,7 +70,7 @@ def plot_registration(anat_nii, div_id, plot_params=None, # Find and replace the figure_1 id. xml_data = etree.fromstring(svg) find_text = etree.ETXPath("//{%s}g[@id='figure_1']" % SVGNS) - find_text(xml_data)[0].set('id', '%s-%s-%s' % (div_id, mode, uuid4())) + find_text(xml_data)[0].set("id", "%s-%s-%s" % (div_id, mode, uuid4())) svg_fig = SVGFigure() svg_fig.root = xml_data @@ -88,10 +96,12 @@ def coolwarm_transparent(max_alpha=0.7, opaque_perc=30, transparent_perc=8): _10perc = (cmap.N * transparent_perc) // 100 # Set alpha alpha = np.ones(cmap.N) * max_alpha - alpha[midpoint - _10perc:midpoint + _10perc] = 0 - alpha[_20perc:midpoint - _10perc - 1] = np.linspace( - max_alpha, 0, len(alpha[_20perc:midpoint - _10perc - 1])) - alpha[midpoint + _10perc:-_20perc] = np.linspace( - 0, max_alpha, len(alpha[midpoint + _10perc:-_20perc])) + alpha[midpoint - _10perc : midpoint + _10perc] = 0 + alpha[_20perc : midpoint - _10perc - 1] = np.linspace( + max_alpha, 0, len(alpha[_20perc : midpoint - _10perc - 1]) + ) + alpha[midpoint + _10perc : -_20perc] = np.linspace( + 0, max_alpha, len(alpha[midpoint + _10perc : -_20perc]) + ) my_cmap[:, -1] = alpha return ListedColormap(my_cmap) diff --git a/sdcflows/workflows/apply/tests/test_correct.py b/sdcflows/workflows/apply/tests/test_correct.py index 3774eeefb5..56acf1f0cd 100644 --- a/sdcflows/workflows/apply/tests/test_correct.py +++ b/sdcflows/workflows/apply/tests/test_correct.py @@ -22,11 +22,7 @@ def test_unwarp_wf(tmpdir, datadir, workdir, outdir): ) magnitude = ( - datadir - / "HCP101006" - / "sub-101006" - / "fmap" - / "sub-101006_magnitude1.nii.gz" + datadir / "HCP101006" / "sub-101006" / "fmap" / "sub-101006_magnitude1.nii.gz" ) fmap_ref_wf = init_magnitude_wf(2, name="fmap_ref_wf") fmap_ref_wf.inputs.inputnode.magnitude = magnitude diff --git a/sdcflows/workflows/apply/tests/test_registration.py b/sdcflows/workflows/apply/tests/test_registration.py index bca74d30f3..592ec762a4 100644 --- a/sdcflows/workflows/apply/tests/test_registration.py +++ b/sdcflows/workflows/apply/tests/test_registration.py @@ -22,11 +22,7 @@ def test_registration_wf(tmpdir, datadir, workdir, outdir): ) magnitude = ( - datadir - / "HCP101006" - / "sub-101006" - / "fmap" - / "sub-101006_magnitude1.nii.gz" + datadir / "HCP101006" / "sub-101006" / "fmap" / "sub-101006_magnitude1.nii.gz" ) fmap_ref_wf = init_magnitude_wf(2, name="fmap_ref_wf") fmap_ref_wf.inputs.inputnode.magnitude = magnitude diff --git a/setup.cfg b/setup.cfg index 4e54959055..1b19ddd21a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -85,6 +85,7 @@ doctests = False ignore = W503 E231 + E203 exclude = *build/ docs/sphinxext/ diff --git a/setup.py b/setup.py index 0e0c6596e0..2e2abd09f2 100644 --- a/setup.py +++ b/setup.py @@ -8,8 +8,9 @@ def main(): """Install entry-point.""" from setuptools import setup - setup(name='sdcflows') + setup(name="sdcflows") -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/tools/cache_templateflow.py b/tools/cache_templateflow.py index 18458df156..4397b2e3e9 100644 --- a/tools/cache_templateflow.py +++ b/tools/cache_templateflow.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 from templateflow import api as tfapi -tfapi.get('MNI152NLin2009cAsym', resolution=2, desc='brain', suffix='mask') -tfapi.get('MNI152NLin2009cAsym', resolution=1, label='brain', suffix='probseg') -tfapi.get('MNI152NLin2009cAsym', resolution=2, desc='fMRIPrep', suffix='boldref') +tfapi.get("MNI152NLin2009cAsym", resolution=2, desc="brain", suffix="mask") +tfapi.get("MNI152NLin2009cAsym", resolution=1, label="brain", suffix="probseg") +tfapi.get("MNI152NLin2009cAsym", resolution=2, desc="fMRIPrep", suffix="boldref") diff --git a/tools/update_requirements.py b/tools/update_requirements.py index 8eb202b8fa..9dd4e9af20 100755 --- a/tools/update_requirements.py +++ b/tools/update_requirements.py @@ -11,8 +11,10 @@ config = ConfigParser() config.read(setup_cfg) -requirements = [Requirement(req) - for req in config.get("options", "install_requires").strip().splitlines()] +requirements = [ + Requirement(req) + for req in config.get("options", "install_requires").strip().splitlines() +] script_name = Path(__file__).relative_to(repo_root) @@ -20,8 +22,8 @@ def to_min(req): if req.specifier: req = copy(req) - min_spec = [spec for spec in req.specifier if spec.operator in ('>=', '~=')][0] - min_spec._spec = ('==', ) + min_spec._spec[1:] + min_spec = [spec for spec in req.specifier if spec.operator in (">=", "~=")][0] + min_spec._spec = ("==",) + min_spec._spec[1:] req.specifier = SpecifierSet(str(min_spec)) return req From d33f7fc7a46f6a288052b3563778da1dae5e0438 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Fri, 4 Dec 2020 14:32:36 +0100 Subject: [PATCH 66/68] rel(1.4.0rc1): Update CHANGES.rst --- CHANGES.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 4fda05ae9d..2d6dc3de16 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -43,12 +43,16 @@ Some of the most prominent pull-requests conducive to this release are: * FIX: Convert SEI fieldmaps given in rad/s into Hz (#127) * FIX: Limit ``3dQwarp`` to maximum 4 CPUs for stability reasons (#128) +* ENH: Putting the new API together on a base workflow (#143) +* ENH: Autogenerate ``B0FieldIdentifiers`` from ``IntendedFor`` (#142) * ENH: Finalizing the API overhaul (#132) * ENH: Keep a registry of already-used identifiers (and auto-generate new) (#130) * ENH: New objects for better representation of fieldmap estimation (#114) * ENH: Show FieldmapReportlet oriented aligned with cardinal axes (#120) * ENH: New estimation API (featuring a TOPUP implementation!) (#115) * DOC: Enable NiPype's sphinx-extension to better render Interfaces (#131) +* MAINT: Migrate TravisCI -> GH Actions (completion) (#138) +* MAINT: Migrate TravisCI -> GH Actions (#137) * MAINT: Minimal unit test and refactor of pepolar workflow node (#133) * MAINT: Collect code coverage from tests on Circle (#122) * MAINT: Test minimum dependencies with TravisCI (#96) From afd0cf4363105d99bd6367fd31ead75bb0af379d Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Fri, 4 Dec 2020 14:50:57 +0100 Subject: [PATCH 67/68] maint: update crediting and contribution recognition [skip ci] --- .mailmap | 68 ++++++++++++++++++ .maint/contributors.json | 32 +++++++++ .maint/developers.json | 32 +++++++++ .maint/former.json | 5 ++ .maint/paper_author_list.py | 69 ++++++++++++++++++ .maint/update_zenodo.py | 135 ++++++++++++++++++++++++++++++++++++ .zenodo.json | 59 ++++++++-------- CHANGES.rst | 19 +++++ CONTRIBUTING.md | 1 + 9 files changed, 393 insertions(+), 27 deletions(-) create mode 100644 .mailmap create mode 100644 .maint/contributors.json create mode 100644 .maint/developers.json create mode 100644 .maint/former.json create mode 100644 .maint/paper_author_list.py create mode 100755 .maint/update_zenodo.py create mode 100644 CONTRIBUTING.md diff --git a/.mailmap b/.mailmap new file mode 100644 index 0000000000..72bb85757b --- /dev/null +++ b/.mailmap @@ -0,0 +1,68 @@ +Adriana Rivera-Dompenciel +Adriana Rivera-Dompenciel <34017586+ariveradompenciel@users.noreply.github.com> +Alejandro de la Vega +Alejandro de la Vega delavega4 +Anibal Sólon Heinsfeld +Asier Erramuzpe +Basile Pinsard +Basile Pinsard basile +Basile Pinsard +Blaise Frederick +Blaise Frederick bbfrederick +Christopher J. Markiewicz +Christopher J. Markiewicz +Christopher J. Markiewicz +Daniel J. Lurie +Elizabeth DuPre +Franklin Feingold +Franklin Feingold <35307458+franklin-feingold@users.noreply.github.com> +Franz Liem +Gabriel A. Devenyi +Ilkay Isik +James D. Kent +James D. Kent +James D. Kent Fred Mertz +Jarod Roland +Joseph B. Wexler +Joseph B. Wexler Joe B. Wexler +Joseph B. Wexler jbwexler +Karolina Finc +Karolina Finc kfinc +Kevin Sitek +Kevin Sitek sitek +Krzysztof J. Gorgolewski +Krzysztof J. Gorgolewski +Krzysztof J. Gorgolewski +Krzysztof J. Gorgolewski +Krzysztof J. Gorgolewski +Krzysztof J. Gorgolewski +Krzysztof J. Gorgolewski +Marc Bue <44844855+marcbue@users.noreply.github.com> +Markus H. Sneve <44242647+markushs@users.noreply.github.com> +Mathias Goncalves +Mathias Goncalves +Mathias Goncalves +Matteo Visconti di Oleggio Castello +Marc Bue <44844855+marcbue@users.noreply.github.com> +Marc Bue <44844855+marcbue@users.noreply.github.com> marcbue <44844855+marcbue@users.noreply.github.com> +Mikael Naveau +Nir Jacoby +Nir Jacoby +Noah C. Benson +Noah C. Benson noahbenson +Oscar Esteban +Oscar Esteban +Pablo Velasco +Rastko Ciric +Rastko Ciric +Rastko Ciric +Romain Valabregue +Ross Blair +Ross Blair +Saren Seeley <31291534+sarenseeley@users.noreply.github.com> +Sebastian Urchs +Sebastien Naze +Shoshana Berleant +Soichi Hayashi +William Hedley Thompson +William Hedley Thompson diff --git a/.maint/contributors.json b/.maint/contributors.json new file mode 100644 index 0000000000..8d27caf55b --- /dev/null +++ b/.maint/contributors.json @@ -0,0 +1,32 @@ +[ + { + "affiliation": "Perelman School of Medicine, University of Pennsylvania, PA, USA", + "name": "Adebimpe, Azeez", + "orcid": "0000-0001-9049-0135" + }, + { + "affiliation": "Perelman School of Medicine, University of Pennsylvania, PA, USA", + "name": "Cieslak, Matthew", + "orcid": "0000-0002-1931-4734" + }, + { + "affiliation": "Cyceron, UMS 3408 (CNRS - UCBN), France", + "name": "Naveau, Mikaël", + "orcid": "0000-0001-6948-9068" + }, + { + "affiliation": "Perelman School of Medicine, University of Pennsylvania, PA, USA", + "name": "Satterthwaite, Theodore D.", + "orcid": "0000-0001-7072-9399" + }, + { + "affiliation": "Speech & Hearing Bioscience & Technology Program, Harvard University", + "name": "Sitek, Kevin R.", + "orcid": "0000-0002-2172-5786" + }, + { + "affiliation": "Center for Lifespan Changes in Brain and Cognition, University of Oslo", + "name": "Sneve, Markus H.", + "orcid": "0000-0001-7644-7915" + } +] diff --git a/.maint/developers.json b/.maint/developers.json new file mode 100644 index 0000000000..26fc887bdb --- /dev/null +++ b/.maint/developers.json @@ -0,0 +1,32 @@ +[ + { + "affiliation": "Department of Psychology, Stanford University", + "name": "Blair, Ross W.", + "orcid": "0000-0003-3007-1056" + }, + { + "affiliation": "Dept. of Radiology, Lausanne University Hospital, University of Lausanne", + "name": "Esteban, Oscar", + "orcid": "0000-0001-8435-6191" + }, + { + "affiliation": "Department of Psychology, Stanford University", + "name": "Goncalves, Mathias", + "orcid": "0000-0002-7252-7771" + }, + { + "affiliation": "Department of Psychology, Stanford University", + "name": "Gorgolewski, Krzysztof J.", + "orcid": "0000-0003-3321-7583" + }, + { + "affiliation": "Department of Psychology, Stanford University", + "name": "Markiewicz, Christopher J.", + "orcid": "0000-0002-6533-164X" + }, + { + "affiliation": "Department of Psychology, Stanford University", + "name": "Poldrack, Russell A.", + "orcid": "0000-0001-6755-0259" + } +] \ No newline at end of file diff --git a/.maint/former.json b/.maint/former.json new file mode 100644 index 0000000000..1f1b247aaa --- /dev/null +++ b/.maint/former.json @@ -0,0 +1,5 @@ +[ + { + "name": "Berleant, Shoshana" + } +] \ No newline at end of file diff --git a/.maint/paper_author_list.py b/.maint/paper_author_list.py new file mode 100644 index 0000000000..ccdceaae30 --- /dev/null +++ b/.maint/paper_author_list.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Generate an author list for a new paper or abstract.""" +import sys +from pathlib import Path +import json +from update_zenodo import get_git_lines, sort_contributors + + +# These authors should go last +AUTHORS_LAST = ["Gorgolewski, Krzysztof J.", "Satterthwaite, Theodore D.", "Poldrack, Russell A.", "Esteban, Oscar"] + + +def _aslist(inlist): + if not isinstance(inlist, list): + return [inlist] + return inlist + + +if __name__ == "__main__": + devs = json.loads(Path(".maint/developers.json").read_text()) + contribs = json.loads(Path(".maint/contributors.json").read_text()) + + author_matches, unmatched = sort_contributors( + devs + contribs, + get_git_lines(), + exclude=json.loads(Path(".maint/former.json").read_text()), + last=AUTHORS_LAST, + ) + # Remove position + affiliations = [] + for item in author_matches: + del item["position"] + for a in _aslist(item.get("affiliation", "Unaffiliated")): + if a not in affiliations: + affiliations.append(a) + + aff_indexes = [ + ", ".join( + [ + "%d" % (affiliations.index(a) + 1) + for a in _aslist(author.get("affiliation", "Unaffiliated")) + ] + ) + for author in author_matches + ] + + print( + "Some people made commits, but are missing in .maint/ " + "files: %s." % ", ".join(unmatched), + file=sys.stderr, + ) + + print("Authors (%d):" % len(author_matches)) + print( + "%s." + % "; ".join( + [ + "%s \\ :sup:`%s`\\ " % (i["name"], idx) + for i, idx in zip(author_matches, aff_indexes) + ] + ) + ) + + print( + "\n\nAffiliations:\n%s" + % "\n".join( + ["{0: >2}. {1}".format(i + 1, a) for i, a in enumerate(affiliations)] + ) + ) diff --git a/.maint/update_zenodo.py b/.maint/update_zenodo.py new file mode 100755 index 0000000000..d238b47646 --- /dev/null +++ b/.maint/update_zenodo.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Update and sort the creators list of the zenodo record.""" +import sys +from pathlib import Path +import json +from fuzzywuzzy import fuzz, process + +# These ORCIDs should go last +CREATORS_LAST = ["Poldrack, Russell A.", "Esteban, Oscar"] +CONTRIBUTORS_LAST = ["Satterthwaite, Theodore D."] + + +def sort_contributors(entries, git_lines, exclude=None, last=None): + """Return a list of author dictionaries, ordered by contribution.""" + last = last or [] + sorted_authors = sorted(entries, key=lambda i: i["name"]) + + first_last = [ + " ".join(val["name"].split(",")[::-1]).strip() for val in sorted_authors + ] + first_last_excl = [ + " ".join(val["name"].split(",")[::-1]).strip() for val in exclude or [] + ] + + unmatched = [] + author_matches = [] + position = 1 + for ele in git_lines: + matches = process.extract( + ele, first_last, scorer=fuzz.token_sort_ratio, limit=2 + ) + # matches is a list [('First match', % Match), ('Second match', % Match)] + if matches[0][1] > 80: + val = sorted_authors[first_last.index(matches[0][0])] + else: + # skip unmatched names + if ele not in first_last_excl: + unmatched.append(ele) + continue + + if val not in author_matches: + val["position"] = position + author_matches.append(val) + position += 1 + + names = {" ".join(val["name"].split(",")[::-1]).strip() for val in author_matches} + for missing_name in first_last: + if missing_name not in names: + missing = sorted_authors[first_last.index(missing_name)] + missing["position"] = position + author_matches.append(missing) + position += 1 + + all_names = [val["name"] for val in author_matches] + for last_author in last: + author_matches[all_names.index(last_author)]["position"] = position + position += 1 + + author_matches = sorted(author_matches, key=lambda k: k["position"]) + + return author_matches, unmatched + + +def get_git_lines(fname="line-contributors.txt"): + """Run git-line-summary.""" + import shutil + import subprocess as sp + + contrib_file = Path(fname) + + lines = [] + if contrib_file.exists(): + print("WARNING: Reusing existing line-contributors.txt file.", file=sys.stderr) + lines = contrib_file.read_text().splitlines() + + git_line_summary_path = shutil.which("git-line-summary") + if not lines and git_line_summary_path: + print("Running git-line-summary on repo") + lines = sp.check_output([git_line_summary_path]).decode().splitlines() + lines = [l for l in lines if "Not Committed Yet" not in l] + contrib_file.write_text("\n".join(lines)) + + if not lines: + raise RuntimeError( + """\ +Could not find line-contributors from git repository.%s""" + % """ \ +git-line-summary not found, please install git-extras. """ + * (git_line_summary_path is None) + ) + return [" ".join(line.strip().split()[1:-1]) for line in lines if "%" in line] + + +if __name__ == "__main__": + data = get_git_lines() + + zenodo_file = Path(".zenodo.json") + zenodo = json.loads(zenodo_file.read_text()) + + creators = json.loads(Path(".maint/developers.json").read_text()) + zen_creators, miss_creators = sort_contributors( + creators, + data, + exclude=json.loads(Path(".maint/former.json").read_text()), + last=CREATORS_LAST, + ) + contributors = json.loads(Path(".maint/contributors.json").read_text()) + zen_contributors, miss_contributors = sort_contributors( + contributors, + data, + exclude=json.loads(Path(".maint/former.json").read_text()), + last=CONTRIBUTORS_LAST, + ) + zenodo["creators"] = zen_creators + zenodo["contributors"] = zen_contributors + + print( + "Some people made commits, but are missing in .maint/ " + "files: %s." % ", ".join(set(miss_creators).intersection(miss_contributors)), + file=sys.stderr, + ) + + # Remove position + for creator in zenodo["creators"]: + del creator["position"] + if isinstance(creator["affiliation"], list): + creator["affiliation"] = creator["affiliation"][0] + + for creator in zenodo["contributors"]: + creator["type"] = "Researcher" + del creator["position"] + if isinstance(creator["affiliation"], list): + creator["affiliation"] = creator["affiliation"][0] + + zenodo_file.write_text("%s\n" % json.dumps(zenodo, indent=2)) diff --git a/.zenodo.json b/.zenodo.json index c2f9c85dc9..2db7e6cf96 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -3,65 +3,70 @@ "description": "

Nipype-based neuroimaging workflows for susceptibility distortion correction (SDC) of echo-planar MRI images.

", "creators": [ { - "affiliation": "Department of Psychology, Stanford University, CA, USA", - "name": "Esteban, Oscar", - "orcid": "0000-0001-8435-6191" - }, - { - "affiliation": "Department of Psychology, Stanford University, CA, USA", + "affiliation": "Department of Psychology, Stanford University", "name": "Markiewicz, Christopher J.", "orcid": "0000-0002-6533-164X" }, { - "affiliation": "Department of Psychology, Stanford University, CA, USA", + "affiliation": "Department of Psychology, Stanford University", + "name": "Goncalves, Mathias", + "orcid": "0000-0002-7252-7771" + }, + { + "affiliation": "Department of Psychology, Stanford University", "name": "Blair, Ross W.", "orcid": "0000-0003-3007-1056" }, { - "affiliation": "Department of Psychology, Stanford University, CA, USA", + "affiliation": "Department of Psychology, Stanford University", + "name": "Gorgolewski, Krzysztof J.", + "orcid": "0000-0003-3321-7583" + }, + { + "affiliation": "Department of Psychology, Stanford University", "name": "Poldrack, Russell A.", "orcid": "0000-0001-6755-0259" }, { - "affiliation": "Department of Psychology, Stanford University, CA, USA", - "name": "Gorgolewski, Krzysztof J.", - "orcid": "0000-0003-3321-7583" + "affiliation": "Dept. of Radiology, Lausanne University Hospital, University of Lausanne", + "name": "Esteban, Oscar", + "orcid": "0000-0001-8435-6191" } ], "contributors": [ { - "name": "Kent, James D.", - "affiliation": "Neuroscience Program, University of Iowa, IA, USA", - "orcid": "0000-0002-4892-2659", + "affiliation": "Perelman School of Medicine, University of Pennsylvania, PA, USA", + "name": "Adebimpe, Azeez", + "orcid": "0000-0001-9049-0135", "type": "Researcher" }, { - "name": "Sitek, Kevin R.", - "affiliation": "Speech & Hearing Bioscience & Technology Program, Harvard University, MA, USA", - "orcid": "0000-0002-2172-5786", + "affiliation": "Perelman School of Medicine, University of Pennsylvania, PA, USA", + "name": "Cieslak, Matthew", + "orcid": "0000-0002-1931-4734", "type": "Researcher" }, { "affiliation": "Cyceron, UMS 3408 (CNRS - UCBN), France", - "name": "Naveau, Mikaël", + "name": "Naveau, Mika\u00ebl", "orcid": "0000-0001-6948-9068", "type": "Researcher" }, { - "name": "Cieslak, Matthew", - "affiliation": "Department of Neuropsychiatry, University of Pennsylvania, PA, USA", - "orcid": "0000-0002-1931-4734", + "affiliation": "Speech & Hearing Bioscience & Technology Program, Harvard University", + "name": "Sitek, Kevin R.", + "orcid": "0000-0002-2172-5786", "type": "Researcher" }, { - "name": "Adebimpe, Azeez", - "affiliation": "Department of Neuropsychiatry, University of Pennsylvania, PA, USA", - "orcid": "0000-0001-9049-0135", + "affiliation": "Center for Lifespan Changes in Brain and Cognition, University of Oslo", + "name": "Sneve, Markus H.", + "orcid": "0000-0001-7644-7915", "type": "Researcher" }, { - "name": "Satterthwaite, Theodore", - "affiliation": "Department of Neuropsychiatry, University of Pennsylvania, PA, USA", + "affiliation": "Perelman School of Medicine, University of Pennsylvania, PA, USA", + "name": "Satterthwaite, Theodore D.", "orcid": "0000-0001-7072-9399", "type": "Researcher" } @@ -88,4 +93,4 @@ } ], "upload_type": "software" -} \ No newline at end of file +} diff --git a/CHANGES.rst b/CHANGES.rst index 2d6dc3de16..cc89b66966 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -61,6 +61,25 @@ Some of the most prominent pull-requests conducive to this release are: A complete list of issues addressed by the release is found `in the GitHub repo `__. +.. admonition:: Author list for papers based on *SDCFlows* 2.0.x series + + As described in the `Contributor Guidelines + `__, + anyone listed as developer or contributor may write and submit manuscripts + about *SDCFlows*. + To do so, please move the author(s) name(s) to the front of the following list: + + Markiewicz, Christopher J. \ :sup:`1`\ ; Goncalves, Mathias \ :sup:`1`\ ; Adebimpe, Azeez \ :sup:`2`\ ; Blair, Ross W. \ :sup:`1`\ ; Cieslak, Matthew \ :sup:`2`\ ; Naveau, Mikaël \ :sup:`3`\ ; Sitek, Kevin R. \ :sup:`4`\ ; Sneve, Markus H. \ :sup:`5`\ ; Gorgolewski, Krzysztof J. \ :sup:`1`\ ; Satterthwaite, Theodore D. \ :sup:`2`\ ; Poldrack, Russell A. \ :sup:`1`\ ; Esteban, Oscar \ :sup:`6`\ . + + Affiliations: + + 1. Department of Psychology, Stanford University + 2. Perelman School of Medicine, University of Pennsylvania, PA, USA + 3. Cyceron, UMS 3408 (CNRS - UCBN), France + 4. Speech & Hearing Bioscience & Technology Program, Harvard University + 5. Center for Lifespan Changes in Brain and Cognition, University of Oslo + 6. Dept. of Radiology, Lausanne University Hospital, University of Lausanne + 1.3.x series ============ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000..f0083763da --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1 @@ +Please check the [contributing guidelines of the NiPreps organization](https://www.nipreps.org/community/CONTRIBUTING) From 91d9d7eb3e803d0725f6b68155974b368fd5566c Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Fri, 4 Dec 2020 06:19:30 -0800 Subject: [PATCH 68/68] rel(2.0.0rc1): Update version to 2.x in CHANGES [skip ci] --- CHANGES.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index cc89b66966..86468d36f5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,6 @@ -1.4.0 (TBD) +2.0.0 (TBD) =========== -The *SDCFlows* 1.4.x series are release with a comprehensive overhaul of the software's API. +The *SDCFlows* 2.0.x series are release with a comprehensive overhaul of the software's API. This overhaul has the vision of converting *SDCFlows* into some sort of subordinate pipeline to other *d/fMRIPrep*, inline with *sMRIPrep*'s approach. The idea is to consider fieldmaps a first-citizen input, for which derivatives are generated