ME-ICA · handwerkerd · Dec 11, 2019 · Dec 13, 2019 · Feb 21, 2020 · tsalo
diff --git a/.all-contributorsrc b/.all-contributorsrc
diff --git a/.circleci/config.yml b/.circleci/config.yml
diff --git a/.codecov.yml b/.codecov.yml
diff --git a/.dockerignore b/.dockerignore
diff --git a/.gitattributes b/.gitattributes
diff --git a/.github/ISSUE_TEMPLATE/issue_template.md b/.github/ISSUE_TEMPLATE/issue_template.md
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml
diff --git a/.github/stale.yml b/.github/stale.yml
diff --git a/.gitignore b/.gitignore
diff --git a/.zenodo.json b/.zenodo.json
diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
diff --git a/Dockerfile_dev b/Dockerfile_dev
diff --git a/LICENSE b/LICENSE
diff --git a/MANIFEST.in b/MANIFEST.in
diff --git a/Makefile b/Makefile
diff --git a/README.md b/README.md
diff --git a/dev_requirements.txt b/dev_requirements.txt
diff --git a/dev_tools/run_tests.sh b/dev_tools/run_tests.sh
diff --git a/docs/Makefile b/docs/Makefile
diff --git a/docs/_static/a01_echo_timeseries.png b/docs/_static/a01_echo_timeseries.png
diff --git a/docs/_static/a02_echo_value_distributions.png b/docs/_static/a02_echo_value_distributions.png
diff --git a/docs/_static/a03_adaptive_mask.png b/docs/_static/a03_adaptive_mask.png
diff --git a/docs/_static/a04_echo_log_value_distributions.png b/docs/_static/a04_echo_log_value_distributions.png
diff --git a/docs/_static/a05_loglinear_regression.png b/docs/_static/a05_loglinear_regression.png
diff --git a/docs/_static/a06_monoexponential_decay_model.png b/docs/_static/a06_monoexponential_decay_model.png
diff --git a/docs/_static/a07_monoexponential_decay_model_with_t2.png b/docs/_static/a07_monoexponential_decay_model_with_t2.png
diff --git a/docs/_static/a08_optimal_combination_echo_weights.png b/docs/_static/a08_optimal_combination_echo_weights.png
diff --git a/docs/_static/a09_optimal_combination_value_distributions.png b/docs/_static/a09_optimal_combination_value_distributions.png
diff --git a/docs/_static/a10_optimal_combination_timeseries.png b/docs/_static/a10_optimal_combination_timeseries.png
diff --git a/docs/_static/a11_pca_component_timeseries.png b/docs/_static/a11_pca_component_timeseries.png
diff --git a/docs/_static/a12_pca_reduced_data.png b/docs/_static/a12_pca_reduced_data.png
diff --git a/docs/_static/a13_ica_component_timeseries.png b/docs/_static/a13_ica_component_timeseries.png
diff --git a/docs/_static/a14_te_dependence_models_component_0.png b/docs/_static/a14_te_dependence_models_component_0.png
diff --git a/docs/_static/a14_te_dependence_models_component_1.png b/docs/_static/a14_te_dependence_models_component_1.png
diff --git a/docs/_static/a14_te_dependence_models_component_2.png b/docs/_static/a14_te_dependence_models_component_2.png
diff --git a/docs/_static/a15_denoised_data_timeseries.png b/docs/_static/a15_denoised_data_timeseries.png
diff --git a/docs/_static/a16_t1c_denoised_data_timeseries.png b/docs/_static/a16_t1c_denoised_data_timeseries.png
diff --git a/docs/_static/b01_simulated_fluctuations.png b/docs/_static/b01_simulated_fluctuations.png
diff --git a/docs/_static/b02_model_fits.png b/docs/_static/b02_model_fits.png
diff --git a/docs/_static/b03_component_timeseries.png b/docs/_static/b03_component_timeseries.png
diff --git a/docs/_static/b04_component_model_fits.png b/docs/_static/b04_component_model_fits.png
diff --git a/docs/_static/b04_echo_timeseries.png b/docs/_static/b04_echo_timeseries.png
diff --git a/docs/_static/b05_component_model_fits.png b/docs/_static/b05_component_model_fits.png
diff --git a/docs/_static/example_Component_Overview.png b/docs/_static/example_Component_Overview.png
diff --git a/docs/_static/example_Kappa_vs_Rho_Scatter.png b/docs/_static/example_Kappa_vs_Rho_Scatter.png
diff --git a/docs/_static/example_bad_component.png b/docs/_static/example_bad_component.png
diff --git a/docs/_static/example_good_component.png b/docs/_static/example_good_component.png
diff --git a/docs/_static/physics_kundu_2017_TE_dependence.jpg b/docs/_static/physics_kundu_2017_TE_dependence.jpg
diff --git a/docs/_static/physics_kundu_2017_multiple_echoes.jpg b/docs/_static/physics_kundu_2017_multiple_echoes.jpg
diff --git a/docs/_static/tedana-ohbm2018-poster.png b/docs/_static/tedana-ohbm2018-poster.png
diff --git a/docs/_static/tedana-ohbm2019-poster.png b/docs/_static/tedana-ohbm2019-poster.png
diff --git a/docs/_static/tedana-workflow.png b/docs/_static/tedana-workflow.png
diff --git a/docs/_static/tedana_favicon.png b/docs/_static/tedana_favicon.png
diff --git a/docs/_static/theme_overrides.css b/docs/_static/theme_overrides.css
diff --git a/docs/_templates/function.rst b/docs/_templates/function.rst
diff --git a/docs/_templates/module.rst b/docs/_templates/module.rst
diff --git a/docs/api.rst b/docs/api.rst
diff --git a/docs/approach.rst b/docs/approach.rst
diff --git a/docs/conf.py b/docs/conf.py
diff --git a/docs/considerations.rst b/docs/considerations.rst
diff --git a/docs/contributing.rst b/docs/contributing.rst
diff --git a/docs/dependence_metrics.rst b/docs/dependence_metrics.rst
diff --git a/docs/faq.rst b/docs/faq.rst
diff --git a/docs/index.rst b/docs/index.rst
diff --git a/docs/installation.rst b/docs/installation.rst
diff --git a/docs/make.bat b/docs/make.bat
diff --git a/docs/multi-echo.rst b/docs/multi-echo.rst
diff --git a/docs/outputs.rst b/docs/outputs.rst
diff --git a/docs/publications.rst b/docs/publications.rst
diff --git a/docs/roadmap.rst b/docs/roadmap.rst
diff --git a/docs/sphinxext/github_link.py b/docs/sphinxext/github_link.py
diff --git a/docs/support.rst b/docs/support.rst
diff --git a/docs/usage.rst b/docs/usage.rst
diff --git a/examples/plot_approach_figures.ipynb b/examples/plot_approach_figures.ipynb
diff --git a/examples/plot_metric_simulations.ipynb b/examples/plot_metric_simulations.ipynb
diff --git a/requirements.txt b/requirements.txt
diff --git a/setup.cfg b/setup.cfg
diff --git a/setup.py b/setup.py
diff --git a/tedana/__init__.py b/tedana/__init__.py
diff --git a/tedana/_version.py b/tedana/_version.py
diff --git a/tedana/combine.py b/tedana/combine.py
diff --git a/tedana/decay.py b/tedana/decay.py
diff --git a/tedana/decomposition/__init__.py b/tedana/decomposition/__init__.py
diff --git a/tedana/decomposition/_utils.py b/tedana/decomposition/_utils.py
diff --git a/tedana/decomposition/ica.py b/tedana/decomposition/ica.py
diff --git a/tedana/decomposition/ma_pca.py b/tedana/decomposition/ma_pca.py
diff --git a/tedana/decomposition/pca.py b/tedana/decomposition/pca.py
@@ -10,7 +10,7 @@
 from sklearn.decomposition import PCA
 
 from tedana import metrics, utils, io
-from tedana.decomposition import (ma_pca, _utils)
+from tedana.decomposition import ma_pca
 from tedana.stats import computefeats2
 from tedana.selection import kundu_tedpca
 from tedana.due import due, BibTeX
@@ -255,17 +255,15 @@ def tedpca(data_cat, data_oc, combmode, mask, t2s, t2sG,
         LGR.info('Computing PCA of echo #{0}'.format(','.join([str(ee) for ee in source_tes])))
         data = np.stack([data_cat[mask, ee, :] for ee in source_tes - 1], axis=1)
 
-    eim = np.squeeze(_utils.eimask(data))
-    data = np.squeeze(data[eim])
+    data = np.squeeze(data)
 
     data_z = ((data.T - data.T.mean(axis=0)) / data.T.std(axis=0)).T  # var normalize ts
     data_z = (data_z - data_z.mean()) / data_z.std()  # var normalize everything
 
     if algorithm in ['mdl', 'aic', 'kic']:
         data_img = io.new_nii_like(
-            ref_img, utils.unmask(utils.unmask(data, eim), mask))
-        mask_img = io.new_nii_like(ref_img,
-                                   utils.unmask(eim, mask).astype(int))
+            ref_img, utils.unmask(data, mask))
+        mask_img = io.new_nii_like(ref_img, mask.astype(int))
         voxel_comp_weights, varex, varex_norm, comp_ts = ma_pca.ma_pca(
             data_img, mask_img, algorithm)
     elif algorithm == 'mle':
@@ -283,12 +281,6 @@ def tedpca(data_cat, data_oc, combmode, mask, t2s, t2sG,
         varex_norm = varex / varex.sum()
 
     # Compute Kappa and Rho for PCA comps
-    eimum = np.atleast_2d(eim)
-    eimum = np.transpose(eimum, np.argsort(eimum.shape)[::-1])
-    eimum = eimum.prod(axis=1)
-    o = np.zeros((mask.shape[0], *eimum.shape[1:]))
-    o[mask, ...] = eimum
-    eimum = np.squeeze(o).astype(bool)
 
     # Normalize each component's time series
     vTmixN = stats.zscore(comp_ts, axis=0)

diff --git a/tedana/due.py b/tedana/due.py
diff --git a/tedana/gscontrol.py b/tedana/gscontrol.py
diff --git a/tedana/info.py b/tedana/info.py
diff --git a/tedana/io.py b/tedana/io.py
diff --git a/tedana/metrics/__init__.py b/tedana/metrics/__init__.py
diff --git a/tedana/metrics/kundu_fit.py b/tedana/metrics/kundu_fit.py
@@ -68,6 +68,7 @@ def dependence_metrics(catd, tsoc, mmix, t2s, tes, ref_img,
     betas : :obj:`numpy.ndarray`
     mmix_new : :obj:`numpy.ndarray`
     """
+
     # Use t2s as mask
     mask = t2s != 0
     if not (catd.shape[0] == t2s.shape[0] == mask.shape[0] == tsoc.shape[0]):

diff --git a/tedana/selection/__init__.py b/tedana/selection/__init__.py
diff --git a/tedana/selection/_utils.py b/tedana/selection/_utils.py
diff --git a/tedana/selection/tedica.py b/tedana/selection/tedica.py
diff --git a/tedana/selection/tedpca.py b/tedana/selection/tedpca.py
diff --git a/tedana/stats.py b/tedana/stats.py
diff --git a/tedana/tests/__init__.py b/tedana/tests/__init__.py
diff --git a/tedana/tests/conftest.py b/tedana/tests/conftest.py
diff --git a/tedana/tests/data/cornell_three_echo_outputs.txt b/tedana/tests/data/cornell_three_echo_outputs.txt
diff --git a/tedana/tests/data/echo1.nii.gz b/tedana/tests/data/echo1.nii.gz
diff --git a/tedana/tests/data/echo2.nii.gz b/tedana/tests/data/echo2.nii.gz
diff --git a/tedana/tests/data/echo3.nii.gz b/tedana/tests/data/echo3.nii.gz
diff --git a/tedana/tests/data/fdist.npy b/tedana/tests/data/fdist.npy
diff --git a/tedana/tests/data/fiu_four_echo_outputs.txt b/tedana/tests/data/fiu_four_echo_outputs.txt
diff --git a/tedana/tests/data/mask.nii.gz b/tedana/tests/data/mask.nii.gz
diff --git a/tedana/tests/data/nih_five_echo_outputs_verbose.txt b/tedana/tests/data/nih_five_echo_outputs_verbose.txt
diff --git a/tedana/tests/test_combine.py b/tedana/tests/test_combine.py
diff --git a/tedana/tests/test_decay.py b/tedana/tests/test_decay.py
diff --git a/tedana/tests/test_gscontrol.py b/tedana/tests/test_gscontrol.py
diff --git a/tedana/tests/test_integration.py b/tedana/tests/test_integration.py
diff --git a/tedana/tests/test_io.py b/tedana/tests/test_io.py
diff --git a/tedana/tests/test_mapca.py b/tedana/tests/test_mapca.py
diff --git a/tedana/tests/test_model_fit_dependence_metrics.py b/tedana/tests/test_model_fit_dependence_metrics.py
diff --git a/tedana/tests/test_model_kundu_metrics.py b/tedana/tests/test_model_kundu_metrics.py
diff --git a/tedana/tests/test_selection.py b/tedana/tests/test_selection.py
diff --git a/tedana/tests/test_stats.py b/tedana/tests/test_stats.py
diff --git a/tedana/tests/test_t2smap.py b/tedana/tests/test_t2smap.py
diff --git a/tedana/tests/test_utils.py b/tedana/tests/test_utils.py
diff --git a/tedana/tests/test_viz.py b/tedana/tests/test_viz.py
diff --git a/tedana/tests/utils.py b/tedana/tests/utils.py
diff --git a/tedana/utils.py b/tedana/utils.py
@@ -5,7 +5,7 @@
 
 import numpy as np
 import nibabel as nib
-from scipy import ndimage
+from scipy import ndimage, stats
 from nilearn._utils import check_niimg
 from sklearn.utils import check_array
 
@@ -41,6 +41,45 @@ def load_image(data):
     return fdata
 
 
+def extreme_percentile_mask(dd, ees=None):
+    """
+    Returns mask for data between [0.001, 5] * 98th percentile of dd
+
+    Parameters
+    ----------
+    dd : (S x E x T) array_like
+        Input data, where `S` is samples, `E` is echos, and `T` is time
+    ees : (N,) :obj:`list`
+        Indices of echos to assess from `dd` in calculating output
+
+    Returns
+    -------
+    imask : (S x N) :obj:`numpy.ndarray`
+        0 1 int array denoting
+    """
+
+    if ees is None:
+        ees = range(dd.shape[1])
+    imask = np.zeros((dd.shape[0], len(ees)), dtype=bool)
+    for ee in ees:
+        if len(ees) == 1:
+            LGR.debug('Creating eimask for optimal combination')
+        else:
+            LGR.debug('Creating eimask for echo {}'.format(ee))
+        perc98 = stats.scoreatpercentile(dd[:, ee, :].flatten(), 98,
+                                         interpolation_method='lower')
+        lthr, hthr = 0.001 * perc98, 5 * perc98
+        LGR.debug('Eimask threshold boundaries: '
+                  '{:.03f} {:.03f}'.format(lthr, hthr))
+        m = dd[:, ee, :].mean(axis=1)
+        imask[np.logical_and(m > lthr, m < hthr), ee] = True
+
+    # if any echo has an outlier set the mask for that voxel to False
+
+    mask = imask.all(axis=1).astype(int)
+    return mask
+
+
 def make_adaptive_mask(data, mask=None, getsum=False):
     """
     Makes map of `data` specifying longest echo a voxel can be sampled with
@@ -68,6 +107,10 @@ def make_adaptive_mask(data, mask=None, getsum=False):
     RepLGR.info("An adaptive mask was then generated, in which each voxel's "
                 "value reflects the number of echoes with 'good' data.")
 
+    # TODO: This function is a combination of several arbtirary thresholds
+    #       It would be nice to modularize each threshold so that this is slightly
+    #       easier to tweak the arbirary levels.
+
     # take temporal mean of echos and extract non-zero values in first echo
     echo_means = data.mean(axis=-1)  # temporal mean of echos
     first_echo = echo_means[echo_means[:, 0] != 0, 0]
@@ -87,21 +130,29 @@ def make_adaptive_mask(data, mask=None, getsum=False):
 
     # determine samples where absolute value is greater than echo-specific thresholds
     # and count # of echos that pass criterion
-    masksum = (np.abs(echo_means) > lthrs).sum(axis=-1)
+    masksum1 = ((np.abs(echo_means) > lthrs).sum(axis=-1))
+
+    masksum2 = extreme_percentile_mask(data)
+
+    masksum = masksum1 * masksum2
 
     if mask is None:
         # make it a boolean mask to (where we have at least 1 echo with good signal)
         mask = masksum.astype(bool)
     else:
         # if the user has supplied a binary mask
+        # Note: The way this function is currently called in tedana.py means either
+        #  a user-provided mask or a mask defined by compute_epi_mask is the input
+        #  The warning message below was changed to account for both of these scenarios
         mask = load_image(mask).astype(bool)
         masksum = masksum * mask
         # reduce mask based on masksum
         # TODO: Use visual report to make checking the reduced mask easier
         if np.any(masksum[mask] == 0):
             n_bad_voxels = np.sum(masksum[mask] == 0)
-            LGR.warning('{0} voxels in user-defined mask do not have good '
-                        'signal. Removing voxels from mask.'.format(n_bad_voxels))
+            LGR.warning(('make_adaptive_mask removed an additional {} voxels from the {} voxels '
+                         'in the inputted mask.').format(
+                n_bad_voxels, mask.sum()))
             mask = masksum.astype(bool)
 
     if getsum:

diff --git a/tedana/viz.py b/tedana/viz.py
diff --git a/tedana/workflows/__init__.py b/tedana/workflows/__init__.py
diff --git a/tedana/workflows/parser_utils.py b/tedana/workflows/parser_utils.py
diff --git a/tedana/workflows/t2smap.py b/tedana/workflows/t2smap.py
diff --git a/tedana/workflows/tedana.py b/tedana/workflows/tedana.py
@@ -438,7 +438,7 @@ def tedana_workflow(data, tes, mask=None, mixm=None, ctab=None, manacc=None,
         # TODO: add affine check
         LGR.info('Using user-defined mask')
         RepLGR.info("A user-defined mask was applied to the data.")
-
+    LGR.info('Checking for additional outlier voxels to mask')
     mask, masksum = utils.make_adaptive_mask(catd, mask=mask, getsum=True)
     LGR.debug('Retaining {}/{} samples'.format(mask.sum(), n_samp))
     io.filewrite(masksum, op.join(out_dir, 'adaptive_mask.nii'), ref_img)

diff --git a/update_changes.sh b/update_changes.sh
diff --git a/versioneer.py b/versioneer.py