diff --git a/.all-contributorsrc b/.all-contributorsrc old mode 100644 new mode 100755 diff --git a/.circleci/config.yml b/.circleci/config.yml old mode 100644 new mode 100755 diff --git a/.codecov.yml b/.codecov.yml old mode 100644 new mode 100755 diff --git a/.dockerignore b/.dockerignore old mode 100644 new mode 100755 diff --git a/.gitattributes b/.gitattributes old mode 100644 new mode 100755 diff --git a/.github/ISSUE_TEMPLATE/issue_template.md b/.github/ISSUE_TEMPLATE/issue_template.md old mode 100644 new mode 100755 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md old mode 100644 new mode 100755 diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml old mode 100644 new mode 100755 diff --git a/.github/stale.yml b/.github/stale.yml old mode 100644 new mode 100755 diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 diff --git a/.zenodo.json b/.zenodo.json old mode 100644 new mode 100755 diff --git a/dev_tools/run_tests.sh b/dev_tools/run_tests.sh old mode 100644 new mode 100755 diff --git a/docs/Makefile b/docs/Makefile old mode 100644 new mode 100755 diff --git a/docs/_static/a01_echo_timeseries.png b/docs/_static/a01_echo_timeseries.png old mode 100644 new mode 100755 diff --git a/docs/_static/a02_echo_value_distributions.png b/docs/_static/a02_echo_value_distributions.png old mode 100644 new mode 100755 diff --git a/docs/_static/a03_adaptive_mask.png b/docs/_static/a03_adaptive_mask.png old mode 100644 new mode 100755 diff --git a/docs/_static/a04_echo_log_value_distributions.png b/docs/_static/a04_echo_log_value_distributions.png old mode 100644 new mode 100755 diff --git a/docs/_static/a05_loglinear_regression.png b/docs/_static/a05_loglinear_regression.png old mode 100644 new mode 100755 diff --git a/docs/_static/a06_monoexponential_decay_model.png b/docs/_static/a06_monoexponential_decay_model.png old mode 100644 new mode 100755 diff --git a/docs/_static/a07_monoexponential_decay_model_with_t2.png b/docs/_static/a07_monoexponential_decay_model_with_t2.png old mode 100644 new mode 100755 diff --git a/docs/_static/a08_optimal_combination_echo_weights.png b/docs/_static/a08_optimal_combination_echo_weights.png old mode 100644 new mode 100755 diff --git a/docs/_static/a09_optimal_combination_value_distributions.png b/docs/_static/a09_optimal_combination_value_distributions.png old mode 100644 new mode 100755 diff --git a/docs/_static/a10_optimal_combination_timeseries.png b/docs/_static/a10_optimal_combination_timeseries.png old mode 100644 new mode 100755 diff --git a/docs/_static/a11_pca_component_timeseries.png b/docs/_static/a11_pca_component_timeseries.png old mode 100644 new mode 100755 diff --git a/docs/_static/a12_pca_reduced_data.png b/docs/_static/a12_pca_reduced_data.png old mode 100644 new mode 100755 diff --git a/docs/_static/a13_ica_component_timeseries.png b/docs/_static/a13_ica_component_timeseries.png old mode 100644 new mode 100755 diff --git a/docs/_static/a14_te_dependence_models_component_0.png b/docs/_static/a14_te_dependence_models_component_0.png old mode 100644 new mode 100755 diff --git a/docs/_static/a14_te_dependence_models_component_1.png b/docs/_static/a14_te_dependence_models_component_1.png old mode 100644 new mode 100755 diff --git a/docs/_static/a14_te_dependence_models_component_2.png b/docs/_static/a14_te_dependence_models_component_2.png old mode 100644 new mode 100755 diff --git a/docs/_static/a15_denoised_data_timeseries.png b/docs/_static/a15_denoised_data_timeseries.png old mode 100644 new mode 100755 diff --git a/docs/_static/a16_t1c_denoised_data_timeseries.png b/docs/_static/a16_t1c_denoised_data_timeseries.png old mode 100644 new mode 100755 diff --git a/docs/_static/b01_simulated_fluctuations.png b/docs/_static/b01_simulated_fluctuations.png old mode 100644 new mode 100755 diff --git a/docs/_static/b02_model_fits.png b/docs/_static/b02_model_fits.png old mode 100644 new mode 100755 diff --git a/docs/_static/b03_component_timeseries.png b/docs/_static/b03_component_timeseries.png old mode 100644 new mode 100755 diff --git a/docs/_static/b04_component_model_fits.png b/docs/_static/b04_component_model_fits.png old mode 100644 new mode 100755 diff --git a/docs/_static/b04_echo_timeseries.png b/docs/_static/b04_echo_timeseries.png old mode 100644 new mode 100755 diff --git a/docs/_static/b05_component_model_fits.png b/docs/_static/b05_component_model_fits.png old mode 100644 new mode 100755 diff --git a/docs/_static/example_Component_Overview.png b/docs/_static/example_Component_Overview.png old mode 100644 new mode 100755 diff --git a/docs/_static/example_Kappa_vs_Rho_Scatter.png b/docs/_static/example_Kappa_vs_Rho_Scatter.png old mode 100644 new mode 100755 diff --git a/docs/_static/example_bad_component.png b/docs/_static/example_bad_component.png old mode 100644 new mode 100755 diff --git a/docs/_static/example_good_component.png b/docs/_static/example_good_component.png old mode 100644 new mode 100755 diff --git a/docs/_static/physics_kundu_2017_TE_dependence.jpg b/docs/_static/physics_kundu_2017_TE_dependence.jpg old mode 100644 new mode 100755 diff --git a/docs/_static/physics_kundu_2017_multiple_echoes.jpg b/docs/_static/physics_kundu_2017_multiple_echoes.jpg old mode 100644 new mode 100755 diff --git a/docs/_static/tedana-ohbm2018-poster.png b/docs/_static/tedana-ohbm2018-poster.png old mode 100644 new mode 100755 diff --git a/docs/_static/tedana-ohbm2019-poster.png b/docs/_static/tedana-ohbm2019-poster.png old mode 100644 new mode 100755 diff --git a/docs/_static/tedana-workflow.png b/docs/_static/tedana-workflow.png old mode 100644 new mode 100755 diff --git a/docs/_static/tedana_favicon.png b/docs/_static/tedana_favicon.png old mode 100644 new mode 100755 diff --git a/docs/_static/theme_overrides.css b/docs/_static/theme_overrides.css old mode 100644 new mode 100755 diff --git a/docs/_templates/function.rst b/docs/_templates/function.rst old mode 100644 new mode 100755 diff --git a/docs/_templates/module.rst b/docs/_templates/module.rst old mode 100644 new mode 100755 diff --git a/docs/api.rst b/docs/api.rst old mode 100644 new mode 100755 diff --git a/docs/approach.rst b/docs/approach.rst old mode 100644 new mode 100755 diff --git a/docs/conf.py b/docs/conf.py old mode 100644 new mode 100755 diff --git a/docs/considerations.rst b/docs/considerations.rst old mode 100644 new mode 100755 diff --git a/docs/contributing.rst b/docs/contributing.rst old mode 100644 new mode 100755 diff --git a/docs/dependence_metrics.rst b/docs/dependence_metrics.rst old mode 100644 new mode 100755 diff --git a/docs/faq.rst b/docs/faq.rst old mode 100644 new mode 100755 diff --git a/docs/index.rst b/docs/index.rst old mode 100644 new mode 100755 diff --git a/docs/installation.rst b/docs/installation.rst old mode 100644 new mode 100755 diff --git a/docs/make.bat b/docs/make.bat old mode 100644 new mode 100755 diff --git a/docs/multi-echo.rst b/docs/multi-echo.rst old mode 100644 new mode 100755 diff --git a/docs/outputs.rst b/docs/outputs.rst old mode 100644 new mode 100755 diff --git a/docs/publications.rst b/docs/publications.rst old mode 100644 new mode 100755 diff --git a/docs/roadmap.rst b/docs/roadmap.rst old mode 100644 new mode 100755 diff --git a/docs/sphinxext/github_link.py b/docs/sphinxext/github_link.py old mode 100644 new mode 100755 diff --git a/docs/support.rst b/docs/support.rst old mode 100644 new mode 100755 diff --git a/docs/usage.rst b/docs/usage.rst old mode 100644 new mode 100755 diff --git a/examples/plot_approach_figures.ipynb b/examples/plot_approach_figures.ipynb old mode 100644 new mode 100755 diff --git a/examples/plot_metric_simulations.ipynb b/examples/plot_metric_simulations.ipynb old mode 100644 new mode 100755 diff --git a/tedana/__init__.py b/tedana/__init__.py old mode 100644 new mode 100755 diff --git a/tedana/_version.py b/tedana/_version.py old mode 100644 new mode 100755 diff --git a/tedana/combine.py b/tedana/combine.py old mode 100644 new mode 100755 diff --git a/tedana/decay.py b/tedana/decay.py old mode 100644 new mode 100755 diff --git a/tedana/decomposition/__init__.py b/tedana/decomposition/__init__.py old mode 100644 new mode 100755 diff --git a/tedana/decomposition/_utils.py b/tedana/decomposition/_utils.py deleted file mode 100644 index 115751eb5..000000000 --- a/tedana/decomposition/_utils.py +++ /dev/null @@ -1,47 +0,0 @@ -""" -Utility functions for tedana decomposition -""" -import logging - -import numpy as np -from scipy import stats - -LGR = logging.getLogger(__name__) -RepLGR = logging.getLogger('REPORT') -RefLGR = logging.getLogger('REFERENCES') - - -def eimask(dd, ees=None): - """ - Returns mask for data between [0.001, 5] * 98th percentile of dd - - Parameters - ---------- - dd : (S x E x T) array_like - Input data, where `S` is samples, `E` is echos, and `T` is time - ees : (N,) :obj:`list` - Indices of echos to assess from `dd` in calculating output - - Returns - ------- - imask : (S x N) :obj:`numpy.ndarray` - Boolean array denoting - """ - - if ees is None: - ees = range(dd.shape[1]) - imask = np.zeros((dd.shape[0], len(ees)), dtype=bool) - for ee in ees: - if len(ees) == 1: - LGR.debug('Creating eimask for optimal combination') - else: - LGR.debug('Creating eimask for echo {}'.format(ee)) - perc98 = stats.scoreatpercentile(dd[:, ee, :].flatten(), 98, - interpolation_method='lower') - lthr, hthr = 0.001 * perc98, 5 * perc98 - LGR.debug('Eimask threshold boundaries: ' - '{:.03f} {:.03f}'.format(lthr, hthr)) - m = dd[:, ee, :].mean(axis=1) - imask[np.logical_and(m > lthr, m < hthr), ee] = True - - return imask diff --git a/tedana/decomposition/ica.py b/tedana/decomposition/ica.py old mode 100644 new mode 100755 diff --git a/tedana/decomposition/ma_pca.py b/tedana/decomposition/ma_pca.py old mode 100644 new mode 100755 diff --git a/tedana/decomposition/pca.py b/tedana/decomposition/pca.py old mode 100644 new mode 100755 index f9b4f9a69..f2a3329e3 --- a/tedana/decomposition/pca.py +++ b/tedana/decomposition/pca.py @@ -10,7 +10,7 @@ from sklearn.decomposition import PCA from tedana import metrics, utils, io -from tedana.decomposition import (ma_pca, _utils) +from tedana.decomposition import ma_pca from tedana.stats import computefeats2 from tedana.selection import kundu_tedpca from tedana.due import due, BibTeX @@ -255,17 +255,15 @@ def tedpca(data_cat, data_oc, combmode, mask, t2s, t2sG, LGR.info('Computing PCA of echo #{0}'.format(','.join([str(ee) for ee in source_tes]))) data = np.stack([data_cat[mask, ee, :] for ee in source_tes - 1], axis=1) - eim = np.squeeze(_utils.eimask(data)) - data = np.squeeze(data[eim]) + data = np.squeeze(data) data_z = ((data.T - data.T.mean(axis=0)) / data.T.std(axis=0)).T # var normalize ts data_z = (data_z - data_z.mean()) / data_z.std() # var normalize everything if algorithm in ['mdl', 'aic', 'kic']: data_img = io.new_nii_like( - ref_img, utils.unmask(utils.unmask(data, eim), mask)) - mask_img = io.new_nii_like(ref_img, - utils.unmask(eim, mask).astype(int)) + ref_img, utils.unmask(data, mask)) + mask_img = io.new_nii_like(ref_img, mask.astype(int)) voxel_comp_weights, varex, varex_norm, comp_ts = ma_pca.ma_pca( data_img, mask_img, algorithm) elif algorithm == 'mle': @@ -283,12 +281,6 @@ def tedpca(data_cat, data_oc, combmode, mask, t2s, t2sG, varex_norm = varex / varex.sum() # Compute Kappa and Rho for PCA comps - eimum = np.atleast_2d(eim) - eimum = np.transpose(eimum, np.argsort(eimum.shape)[::-1]) - eimum = eimum.prod(axis=1) - o = np.zeros((mask.shape[0], *eimum.shape[1:])) - o[mask, ...] = eimum - eimum = np.squeeze(o).astype(bool) # Normalize each component's time series vTmixN = stats.zscore(comp_ts, axis=0) diff --git a/tedana/due.py b/tedana/due.py old mode 100644 new mode 100755 diff --git a/tedana/gscontrol.py b/tedana/gscontrol.py old mode 100644 new mode 100755 diff --git a/tedana/info.py b/tedana/info.py old mode 100644 new mode 100755 diff --git a/tedana/io.py b/tedana/io.py old mode 100644 new mode 100755 diff --git a/tedana/metrics/__init__.py b/tedana/metrics/__init__.py old mode 100644 new mode 100755 diff --git a/tedana/metrics/kundu_fit.py b/tedana/metrics/kundu_fit.py old mode 100644 new mode 100755 index d3ca732d9..0b7800d70 --- a/tedana/metrics/kundu_fit.py +++ b/tedana/metrics/kundu_fit.py @@ -68,6 +68,7 @@ def dependence_metrics(catd, tsoc, mmix, t2s, tes, ref_img, betas : :obj:`numpy.ndarray` mmix_new : :obj:`numpy.ndarray` """ + # Use t2s as mask mask = t2s != 0 if not (catd.shape[0] == t2s.shape[0] == mask.shape[0] == tsoc.shape[0]): diff --git a/tedana/selection/__init__.py b/tedana/selection/__init__.py old mode 100644 new mode 100755 diff --git a/tedana/selection/_utils.py b/tedana/selection/_utils.py old mode 100644 new mode 100755 diff --git a/tedana/selection/tedica.py b/tedana/selection/tedica.py old mode 100644 new mode 100755 diff --git a/tedana/selection/tedpca.py b/tedana/selection/tedpca.py old mode 100644 new mode 100755 diff --git a/tedana/stats.py b/tedana/stats.py old mode 100644 new mode 100755 diff --git a/tedana/tests/__init__.py b/tedana/tests/__init__.py old mode 100644 new mode 100755 diff --git a/tedana/tests/conftest.py b/tedana/tests/conftest.py old mode 100644 new mode 100755 diff --git a/tedana/tests/data/cornell_three_echo_outputs.txt b/tedana/tests/data/cornell_three_echo_outputs.txt old mode 100644 new mode 100755 diff --git a/tedana/tests/data/echo1.nii.gz b/tedana/tests/data/echo1.nii.gz old mode 100644 new mode 100755 diff --git a/tedana/tests/data/echo2.nii.gz b/tedana/tests/data/echo2.nii.gz old mode 100644 new mode 100755 diff --git a/tedana/tests/data/echo3.nii.gz b/tedana/tests/data/echo3.nii.gz old mode 100644 new mode 100755 diff --git a/tedana/tests/data/fdist.npy b/tedana/tests/data/fdist.npy old mode 100644 new mode 100755 diff --git a/tedana/tests/data/fiu_four_echo_outputs.txt b/tedana/tests/data/fiu_four_echo_outputs.txt old mode 100644 new mode 100755 diff --git a/tedana/tests/data/mask.nii.gz b/tedana/tests/data/mask.nii.gz old mode 100644 new mode 100755 diff --git a/tedana/tests/data/nih_five_echo_outputs_verbose.txt b/tedana/tests/data/nih_five_echo_outputs_verbose.txt old mode 100644 new mode 100755 diff --git a/tedana/tests/test_combine.py b/tedana/tests/test_combine.py old mode 100644 new mode 100755 diff --git a/tedana/tests/test_decay.py b/tedana/tests/test_decay.py old mode 100644 new mode 100755 diff --git a/tedana/tests/test_gscontrol.py b/tedana/tests/test_gscontrol.py old mode 100644 new mode 100755 diff --git a/tedana/tests/test_integration.py b/tedana/tests/test_integration.py old mode 100644 new mode 100755 diff --git a/tedana/tests/test_io.py b/tedana/tests/test_io.py old mode 100644 new mode 100755 diff --git a/tedana/tests/test_mapca.py b/tedana/tests/test_mapca.py old mode 100644 new mode 100755 diff --git a/tedana/tests/test_model_fit_dependence_metrics.py b/tedana/tests/test_model_fit_dependence_metrics.py old mode 100644 new mode 100755 diff --git a/tedana/tests/test_model_kundu_metrics.py b/tedana/tests/test_model_kundu_metrics.py old mode 100644 new mode 100755 diff --git a/tedana/tests/test_selection.py b/tedana/tests/test_selection.py old mode 100644 new mode 100755 diff --git a/tedana/tests/test_stats.py b/tedana/tests/test_stats.py old mode 100644 new mode 100755 diff --git a/tedana/tests/test_t2smap.py b/tedana/tests/test_t2smap.py old mode 100644 new mode 100755 diff --git a/tedana/tests/test_utils.py b/tedana/tests/test_utils.py old mode 100644 new mode 100755 diff --git a/tedana/tests/test_viz.py b/tedana/tests/test_viz.py old mode 100644 new mode 100755 diff --git a/tedana/tests/utils.py b/tedana/tests/utils.py old mode 100644 new mode 100755 diff --git a/tedana/utils.py b/tedana/utils.py old mode 100644 new mode 100755 index 94e555d74..1719b2b1d --- a/tedana/utils.py +++ b/tedana/utils.py @@ -5,7 +5,7 @@ import numpy as np import nibabel as nib -from scipy import ndimage +from scipy import ndimage, stats from nilearn._utils import check_niimg from sklearn.utils import check_array @@ -41,6 +41,45 @@ def load_image(data): return fdata +def extreme_percentile_mask(dd, ees=None): + """ + Returns mask for data between [0.001, 5] * 98th percentile of dd + + Parameters + ---------- + dd : (S x E x T) array_like + Input data, where `S` is samples, `E` is echos, and `T` is time + ees : (N,) :obj:`list` + Indices of echos to assess from `dd` in calculating output + + Returns + ------- + imask : (S x N) :obj:`numpy.ndarray` + 0 1 int array denoting + """ + + if ees is None: + ees = range(dd.shape[1]) + imask = np.zeros((dd.shape[0], len(ees)), dtype=bool) + for ee in ees: + if len(ees) == 1: + LGR.debug('Creating eimask for optimal combination') + else: + LGR.debug('Creating eimask for echo {}'.format(ee)) + perc98 = stats.scoreatpercentile(dd[:, ee, :].flatten(), 98, + interpolation_method='lower') + lthr, hthr = 0.001 * perc98, 5 * perc98 + LGR.debug('Eimask threshold boundaries: ' + '{:.03f} {:.03f}'.format(lthr, hthr)) + m = dd[:, ee, :].mean(axis=1) + imask[np.logical_and(m > lthr, m < hthr), ee] = True + + # if any echo has an outlier set the mask for that voxel to False + + mask = imask.all(axis=1).astype(int) + return mask + + def make_adaptive_mask(data, mask=None, getsum=False): """ Makes map of `data` specifying longest echo a voxel can be sampled with @@ -68,6 +107,10 @@ def make_adaptive_mask(data, mask=None, getsum=False): RepLGR.info("An adaptive mask was then generated, in which each voxel's " "value reflects the number of echoes with 'good' data.") + # TODO: This function is a combination of several arbtirary thresholds + # It would be nice to modularize each threshold so that this is slightly + # easier to tweak the arbirary levels. + # take temporal mean of echos and extract non-zero values in first echo echo_means = data.mean(axis=-1) # temporal mean of echos first_echo = echo_means[echo_means[:, 0] != 0, 0] @@ -87,21 +130,29 @@ def make_adaptive_mask(data, mask=None, getsum=False): # determine samples where absolute value is greater than echo-specific thresholds # and count # of echos that pass criterion - masksum = (np.abs(echo_means) > lthrs).sum(axis=-1) + masksum1 = ((np.abs(echo_means) > lthrs).sum(axis=-1)) + + masksum2 = extreme_percentile_mask(data) + + masksum = masksum1 * masksum2 if mask is None: # make it a boolean mask to (where we have at least 1 echo with good signal) mask = masksum.astype(bool) else: # if the user has supplied a binary mask + # Note: The way this function is currently called in tedana.py means either + # a user-provided mask or a mask defined by compute_epi_mask is the input + # The warning message below was changed to account for both of these scenarios mask = load_image(mask).astype(bool) masksum = masksum * mask # reduce mask based on masksum # TODO: Use visual report to make checking the reduced mask easier if np.any(masksum[mask] == 0): n_bad_voxels = np.sum(masksum[mask] == 0) - LGR.warning('{0} voxels in user-defined mask do not have good ' - 'signal. Removing voxels from mask.'.format(n_bad_voxels)) + LGR.warning(('make_adaptive_mask removed an additional {} voxels from the {} voxels ' + 'in the inputted mask.').format( + n_bad_voxels, mask.sum())) mask = masksum.astype(bool) if getsum: diff --git a/tedana/viz.py b/tedana/viz.py old mode 100644 new mode 100755 diff --git a/tedana/workflows/__init__.py b/tedana/workflows/__init__.py old mode 100644 new mode 100755 diff --git a/tedana/workflows/parser_utils.py b/tedana/workflows/parser_utils.py old mode 100644 new mode 100755 diff --git a/tedana/workflows/t2smap.py b/tedana/workflows/t2smap.py old mode 100644 new mode 100755 diff --git a/tedana/workflows/tedana.py b/tedana/workflows/tedana.py old mode 100644 new mode 100755 index dc3b38153..53c1d2111 --- a/tedana/workflows/tedana.py +++ b/tedana/workflows/tedana.py @@ -438,7 +438,7 @@ def tedana_workflow(data, tes, mask=None, mixm=None, ctab=None, manacc=None, # TODO: add affine check LGR.info('Using user-defined mask') RepLGR.info("A user-defined mask was applied to the data.") - + LGR.info('Checking for additional outlier voxels to mask') mask, masksum = utils.make_adaptive_mask(catd, mask=mask, getsum=True) LGR.debug('Retaining {}/{} samples'.format(mask.sum(), n_samp)) io.filewrite(masksum, op.join(out_dir, 'adaptive_mask.nii'), ref_img)