Adding regional and optimization modules (#30)

* docs: fix some docstrings * feat: add best_spline_cv function * feat: add eq_sources_score function * feat: add synthetic regional field function * feat: add optimization module * feat: add regional module * docs: add module descriptions to overview * docs: add testing instructions to contrib guide * build: add new dependencies * feat: add test files for all modules * docs: add estimating regional field user guide * fix: typos and formatting * fix: replace psutil cpu affinity with new function used a function from a stack overlow answer which is able to get cpu core numbers for many operating systems since psutil doesn't seem to work for MacOS or Windows. * build: alter dependency versions * fix: use lock_obj to fix file store for windows * chore: add sphinx-apidoc call to ci.yml * docs: add rst files for new modules * chore: remove build api docs to ci.yml * chore: seperate RTD env and testing env * chore: fix env.yml files * chore: move changelog template location * docs: customize jinja changelog template * build: add optimization optional deps * build: combine optional dependencies into `all` * build: restrict xrft version Seems to be cause issues in conda-forge feedstock, harmonica requires >= 1.0, match this * chore: numba-progress install from pip to conda * docs: replace `Unknown` with `Other` in changelog * chore: add Make changelog command * fix: make optimization dep imports optional * fix: optional optuna Journal import
mdtanker · Nov 30, 2023 · daf3c9c · daf3c9c
1 parent cade158
commit daf3c9c
Show file tree

Hide file tree

Showing 29 changed files with 2,509 additions and 67 deletions.
diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
@@ -86,6 +86,18 @@ Use pytest to run the unit checks and generate coverage reports:
 make test
 ```
 
+To test just one of the modules:
+
+```
+pytest tests/test_<MODULE>.py
+```
+
+To test just one function:
+
+```
+pytest tests/test_<MODULE>.py::<FUNCTION>
+```
+
 # Formatting
 
 Use the Makefile commands to locally format and check the code with `ruff` and

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -40,6 +40,7 @@ jobs:
         run: |
           echo "::add-matcher::$GITHUB_WORKSPACE/.github/matchers/pylint.json"
           pipx run nox -s pylint
+
   # Run tests and upload to codecov
   test:
     name: Check Python ${{ matrix.python-version }} on ${{ matrix.runs-on }}
@@ -72,7 +73,7 @@ jobs:
         uses: conda-incubator/setup-miniconda@v2
         with:
           activate-environment: invert4geom
-          environment-file: environment.yml
+          environment-file: env/test_env.yml
           python-version: ${{ matrix.python-version }}
           channels: conda-forge,nodefaults
           channel-priority: strict

diff --git a/.gitignore b/.gitignore
@@ -107,7 +107,7 @@ celerybeat.pid
 # Environments
 .env
 .venv
-env/
+# env/
 venv/
 ENV/
 env.bak/

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -11,6 +11,8 @@ exclude: |
       docs/api/invert4geom.synthetic.rst|
       docs/api/invert4geom.utils.rst|
       docs/api/invert4geom.cross_validation.rst|
+      docs/api/invert4geom.regional.rst|
+      docs/api/invert4geom.optimization.rst|
       CHANGELOG.md|
   )$
 

diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -12,4 +12,4 @@ sphinx:
   configuration: docs/conf.py
 
 conda:
-  environment: environment.yml
+  environment: env/RTD_env.yml
diff --git a/Makefile b/Makefile
@@ -8,10 +8,10 @@ create_test_env:
 	mamba create --name test --yes python=3.11
 
 install:
-	pip install -e .[viz,test,dev,docs]
+	pip install -e .[all]
 
 install_test:
-	pip install invert4geom[viz,test,dev,docs]
+	pip install invert4geom[all]
 
 remove:
 	mamba remove --name $(PROJECT) --all
@@ -40,3 +40,6 @@ style: format check lint pylint
 
 release_check:
 	semantic-release --noop version
+
+changelog:
+	semantic-release changelog
diff --git a/docs/api/invert4geom.optimization.rst b/docs/api/invert4geom.optimization.rst
@@ -0,0 +1,6 @@
+invert4geom.optimization
+------------------------
+.. automodule:: invert4geom.optimization
+   :members:
+   :undoc-members:
+   :show-inheritance:
diff --git a/docs/api/invert4geom.regional.rst b/docs/api/invert4geom.regional.rst
@@ -0,0 +1,6 @@
+invert4geom.regional
+--------------------
+.. automodule:: invert4geom.regional
+   :members:
+   :undoc-members:
+   :show-inheritance:
diff --git a/docs/api/invert4geom.rst b/docs/api/invert4geom.rst
@@ -7,6 +7,8 @@ API Reference
 
    invert4geom.cross_validation
    invert4geom.inversion
+   invert4geom.optimization
    invert4geom.plotting
+   invert4geom.regional
    invert4geom.synthetic
    invert4geom.utils
diff --git a/docs/conf.py b/docs/conf.py
@@ -54,7 +54,7 @@
     # nptyping
     # numba_progress
     # "tqdm": ("https://tqdm.github.io/", None),
-    # "pygmt": ("https://www.pygmt.org/latest/", None),
+    "pygmt": ("https://www.pygmt.org/latest/", None),
     #
     # Viz deps
     #
@@ -63,6 +63,10 @@
     # ipywidgets
     "matplotlib": ("https://matplotlib.org/", None),
     "seaborn": ("https://seaborn.pydata.org/", None),
+    #
+    # Opti deps
+    #
+    "optuna": ("https://optuna.readthedocs.io/en/stable/", None),
 }
 
 nitpick_ignore = [

diff --git a/docs/index.md b/docs/index.md
@@ -18,6 +18,7 @@ user_guide/reference_level_cross_validation
 user_guide/including_starting_model
 user_guide/adhering_to_constraints
 user_guide/combining_it_all
+user_guide/estimating_regional_field
 ```
 
 ```{toctree}

diff --git a/docs/install.md b/docs/install.md
@@ -37,7 +37,7 @@ pip install invert4geom
 
 ```{note}
 to install the optional dependencies, use this instead:
-`pip install [viz,test,dev,docs]`
+`pip install [all]`
 ```
 
 ## Locally
@@ -68,5 +68,5 @@ same with the following:
 ```
 conda create --name invert4geom --yes --force antarctic-plots python=3.11
 conda activate invert4geom
-pip install -e .[viz,test,dev,docs]
+pip install -e .[all]
 ```
diff --git a/docs/overview.md b/docs/overview.md
@@ -31,3 +31,74 @@ chapter 3 and 4 of my thesis, available
 [here](https://doi.org/10.26686/wgtn.24408304). The code was originally included
 in [this GitHub repository](https://github.com/mdtanker/RIS_gravity_inversion),
 but much of it has been migrated here.
+
+## Modules
+
+**Invert4Geom** consists of 7 modules:
+
+### Inversion
+
+This contains the core tools for performing the inversion. These tools allow for
+creating the Jacobian matrix, performing the least squares solution, running the
+inversion, determining when to end the inversion, and updating the misfit and
+gravity values between each iteration.
+
+## Cross Validation
+
+This module contains the code necessary for performing the various
+cross-validation routines. These routines are split into 2 categories; _gravity_
+and _constraints_ cross validation.
+
+### Gravity cross validations
+
+The _gravity cross validations_ are those which split the gravity data into
+testing and training sets, perform the inversion with the training set, and
+compare the forward gravity of the inverted topography with the un-used testing
+data. This is a Generalized Cross Validation, specifically a hold-out
+cross-validation, as described in
+[Uieda & Barbosa (2017)](https://academic.oup.com/gji/article-lookup/doi/10.1093/gji/ggw390).
+This is used here for estimated the optimal regularization damping parameter
+during the inversion.
+
+### Constraint cross validations
+
+The _constraint cross validations_ are those which use _apriori_ points of known
+elevation of the surface of in interest to determine optimal inversion
+parameters. The inversion is performed without including these constraint
+points, and the inverted surface is compared with the elevations of these
+constraints points to give a score. This style of validation is used here for
+estimating the optimal reference level and density contrast of the surface of
+interest.
+
+## Regional
+
+This module contains tools for estimating the regional gravity field. In many
+styles of inversions, the regional component of the gravity misfit should be
+removed to help isolate the gravity effects of the layer of interest. This is
+common in inversions for topography, bathymetry, or sedimentary basins. We
+provide four methods of estimating the regional component. 1) Low pass filtering
+the data, 2) fitting a surface to the data with a user defined trend, 3) fitting
+a set of deep equivalent sources to the data and predicting the gravity effect
+of those sources, and 4) using _apriori_ constraints to determine the regional
+field, by assuming the residual component is low at the those points.
+
+## Optimization
+
+Functions for performing optimizations. This uses the package _Optuna_ which
+given a parameter space, an objective function, and a sampling routine, will
+perform optimization to minimize (or maximize) the object function. This is used
+in the **Regional** module for finding the optimal regional separation methods.
+
+## Plotting
+
+Various function for plotting maps and graphs used throughout the other modules
+
+## Utils
+
+Utilities used throughout the other modules
+
+## Synthetic
+
+This module has tools for creating synthetic topography and contaminating data
+with random Gaussian noise. This is mostly used in testing and in the User Guide
+notebooks.
diff --git a/docs/user_guide/estimating_regional_field.ipynb b/docs/user_guide/estimating_regional_field.ipynb
diff --git a/env/RTD_env.yml b/env/RTD_env.yml
@@ -0,0 +1,41 @@
+# Requirements for running the github action: test only need run-time dependencies
+name: invert4geom
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - python>=3.8
+  - pip
+  # Run-time
+  - numpy
+  - pandas
+  - xarray
+  - verde>=1.8.0
+  - rioxarray
+  - pykdtree
+  - xrft>=1.0
+  - harmonica>=0.6.0
+  - antarctic-plots>=0.0.6
+  - numba
+  - scipy
+  - nptyping
+  - numba-progress
+  - tqdm
+  - pygmt
+  - dask
+  # opti
+  - optuna>=3.1.0
+  - botorch>=0.4.0
+  - psutil
+  - joblib
+  # docs
+  - sphinx>=4.0
+  - myst-parser
+  - sphinx-copybutton
+  - sphinx-autodoc-typehints
+  - sphinx-book-theme>=0.3
+  - sphinx-design
+  - nbsphinx
+  - pip:
+      - tqdm_joblib
+      - ../.
diff --git a/environment.yml → env/test_env.yml b/environment.yml → env/test_env.yml
@@ -13,13 +13,21 @@ dependencies:
   - verde>=1.8.0
   - rioxarray
   - pykdtree
-  - xrft
+  - xrft>=1.0
   - harmonica>=0.6.0
   - antarctic-plots>=0.0.6
   - numba
   - scipy
   - nptyping
+  - numba-progress
   - tqdm
+  - pygmt
+  - dask
+  # opti
+  - optuna>=3.1.0
+  - botorch>=0.4.0
+  - psutil
+  - joblib
   # viz
   - pyvista
   - trame
@@ -38,5 +46,5 @@ dependencies:
   - sphinx-design
   - nbsphinx
   - pip:
-      - numba-progress
-      - .
+      - tqdm_joblib
+      - ../.
diff --git a/noxfile.py b/noxfile.py
@@ -80,7 +80,7 @@ def docs(session: nox.Session) -> None:
 
     extra_installs = ["sphinx-autobuild"] if args.serve else []
 
-    session.install("-e.[docs,viz]", *extra_installs)
+    session.install("-e.[opti,docs,viz]", *extra_installs)
     session.chdir("docs")
 
     if args.builder == "linkcheck":

diff --git a/pyproject.toml b/pyproject.toml
@@ -35,22 +35,26 @@ dependencies = [
   "verde>=1.8.0",
   "rioxarray",
   "pykdtree",
-  "xrft",
+  "xrft>=1.0",
   "harmonica>=0.6.0",
   "antarctic_plots>=0.0.6",
   "numba",
   "scipy",
   "nptyping",
   "numba_progress",
   "tqdm",
-
-#  "pygmt>=0.9.0",
-#  "geopandas",
-#  "boule",
-#  "numba-progress",
+  "pygmt",
+  "dask",
 ]
 
 [project.optional-dependencies]
+opti = [
+  "optuna>=3.1.0", # need JournalStorage
+  "botorch>=0.4.0",
+  "joblib",
+  "psutil",
+  "tqdm_joblib",
+]
 viz = [
   "pyvista",
   "trame",
@@ -62,23 +66,28 @@ test = [
   "pytest >=6",
   "pytest-cov >=3",
 ]
+docs = [
+  "sphinx>=4.0",
+  "myst_parser",
+  "sphinx_copybutton",
+  "sphinx_autodoc_typehints",
+  "sphinx-book-theme>=0.3",
+  "sphinx_design",
+  "nbsphinx",
+]
 dev = [
+  "invert4geom[opti,viz,test,docs]",
   "pytest >=6",
   "pytest-cov >=3",
   "ruff",
   "nox",
   "pre-commit",
   "pylint",
   "python-semantic-release >=8",
+  "ipykernel",
 ]
-docs = [
-  "sphinx>=4.0",
-  "myst_parser",
-  "sphinx_copybutton",
-  "sphinx_autodoc_typehints",
-  "sphinx-book-theme>=0.3",
-  "sphinx_design",
-  "nbsphinx",
+all = [
+  "invert4geom[opti,viz,test,dev,docs]",
 ]
 
 [project.urls]
@@ -227,6 +236,9 @@ prerelease = true
 [tool.semantic_release.changelog]
 template_dir = "templates"
 
+[tool.semantic_release.changelog.environment]
+trim_blocks = true
+
 [tool.semantic_release.commit_parser_options]
 allowed_tags = [
     "build",    # Changes that affect the build system or external dependencies