Merge pull request #220 from MIT-AI-Accelerator/gnadt

Gnadt

Dockerfile

@@ -1,79 +1,26 @@
-# Adapted from https://github.com/andferrari/julia_notebook,
-# which was derived from https://github.com/jupyter/docker-stacks
-# 
-# Licensing terms for this Dockerfile: 
-# -------------------------------------------------------------------------------- 
-# This project is licensed under the terms of the Modified BSD License
-# (also known as New or Revised or 3-Clause BSD), as follows:
-# 
-# - Copyright (c) 2001-2015, IPython Development Team
-# - Copyright (c) 2015-, Jupyter Development Team
-# 
-# All rights reserved.
-# 
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-# 
-# Redistributions of source code must retain the above copyright notice, this
-# list of conditions and the following disclaimer.
-# 
-# Redistributions in binary form must reproduce the above copyright notice, this
-# list of conditions and the following disclaimer in the documentation and/or
-# other materials provided with the distribution.
-# 
-# Neither the name of the Jupyter Development Team nor the names of its
-# contributors may be used to endorse or promote products derived from this
-# software without specific prior written permission.
-# 
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-# -------------------------------------------------------------------------------- 
-
+# ------------------------------------------------------------------------------
+# Build a Docker image containing Julia, Jupyter, Pluto, MagNav.jl, & examples.
+# It will take a while to build. The image is large, about 6 GB. This uses:
+# https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#jupyter-julia-notebook
+# ------------------------------------------------------------------------------
 # Build/Run commands via Docker
-# -------------------------------------------------------------------------------- 
-# This will take a while, as it installs Julia and MagNav.jl into the image
-# and downloads support files. The image is large, about 10 GB.
 # docker build --tag magnav .
 # docker run -p 8888:8888 magnav
+# ------------------------------------------------------------------------------
 
-FROM "jupyter/minimal-notebook"
-
-USER root
-
-ENV JULIA_VERSION=1.9.3
-
-# Install Julia
-RUN mkdir /opt/julia-${JULIA_VERSION} && \
-    cd /tmp && \
-    wget -q https://julialang-s3.julialang.org/bin/linux/x64/`echo ${JULIA_VERSION} | cut -d. -f 1,2`/julia-${JULIA_VERSION}-linux-x86_64.tar.gz && \
-    tar xzf julia-${JULIA_VERSION}-linux-x86_64.tar.gz -C /opt/julia-${JULIA_VERSION} --strip-components=1 && \
-    rm /tmp/julia-${JULIA_VERSION}-linux-x86_64.tar.gz
-
-RUN ln -fs /opt/julia-*/bin/julia /usr/local/bin/julia
-
-USER $NB_UID
+# Get Julia, Jupyter, & Pluto image, might not use latest Julia
+FROM jupyter/julia-notebook:latest
 
-# Add packages and precompile
-RUN julia -e 'import Pkg; Pkg.update()' && \
-    julia -e 'import Pkg; Pkg.add("CSV"); using CSV' && \
-    julia -e 'import Pkg; Pkg.add("DataFrames"); using DataFrames' && \
-    julia -e 'import Pkg; Pkg.add("IJulia"); using IJulia' && \
-    julia -e 'import Pkg; Pkg.add("MagNav"); using MagNav' && \
-    julia -e 'import Pkg; Pkg.add("Plots"); using Plots'
+# Add packages & precompile
+RUN julia -e 'import Pkg; Pkg.update(); \
+    Pkg.add(["MagNav","CSV","DataFrames","Plots"]); \
+    Pkg.precompile();'
 
 # Download examples
-RUN git clone "https://github.com/MIT-AI-Accelerator/MagNav.jl" /home/$NB_USER/work/MagNav.jl
-RUN cp -r /home/$NB_USER/work/MagNav.jl/examples/dataframes          /home/$NB_USER/work
-RUN cp -r /home/$NB_USER/work/MagNav.jl/examples/dataframes_setup.jl /home/$NB_USER/work
-RUN cp -r /home/$NB_USER/work/MagNav.jl/examples/demo.ipynb          /home/$NB_USER/work
-RUN rm -r /home/$NB_USER/work/MagNav.jl
+RUN git clone "https://github.com/MIT-AI-Accelerator/MagNav.jl" $HOME/MagNav.jl && \
+    cp -r $HOME/MagNav.jl/examples/. $HOME && \
+    rm -r $HOME/MagNav.jl && \
+    rm -r $HOME/work
 
-RUN fix-permissions /home/$NB_USER
+# Fix permissions
+RUN fix-permissions $HOME

README.md

@@ -4,7 +4,7 @@
 [![Codecov](https://codecov.io/gh/MIT-AI-Accelerator/MagNav.jl/branch/master/graph/badge.svg)](https://app.codecov.io/gh/MIT-AI-Accelerator/MagNav.jl)
 [![docs-stable](https://img.shields.io/badge/docs-stable-blue.svg)](https://mit-ai-accelerator.github.io/MagNav.jl/stable/)
 
-MagNav.jl contains a full suite of tools for airborne Magnetic anomaly Navigation (MagNav), including flight path & INS data import or simulation, mapping, aeromagnetic compensation, and navigation. This package was developed as part of the [DAF-MIT Artificial Intelligence Accelerator](https://aia.mit.edu/). More information on this effort, including a list of relevant publications, is provided on the [challenge problem website](https://magnav.mit.edu/). The package has been tested on the long-term support (LTS) and latest stable versions of Julia, which may be downloaded from [here](https://julialang.org/downloads/). The recommended IDE for Julia is [Visual Studio Code](https://code.visualstudio.com/).
+MagNav.jl contains a full suite of tools for airborne Magnetic anomaly Navigation (MagNav), including flight path & INS data import or simulation, mapping, aeromagnetic compensation, and navigation. This package was developed as part of the [DAF-MIT Artificial Intelligence Accelerator](https://aia.mit.edu/). More information on this effort, including a list of relevant publications, is provided on the [challenge problem website](https://magnav.mit.edu/). The package has been tested on the long-term support (LTS) and latest stable versions of Julia, which may be downloaded from [here](https://julialang.org/downloads/). The recommended IDE for Julia is [Visual Studio Code](https://code.visualstudio.com/) (with the Julia extension).
 
 ## Installation
 
@@ -40,7 +40,9 @@ For general usage, run:
 julia> using MagNav
 ```
 
-Multiple examples (Pluto notebooks) are in the [`examples`](examples) folder. To start Pluto in a web browser, run:
+### Examples
+
+Multiple example Jupyter and Pluto notebooks are in the [`examples`](examples) folder. Jupyter can be run directly in [Visual Studio Code](https://code.visualstudio.com/) (with the Jupyter extension). To start Pluto in a web browser, run:
 
 ```julia
 julia> using Pkg
@@ -49,21 +51,20 @@ julia> using Pluto
 julia> Pluto.run()
 ```
 
-In Pluto, open the desired Pluto notebook file and it should run automatically.
+In Pluto, select and open the desired Pluto notebook, and it should run automatically.
 
-### Docker Demonstration Notebook
+### Docker
 
-A Docker image is available that contains an example usage of the package. Install [Docker Desktop](https://www.docker.com/products/docker-desktop/), search and pull `jtaylormit/magnav`, and run with the host port set to `8888`. Alternatively, from the command line, run:
+A Docker image is available, which contains Julia, MagNav.jl, and the example notebooks. Install [Docker Desktop](https://www.docker.com/products/docker-desktop/), search and pull `jtaylormit/magnav`, and run with the host port set to `8888`. Alternatively, from the command line, run:
 
 ```
 docker pull jtaylormit/magnav
 docker run -p 8888:8888 jtaylormit/magnav
 ```
 
-A Docker container will spin up and provide a URL to copy into a local browser. It will look something like this: `http://127.0.0.1:8888/lab?token=###`. Note that changes to the notebook occur inside
-the container only. The notebook must be manually downloaded from the browser to be saved.
+A Docker container will spin up and provide a URL that then must be opened in a local browser. It will look something like this: `http://127.0.0.1:8888/lab?token=###`. Jupyter notebooks can be opened directly, but for Pluto notebooks, click `Pluto Notebook` from the Launcher, then select and open the desired Pluto notebook. Note that any changes to the notebooks occur inside the Docker container only, so a notebook must be manually downloaded to be saved.
 
-The above image is [hosted on Docker Hub](https://hub.docker.com/r/jtaylormit/magnav), and it is manually and sporadically updated. For the most recent image, run:
+The above Docker image is [hosted on Docker Hub](https://hub.docker.com/r/jtaylormit/magnav), and it is manually and sporadically updated. For the most recent Docker image, run:
 
 ```
 docker pull ghcr.io/mit-ai-accelerator/magnav.jl

docs/src/structs.md

@@ -15,10 +15,18 @@ MagNav.MapS
 MagNav.MapSd
 ```
 
+```@docs
+MagNav.MapS3D
+```
+
 ```@docs
 MagNav.MapV
 ```
 
+```@docs
+MagNav.Map_Cache
+```
+
 ## Vector Magnetometer
 ```@docs
 MagNav.MagV
@@ -75,7 +83,12 @@ MagNav.INSout
 MagNav.FILTout
 ```
 
-## Aeromagnetic Compensation Model Parameters
+## Real-time EKF
+```@docs
+MagNav.EKF_RT
+```
+
+## Compensation Parameters
 ```@docs
 MagNav.CompParams
 ```

examples/demo.ipynb → examples/jupyter_demo.ipynb

@@ -230,9 +230,7 @@
    },
    "source": [
     "## Navigation\n",
-    "Here, we pull out the trajectory (`Traj`) and `INS` structures from the `XYZ` structure for convenience, load the map data into a map (`MapS`) structure, and then \"upward continue\" (via a Fast Fourier Transform) the map data from where it was collected to the average altitude of the trajectory of interest. \n",
-    "\n",
-    "The map data is then outfitted with an interpolation function, `itp_mapS`, that can operate on any trajectory data provided to it. Finally, the expected scalar magnetometer reading along the flight path, `map_val`, can be computed by performing the interpolation and correcting it for the core magnetic field (IGRF) and diurnal effects."
+    "Here, the trajectory (`Traj`) and `INS` structures are pulled from the `XYZ` structure for convenience, and map data is loaded into a map (`MapS`) structure. The map is then \"upward continued\" (via a Fast Fourier Transform) to the trajectory altitude and outfitted with an interpolation function (`itp_mapS`). Finally, the expected scalar magnetometer reading along the flight path (`map_val`) is computed using the interpolation function, as done in `get_map_val()`, then corrected for diurnal effects and the core magnetic field (IGRF)."
    ]
   },
   {
@@ -242,12 +240,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "traj = get_traj(xyz,ind) # get trajectory (gps) struct\n",
-    "ins  = get_ins(xyz,ind;N_zero_ll=1) # get INS struct, \"zero\" the lat/lon to match `traj` for 1 data point\n",
+    "traj = get_traj(xyz,ind) # trajectory (GPS) struct\n",
+    "ins  = get_ins( xyz,ind;N_zero_ll=1) # INS struct, \"zero\" lat/lon to match `traj` for 1 data point\n",
     "mapS = get_map(map_name,df_map) # load map data\n",
-    "all(mapS.alt .> 0) && (mapS = upward_fft(mapS,mean(traj.alt))) # upward continue, unless drape map (alt = -1)\n",
-    "itp_mapS = map_interpolate(mapS) # get interpolation\n",
-    "map_val  = itp_mapS.(traj.lon,traj.lat) + (xyz.igrf + xyz.diurnal)[ind]; # add in core & diurnal\n",
+    "# get map values & map interpolation function\n",
+    "(map_val,itp_mapS) = get_map_val(mapS,traj;return_itp=true)\n",
+    "map_val += (xyz.diurnal + xyz.igrf)[ind] # add in diurnal & IGRF (core)\n",
     "println(\"Error for scalar Mag 4: \",round(std(map_val-mag_4_c),digits=2),\" nT\")"
    ]
   },
@@ -268,7 +266,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "plot_autocor(mag_4_c-map_val);"
+    "(sigma, tau) = get_autocor(mag_4_c-map_val)"
    ]
   },
   {
@@ -532,7 +530,7 @@
     "mag_use .+= map_val[1]-mag_use[1] # remove initial DC offset\n",
     "println(\"TL + NN σ: \",round(std(map_val-mag_use),digits=2))\n",
     "println(\"TL σ: \",round(std(mag_4_c.+(map_val[1]-mag_4_c[1])-map_val),digits=2))\n",
-    "plot_autocor(mag_use-map_val);"
+    "(sigma, tau) = get_autocor(mag_use-map_val)"
    ]
   },
   {
@@ -652,7 +650,7 @@
     "mag_use .+= map_val[1]-mag_use[1];\n",
     "println(\"NN σ: \",round(std(map_val-mag_use),digits=2))\n",
     "println(\"TL σ: \",round(std(mag_4_c.+(map_val[1]-mag_4_c[1])-map_val),digits=2))\n",
-    "plot_autocor(mag_use-map_val);"
+    "(sigma, tau) = get_autocor(mag_use-map_val)"
    ]
   },
   {

examples/example_model3.jl → examples/pluto_model3.jl

@@ -196,16 +196,14 @@ Ultimately, we'd like to use the compensated magnetometer values in a navigation
 * Initialized covariance & noise matrices for the prediction & measurement steps, (`P0`, `Qd`, `R`)
 "
 
-
 # ╔═╡ ed1f8baa-d228-4f66-a979-527f68acccdc
 begin
 	map_name = df_nav[df_nav.line .== line_test, :].map_name[1]
-	traj = get_traj(xyz_test,ind_test) # get trajectory (GPS) struct
-	ins  = get_ins(xyz_test,ind_test;N_zero_ll=1) # get INS struct
+	traj = get_traj(xyz_test,ind_test) # trajectory (GPS) struct
+	ins  = get_ins(xyz_test,ind_test;N_zero_ll=1) # INS struct
 	mapS = get_map(map_name,df_map) # load map data
-	all(mapS.alt .> 0) && (mapS = upward_fft(mapS,mean(traj.alt)))
-	itp_mapS = map_interpolate(mapS) # get interpolation
-	map_val  = itp_mapS.(traj.lon,traj.lat)
+    # get map values & map interpolation function
+    (map_val,itp_mapS) = get_map_val(mapS,traj;return_itp=true)
 end;
 
 # ╔═╡ 4e01e107-5d58-409d-b08c-23135b4c28ba
@@ -217,7 +215,7 @@ We next compute the compensated magnetometer values consistent with the `:d` set
 begin
 	# perform the compensation from the neural network
 	mag_4_c  = xyz_test.mag_4_uc[ind_test] - y_hat
-	# Remove an assumed constant map-magnetometer bias, which is approximated at the first measurement. The diurnal & IGRF core fields must be included to leave only the nearly constant bias, if any.
+	# Remove an assumed constant map-magnetometer bias, which is approximated at the first measurement. The diurnal & IGRF (core) fields must be included to leave only the nearly constant bias, if any.
 	mag_4_c .+= (map_val + (xyz_test.diurnal + xyz_test.igrf)[ind_test] - mag_4_c)[1]
 
 	# These first-order Gauss-Markov noise values can also be estimated from the training output without resorting to the known map values.

examples/example_sgl.jl → examples/pluto_sgl.jl

@@ -191,12 +191,12 @@ md"Prepare the flight data for the navigation filter, load the map data, & get t
 
 # ╔═╡ 3f2dd431-6c5b-403f-9a96-320d8ab6ef17
 begin
-	traj = get_traj(xyz,ind) # trajectory (gps) struct
-	ins  = get_ins(xyz,ind;N_zero_ll=1) # INS struct
+	traj = get_traj(xyz,ind) # trajectory (GPS) struct
+	ins  = get_ins( xyz,ind;N_zero_ll=1) # INS struct
 	mapS = get_map(map_name,df_map) # load map data
 	# get map values & map interpolation function
 	(map_val,itp_mapS) = get_map_val(mapS,traj;return_itp=true)
-	map_val += (xyz.diurnal + xyz.igrf)[ind] # add in diurnal & IGRF
+	map_val += (xyz.diurnal + xyz.igrf)[ind] # add in diurnal & IGRF (core)
 end;
 
 # ╔═╡ 50a6302c-b1ed-4be3-8af7-1c641715b25f

examples/example_sim.jl → examples/pluto_sim.jl

@@ -19,7 +19,7 @@ begin
 end;
 
 # ╔═╡ d9ac0df2-3d79-11ee-0869-73b7f6649d95
-md"# Using the MagNav Package with Real Simulated Data Example
+md"# Using the MagNav Package with Simulated Data Example
 This file is best viewed in a [Pluto](https://plutojl.org/) notebook. To run it this way, from the MagNav.jl directory, do:
 ```julia
 julia> using Pluto
@@ -120,10 +120,10 @@ md"Display the map or flight paths in Google Earth by uncommenting below to gene
 
 # ╔═╡ 26810eff-0812-43cc-b1bc-d4f5d7c9542d
 begin
-	# map2kmz(mapS,"example_sim_map")
-	# path2kml(traj,"example_sim_gps")
-	# path2kml(ins,"example_sim_ins")
-	# path2kml(filt_out,"example_sim_filt")
+	# map2kmz(mapS,"pluto_sim_map")
+	# path2kml(traj,"pluto_sim_gps")
+	# path2kml(ins,"pluto_sim_ins")
+	# path2kml(filt_out,"pluto_sim_filt")
 end;
 
 # ╔═╡ 00000000-0000-0000-0000-000000000001

src/MagNav.jl

@@ -27,7 +27,7 @@ module MagNav
     using Optim: LBFGS, Options, only_fg!, optimize
     using Parameters: @unpack, @with_kw
     using Pkg.Artifacts: @artifact_str
-    using Plots: mm, plot, plot!
+    using Plots: annotate!, contourf!, heatmap, mm, plot, plot!, scatter
     using Random: rand, randn, randperm, seed!, shuffle
     using RecipesBase: @recipe
     using SatelliteToolboxGeomagneticField: igrf, igrfd
@@ -40,18 +40,18 @@ module MagNav
     project_toml = normpath(joinpath(@__DIR__,"../Project.toml"))
 
     """
-        magnav_version
+        magnav_version :: VersionNumber
 
     MagNav.jl version as recorded in Project.toml.
     """
-    const magnav_version = TOML.parse(open(project_toml))["version"]
+    const magnav_version = VersionNumber(TOML.parse(open(project_toml))["version"])
 
     """
-        const num_mag_max = 10
+        const num_mag_max = 6
 
     Maximum number of scalar & vector magnetometers (each)
     """
-    const num_mag_max = 10
+    const num_mag_max = 6
 
     """
         const e_earth = 0.0818191908426
@@ -133,7 +133,7 @@ module MagNav
     """
         const emm720
 
-    Enhanced Magnetic Model
+    Enhanced Magnetic Model (vector magnetic anomaly map)
     """
     const emm720 = joinpath(artifact"EMM720_World","EMM720_World.h5")
 
@@ -962,7 +962,7 @@ module MagNav
     |`λ_TL`  | ridge parameter, only used for `model_type = :TL, :mod_TL, :map_TL`
     """
     @with_kw struct LinCompParams <: CompParams
-        version          :: String          = magnav_version
+        version          :: String          = string(magnav_version)
         features_setup   :: Vector{Symbol}  = [:mag_1_uc,:TL_A_flux_a]
         features_no_norm :: Vector{Symbol}  = Symbol[]
         model_type       :: Symbol          = :plsr
@@ -1068,7 +1068,7 @@ module MagNav
         - `tanh`  = hyperbolic tan
     """
     @with_kw struct NNCompParams <: CompParams
-        version          :: String          = magnav_version
+        version          :: String          = string(magnav_version)
         features_setup   :: Vector{Symbol}  = [:mag_1_uc,:TL_A_flux_a]
         features_no_norm :: Vector{Symbol}  = Symbol[]
         model_type       :: Symbol          = :m1

src/create_XYZ.jl

@@ -582,7 +582,7 @@ function create_mag_c(lat, lon, mapS::Union{MapS,MapSd,MapS3D} = get_map(namad);
     map_val = get_map_val(mapS,lat,lon,alt;α=200)
 
     # FOGM & white noise
-    @info("adding FOGM & white noise to scalar map value")
+    @info("adding FOGM & white noise to scalar map values")
     mag_c = map_val + fogm(fogm_sigma,fogm_tau,dt,N) + sqrt(meas_var)*randn(N)
 
     return (mag_c)