Merge branch 'main' into bfgs

.github/workflows/Documenter.yml

@@ -24,6 +24,13 @@ jobs:
             Pkg.instantiate()
             Pkg.build()
             Pkg.precompile()'
+          sudo apt-get install imagemagick
+          sudo apt-get install poppler-utils
+          sudo apt-get install texlive-xetex
+          sudo apt-get install texlive-science
+      -name: Generate tikz pictures 
+        run: |
+          make all -C src/tikz
       - name: Run doctests
         run: |
           julia --project=docs -e '

README.md

@@ -1,4 +1,7 @@
-# Geometric Machine Learning
+<p align="center">
+    <img width="700px" src="docs/src/assets/logo.png#gh-light-mode-only"/>
+    <img width="700px" src="docs/src/assets/logo_dark.png#gh-dark-mode-only"/>
+</p>
 
 [![Stable](https://img.shields.io/badge/docs-stable-blue.svg)](https://juliagni.github.io/GeometricMachineLearning.jl/stable)
 [![Latest](https://img.shields.io/badge/docs-latest-blue.svg)](https://juliagni.github.io/GeometricMachineLearning.jl/latest)

docs/Makefile

@@ -0,0 +1,15 @@
+
+.PHONY: documenter images
+
+all: images documenter
+
+documenter:
+	julia --color=yes --project=. make.jl
+
+images:
+	$(MAKE) all   -C src/tikz
+
+clean:
+	$(MAKE) empty -C src/tikz
+	rm -Rf build
+	rm -Rf src/tutorial

docs/make.jl

@@ -1,8 +1,15 @@
 using GeometricMachineLearning
 using Documenter
+using DocumenterCitations
+# using Weave
 
+# this is necessary to avoid warnings. See https://documenter.juliadocs.org/dev/man/syntax/
+ENV["GKSwstype"] = "100"
+
+bib = CitationBibliography(joinpath(@__DIR__, "src", "GeometricMachineLearning.bib"))
 
 makedocs(;
+    plugins=[bib],
     modules=[GeometricMachineLearning],
     authors="Michael Kraus, Benedikt Brantner",
     repo="https://github.com/JuliaGNI/GeometricMachineLearning.jl/blob/{commit}{path}#L{line}",
@@ -11,6 +18,8 @@ makedocs(;
         prettyurls=get(ENV, "CI", "false") == "true",
         canonical="https://juliagni.github.io/GeometricMachineLearning.jl",
         assets=String[],
+        # specifies that we do not display the package name again (it's already in the logo)
+        sidebar_sitename=false,
     ),
     pages=[
         "Home" => "index.md",
@@ -49,9 +58,11 @@ makedocs(;
             "Projection and Reduction Error" => "reduced_order_modeling/projection_reduction_errors.md",
         ],
         "Tutorials" =>[
+            "Sympnets" => "tutorials/sympnet_tutorial.md",
             "Linear Wave Equation" => "tutorials/linear_wave_equation.md",
             "MNIST" => "tutorials/mnist_tutorial.md",
         ],
+        "References" => "references.md",
         "Library" => "library.md",
     ],
 )

docs/src/GeometricMachineLearning.bib

@@ -0,0 +1,40 @@
+@article{brantner2023generalizing,
+  title={Generalizing Adam To Manifolds For Efficiently Training Transformers},
+  author={Brantner, Benedikt},
+  journal={arXiv preprint arXiv:2305.16901},
+  year={2023}
+}
+
+@article{jin2020sympnets,
+  title={SympNets: Intrinsic structure-preserving symplectic networks for identifying Hamiltonian systems},
+  author={Jin, Pengzhan and Zhang, Zhen and Zhu, Aiqing and Tang, Yifa and Karniadakis, George Em},
+  journal={Neural Networks},
+  volume={132},
+  pages={166--179},
+  year={2020},
+  publisher={Elsevier}
+}
+
+@article{jin2022optimal,
+  title={Optimal unit triangular factorization of symplectic matrices},
+  author={Jin, Pengzhan and Lin, Zhangli and Xiao, Bo},
+  journal={Linear Algebra and its Applications},
+  year={2022},
+  publisher={Elsevier}
+}
+
+@book{hairer2006geometric,
+  title={Geometric Numerical integration: structure-preserving algorithms for ordinary differential equations},
+  author={Hairer, Ernst and Lubich, Christian and Wanner, Gerhard},
+  year={2006},
+  publisher={Springer}
+}
+
+
+@book{leimkuhler2004simulating,
+  title={Simulating hamiltonian dynamics},
+  author={Leimkuhler, Benedict and Reich, Sebastian},
+  number={14},
+  year={2004},
+  publisher={Cambridge university press}
+}

docs/src/Optimizer.md

@@ -4,7 +4,7 @@ In order to generalize neural network optimizers to [homogeneous spaces](manifol
 
 Starting from an element of the tangent space $T_Y\mathcal{M}$[^1], we need to perform two mappings to arrive at $\mathfrak{g}^\mathrm{hor}$, which we refer to by $\Omega$ and a red horizontal arrow:
 
-![](images/general_optimization_with_boundary.png)
+![](tikz/general_optimization_with_boundary.png)
 
 Here the mapping $\Omega$ is a [horizontal lift](optimizers/manifold_related/horizontal_lift.md) from the tangent space onto the **horizontal component of the Lie algebra at $Y$**. 
 

docs/src/optimizers/adam_optimizer.md

@@ -19,7 +19,7 @@ If all the weights are on a vector space, then we directly compute updates for $
 
 Here $\eta$ (with default 0.01) is the **learning rate** and $\delta$ (with default $3\cdot10^{-7}$) is a small constant that is added for stability. The division, square root and addition in step 3 are performed element-wise. 
 
-![](../images/adam_optimizer.png)
+![](../tikz/adam_optimizer.png)
 
 ## Weights on manifolds 
 

docs/src/references.md

@@ -0,0 +1,4 @@
+#References 
+
+```@bibliography
+```

docs/src/tikz/Makefile

@@ -0,0 +1,44 @@
+pdf:
+	xelatex -shell-escape adam_optimizer
+	xelatex -shell-escape general_optimization
+	xelatex -shell-escape general_optimization_with_boundary
+	xelatex -shell-escape transformer_encoder
+	xelatex -shell-escape third_degree_spline
+	xelatex -shell-escape sympnet_architecture
+	xelatex -shell-escape structs_visualization
+	xelatex -shell-escape logo 
+
+png:
+	pdftocairo  -png -r 150 -transp -singlefile  adam_optimizer.pdf               	 	adam_optimizer              
+	pdftocairo  -png -r 150 -transp -singlefile  general_optimization.pdf		 	 	general_optimization   
+	pdftocairo  -png -r 150 -transp -singlefile  general_optimization_with_boundary.pdf general_optimization_with_boundary              
+	pdftocairo  -png -r 150 -transp -singlefile  transformer_encoder.pdf		        transformer_encoder   
+	pdftocairo  -png -r 150 -transp -singlefile  third_degree_spline.pdf	           	third_degree_spline          
+	pdftocairo	-png -r 150 -transp -singlefile	 sympnet_architecture.pdf				sympnet_architecture
+	pdftocairo 	-png -r 150 -transp -singlefile  structs_visualization.pdf				structs_visualization
+	pdftocairo 	-png -r 150 -transp -singlefile	 logo.pdf								logo 
+
+logo:
+	xelatex -shell-escape logo_with_name
+	xelatex -shell-escape logo_with_name_dark
+
+	pdftocairo 	-png -r 500 -transp -singlefile	 logo_with_name.pdf						logo_with_name 
+	pdftocairo 	-png -r 500 -transp -singlefile  logo_with_name_dark.pdf 				logo_with_name_dark
+
+	cp logo_with_name.png 		../assets/logo.png
+	cp logo_with_name_dark.png 	../assets/logo_dark.png
+
+clean:
+	rm -f *.aux
+	rm -f *.fdb_latexmk
+	rm -f *.fls
+	rm -f *.log
+	rm -f *.pdf
+	rm -f *.xdv
+
+empty: clean
+	rm -f *.png
+	rm -f *.svg
+	rm -f ../assets/*.png
+
+all: pdf png logo clean

docs/src/tikz/logo.tex

@@ -0,0 +1,53 @@
+\documentclass[tikz]{standalone}
+
+\definecolor{morange}{RGB}{255,127,14}
+\definecolor{mblue}{RGB}{31,119,180}
+\definecolor{mred}{RGB}{214,39,40}
+\definecolor{mpurple}{RGB}{148,103,189}
+\definecolor{mgreen}{RGB}{44,160,44}
+
+\usepackage{neuralnetwork}
+
+\newcommand{\symplecticlayer}[1][] { \layer[bias=true,nodeclass={symplectic neuron},#1] }
+\newcommand{\autoencoderlayer}[1][] { \layer[bias=true,nodeclass={autoencoder neuron},#1] }
+
+\newcommand{\xin}[2]{$x_#2$}
+\newcommand{\xout}[2]{$\hat x_#2$}
+
+\begin{document}
+
+\begin{neuralnetwork}[height=8, layertitleheight=30, toprow=true]
+
+  \tikzstyle{input neuron}=[neuron, fill=morange];
+  \tikzstyle{hidden neuron}=[neuron, fill=mred];
+  \tikzstyle{output neuron}=[neuron, fill=mgreen];
+  \tikzstyle{symplectic neuron}=[neuron, fill=mblue];
+  \tikzstyle{autoencoder neuron}=[neuron, fill=mpurple];
+
+  \inputlayer[count=8, bias=false, text=\xin]%, title=Input Layer
+
+  \symplecticlayer[count=8, bias=false]
+  \linklayers%[title=$\psi^1_\mathrm{symp}$]
+
+  \autoencoderlayer[count=4, bias=false]
+  \linklayers%[title=$A_\mathrm{Stiefel}$]
+
+  \symplecticlayer[count=4, bias=false]
+  \linklayers%[title=$\psi^2_\mathrm{symp}$]
+
+  \hiddenlayer[count=2, bias=false]%, title=\vspace{4cm}\\{Reduced Basis}]
+  \linklayers%[title=$\psi^2_\mathrm{symp}$]
+
+  \symplecticlayer[count=4, bias=false]
+  \linklayers%[title={$\tilde{A}_\mathrm{Stiefel}$}]
+
+  \autoencoderlayer[count=4, bias=false]
+  \linklayers%[title=$\tilde{\psi}^1_\mathrm{symp}$]
+
+  \symplecticlayer[count=8, bias=false]
+  \linklayers%[title={$\tilde{A}_\mathrm{Stiefel}$}]
+
+  \outputlayer[count=8, text=\xout]%, title=Output Layer
+  \linklayers%[title=$\tilde{\psi}^2_\mathrm{symp}$]
+\end{neuralnetwork}
+\end{document}

docs/src/tikz/logo_with_name.tex

@@ -0,0 +1,58 @@
+\documentclass[tikz]{standalone}
+
+\definecolor{morange}{RGB}{255,127,14}
+\definecolor{mblue}{RGB}{31,119,180}
+\definecolor{mred}{RGB}{214,39,40}
+\definecolor{mpurple}{RGB}{148,103,189}
+\definecolor{mgreen}{RGB}{44,160,44}
+
+\usepackage{neuralnetwork}
+
+% for bigger font size
+\usepackage{mathptmx}
+
+\newcommand{\symplecticlayer}[1][] { \layer[bias=true,nodeclass={symplectic neuron},#1] }
+\newcommand{\autoencoderlayer}[1][] { \layer[bias=true,nodeclass={autoencoder neuron},#1] }
+
+\newcommand{\xin}[2]{$x_#2$}
+\newcommand{\xout}[2]{$\hat x_#2$}
+
+\begin{document}
+
+\begin{neuralnetwork}[height=8, layertitleheight=30, toprow=true]
+
+  \tikzstyle{input neuron}=[neuron, fill=morange];
+  \tikzstyle{hidden neuron}=[neuron, fill=mred];
+  \tikzstyle{output neuron}=[neuron, fill=mgreen];
+  \tikzstyle{symplectic neuron}=[neuron, fill=mblue];
+  \tikzstyle{autoencoder neuron}=[neuron, fill=mpurple];
+
+  \inputlayer[count=8, bias=false, text=\xin]%, title=Input Layer
+
+  \symplecticlayer[count=8, bias=false]
+  \linklayers%[title=$\psi^1_\mathrm{symp}$]
+
+  \autoencoderlayer[count=4, bias=false]
+  \linklayers%[title=$A_\mathrm{Stiefel}$]
+
+  \symplecticlayer[count=4, bias=false]
+  \linklayers%[title=$\psi^2_\mathrm{symp}$]
+
+  \hiddenlayer[count=2, bias=false]%, title=\vspace{4cm}\\{Reduced Basis}]
+  \linklayers%[title=$\psi^2_\mathrm{symp}$]
+
+  \symplecticlayer[count=4, bias=false]
+  \linklayers%[title={$\tilde{A}_\mathrm{Stiefel}$}]
+
+  \autoencoderlayer[count=4, bias=false]
+  \linklayers%[title=$\tilde{\psi}^1_\mathrm{symp}$]
+
+  \symplecticlayer[count=8, bias=false]
+  \linklayers%[title={$\tilde{A}_\mathrm{Stiefel}$}]
+
+  \outputlayer[count=8, text=\xout]%, title=Output Layer
+  \linklayers%[title=$\tilde{\psi}^2_\mathrm{symp}$]
+
+  \node[xshift=10cm, yshift=-10cm] (gml) {\fontsize{50}{30} \texttt{{\color{morange}G}eometric{\color{mblue}M}achine{\color{mred}L}earning.{\color{mpurple}j}{\color{mgreen}l}}};
+\end{neuralnetwork}
+\end{document}

docs/src/tikz/logo_with_name_dark.tex

@@ -0,0 +1,58 @@
+\documentclass[tikz]{standalone}
+
+\definecolor{morange}{RGB}{255,127,14}
+\definecolor{mblue}{RGB}{31,119,180}
+\definecolor{mred}{RGB}{214,39,40}
+\definecolor{mpurple}{RGB}{148,103,189}
+\definecolor{mgreen}{RGB}{44,160,44}
+
+\usepackage{neuralnetwork}
+
+% for bigger font size
+\usepackage{mathptmx}
+
+\newcommand{\symplecticlayer}[1][] { \layer[bias=true,nodeclass={symplectic neuron},#1] }
+\newcommand{\autoencoderlayer}[1][] { \layer[bias=true,nodeclass={autoencoder neuron},#1] }
+
+\newcommand{\xin}[2]{$x_#2$}
+\newcommand{\xout}[2]{$\hat x_#2$}
+
+\begin{document}
+
+\begin{neuralnetwork}[height=8, layertitleheight=30, toprow=true]
+
+  \tikzstyle{input neuron}=[neuron, fill=morange];
+  \tikzstyle{hidden neuron}=[neuron, fill=mred];
+  \tikzstyle{output neuron}=[neuron, fill=mgreen];
+  \tikzstyle{symplectic neuron}=[neuron, fill=mblue];
+  \tikzstyle{autoencoder neuron}=[neuron, fill=mpurple];
+
+  \inputlayer[count=8, bias=false, text=\xin]%, title=Input Layer
+
+  \symplecticlayer[count=8, bias=false]
+  \linklayers%[title=$\psi^1_\mathrm{symp}$]
+
+  \autoencoderlayer[count=4, bias=false]
+  \linklayers%[title=$A_\mathrm{Stiefel}$]
+
+  \symplecticlayer[count=4, bias=false]
+  \linklayers%[title=$\psi^2_\mathrm{symp}$]
+
+  \hiddenlayer[count=2, bias=false]%, title=\vspace{4cm}\\{Reduced Basis}]
+  \linklayers%[title=$\psi^2_\mathrm{symp}$]
+
+  \symplecticlayer[count=4, bias=false]
+  \linklayers%[title={$\tilde{A}_\mathrm{Stiefel}$}]
+
+  \autoencoderlayer[count=4, bias=false]
+  \linklayers%[title=$\tilde{\psi}^1_\mathrm{symp}$]
+
+  \symplecticlayer[count=8, bias=false]
+  \linklayers%[title={$\tilde{A}_\mathrm{Stiefel}$}]
+
+  \outputlayer[count=8, text=\xout]%, title=Output Layer
+  \linklayers%[title=$\tilde{\psi}^2_\mathrm{symp}$]
+
+  \node[xshift=10cm, yshift=-10cm] (gml) {\fontsize{50}{30} \texttt{{\color{morange}G}{\color{white}eometric}{\color{mblue}M}{\color{white}achine}{\color{mred}L}{\color{white}earning.}{\color{mpurple}j}{\color{mgreen}l}}};
+\end{neuralnetwork}
+\end{document}