Merge pull request #96 from JuliaGNI/remove_unnecessary_image_files

Removed unnecessary image files (`png`s)

docs/src/index.md

@@ -15,6 +15,8 @@ GeometricMachineLearning.jl implements various scientific machine learning model
 
 ## Architectures
 
+There are several architectures tailored towards problems in scientific machine learning implemented in `GeometricMachineLearning`.
+
 ```@contents
 Pages = [
     "architectures/sympnet.md",
@@ -23,9 +25,44 @@ Pages = [
 
 ## Manifolds
 
+`GeometricMachineLearning` supports putting neural network weights on manifolds. These include:
+
 ```@contents
 Pages = [
     "manifolds/grassmann_manifold.md",
     "manifolds/stiefel_manifold.md",
 ]
 ```
+
+## Special Neural Network Layer
+
+Many layers have been adapted in order to be used for problems in scientific machine learning. Including:
+
+```@contents
+Pages = [
+    "layers/attention_layer.md",
+]
+```
+
+## Tutorials 
+
+Tutorials for using `GeometricMachineLearning` are: 
+
+```@contents
+Pages = [
+    "tutorials/sympnet_tutorial.md",
+    "tutorials/mnist_tutorial.md",
+]
+```
+
+## Reduced Order Modeling
+
+A short description of the key concepts in **reduced order modeling** (where `GeometricMachineLearning` can be used) are in:
+
+```@contents
+Pages = [
+    "reduced_order_modeling/autoencoder.md",
+    "reduced_order_modeling/symplectic_autoencoder.md",
+    "reduced_order_modeling/kolmogorov_n_width.md",
+]
+```

docs/src/layers/multihead_attention_layer.md

@@ -19,7 +19,7 @@ where $n$ is the dimension of the vectors in $V$, $Q$ and $K$. The softmax activ
 
 The transformer contains a **self-attention mechanism**, i.e. takes an input $X$ and then transforms it linearly to $V$, $Q$ and $K$, i.e. $V = P^VX$, $Q = P^QX$ and $K = P^KX$. What distinguishes the multihead attention layer from the singlehead attention layer, is that there is not just one $P^V$, $P^Q$ and $P^K$, but there are several: one for each **head** of the multihead attention layer. After computing the individual values, queries and vectors, and after applying the softmax, the outputs are then concatenated together in order to obtain again an array that is of the same size as the input array:
 
-![](../images/mha.png)
+![](../tikz/mha.png)
 
 Here the various $P$ matrices can be interpreted as being projections onto lower-dimensional subspaces, hence the designation by the letter $P$. Because of this interpretation as projection matrices onto smaller spaces that should **capture features in the input data** it makes sense to constrain these elements to be part of the Stiefel manifold.   
 

docs/src/reduced_order_modeling/autoencoder.md

@@ -14,7 +14,7 @@ To any parametric PDE we associate a **solution manifold**:
 \mathcal{M} = \{z(\mu):F(z(\mu);\mu)=0, \mu\in\mathbb{P}\}.
 ```
 
-![](../images/solution_manifold_2.png)
+![](../tikz/solution_manifold_2.png)
 
 In the image above a 2-dimensional solution manifold is visualized as a sub-manifold in 3-dimensional space. In general the embedding space is an infinite-dimensional function space.
 

docs/src/reduced_order_modeling/symplectic_autoencoder.md

@@ -77,7 +77,7 @@ PSD suffers from the similar shortcomings as regular POD: it is a linear map and
 
 In order to overcome this difficulty we use neural networks, more specifically [SympNets](../architectures/sympnet.md), together with cotangent lift-like matrices. The resulting architecture, symplectic autoencoders, are demonstrated in the following image: 
 
-![](../images/symplectic_autoencoder.png)
+![](../tikz/symplectic_autoencoder.png)
 
 So we alternate between SympNet and PSD layers. Because all the PSD layers are based on matrices $\Phi\in{}St(n,N)$ we have to [optimize on the Stiefel manifold](../Optimizer.md).
 

docs/src/tikz/Makefile

@@ -7,6 +7,9 @@ pdf:
 	xelatex -shell-escape sympnet_architecture
 	xelatex -shell-escape structs_visualization
 	xelatex -shell-escape logo 
+	xelatex -shell-escape symplectic_autoencoder
+	xelatex -shell-escape solution_manifold_2
+	xelatex -shell-escape mha
 
 png:
 	pdftocairo  -png -r 150 -transp -singlefile  adam_optimizer.pdf               	 	adam_optimizer              
@@ -17,6 +20,9 @@ png:
 	pdftocairo	-png -r 150 -transp -singlefile	 sympnet_architecture.pdf				sympnet_architecture
 	pdftocairo 	-png -r 150 -transp -singlefile  structs_visualization.pdf				structs_visualization
 	pdftocairo 	-png -r 150 -transp -singlefile	 logo.pdf								logo 
+	pdftocairo  -png -r 150 -transp -singlefile  symplectic_autoencoder.pdf				symplectic_autoencoder
+	pdftocairo  -png -r 150 -transp -singlefile  solution_manifold_2.pdf				solution_manifold_2
+	pdftocairo  -png -r 150 -transp -singlefile  mha.pdf 								mha
 
 logo:
 	xelatex -shell-escape logo_with_name
@@ -25,6 +31,7 @@ logo:
 	pdftocairo 	-png -r 500 -transp -singlefile	 logo_with_name.pdf						logo_with_name 
 	pdftocairo 	-png -r 500 -transp -singlefile  logo_with_name_dark.pdf 				logo_with_name_dark
 
+	mkdir -p ../assets
 	cp logo_with_name.png 		../assets/logo.png
 	cp logo_with_name_dark.png 	../assets/logo-dark.png
 
@@ -40,5 +47,6 @@ empty: clean
 	rm -f *.png
 	rm -f *.svg
 	rm -f ../assets/*.png
+	rm -f ../assets
 
 all: pdf png logo clean

docs/src/tikz/mha.tex

@@ -0,0 +1,45 @@
+\documentclass[tikz]{standalone}
+
+\usepackage{xcolor}
+\definecolor{morange}{RGB}{255,127,14}
+\definecolor{mblue}{RGB}{31,119,180}
+\definecolor{mred}{RGB}{214,39,40}
+\definecolor{mpurple}{RGB}{148,103,189}
+\definecolor{mgreen}{RGB}{44,160,44}
+
+\usepackage{tikz}
+\usetikzlibrary{positioning}
+\usetikzlibrary{calc}
+\usetikzlibrary{fit}
+\usetikzlibrary{shadows}
+\usetikzlibrary{decorations}
+
+\begin{document}
+\begin{tikzpicture}[module/.style={draw, very thick, rounded corners, minimum width=15ex},
+    attmodule/.style={module, fill=morange!30},
+    linear/.style={module, fill=mblue!30},
+    concat/.style={module, fill=mpurple!30},
+    arrow/.style={-stealth, thick, rounded corners},
+]
+\node (input) {Input};
+\node[above of=input, linear, align=center, double copy shadow={opacity=.5, shadow yshift=1.7ex, shadow xshift=1.2ex}] (linear_center) {Linear};
+\node[left of=linear_center, linear, align=center, xshift=-1.5cm, double copy shadow={opacity=.5, shadow yshift=1.7ex, shadow xshift=1.2ex}] (linear_left) {Linear};
+\node[right of=linear_center, linear, align=center, xshift=1.5cm, double copy shadow={opacity=.5, shadow yshift=1.7ex, shadow xshift=1.2ex}] (linear_right) {Linear};
+\node[above of=linear_center, attmodule, align=center, yshift=.5cm, double copy shadow={opacity=.5, shadow yshift=1.7ex, shadow xshift=1.2ex}] (sha) {Attention Mechanism};
+\node[above of=sha, concat, yshift=.4cm] (concat) {Concatenate};
+\node[above of=concat] (output) {Output};
+
+% to have the shadows displayed in the final pdf
+\node[right of=linear_right, xshift=.4cm] (right_of_linear_right) {};
+
+\draw[arrow, double copy shadow={opacity=.5, shadow yshift=1.7ex, shadow xshift=1.2ex}] (input) -- (linear_left);
+\draw[arrow, double copy shadow={opacity=.2, shadow yshift=1.7ex, shadow xshift=1.2ex}] (input) -- (linear_center); 
+\draw[arrow, double copy shadow={opacity=.5, shadow yshift=1.7ex, shadow xshift=1.2ex}] (input) -- (linear_right);
+\draw[arrow, double copy shadow={opacity=.2, shadow yshift=1.7ex, shadow xshift=1.2ex}] (linear_left) -- (sha.south west);
+\draw[arrow, double copy shadow={opacity=.2, shadow yshift=1.7ex, shadow xshift=1.2ex}] (linear_center) -- (sha.south);
+\draw[arrow, double copy shadow={opacity=.5, shadow yshift=1.7ex, shadow xshift=1.2ex}] (linear_right) -- (sha.south east); 
+\draw[arrow, double copy shadow={opacity=.2, shadow yshift=1.7ex, shadow xshift=1.2ex}] (sha) -- (concat.south);
+\draw[arrow] (concat) -- (output);
+
+\end{tikzpicture}
+\end{document}

docs/src/tikz/solution_manifold_2.tex

@@ -0,0 +1,50 @@
+\documentclass[border = .2cm]{standalone}
+
+\usepackage{pgfplots}
+
+\pgfplotsset{compat = newest}
+
+\definecolor{mblue}{RGB}{66,164,233}
+
+\pgfplotsset{
+    colormap={cmblue}{
+            rgb255(0cm)=(132,233,233);
+            rgb255(1cm)=(66,164,233);
+            rgb255(2cm)=(132,233,233);
+        },
+    colormap={whiteblack}{color=(black) color=(white)},
+    tick label style={color=white},
+}
+
+\begin{document}
+ \begin{tikzpicture}
+
+    \begin{axis}[
+        view = {-25}{25},
+        mesh/interior colormap name=cmblue, colormap name=whiteblack,
+        xmax = +3,
+        xmin = -3,
+        ymax = +3, 
+        ymin = -3, 
+        zmax = +10, 
+        zmin = -10,
+        xtick = {-3,-2,-1,0,1,2,3},
+        ytick = {-3,-2,-1,0,1,2,3},
+        ztick = {-10,-5,0,5,10},
+        grid,
+        3d box = complete*
+        ]
+
+    \addplot3 [
+        domain = -3:3,
+        domain y = -3:3,
+        samples = 50,
+        samples y = 50,
+        surf,
+        shader = flat
+        ] {x^2 - y^2};
+
+\end{axis}
+
+\end{tikzpicture}
+\end{document}

docs/src/tikz/symplectic_autoencoder.tex

@@ -0,0 +1,128 @@
+\documentclass[tikz]{standalone}
+
+\definecolor{morange}{RGB}{255,127,14}
+\definecolor{mblue}{RGB}{31,119,180}
+\definecolor{mred}{RGB}{214,39,40}
+\definecolor{mpurple}{RGB}{148,103,189}
+\definecolor{mgreen}{RGB}{44,160,44}
+
+\newcommand*{\colIn}{morange}%
+\newcommand*{\colEnc}{mpurple}%
+\newcommand*{\colIntr}{mred}%
+\newcommand*{\colDec}{mpurple}%
+\newcommand*{\colOut}{mgreen}%
+\newcommand*{\colSym}{mblue}%
+
+\newcommand*{\arrowLength}{1}%
+
+\newcommand*{\xMin}{1}%
+\newcommand*{\xMax}{8}%
+\newcommand*{\xMaxEnc}{4}%
+\newcommand*{\xMaxIntr}{2}%
+\newcommand*{\xMaxDec}{4}%
+
+\newcommand*{\xin}{0}%
+\newcommand*{\xinsym}{\xin+\arrowLength+2}%
+\newcommand*{\xenc}{\xinsym+\arrowLength+2}%
+\newcommand*{\xencsym}{\xenc+\arrowLength+2}%
+\newcommand*{\xintr}{\xencsym+\arrowLength+2}%
+\newcommand*{\xdecsym}{\xintr+\arrowLength+2}%
+\newcommand*{\xdec}{\xdecsym+\arrowLength+2}%
+\newcommand*{\xoutsym}{\xdec+\arrowLength+2}%
+\newcommand*{\xout}{\xoutsym+\arrowLength+2}%
+
+
+\begin{document}
+\begin{tikzpicture}[scale=0.5]
+
+    \begin{scope}
+        \foreach \i in {\xMin,...,\xMax} {
+            \draw [rectangle, draw, fill=\colIn] (\xin,\xMax/2-\i) rectangle (\xin+1,\xMax/2-\i+1) node [pos=.5] {$x_\i$};
+        }
+    \end{scope}
+
+    \path [draw,->] (\xin+1.5,0) -- (\xinsym-0.5,0) node [pos=0.5,above,align=left] {$\psi_1$};
+
+    \begin{scope}
+        \foreach \i in {\xMin,...,\xMax} {
+            \draw [rectangle, draw, fill=\colSym] (\xinsym,\xMax/2-\i) rectangle (\xinsym+1,\xMax/2-\i+1) node [pos=.5] {};
+        }
+    \end{scope}
+
+    \path [draw,->] (\xinsym+1.5,0) -- (\xenc-0.5,0) node [pos=0.5,above,align=left] {$A_1$};
+
+    \begin{scope}
+        \foreach \i in {\xMin,...,\xMaxEnc} {
+            \draw [rectangle, draw, fill=\colEnc] (\xenc,\xMaxEnc/2-\i) rectangle (\xenc+1,\xMaxEnc/2-\i+1) node {};
+        }
+    \end{scope}
+
+    \path [draw,->] (\xenc+1.5,0) -- (\xencsym-0.5,0) node [pos=0.5,above,align=left] {$\psi_2$};
+
+    \begin{scope}
+        \foreach \i in {\xMin,...,\xMaxEnc} {
+            \draw [rectangle, draw, fill=\colSym] (\xencsym,\xMaxEnc/2-\i) rectangle (\xencsym+1,\xMaxEnc/2-\i+1) node {};
+        }
+    \end{scope}
+
+    \path [draw,->] (\xencsym+1.5,0) -- (\xintr-0.5,0) node [pos=0.5,above,align=left] {$A_2$};
+
+    \begin{scope}
+        \foreach \i in {\xMin,...,\xMaxIntr} {
+            \draw [rectangle, draw, fill=\colIntr] (\xintr,\xMaxIntr/2-\i) rectangle (\xintr+1,\xMaxIntr/2-\i+1) node {};
+        }
+    \end{scope}
+
+    \path [draw,->] (\xintr+1.5,0) -- (\xdecsym-0.5,0) node [pos=0.5,above,align=left] {$A_3^+$};
+
+    \begin{scope}
+        \foreach \i in {\xMin,...,\xMaxEnc} {
+            \draw [rectangle, draw, fill=\colSym] (\xdecsym,\xMaxEnc/2-\i) rectangle (\xdecsym+1,\xMaxEnc/2-\i+1) node {};
+        }
+    \end{scope}
+
+    \path [draw,->] (\xdecsym+1.5,0) -- (\xdec-0.5,0) node [pos=0.5,above,align=left] {$\psi_3$};
+
+    \begin{scope}
+        \foreach \i in {\xMin,...,\xMaxDec} {
+            \draw [rectangle, draw, fill=\colDec] (\xdec,\xMaxDec/2-\i) rectangle (\xdec+1,\xMaxDec/2-\i+1) node {};
+        }
+    \end{scope}
+
+    \path [draw,->] (\xdec+1.5,0) -- (\xoutsym-0.5,0) node [pos=0.5,above,align=left] {$A_4^+$};
+
+    \begin{scope}
+        \foreach \i in {\xMin,...,\xMax} {
+            \draw [rectangle, draw, fill=\colSym] (\xoutsym,\xMax/2-\i) rectangle (\xoutsym+1,\xMax/2-\i+1) node [pos=.5] {};
+        }
+    \end{scope}
+
+    \path [draw,->] (\xoutsym+1.5,0) -- (\xout-0.5,0) node [pos=0.5,above,align=left] {$\psi_4$};
+
+    \begin{scope}
+        \foreach \i in {\xMin,...,\xMax} {
+            \draw [rectangle, draw, fill=\colOut] (\xout,\xMax/2-\i) rectangle (\xout+1,\xMax/2-\i+1) node [pos=.5] {$\tilde{x}_\i$};
+        }
+    \end{scope}
+
+
+%   \node[rectangle,draw,fill=teal!,text width=.3cm] (11) at (0,0) {$x_1$ \\ $x_2$ \\ $x_3$ \\ $x_4$ \\ $x_5$ \\ $x_6$ \\ $x_7$ \\ $x_8$};
+%   \node[rectangle,draw,fill=teal!,text width=.3cm] (12) at (1,0) {$x_1$ \\ $x_2$ \\ $x_3$ \\ $x_4$ \\ $x_5$ \\ $x_6$ \\ $x_7$ \\ $x_8$};
+%   \node[rectangle,draw,fill=olive!,text width=.3cm] (21) at (2,0) {$x_1$ \\ $x_2$ \\ $x_3$ \\ $x_4$ };
+%   \node[rectangle,draw,fill=olive!,text width=.3cm] (22) at (3,0) {$x_1$ \\ $x_2$ \\ $x_3$ \\ $x_4$ };
+%   \node[rectangle,draw,fill=lightgray!,text width=.3cm] (3) at (4,0) {$x_1$ \\ $x_2$ };
+%   \node[rectangle,draw,fill=olive!,text width=.3cm] (41) at (5,0) {$x_1$ \\ $x_2$ \\ $x_3$ \\ $x_4$ };
+%   \node[rectangle,draw,fill=olive!,text width=.3cm] (42) at (6,0) {$x_1$ \\ $x_2$ \\ $x_3$ \\ $x_4$ };
+%   \node[rectangle,draw,fill=teal!,text width=.3cm] (51) at (7,0) {$x_1$ \\ $x_2$ \\ $x_3$ \\ $x_4$ \\ $x_5$ \\ $x_6$ \\ $x_7$ \\ $x_8$};
+%   \node[rectangle,draw,fill=teal!,text width=.3cm] (52) at (8,0) {$x_1$ \\ $x_2$ \\ $x_3$ \\ $x_4$ \\ $x_5$ \\ $x_6$ \\ $x_7$ \\ $x_8$};
+%   \path[draw,->] (11.east) -- (12.west) node[pos=0.5,above,align=left] {$\psi_1$};
+%   \path[draw,->] (21.east) -- (22.west) node[pos=0.5,above,align=left] {$\psi_2$};
+%   \path[draw,->] (41.east) -- (42.west) node[pos=0.5,above,align=left] {$\psi_3$};
+%   \path[draw,->] (51.east) -- (52.west) node[pos=0.5,above,align=left] {$\psi_4$};
+%   \path[draw,->] (12.east) -- (21.west) node[pos=0.5,above,align=left] {$A_1$};
+%   \path[draw,->] (22.east) -- (3.west) node[pos=0.5,above,align=left] {$A_2$};
+%   \path[draw,->] (3.east) -- (41.west) node[pos=0.5,above,align=left] {$A_3^+$};
+%   \path[draw,->] (42.east) -- (51.west) node[pos=0.5,above,align=left] {$A_4^+$};
+
+\end{tikzpicture}
+\end{document}

docs/src/tikz/transformer_encoder.tex

@@ -15,14 +15,18 @@
 
 \tikzset{set/.style={draw,circle,inner sep=0pt,align=center}}
 
+\definecolor{morange}{RGB}{255,127,14}
+\definecolor{mblue}{RGB}{31,119,180}
 \definecolor{mred}{RGB}{214,39,40}
+\definecolor{mpurple}{RGB}{148,103,189}
 \definecolor{mgreen}{RGB}{44,160,44}
 
+
 \begin{document}
 \begin{tikzpicture}[module/.style={draw, very thick, rounded corners, minimum width=15ex},
-    attmodule/.style={module, fill=orange!20},
-    ffnnmodule/.style={module, fill=cyan!20},
-    addmodule/.style={module, fill=yellow!20},
+    attmodule/.style={module, fill=morange!30},
+    ffnnmodule/.style={module, fill=mblue!30},
+    addmodule/.style={module, fill=mpurple!30},
     arrow/.style={-stealth, thick, rounded corners},
 ]
 \node (input) {Input};