add visualization chapter

MerkulovDaniil · Nov 28, 2024 · 93a1f75 · 93a1f75
1 parent 7484222
commit 93a1f75
Show file tree

Hide file tree

Showing 4 changed files with 148 additions and 17 deletions.
diff --git a/_quarto.yml b/_quarto.yml
@@ -12,28 +12,21 @@ project:
     - "assets/js/**"
     - "assets/files/*.csv"
 
-format: fminxyz-html
-
-formats:
+format: 
+  fminxyz-html: default
   # pdf:
-  #   toc: false
-  #   geometry:
-  #   - left=10mm
-  #   - right=10mm
-  gfm:
-    toc: false
-  # beamer:
-  #   pdf-engine: pdflatex
-  #   aspectratio: 169
-  #   fontsize: 9pt
-  #   section-titles: true
-  #   incremental: true
-  #   include-in-header: header.tex  # Custom LaTeX commands and preamble
+  #   pdf-engine: xelatex
+  #   number-sections: true
+  #   number-depth: 1
+  #   keep-tex: true
+  gfm: default
 
 format-links:
   - fminxyz-html
   - format: gfm
     text: Markdown
+  # - format: pdf
+    # text: PDF
 
 execute:
   freeze: auto

diff --git a/assets/graphon.drawio b/assets/graphon.drawio
@@ -1,4 +1,4 @@
-<mxfile host="Electron" agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/24.7.8 Chrome/128.0.6613.36 Electron/32.0.1 Safari/537.36" version="24.7.8" pages="176">
+<mxfile host="Electron" agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/24.7.17 Chrome/128.0.6613.36 Electron/32.0.1 Safari/537.36" version="24.7.17" pages="177">
   <diagram id="Tc_C-4fhjzrIpTlicYjt" name="fmin_logo">
     <mxGraphModel dx="1546" dy="1087" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1830" pageHeight="430" background="none" math="1" shadow="0">
       <root>
@@ -16146,4 +16146,130 @@
       </root>
     </mxGraphModel>
   </diagram>
+  <diagram id="1-F7JSGl7FbFxNFHqWIN" name="gpt2_memory">
+    <mxGraphModel dx="1026" dy="353" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="700" pageHeight="220" math="0" shadow="0">
+      <root>
+        <mxCell id="0" />
+        <mxCell id="1" parent="0" />
+        <mxCell id="bcFgxOS0jOe-36kQjctr-1" value="" style="shape=table;startSize=0;container=1;collapsible=0;childLayout=tableLayout;flipV=1;flipH=1;direction=east;swimlaneFillColor=none;strokeColor=default;perimeterSpacing=17;strokeWidth=2;fontSize=18;align=center;labelBackgroundColor=none;shadow=0;rounded=0;swimlaneLine=1;" vertex="1" parent="1">
+          <mxGeometry x="40" y="160" width="820" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="bcFgxOS0jOe-36kQjctr-2" value="" style="shape=tableRow;horizontal=0;startSize=0;swimlaneHead=0;swimlaneBody=0;strokeColor=inherit;top=0;left=0;bottom=0;right=0;collapsible=0;dropTarget=0;fillColor=none;points=[[0,0.5],[1,0.5]];portConstraint=eastwest;" vertex="1" parent="bcFgxOS0jOe-36kQjctr-1">
+          <mxGeometry width="820" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="bcFgxOS0jOe-36kQjctr-3" value="3 GB" style="shape=partialRectangle;html=1;whiteSpace=wrap;connectable=0;strokeColor=#666666;overflow=hidden;fillColor=#f5f5f5;top=0;left=0;bottom=0;right=0;pointerEvents=1;fontColor=#333333;fontSize=23;fontFamily=Palatino;fontSource=https%3A%2F%2Ffonts.googleapis.com%2Fcss%3Ffamily%3DPalatino;strokeWidth=20;perimeterSpacing=3;" vertex="1" parent="bcFgxOS0jOe-36kQjctr-2">
+          <mxGeometry width="60" height="60" as="geometry">
+            <mxRectangle width="60" height="60" as="alternateBounds" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="bcFgxOS0jOe-36kQjctr-4" value="6 GB" style="shape=partialRectangle;html=1;whiteSpace=wrap;connectable=0;strokeColor=#d6b656;overflow=hidden;fillColor=#fff2cc;top=0;left=0;bottom=0;right=0;pointerEvents=1;fontSize=23;fontFamily=Palatino;fontSource=https%3A%2F%2Ffonts.googleapis.com%2Fcss%3Ffamily%3DPalatino;strokeWidth=20;perimeterSpacing=3;" vertex="1" parent="bcFgxOS0jOe-36kQjctr-2">
+          <mxGeometry x="60" width="120" height="60" as="geometry">
+            <mxRectangle width="120" height="60" as="alternateBounds" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="bcFgxOS0jOe-36kQjctr-5" value="8 GB" style="shape=partialRectangle;html=1;whiteSpace=wrap;connectable=0;strokeColor=#82b366;overflow=hidden;fillColor=#d5e8d4;top=0;left=0;bottom=0;right=0;pointerEvents=1;fontSize=23;fontFamily=Palatino;fontSource=https%3A%2F%2Ffonts.googleapis.com%2Fcss%3Ffamily%3DPalatino;strokeWidth=20;perimeterSpacing=3;" vertex="1" parent="bcFgxOS0jOe-36kQjctr-2">
+          <mxGeometry x="180" width="160" height="60" as="geometry">
+            <mxRectangle width="160" height="60" as="alternateBounds" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="bcFgxOS0jOe-36kQjctr-6" value="6 GB" style="shape=partialRectangle;html=1;whiteSpace=wrap;connectable=0;strokeColor=#b85450;overflow=hidden;fillColor=#f8cecc;top=0;left=0;bottom=0;right=0;pointerEvents=1;fontSize=23;fontFamily=Palatino;fontSource=https%3A%2F%2Ffonts.googleapis.com%2Fcss%3Ffamily%3DPalatino;strokeWidth=20;perimeterSpacing=3;" vertex="1" parent="bcFgxOS0jOe-36kQjctr-2">
+          <mxGeometry x="340" width="120" height="60" as="geometry">
+            <mxRectangle width="120" height="60" as="alternateBounds" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="bcFgxOS0jOe-36kQjctr-7" value="6 GB" style="shape=partialRectangle;html=1;whiteSpace=wrap;connectable=0;strokeColor=#b85450;overflow=hidden;fillColor=#f8cecc;top=0;left=0;bottom=0;right=0;pointerEvents=1;fontSize=23;fontFamily=Palatino;fontSource=https%3A%2F%2Ffonts.googleapis.com%2Fcss%3Ffamily%3DPalatino;strokeWidth=20;perimeterSpacing=3;" vertex="1" parent="bcFgxOS0jOe-36kQjctr-2">
+          <mxGeometry x="460" width="120" height="60" as="geometry">
+            <mxRectangle width="120" height="60" as="alternateBounds" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="bcFgxOS0jOe-36kQjctr-8" value="6 GB" style="shape=partialRectangle;html=1;whiteSpace=wrap;connectable=0;strokeColor=#b85450;overflow=hidden;fillColor=#f8cecc;top=0;left=0;bottom=0;right=0;pointerEvents=1;fontSize=23;fontFamily=Palatino;fontSource=https%3A%2F%2Ffonts.googleapis.com%2Fcss%3Ffamily%3DPalatino;strokeWidth=20;perimeterSpacing=3;" vertex="1" parent="bcFgxOS0jOe-36kQjctr-2">
+          <mxGeometry x="580" width="120" height="60" as="geometry">
+            <mxRectangle width="120" height="60" as="alternateBounds" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="bcFgxOS0jOe-36kQjctr-9" value="3 GB" style="shape=partialRectangle;html=1;whiteSpace=wrap;connectable=0;strokeColor=#6c8ebf;overflow=hidden;fillColor=#dae8fc;top=0;left=0;bottom=0;right=0;pointerEvents=1;fontSize=23;fontFamily=Palatino;fontSource=https%3A%2F%2Ffonts.googleapis.com%2Fcss%3Ffamily%3DPalatino;strokeWidth=20;perimeterSpacing=3;labelBorderColor=none;" vertex="1" parent="bcFgxOS0jOe-36kQjctr-2">
+          <mxGeometry x="700" width="60" height="60" as="geometry">
+            <mxRectangle width="60" height="60" as="alternateBounds" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="bcFgxOS0jOe-36kQjctr-10" value="3 GB" style="shape=partialRectangle;html=1;whiteSpace=wrap;connectable=0;strokeColor=default;overflow=hidden;fillColor=#dae8fc;top=0;left=0;bottom=0;right=0;pointerEvents=1;labelBackgroundColor=none;perimeterSpacing=3;fontSize=23;fontFamily=Palatino;fontSource=https%3A%2F%2Ffonts.googleapis.com%2Fcss%3Ffamily%3DPalatino;strokeWidth=20;fillStyle=auto;" vertex="1" parent="bcFgxOS0jOe-36kQjctr-2">
+          <mxGeometry x="760" width="60" height="60" as="geometry">
+            <mxRectangle width="60" height="60" as="alternateBounds" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="bcFgxOS0jOe-36kQjctr-14" value="&lt;p class=&quot;p1&quot; style=&quot;margin: 0px; font-variant-numeric: normal; font-variant-east-asian: normal; font-variant-alternates: normal; font-size-adjust: none; font-kerning: auto; font-optical-sizing: auto; font-feature-settings: normal; font-variation-settings: normal; font-variant-position: normal; font-stretch: normal; font-size: 19px; line-height: normal;&quot;&gt;Parameters (fp16)&lt;/p&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontFamily=Palatino;fontSource=https%3A%2F%2Ffonts.googleapis.com%2Fcss%3Ffamily%3DPalatino;fontSize=19;labelBackgroundColor=none;" vertex="1" parent="1">
+          <mxGeometry x="660" y="80" width="200" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="bcFgxOS0jOe-36kQjctr-15" value="" style="endArrow=none;html=1;rounded=0;curved=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" edge="1" parent="1" source="bcFgxOS0jOe-36kQjctr-14">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="970" y="290" as="sourcePoint" />
+            <mxPoint x="830" y="150" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="bcFgxOS0jOe-36kQjctr-16" value="&lt;p class=&quot;p1&quot; style=&quot;margin: 0px; font-variant-numeric: normal; font-variant-east-asian: normal; font-variant-alternates: normal; font-size-adjust: none; font-kerning: auto; font-optical-sizing: auto; font-feature-settings: normal; font-variation-settings: normal; font-variant-position: normal; font-stretch: normal; font-size: 19px; line-height: normal;&quot;&gt;Gradients (fp16)&lt;/p&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontFamily=Palatino;fontSource=https%3A%2F%2Ffonts.googleapis.com%2Fcss%3Ffamily%3DPalatino;fontSize=19;labelBackgroundColor=none;" vertex="1" parent="1">
+          <mxGeometry x="460" y="80" width="200" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="bcFgxOS0jOe-36kQjctr-17" value="&lt;p class=&quot;p1&quot; style=&quot;margin: 0px; font-variant-numeric: normal; font-variant-east-asian: normal; font-variant-alternates: normal; font-size-adjust: none; font-kerning: auto; font-optical-sizing: auto; font-feature-settings: normal; font-variation-settings: normal; font-variant-position: normal; font-stretch: normal; font-size: 19px; line-height: normal;&quot;&gt;Optimizer States (fp32 Parameters)&lt;/p&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontFamily=Palatino;fontSource=https%3A%2F%2Ffonts.googleapis.com%2Fcss%3Ffamily%3DPalatino;fontSize=19;labelBackgroundColor=none;" vertex="1" parent="1">
+          <mxGeometry x="660" y="280" width="200" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="bcFgxOS0jOe-36kQjctr-18" value="&lt;p class=&quot;p1&quot; style=&quot;margin: 0px; font-variant-numeric: normal; font-variant-east-asian: normal; font-variant-alternates: normal; font-size-adjust: none; font-kerning: auto; font-optical-sizing: auto; font-feature-settings: normal; font-variation-settings: normal; font-variant-position: normal; font-stretch: normal; font-size: 19px; line-height: normal;&quot;&gt;Optimizer States (fp32 Momentum)&lt;/p&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontFamily=Palatino;fontSource=https%3A%2F%2Ffonts.googleapis.com%2Fcss%3Ffamily%3DPalatino;fontSize=19;labelBackgroundColor=none;" vertex="1" parent="1">
+          <mxGeometry x="457" y="280" width="200" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="bcFgxOS0jOe-36kQjctr-19" value="&lt;p class=&quot;p1&quot; style=&quot;margin: 0px; font-variant-numeric: normal; font-variant-east-asian: normal; font-variant-alternates: normal; font-size-adjust: none; font-kerning: auto; font-optical-sizing: auto; font-feature-settings: normal; font-variation-settings: normal; font-variant-position: normal; font-stretch: normal; font-size: 19px; line-height: normal;&quot;&gt;Optimizer States (fp32 Variance)&lt;/p&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontFamily=Palatino;fontSource=https%3A%2F%2Ffonts.googleapis.com%2Fcss%3Ffamily%3DPalatino;fontSize=19;labelBackgroundColor=none;" vertex="1" parent="1">
+          <mxGeometry x="243" y="280" width="200" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="bcFgxOS0jOe-36kQjctr-20" value="&lt;p class=&quot;p1&quot; style=&quot;margin: 0px; font-variant-numeric: normal; font-variant-east-asian: normal; font-variant-alternates: normal; font-size-adjust: none; font-kerning: auto; font-optical-sizing: auto; font-feature-settings: normal; font-variation-settings: normal; font-variant-position: normal; font-stretch: normal; font-size: 19px; line-height: normal;&quot;&gt;Activations (with checkpointing)&lt;/p&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontFamily=Palatino;fontSource=https%3A%2F%2Ffonts.googleapis.com%2Fcss%3Ffamily%3DPalatino;fontSize=19;labelBackgroundColor=none;" vertex="1" parent="1">
+          <mxGeometry x="40" y="280" width="200" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="bcFgxOS0jOe-36kQjctr-21" value="&lt;p class=&quot;p1&quot; style=&quot;margin: 0px; font-variant-numeric: normal; font-variant-east-asian: normal; font-variant-alternates: normal; font-size-adjust: none; font-kerning: auto; font-optical-sizing: auto; font-feature-settings: normal; font-variation-settings: normal; font-variant-position: normal; font-stretch: normal; font-size: 19px; line-height: normal;&quot;&gt;Temporary Buffers (fp32)&lt;/p&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontFamily=Palatino;fontSource=https%3A%2F%2Ffonts.googleapis.com%2Fcss%3Ffamily%3DPalatino;fontSize=19;labelBackgroundColor=none;" vertex="1" parent="1">
+          <mxGeometry x="257" y="80" width="200" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="bcFgxOS0jOe-36kQjctr-22" value="&lt;p class=&quot;p1&quot; style=&quot;margin: 0px; font-variant-numeric: normal; font-variant-east-asian: normal; font-variant-alternates: normal; font-size-adjust: none; font-kerning: auto; font-optical-sizing: auto; font-feature-settings: normal; font-variation-settings: normal; font-variant-position: normal; font-stretch: normal; font-size: 19px; line-height: normal;&quot;&gt;Fragmentation Overhead (Variable)&lt;/p&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontFamily=Palatino;fontSource=https%3A%2F%2Ffonts.googleapis.com%2Fcss%3Ffamily%3DPalatino;fontSize=19;labelBackgroundColor=none;" vertex="1" parent="1">
+          <mxGeometry x="40" y="80" width="200" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="bcFgxOS0jOe-36kQjctr-23" value="" style="endArrow=none;html=1;rounded=0;curved=0;exitX=0.75;exitY=1;exitDx=0;exitDy=0;" edge="1" parent="1" source="bcFgxOS0jOe-36kQjctr-16">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="1110" y="295" as="sourcePoint" />
+            <mxPoint x="770" y="150" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="bcFgxOS0jOe-36kQjctr-24" value="" style="endArrow=none;html=1;rounded=0;curved=0;exitX=0.5;exitY=0;exitDx=0;exitDy=0;" edge="1" parent="1" source="bcFgxOS0jOe-36kQjctr-17">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="924" y="290" as="sourcePoint" />
+            <mxPoint x="680" y="230" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="bcFgxOS0jOe-36kQjctr-25" value="" style="endArrow=none;html=1;rounded=0;curved=0;exitX=0.5;exitY=0;exitDx=0;exitDy=0;" edge="1" parent="1" source="bcFgxOS0jOe-36kQjctr-18">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="751" y="290" as="sourcePoint" />
+            <mxPoint x="560" y="230" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="bcFgxOS0jOe-36kQjctr-26" value="" style="endArrow=none;html=1;rounded=0;curved=0;exitX=0.5;exitY=0;exitDx=0;exitDy=0;" edge="1" parent="1" source="bcFgxOS0jOe-36kQjctr-19">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="440" y="280" as="sourcePoint" />
+            <mxPoint x="440" y="230" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="bcFgxOS0jOe-36kQjctr-27" value="" style="endArrow=none;html=1;rounded=0;curved=0;exitX=0.5;exitY=0;exitDx=0;exitDy=0;" edge="1" parent="1" source="bcFgxOS0jOe-36kQjctr-20">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="385" y="290" as="sourcePoint" />
+            <mxPoint x="300" y="230" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="bcFgxOS0jOe-36kQjctr-28" value="" style="endArrow=none;html=1;rounded=0;curved=0;exitX=0.25;exitY=1;exitDx=0;exitDy=0;" edge="1" parent="1" source="bcFgxOS0jOe-36kQjctr-21">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="196" y="290" as="sourcePoint" />
+            <mxPoint x="160" y="150" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="bcFgxOS0jOe-36kQjctr-29" value="" style="endArrow=none;html=1;rounded=0;curved=0;" edge="1" parent="1" source="bcFgxOS0jOe-36kQjctr-22">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="6" y="290" as="sourcePoint" />
+            <mxPoint x="70" y="150" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+      </root>
+    </mxGraphModel>
+  </diagram>
 </mxfile>
diff --git a/docs/visualizations/gd_lls.md b/docs/visualizations/gd_lls.md
@@ -0,0 +1,12 @@
+---
+title: "Gradient descent for linear regression"
+---
+
+:::{.video}
+gd_lls.mp4
+:::
+
+What could be simpler than that? However, I'm using this animation as an illustration for people unfamiliar with optimization algorithms how minimizing the loss function corresponds to training a machine learning model ( even though there are only two trainable parameters).
+
+
+[\faPython\ Code](https://colab.research.google.com/github/MerkulovDaniil/optim/blob/master/assets/Notebooks/GD_2d_visualization.ipynb)
diff --git a/docs/visualizations/gd_lls.mp4 b/docs/visualizations/gd_lls.mp4