build based on d4562e3

dev/adjoints/index.html

@@ -100,4 +100,4 @@
 1 levels of nesting
 
 julia> grad(x -> x*grad(f, x), 1);
-2 levels of nesting</code></pre></article><nav class="docs-footer"><a class="docs-footer-prevpage" href="../limitations/">« Limitations</a><a class="docs-footer-nextpage" href="../utils/">Utilities »</a></nav></div><div class="modal" id="documenter-settings"><div class="modal-background"></div><div class="modal-card"><header class="modal-card-head"><p class="modal-card-title">Settings</p><button class="delete"></button></header><section class="modal-card-body"><p><label class="label">Theme</label><div class="select"><select id="documenter-themepicker"><option value="documenter-light">documenter-light</option><option value="documenter-dark">documenter-dark</option></select></div></p><hr/><p>This document was generated with <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> on <span class="colophon-date" title="Thursday 10 August 2023 22:30">Thursday 10 August 2023</span>. Using Julia version 1.9.2.</p></section><footer class="modal-card-foot"></footer></div></div></div></body></html>
+2 levels of nesting</code></pre></article><nav class="docs-footer"><a class="docs-footer-prevpage" href="../limitations/">« Limitations</a><a class="docs-footer-nextpage" href="../utils/">Utilities »</a></nav></div><div class="modal" id="documenter-settings"><div class="modal-background"></div><div class="modal-card"><header class="modal-card-head"><p class="modal-card-title">Settings</p><button class="delete"></button></header><section class="modal-card-body"><p><label class="label">Theme</label><div class="select"><select id="documenter-themepicker"><option value="documenter-light">documenter-light</option><option value="documenter-dark">documenter-dark</option></select></div></p><hr/><p>This document was generated with <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> on <span class="colophon-date" title="Thursday 17 August 2023 14:40">Thursday 17 August 2023</span>. Using Julia version 1.9.2.</p></section><footer class="modal-card-foot"></footer></div></div></div></body></html>

dev/complex/index.html

@@ -27,4 +27,4 @@
 (8.0 + 12.0im, 0.0 + 0.0im)
 
 julia> wirtinger(x -> abs2(x), 1+2im)
-(1.0 - 2.0im, 1.0 + 2.0im)</code></pre></article><nav class="docs-footer"><a class="docs-footer-prevpage" href="../utils/">« Utilities</a><a class="docs-footer-nextpage" href="../profiling/">Profiling »</a></nav></div><div class="modal" id="documenter-settings"><div class="modal-background"></div><div class="modal-card"><header class="modal-card-head"><p class="modal-card-title">Settings</p><button class="delete"></button></header><section class="modal-card-body"><p><label class="label">Theme</label><div class="select"><select id="documenter-themepicker"><option value="documenter-light">documenter-light</option><option value="documenter-dark">documenter-dark</option></select></div></p><hr/><p>This document was generated with <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> on <span class="colophon-date" title="Thursday 10 August 2023 22:30">Thursday 10 August 2023</span>. Using Julia version 1.9.2.</p></section><footer class="modal-card-foot"></footer></div></div></div></body></html>
+(1.0 - 2.0im, 1.0 + 2.0im)</code></pre></article><nav class="docs-footer"><a class="docs-footer-prevpage" href="../utils/">« Utilities</a><a class="docs-footer-nextpage" href="../profiling/">Profiling »</a></nav></div><div class="modal" id="documenter-settings"><div class="modal-background"></div><div class="modal-card"><header class="modal-card-head"><p class="modal-card-title">Settings</p><button class="delete"></button></header><section class="modal-card-body"><p><label class="label">Theme</label><div class="select"><select id="documenter-themepicker"><option value="documenter-light">documenter-light</option><option value="documenter-dark">documenter-dark</option></select></div></p><hr/><p>This document was generated with <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> on <span class="colophon-date" title="Thursday 17 August 2023 14:40">Thursday 17 August 2023</span>. Using Julia version 1.9.2.</p></section><footer class="modal-card-foot"></footer></div></div></div></body></html>

dev/index.html

@@ -79,7 +79,7 @@
          p = size(x, d)
          sum(x.^p .+ y)
        end
-([14.0, 22.0], 2.0, nothing)</code></pre></div><a class="docs-sourcelink" target="_blank" href="https://github.com/FluxML/Zygote.jl/blob/547be707a59e079ff1cc8f079bd8fc9298b93fa3/src/compiler/interface.jl#L70-L94">source</a></section></article><pre><code class="language-julia">julia&gt; linear(θ, x) = θ[:W] * x .+ θ[:b]
+([14.0, 22.0], 2.0, nothing)</code></pre></div><a class="docs-sourcelink" target="_blank" href="https://github.com/FluxML/Zygote.jl/blob/d4562e330d588cb986604bb4f1942bf9fca8ecc5/src/compiler/interface.jl#L70-L94">source</a></section></article><pre><code class="language-julia">julia&gt; linear(θ, x) = θ[:W] * x .+ θ[:b]
 linear (generic function with 1 method)
 
 julia&gt; x = rand(5);
@@ -121,7 +121,7 @@
  8.0  80.0  800.0
 
 julia&gt; haskey(g, z)  # only x and y are parameters
-false</code></pre></div><a class="docs-sourcelink" target="_blank" href="https://github.com/FluxML/Zygote.jl/blob/547be707a59e079ff1cc8f079bd8fc9298b93fa3/src/compiler/interface.jl#L170-L194">source</a></section></article><pre><code class="language-julia">julia&gt; W = rand(2, 5); b = rand(2);
+false</code></pre></div><a class="docs-sourcelink" target="_blank" href="https://github.com/FluxML/Zygote.jl/blob/d4562e330d588cb986604bb4f1942bf9fca8ecc5/src/compiler/interface.jl#L170-L194">source</a></section></article><pre><code class="language-julia">julia&gt; W = rand(2, 5); b = rand(2);
 
 julia&gt; linear(x) = W * x .+ b
 linear (generic function with 2 methods)
@@ -130,4 +130,4 @@
 Grads(...)
 
 julia&gt; grads[W], grads[b] # access gradients using arrays as keys
-([0.652543 … 0.683588], [1.0, 1.0])</code></pre><p>Here <code>grads</code> is a dictionary-like object, whose keys are the same parameters we indicated in <code>Params</code>. (In fact it wraps a dictionary using <code>objectid(W)</code> as keys, which does not change if the values in <code>W</code> are mutated).</p><p>This implicit style is the one presently used by <a href="https://github.com/FluxML/Flux.jl">Flux.jl</a>, a closely related machine learning library. It uses structs like <code>Linear</code> above to define layers, and the function <code>Flux.params(model)</code> returns a <code>Params</code> object containing all the parameters of all layers. See <a href="https://fluxml.ai/Flux.jl/stable/models/basics/">its documentation</a> for more details. When using Zygote for most other purposes, however, the explicit style is usually preferred.</p></article><nav class="docs-footer"><a class="docs-footer-nextpage" href="limitations/">Limitations »</a></nav></div><div class="modal" id="documenter-settings"><div class="modal-background"></div><div class="modal-card"><header class="modal-card-head"><p class="modal-card-title">Settings</p><button class="delete"></button></header><section class="modal-card-body"><p><label class="label">Theme</label><div class="select"><select id="documenter-themepicker"><option value="documenter-light">documenter-light</option><option value="documenter-dark">documenter-dark</option></select></div></p><hr/><p>This document was generated with <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> on <span class="colophon-date" title="Thursday 10 August 2023 22:30">Thursday 10 August 2023</span>. Using Julia version 1.9.2.</p></section><footer class="modal-card-foot"></footer></div></div></div></body></html>
+([0.652543 … 0.683588], [1.0, 1.0])</code></pre><p>Here <code>grads</code> is a dictionary-like object, whose keys are the same parameters we indicated in <code>Params</code>. (In fact it wraps a dictionary using <code>objectid(W)</code> as keys, which does not change if the values in <code>W</code> are mutated).</p><p>This implicit style is the one presently used by <a href="https://github.com/FluxML/Flux.jl">Flux.jl</a>, a closely related machine learning library. It uses structs like <code>Linear</code> above to define layers, and the function <code>Flux.params(model)</code> returns a <code>Params</code> object containing all the parameters of all layers. See <a href="https://fluxml.ai/Flux.jl/stable/models/basics/">its documentation</a> for more details. When using Zygote for most other purposes, however, the explicit style is usually preferred.</p></article><nav class="docs-footer"><a class="docs-footer-nextpage" href="limitations/">Limitations »</a></nav></div><div class="modal" id="documenter-settings"><div class="modal-background"></div><div class="modal-card"><header class="modal-card-head"><p class="modal-card-title">Settings</p><button class="delete"></button></header><section class="modal-card-body"><p><label class="label">Theme</label><div class="select"><select id="documenter-themepicker"><option value="documenter-light">documenter-light</option><option value="documenter-dark">documenter-dark</option></select></div></p><hr/><p>This document was generated with <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> on <span class="colophon-date" title="Thursday 17 August 2023 14:40">Thursday 17 August 2023</span>. Using Julia version 1.9.2.</p></section><footer class="modal-card-foot"></footer></div></div></div></body></html>

dev/internals/index.html

@@ -135,4 +135,4 @@
 julia> y, back = Zygote._pullback(bad, 1);
 
 julia> back(1) # ok, here's our issue. Lather, rinse, repeat.
-ERROR: bad</code></pre><p>Of course, our goal is that you never have to do this, but until Zygote is more mature it can be a useful way to narrow down test cases.</p></article><nav class="docs-footer"><a class="docs-footer-prevpage" href="../profiling/">« Profiling</a><a class="docs-footer-nextpage" href="../glossary/">Glossary »</a></nav></div><div class="modal" id="documenter-settings"><div class="modal-background"></div><div class="modal-card"><header class="modal-card-head"><p class="modal-card-title">Settings</p><button class="delete"></button></header><section class="modal-card-body"><p><label class="label">Theme</label><div class="select"><select id="documenter-themepicker"><option value="documenter-light">documenter-light</option><option value="documenter-dark">documenter-dark</option></select></div></p><hr/><p>This document was generated with <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> on <span class="colophon-date" title="Thursday 10 August 2023 22:30">Thursday 10 August 2023</span>. Using Julia version 1.9.2.</p></section><footer class="modal-card-foot"></footer></div></div></div></body></html>
+ERROR: bad</code></pre><p>Of course, our goal is that you never have to do this, but until Zygote is more mature it can be a useful way to narrow down test cases.</p></article><nav class="docs-footer"><a class="docs-footer-prevpage" href="../profiling/">« Profiling</a><a class="docs-footer-nextpage" href="../glossary/">Glossary »</a></nav></div><div class="modal" id="documenter-settings"><div class="modal-background"></div><div class="modal-card"><header class="modal-card-head"><p class="modal-card-title">Settings</p><button class="delete"></button></header><section class="modal-card-body"><p><label class="label">Theme</label><div class="select"><select id="documenter-themepicker"><option value="documenter-light">documenter-light</option><option value="documenter-dark">documenter-dark</option></select></div></p><hr/><p>This document was generated with <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> on <span class="colophon-date" title="Thursday 17 August 2023 14:40">Thursday 17 August 2023</span>. Using Julia version 1.9.2.</p></section><footer class="modal-card-foot"></footer></div></div></div></body></html>

dev/limitations/index.html

@@ -87,4 +87,4 @@
     tot += x^n  # binds symbol `tot` to new value
   end
   return tot
-end</code></pre><p>However, sometimes such re-binding confuses Zygote, especially if the type of the value changes. Especially if the variable is &quot;boxed&quot;, as will happen if you re-bind from within a closure (such as the function created by a <code>do</code> block).</p><h2 id="Second-derivatives-1"><a class="docs-heading-anchor" href="#Second-derivatives-1">Second derivatives</a><a class="docs-heading-anchor-permalink" href="#Second-derivatives-1" title="Permalink"></a></h2><p>In principle Zygote supports taking derivatives of derivatives. There are, however, a few problems:</p><ul><li>Quite a few of its rules are not written in a way that is itself differentiable. For instance they may work by making an array then writing into it, which is mutation of the sort forbidden above. </li><li>The complexity of the code grows rapidly, as Zygote differentiates its own un-optimised output.</li><li>Reverse mode over reverse mode is seldom the best algorithm.</li></ul><p>The issue tracker has a label for <a href="https://github.com/FluxML/Zygote.jl/issues?q=is%3Aissue+is%3Aopen+label%3A%22second+order%22">second order</a>, which will outline where the bodies are buried.</p><p>Often using a different AD system over Zygote is a better solution. This is what <a href="../utils/#Zygote.hessian"><code>hessian</code></a> does, using ForwardDiff over Zygote, but other combinations are possible. (Note that rules defined here mean that Zygote over ForwardDiff is translated to ForwardDiff over ForwardDiff.)</p></article><nav class="docs-footer"><a class="docs-footer-prevpage" href="../">« Home</a><a class="docs-footer-nextpage" href="../adjoints/">Custom Adjoints »</a></nav></div><div class="modal" id="documenter-settings"><div class="modal-background"></div><div class="modal-card"><header class="modal-card-head"><p class="modal-card-title">Settings</p><button class="delete"></button></header><section class="modal-card-body"><p><label class="label">Theme</label><div class="select"><select id="documenter-themepicker"><option value="documenter-light">documenter-light</option><option value="documenter-dark">documenter-dark</option></select></div></p><hr/><p>This document was generated with <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> on <span class="colophon-date" title="Thursday 10 August 2023 22:30">Thursday 10 August 2023</span>. Using Julia version 1.9.2.</p></section><footer class="modal-card-foot"></footer></div></div></div></body></html>
+end</code></pre><p>However, sometimes such re-binding confuses Zygote, especially if the type of the value changes. Especially if the variable is &quot;boxed&quot;, as will happen if you re-bind from within a closure (such as the function created by a <code>do</code> block).</p><h2 id="Second-derivatives-1"><a class="docs-heading-anchor" href="#Second-derivatives-1">Second derivatives</a><a class="docs-heading-anchor-permalink" href="#Second-derivatives-1" title="Permalink"></a></h2><p>In principle Zygote supports taking derivatives of derivatives. There are, however, a few problems:</p><ul><li>Quite a few of its rules are not written in a way that is itself differentiable. For instance they may work by making an array then writing into it, which is mutation of the sort forbidden above. </li><li>The complexity of the code grows rapidly, as Zygote differentiates its own un-optimised output.</li><li>Reverse mode over reverse mode is seldom the best algorithm.</li></ul><p>The issue tracker has a label for <a href="https://github.com/FluxML/Zygote.jl/issues?q=is%3Aissue+is%3Aopen+label%3A%22second+order%22">second order</a>, which will outline where the bodies are buried.</p><p>Often using a different AD system over Zygote is a better solution. This is what <a href="../utils/#Zygote.hessian"><code>hessian</code></a> does, using ForwardDiff over Zygote, but other combinations are possible. (Note that rules defined here mean that Zygote over ForwardDiff is translated to ForwardDiff over ForwardDiff.)</p></article><nav class="docs-footer"><a class="docs-footer-prevpage" href="../">« Home</a><a class="docs-footer-nextpage" href="../adjoints/">Custom Adjoints »</a></nav></div><div class="modal" id="documenter-settings"><div class="modal-background"></div><div class="modal-card"><header class="modal-card-head"><p class="modal-card-title">Settings</p><button class="delete"></button></header><section class="modal-card-body"><p><label class="label">Theme</label><div class="select"><select id="documenter-themepicker"><option value="documenter-light">documenter-light</option><option value="documenter-dark">documenter-dark</option></select></div></p><hr/><p>This document was generated with <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> on <span class="colophon-date" title="Thursday 17 August 2023 14:40">Thursday 17 August 2023</span>. Using Julia version 1.9.2.</p></section><footer class="modal-card-foot"></footer></div></div></div></body></html>

dev/profiling/index.html

@@ -28,4 +28,4 @@
 │   %2 = (Base.mul_int)(Δ, 1)::Int64
 │   %3 = (Zygote.tuple)(nothing, %1, %2)::PartialTuple(Tuple{Nothing,Int64,Int64}, Any[Const(nothing, false), Int64, Int64])
 └──      return %3
-) =&gt; Tuple{Nothing,Int64,Int64}</code></pre></article><nav class="docs-footer"><a class="docs-footer-prevpage" href="../complex/">« Complex Differentiation</a><a class="docs-footer-nextpage" href="../internals/">Internals »</a></nav></div><div class="modal" id="documenter-settings"><div class="modal-background"></div><div class="modal-card"><header class="modal-card-head"><p class="modal-card-title">Settings</p><button class="delete"></button></header><section class="modal-card-body"><p><label class="label">Theme</label><div class="select"><select id="documenter-themepicker"><option value="documenter-light">documenter-light</option><option value="documenter-dark">documenter-dark</option></select></div></p><hr/><p>This document was generated with <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> on <span class="colophon-date" title="Thursday 10 August 2023 22:30">Thursday 10 August 2023</span>. Using Julia version 1.9.2.</p></section><footer class="modal-card-foot"></footer></div></div></div></body></html>
+) =&gt; Tuple{Nothing,Int64,Int64}</code></pre></article><nav class="docs-footer"><a class="docs-footer-prevpage" href="../complex/">« Complex Differentiation</a><a class="docs-footer-nextpage" href="../internals/">Internals »</a></nav></div><div class="modal" id="documenter-settings"><div class="modal-background"></div><div class="modal-card"><header class="modal-card-head"><p class="modal-card-title">Settings</p><button class="delete"></button></header><section class="modal-card-body"><p><label class="label">Theme</label><div class="select"><select id="documenter-themepicker"><option value="documenter-light">documenter-light</option><option value="documenter-dark">documenter-dark</option></select></div></p><hr/><p>This document was generated with <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> on <span class="colophon-date" title="Thursday 17 August 2023 14:40">Thursday 17 August 2023</span>. Using Julia version 1.9.2.</p></section><footer class="modal-card-foot"></footer></div></div></div></body></html>