From 3e01f05cddd0da718739d42349e8789b79ac36ec Mon Sep 17 00:00:00 2001 From: benedict-96 Date: Wed, 13 Dec 2023 16:36:39 +0100 Subject: [PATCH 01/10] Fixed docs and put type dependency for . --- src/data_loader/tensor_assign.jl | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/data_loader/tensor_assign.jl b/src/data_loader/tensor_assign.jl index 3faad0a99..e2282db83 100644 --- a/src/data_loader/tensor_assign.jl +++ b/src/data_loader/tensor_assign.jl @@ -35,18 +35,22 @@ end end @doc raw""" -The function `assign_output_estimate` is closely related to the transformer. It takes the last prediction_window columns of the output and uses is for the final prediction. +The function `assign_output_estimate` is closely related to the transformer. It takes the last `prediction_window` columns of the output and uses them for the final prediction. i.e. ```math -\mathbb{R}^{N\times\mathtt{pw}}\to\mathbb{R}^{N\times\mathtt{pw}}, \begin{bmatrix} z^{(1)}_1 & \cdots z^{(T)}_1 \\ - \cdots & \cdots \\ - z^{(T - \mathtt{pw})}_n & \cdots & z^{(T})_n\end{bmatrix} \mapsto - \begin{bmatrix} z^{(1)}_1 & \cdots z^{(T)}_1 \\ - \cdots & \cdots \\ - z^{(T - \mathtt{pw})}_n & \cdots & z^{(T})_n\end{bmatrix} +\mathbb{R}^{N\times\mathtt{pw}}\to\mathbb{R}^{N\times\mathtt{pw}}, +\begin{bmatrix} + z^{(1)}_1 & \cdots & z^{(T)}_1 \\ + \cdots & \cdots & \cdots \\ + z^{(1)}_n & \cdots & z^{(T})_n + \end{bmatrix} \mapsto + \begin{bmatrix} + z^{(T - \mathtt{pw})}_1 & \cdots & z^{(T)}_1 \\ + \cdots & \cdots & \cdots \\ + z^{(T - \mathtt{pw})}_n & \cdots & z^{(T})_n\end{bmatrix} ``` """ -function assign_output_estimate(full_output::AbstractArray{T, 3}, prediction_window) where T +function assign_output_estimate(full_output::AbstractArray{T, 3}, prediction_window::Int) where T sys_dim, seq_length, batch_size = size(full_output) backend = KernelAbstractions.get_backend(full_output) output_estimate = KernelAbstractions.allocate(backend, T, sys_dim, prediction_window, batch_size) From 3a1fa6a1f757338d1e6ec8f0d0844ea680bc3055 Mon Sep 17 00:00:00 2001 From: benedict-96 Date: Wed, 13 Dec 2023 17:04:22 +0100 Subject: [PATCH 02/10] Fixed docs for rgrad. --- src/manifolds/stiefel_manifold.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/manifolds/stiefel_manifold.jl b/src/manifolds/stiefel_manifold.jl index ec1c3fb26..a98e22079 100644 --- a/src/manifolds/stiefel_manifold.jl +++ b/src/manifolds/stiefel_manifold.jl @@ -64,7 +64,7 @@ function Base.:*(Y::Adjoint{T, StiefelManifold{T, AT}}, B::AbstractMatrix) where end @doc raw""" -Computes the Riemannian gradient for the Stiefel manifold given an element ``Y\in{}St(N,n)`` and a matrix ``\nabla{}L\in\mahbb{R}^{N\times{}n}`` (the Euclidean gradient). It computes the Riemannian gradient with respect to the canonical metric (see the documentation for the function `metric` for an explanation of this). +Computes the Riemannian gradient for the Stiefel manifold given an element ``Y\in{}St(N,n)`` and a matrix ``\nabla{}L\in\mathbb{R}^{N\times{}n}`` (the Euclidean gradient). It computes the Riemannian gradient with respect to the canonical metric (see the documentation for the function `metric` for an explanation of this). The precise form of the mapping is: ```math \mathtt{rgrad}(Y, \nabla{}L) \mapsto \nabla{}L - Y(\nabla{}L)^TY From 3b8b5a486dd463db0ca2f12dc8f7be1979882b1f Mon Sep 17 00:00:00 2001 From: benedict-96 Date: Wed, 13 Dec 2023 17:12:59 +0100 Subject: [PATCH 03/10] Added additional constructor for the case when the type is the first input argument. --- src/arrays/stiefel_projection.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/arrays/stiefel_projection.jl b/src/arrays/stiefel_projection.jl index 82c04c2ac..236948a09 100644 --- a/src/arrays/stiefel_projection.jl +++ b/src/arrays/stiefel_projection.jl @@ -41,6 +41,8 @@ Outer constructor for `StiefelProjection`. This works with two integers as input """ StiefelProjection(N::Integer, n::Integer, T::Type=Float64) = StiefelProjection(CPU(), T, N, n) +StiefelProjection(T::Type, N::Integer, n::Integer) = StiefelProjection(N, n, T) + Base.size(E::StiefelProjection) = (E.N, E.n) Base.getindex(E::StiefelProjection, i, j) = getindex(E.A, i, j) Base.:+(E::StiefelProjection, A::AbstractMatrix) = E.A + A From b05c66c53f2c5c80d3a572bc2ad7d06467f93e5f Mon Sep 17 00:00:00 2001 From: benedict-96 Date: Wed, 13 Dec 2023 17:13:26 +0100 Subject: [PATCH 04/10] Removed the symplectic arrays. --- test/arrays/array_tests.jl | 45 -------------------------------------- test/runtests.jl | 2 +- 2 files changed, 1 insertion(+), 46 deletions(-) diff --git a/test/arrays/array_tests.jl b/test/arrays/array_tests.jl index fd9269046..cbadf1aba 100644 --- a/test/arrays/array_tests.jl +++ b/test/arrays/array_tests.jl @@ -61,22 +61,6 @@ function skew_mat_mul_test2(n, T=Float64) @test isapprox(AS1, AS2) end -# check if matrix is āˆˆ š”¤ (check if the vector space projection works), addition & subtraction -function sympl_lie_alg_add_sub_test(n) - J = SymplecticPotential(n) - symplectisize(W) = .5*(W - J'*W'*J) - Wā‚ = rand(2*n,2*n) - Sā‚ = SymplecticLieAlgMatrix(Wā‚) - Wā‚‚ = rand(2*n,2*n) - Sā‚‚ = SymplecticLieAlgMatrix(Wā‚‚) - Sā‚ƒ = Sā‚ + Sā‚‚ - Sā‚„ = Sā‚ - Sā‚‚ - @test typeof(Sā‚ƒ) <: SymplecticLieAlgMatrix - @test typeof(Sā‚„) <: SymplecticLieAlgMatrix - @test all(abs.(symplectisize(Wā‚ + Wā‚‚) .- Sā‚ƒ) .< 1e-10) - @test all(abs.(symplectisize(Wā‚ - Wā‚‚) .- Sā‚„) .< 1e-10) -end - # test Stiefel manifold projection test function stiefel_proj_test(N,n) In = I(n) @@ -84,15 +68,6 @@ function stiefel_proj_test(N,n) @test all(abs.((E'*E) .- In) .< 1e-10) end -# test symplectic projection (this is just the E matrix) -function sympl_proj_test(N, n) - JN = SymplecticPotential(N) - Jn = SymplecticPotential(n) - E = SymplecticProjection(N, n, Float64) - @test all(abs.((E'*JN*E) .- Jn) .< 1e-10) -end - - function stiefel_lie_alg_add_sub_test(N, n) E = StiefelProjection(N, n) projection(W::SkewSymMatrix) = W - (I - E*E')*W*(I - E*E') @@ -108,23 +83,6 @@ function stiefel_lie_alg_add_sub_test(N, n) @test all(abs.(projection(Wā‚ - Wā‚‚) .- Sā‚„) .< 1e-10) end -# check if matrix is āˆˆ š”¤ (check if the vector space projection works), addition & subtraction -function sympl_lie_alg_add_sub_test(N, n) - J = SymplecticPotential(n) - E = SymplecticProjection(N, n) - projection(W::SymplecticLieAlgMatrix) = W - (I - E*E')*W*(I - E*E') - Wā‚ = SymplecticLieAlgMatrix(rand(2*N,2*N)) - Sā‚ = SymplecticLieAlgHorMatrix(Wā‚,n) - Wā‚‚ = SymplecticLieAlgMatrix(rand(2*N,2*N)) - Sā‚‚ = SymplecticLieAlgHorMatrix(Wā‚‚,n) - Sā‚ƒ = Sā‚ + Sā‚‚ - Sā‚„ = Sā‚ - Sā‚‚ - @test typeof(Sā‚ƒ) <: SymplecticLieAlgHorMatrix - @test typeof(Sā‚„) <: SymplecticLieAlgHorMatrix - @test all(abs.(projection(Wā‚ + Wā‚‚) .- Sā‚ƒ) .< 1e-10) - @test all(abs.(projection(Wā‚ - Wā‚‚) .- Sā‚„) .< 1e-10) -end - function stiefel_lie_alg_vectorization_test(N, n; T=Float32) A = rand(StiefelLieAlgHorMatrix{T}, N, n) @test isapprox(StiefelLieAlgHorMatrix(vec(A), N, n), A) @@ -149,10 +107,7 @@ for (N, n) āˆˆ zip(N_vec, n_vec) skew_mat_add_sub_test(N) skew_mat_mul_test(N) skew_mat_mul_test2(N) - sympl_lie_alg_add_sub_test(N) stiefel_proj_test(N,n) - sympl_proj_test(N,n) stiefel_lie_alg_add_sub_test(N,n) - sympl_lie_alg_add_sub_test(N,n) stiefel_lie_alg_vectorization_test(N, n) end diff --git a/test/runtests.jl b/test/runtests.jl index af35e6063..848328f7d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3,7 +3,7 @@ using SafeTestsets @safetestset "Check parameterlength " begin include("parameterlength/check_parameterlengths.jl") end @safetestset "Arrays #1 " begin include("arrays/array_tests.jl") end -@safetestset "Arrays #2 " begin include("arrays/array_tests_old.jl") end +@safetestset "Test constructors for custom arrays " begin include("arrays/constructor_tests_for_custom_arrays.jl") end @safetestset "Manifolds (Grassmann): " begin include("manifolds/grassmann_manifold.jl") end @safetestset "Gradient Layer " begin include("layers/gradient_layer_tests.jl") end @safetestset "Test symplecticity of upscaling layer " begin include("layers/sympnet_layers_test.jl") end From 71f791bd1191e6324408c43fc991ae9e9800a555 Mon Sep 17 00:00:00 2001 From: benedict-96 Date: Wed, 13 Dec 2023 17:14:04 +0100 Subject: [PATCH 05/10] Added tests for the various constructors for custom arrays. --- .../constructor_tests_for_custom_arrays.jl | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 test/arrays/constructor_tests_for_custom_arrays.jl diff --git a/test/arrays/constructor_tests_for_custom_arrays.jl b/test/arrays/constructor_tests_for_custom_arrays.jl new file mode 100644 index 000000000..452f850f8 --- /dev/null +++ b/test/arrays/constructor_tests_for_custom_arrays.jl @@ -0,0 +1,37 @@ +using GeometricMachineLearning, Test +using LinearAlgebra: I + +@doc raw""" +This tests various constructor for custom arrays, e.g. if calling `SymmetricMatrix` on a matrix ``A`` does +```math +A \mapsto \frac{1}{2}(A + A^T). +``` +""" +function test_constructors_for_custom_arrays(n::Int, N::Int, T::Type) + A = rand(T, n, n) + B = rand(T, N, N) + + # SymmetricMatrix + @test Matrix{T}(SymmetricMatrix(A)) ā‰ˆ T(.5) * (A + A') + + # SkewSymMatrix + @test Matrix{T}(SkewSymMatrix(A)) ā‰ˆ T(.5) * (A - A') + + # StiefelLieAlgHorMatrix + B_shor = StiefelLieAlgHorMatrix(SkewSymMatrix(B), n) + B_shor2 = Matrix{T}(SkewSymMatrix(B)) + B_shor2[(n+1):N, (n+1):N] .= zero(T) + @test Matrix{T}(B_shor) ā‰ˆ B_shor2 + + # GrassmannLieAlgHorMatrix + B_ghor = GrassmannLieAlgHorMatrix(SkewSymMatrix(B), n) + B_ghor2 = copy(B_shor2) + B_ghor2[1:n, 1:n] .= zero(T) + @test Matrix{T}(B_ghor) ā‰ˆ B_ghor2 + + # StiefelProjection + E = StiefelProjection(T, N, n) + @test Matrix{T}(E) ā‰ˆ vcat(I(n), zeros(T, (N-n), n)) +end + +test_constructors_for_custom_arrays(5, 10, Float32) \ No newline at end of file From cfc1b5fd52fd243f425fad8c68be57dea4cc3c94 Mon Sep 17 00:00:00 2001 From: benedict-96 Date: Wed, 13 Dec 2023 16:36:39 +0100 Subject: [PATCH 06/10] Fixed docs and put type dependency for . --- src/data_loader/tensor_assign.jl | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/data_loader/tensor_assign.jl b/src/data_loader/tensor_assign.jl index 4aac9bfc0..299e52b84 100644 --- a/src/data_loader/tensor_assign.jl +++ b/src/data_loader/tensor_assign.jl @@ -35,18 +35,22 @@ end end @doc raw""" -The function `assign_output_estimate` is closely related to the transformer. It takes the last prediction_window columns of the output and uses is for the final prediction. +The function `assign_output_estimate` is closely related to the transformer. It takes the last `prediction_window` columns of the output and uses them for the final prediction. i.e. ```math -\mathbb{R}^{N\times\mathtt{pw}}\to\mathbb{R}^{N\times\mathtt{pw}}, \begin{bmatrix} z^{(1)}_1 & \cdots z^{(T)}_1 \\ - \cdots & \cdots \\ - z^{(T - \mathtt{pw})}_n & \cdots & z^{(T})_n\end{bmatrix} \mapsto - \begin{bmatrix} z^{(1)}_1 & \cdots z^{(T)}_1 \\ - \cdots & \cdots \\ - z^{(T - \mathtt{pw})}_n & \cdots & z^{(T})_n\end{bmatrix} +\mathbb{R}^{N\times\mathtt{pw}}\to\mathbb{R}^{N\times\mathtt{pw}}, +\begin{bmatrix} + z^{(1)}_1 & \cdots & z^{(T)}_1 \\ + \cdots & \cdots & \cdots \\ + z^{(1)}_n & \cdots & z^{(T})_n + \end{bmatrix} \mapsto + \begin{bmatrix} + z^{(T - \mathtt{pw})}_1 & \cdots & z^{(T)}_1 \\ + \cdots & \cdots & \cdots \\ + z^{(T - \mathtt{pw})}_n & \cdots & z^{(T})_n\end{bmatrix} ``` """ -function assign_output_estimate(full_output::AbstractArray{T, 3}, prediction_window) where T +function assign_output_estimate(full_output::AbstractArray{T, 3}, prediction_window::Int) where T sys_dim, seq_length, batch_size = size(full_output) backend = KernelAbstractions.get_backend(full_output) output_estimate = KernelAbstractions.allocate(backend, T, sys_dim, prediction_window, batch_size) From 9773e453a956b776a376920d16ac68d00b60f0c6 Mon Sep 17 00:00:00 2001 From: benedict-96 Date: Wed, 13 Dec 2023 17:04:22 +0100 Subject: [PATCH 07/10] Fixed docs for rgrad. --- src/manifolds/stiefel_manifold.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/manifolds/stiefel_manifold.jl b/src/manifolds/stiefel_manifold.jl index ec1c3fb26..a98e22079 100644 --- a/src/manifolds/stiefel_manifold.jl +++ b/src/manifolds/stiefel_manifold.jl @@ -64,7 +64,7 @@ function Base.:*(Y::Adjoint{T, StiefelManifold{T, AT}}, B::AbstractMatrix) where end @doc raw""" -Computes the Riemannian gradient for the Stiefel manifold given an element ``Y\in{}St(N,n)`` and a matrix ``\nabla{}L\in\mahbb{R}^{N\times{}n}`` (the Euclidean gradient). It computes the Riemannian gradient with respect to the canonical metric (see the documentation for the function `metric` for an explanation of this). +Computes the Riemannian gradient for the Stiefel manifold given an element ``Y\in{}St(N,n)`` and a matrix ``\nabla{}L\in\mathbb{R}^{N\times{}n}`` (the Euclidean gradient). It computes the Riemannian gradient with respect to the canonical metric (see the documentation for the function `metric` for an explanation of this). The precise form of the mapping is: ```math \mathtt{rgrad}(Y, \nabla{}L) \mapsto \nabla{}L - Y(\nabla{}L)^TY From dbbe3fee8f57df651224554b069f9bfd67cf844f Mon Sep 17 00:00:00 2001 From: benedict-96 Date: Wed, 13 Dec 2023 17:12:59 +0100 Subject: [PATCH 08/10] Added additional constructor for the case when the type is the first input argument. --- src/arrays/stiefel_projection.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/arrays/stiefel_projection.jl b/src/arrays/stiefel_projection.jl index 82c04c2ac..236948a09 100644 --- a/src/arrays/stiefel_projection.jl +++ b/src/arrays/stiefel_projection.jl @@ -41,6 +41,8 @@ Outer constructor for `StiefelProjection`. This works with two integers as input """ StiefelProjection(N::Integer, n::Integer, T::Type=Float64) = StiefelProjection(CPU(), T, N, n) +StiefelProjection(T::Type, N::Integer, n::Integer) = StiefelProjection(N, n, T) + Base.size(E::StiefelProjection) = (E.N, E.n) Base.getindex(E::StiefelProjection, i, j) = getindex(E.A, i, j) Base.:+(E::StiefelProjection, A::AbstractMatrix) = E.A + A From 4e6f2640a3ad0f92b524fee1e434a30e0ae42953 Mon Sep 17 00:00:00 2001 From: benedict-96 Date: Wed, 20 Dec 2023 08:52:39 +0100 Subject: [PATCH 09/10] Resolved merge conflict. --- test/arrays/array_tests.jl | 12 ++++++++++++ test/runtests.jl | 1 + 2 files changed, 13 insertions(+) diff --git a/test/arrays/array_tests.jl b/test/arrays/array_tests.jl index c0da5eba9..abd21fc5f 100644 --- a/test/arrays/array_tests.jl +++ b/test/arrays/array_tests.jl @@ -76,6 +76,14 @@ function stiefel_lie_alg_add_sub_test(N, n) @test all(abs.(projection(Wā‚ - Wā‚‚) .- Sā‚„) .< 1e-10) end +<<<<<<< HEAD +======= +function stiefel_lie_alg_vectorization_test(N, n; T=Float32) + A = rand(StiefelLieAlgHorMatrix{T}, N, n) + @test isapprox(StiefelLieAlgHorMatrix(vec(A), N, n), A) +end + +>>>>>>> b05c66c (Removed the symplectic arrays.) # TODO: tests for ADAM functions # test everything for different n & N values @@ -96,4 +104,8 @@ for (N, n) āˆˆ zip(N_vec, n_vec) skew_mat_mul_test2(N) stiefel_proj_test(N,n) stiefel_lie_alg_add_sub_test(N,n) +<<<<<<< HEAD +======= + stiefel_lie_alg_vectorization_test(N, n) +>>>>>>> b05c66c (Removed the symplectic arrays.) end diff --git a/test/runtests.jl b/test/runtests.jl index 033b04f16..92afd85ef 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,6 +4,7 @@ using SafeTestsets @safetestset "Check parameterlength " begin include("parameterlength/check_parameterlengths.jl") end @safetestset "Arrays #1 " begin include("arrays/array_tests.jl") end @safetestset "Sampling of arrays " begin include("arrays/random_generation_of_custom_arrays.jl") end +@safetestset "Test constructors for custom arrays " begin include("arrays/constructor_tests_for_custom_arrays.jl") end @safetestset "Manifolds (Grassmann): " begin include("manifolds/grassmann_manifold.jl") end @safetestset "Gradient Layer " begin include("layers/gradient_layer_tests.jl") end @safetestset "Test symplecticity of upscaling layer " begin include("layers/sympnet_layers_test.jl") end From 1f04af95a82d187b4ca90da28bccb67c917daf83 Mon Sep 17 00:00:00 2001 From: benedict-96 Date: Wed, 13 Dec 2023 17:14:04 +0100 Subject: [PATCH 10/10] Added tests for the various constructors for custom arrays. --- .../constructor_tests_for_custom_arrays.jl | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 test/arrays/constructor_tests_for_custom_arrays.jl diff --git a/test/arrays/constructor_tests_for_custom_arrays.jl b/test/arrays/constructor_tests_for_custom_arrays.jl new file mode 100644 index 000000000..452f850f8 --- /dev/null +++ b/test/arrays/constructor_tests_for_custom_arrays.jl @@ -0,0 +1,37 @@ +using GeometricMachineLearning, Test +using LinearAlgebra: I + +@doc raw""" +This tests various constructor for custom arrays, e.g. if calling `SymmetricMatrix` on a matrix ``A`` does +```math +A \mapsto \frac{1}{2}(A + A^T). +``` +""" +function test_constructors_for_custom_arrays(n::Int, N::Int, T::Type) + A = rand(T, n, n) + B = rand(T, N, N) + + # SymmetricMatrix + @test Matrix{T}(SymmetricMatrix(A)) ā‰ˆ T(.5) * (A + A') + + # SkewSymMatrix + @test Matrix{T}(SkewSymMatrix(A)) ā‰ˆ T(.5) * (A - A') + + # StiefelLieAlgHorMatrix + B_shor = StiefelLieAlgHorMatrix(SkewSymMatrix(B), n) + B_shor2 = Matrix{T}(SkewSymMatrix(B)) + B_shor2[(n+1):N, (n+1):N] .= zero(T) + @test Matrix{T}(B_shor) ā‰ˆ B_shor2 + + # GrassmannLieAlgHorMatrix + B_ghor = GrassmannLieAlgHorMatrix(SkewSymMatrix(B), n) + B_ghor2 = copy(B_shor2) + B_ghor2[1:n, 1:n] .= zero(T) + @test Matrix{T}(B_ghor) ā‰ˆ B_ghor2 + + # StiefelProjection + E = StiefelProjection(T, N, n) + @test Matrix{T}(E) ā‰ˆ vcat(I(n), zeros(T, (N-n), n)) +end + +test_constructors_for_custom_arrays(5, 10, Float32) \ No newline at end of file