added plots for cross checks (#234)

* added plots for cross checks

* uncomment try/catch

* updated project.toml cross chekcs

cross_checks/Project.toml

@@ -2,11 +2,10 @@
 ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63"
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
+Colors = "5ae59095-9a9b-59fe-a467-6f913c188581"
 DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab"
 DifferentialEquations = "0c46a032-eb83-5123-abaf-570d42b7fbaa"
 FMI = "14a09403-18e3-468f-ad8a-74f8dda2d9ac"
-FMICore = "8af89139-c281-408e-bce2-3005eb87462f"
-FMIImport = "9fcbc62e-52a0-44e9-a616-1359a0008194"
 FMIZoo = "724179cf-c260-40a9-bd27-cccc6fe2f195"
 Git = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2"
 Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"

cross_checks/README.md

@@ -54,6 +54,7 @@ optional arguments:
                         hasn't been executed (e.g. officially not
                         compliant FMUs if they are not skipped)
   --commitfailed        Also commit the result file for failed FMUs
+  --plotfailed          Plot result for failed FMUs (e.g. debugging)
   -h, --help            show this help message and exit
 
 ```

cross_checks/cross_check.jl

@@ -4,8 +4,8 @@
 #
 
 using FMI
+using FMI.FMIImport.FMIBase.FMICore
 using FMIZoo
-using FMICore
 using Plots
 using ArgParse
 using Git
@@ -14,6 +14,7 @@ using DelimitedFiles
 using Tables
 using Statistics
 using DifferentialEquations
+using Plots, Colors
 
 import Base64
 
@@ -27,7 +28,7 @@ getInputValues = function (t, u)
     return nothing
 end
 
-getSolver = function()
+getSolver = function ()
     return Tsit5() # CVODE_BDF() # Rosenbrock23(autodiff=false)
 end
 
@@ -66,6 +67,9 @@ function parse_commandline()
         "--commitfailed"
         help = "Also commit the result file for failed FMUs"
         action = :store_true
+        "--plotfailed"
+        help = "Plot result for failed FMUs"
+        action = :store_true
 
     end
     println("Arguments used for cross check:")
@@ -75,13 +79,14 @@ function parse_commandline()
     return parse_args(s)
 end
 
-function runCrossCheckFmu(
+function runCrossCheckFMU(
     checkPath::String,
     resultPath::String,
     check::FmuCrossCheck,
     skipnotcompliant::Bool,
     commitrejected::Bool,
     commitfailed::Bool,
+    plotfailed::Bool,
 )::FmuCrossCheck
     pathToFMU = joinpath(checkPath, "$(check.fmuCheck).fmu")
 
@@ -147,7 +152,7 @@ function runCrossCheckFmu(
                     simData = simulateME(
                         fmuToCheck,
                         (tStart, tStop);
-                        solver=getSolver(),
+                        solver = getSolver(),
                         reltol = relTol,
                         saveat = fmuRefValues[1],
                         inputFunction = getInputValues,
@@ -170,7 +175,7 @@ function runCrossCheckFmu(
                     simData = simulateME(
                         fmuToCheck,
                         (tStart, tStop);
-                        solver=getSolver(),
+                        solver = getSolver(),
                         reltol = relTol,
                         saveat = fmuRefValues[1],
                         recordValues = fmuRecordValueNames,
@@ -200,11 +205,47 @@ function runCrossCheckFmu(
                 if commitfailed
                     mkpath(resultPath)
                     cd(resultPath)
+
                     rm("passed", force = true)
                     rm("rejected", force = true)
                     rm("README.md", force = true)
                     touch("failed")
                 end
+                if plotfailed
+                    mkpath(resultPath)
+                    cd(resultPath)
+
+                    names = keys(fmuRefValues)
+                    num = length(names) - 1
+                    colors = distinguishable_colors(num)
+
+                    fig = plot()
+                    for j = 1:num
+                        ts = fmuRefValues[1]
+                        vals = fmuRefValues[1+j]
+                        plot!(
+                            fig,
+                            ts,
+                            vals;
+                            style = :solid,
+                            color = colors[j],
+                            label = "$(names[j+1])",
+                        )
+
+                        ts = simData.values.t
+                        vals = collect(u[j] for u in simData.values.saveval)
+                        plot!(
+                            fig,
+                            ts,
+                            vals;
+                            style = :dash,
+                            color = colors[j],
+                            label = :none,
+                        )
+                    end
+                    display(fig)
+                end
+
             end
         else
             check.skipped = true
@@ -248,7 +289,9 @@ function main()
     println("#################### Start FMI Cross checks Run ####################")
     # parsing of cli arguments and setting of configuration
     parsed_args = parse_commandline()
-    unpackPath = haskey(ENV, "crosscheck_tempdir") ? ENV["crosscheck_tempdir"] : parsed_args["tempdir"]
+    unpackPath =
+        haskey(ENV, "crosscheck_tempdir") ? ENV["crosscheck_tempdir"] :
+        parsed_args["tempdir"]
     fmiVersion = parsed_args["fmiversion"]
     crossCheckRepo = parsed_args["ccrepo"]
     crossCheckBranch = parsed_args["ccbranch"]
@@ -258,9 +301,11 @@ function main()
         os = "linux64"
     end
     includeFatals = parsed_args["includefatals"]
-    skipnotcompliant = haskey(ENV, "crosscheck_skipnotcompliant") ? true : parsed_args["skipnotcompliant"]
+    skipnotcompliant =
+        haskey(ENV, "crosscheck_skipnotcompliant") ? true : parsed_args["skipnotcompliant"]
     commitrejected = parsed_args["commitrejected"]
     commitfailed = parsed_args["commitfailed"]
+    plotfailed = haskey(ENV, "crosscheck_plotfailed") ? true : parsed_args["plotfailed"]
 
     # checking of inputs
     if fmiVersion != "2.0"
@@ -307,7 +352,7 @@ function main()
     end
 
     #   Excecute FMUs
-    crossChecks = getFmusToTest(fmiCrossCheckRepoPath, fmiVersion, os)
+    crossChecks = getFMUsToTest(fmiCrossCheckRepoPath, fmiVersion, os)
     if !includeFatals
         crossChecks = filter(c -> (!(c.system in EXCLUDED_SYSTEMS)), crossChecks)
     end
@@ -338,13 +383,14 @@ function main()
         cd(checkPath)
         println("Checking $check for $checkPath and expecting $resultPath")
 
-        check = runCrossCheckFmu(
+        check = runCrossCheckFMU(
             checkPath,
             resultPath,
             check,
             skipnotcompliant,
             commitrejected,
             commitfailed,
+            plotfailed,
         )
         crossChecks[index] = check
     end
@@ -353,7 +399,9 @@ function main()
     # Write Summary of Cross Check run
     println("#################### Start FMI Cross check Summary ####################")
     println("\tTotal Cross checks:\t\t\t$(count(c -> (true), crossChecks))")
-    println("\tSuccessful Cross checks:\t\t$(count(c -> (c.success === true), crossChecks))")
+    println(
+        "\tSuccessful Cross checks:\t\t$(count(c -> (c.success === true), crossChecks))",
+    )
     println(
         "\tFailed Cross checks:\t\t\t$(count(c -> (c.success === false && c.error === nothing && c.skipped === false), crossChecks))",
     )
@@ -367,7 +415,10 @@ function main()
     end
     println("\tList of failed Cross checks")
     for (index, success) in enumerate(
-        filter(c -> (c.success === false && c.error === nothing && c.skipped === false), crossChecks),
+        filter(
+            c -> (c.success === false && c.error === nothing && c.skipped === false),
+            crossChecks,
+        ),
     )
         println("\u001B[33m\t\t$(index):\t$(success)\u001B[0m")
     end

cross_checks/cross_check_lib.jl

@@ -42,7 +42,7 @@ Returns a array of all available FMI Cross Checks
 - `fmiVersion::String`: FMI Version used for running the FMUs. Note: Currently only 2.0 officially supported
 - `os::String`: The operating system that is used for running the FMUs
 """
-function getFmusToTest(
+function getFMUsToTest(
     repoPath::String,
     fmiVersion::String,
     os::String,

test/sim_CS.jl

@@ -72,4 +72,4 @@ end
 # reference values from Simulation in Dymola2020x (Dassl, default settings)
 @test [solution.values.saveval[1]...] == [0.5, 0.0]
 @test sum(abs.([solution.values.saveval[end]...] - [0.613371, 0.188633])) < 0.2
-unloadFMU(fmu)
+unloadFMU(fmu)