From 7481dcc06f66becd6cb69f6f1399bcc417df5fc4 Mon Sep 17 00:00:00 2001
From: Petr Vana <petvana@centrum.cz>
Date: Fri, 23 Dec 2022 23:10:18 +0100
Subject: [PATCH 01/19] Parallelize precompilation script

---
 contrib/generate_precompile.jl | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl
index 53ca9403463b3..50da233e3433c 100644
--- a/contrib/generate_precompile.jl
+++ b/contrib/generate_precompile.jl
@@ -245,15 +245,16 @@ function generate_precompile_statements()
     sysimg = Base.unsafe_string(Base.JLOptions().image_file)
 
     # Extract the precompile statements from the precompile file
-    statements = Set{String}()
+    statements_step1 = Channel{String}(Inf)
+    statements_step2 = Channel{String}(Inf)
 
     # From hardcoded statements
     for statement in split(hardcoded_precompile_statements::String, '\n')
-        push!(statements, statement)
+        push!(statements_step1, statement)
     end
 
     # Collect statements from running the script
-    mktempdir() do prec_path
+    @async mktempdir() do prec_path
         # Also precompile a package here
         pkgname = "__PackagePrecompilationStatementModule"
         mkpath(joinpath(prec_path, pkgname, "src"))
@@ -275,12 +276,13 @@ function generate_precompile_statements()
         for f in (tmp_prec, tmp_proc)
             for statement in split(read(f, String), '\n')
                 occursin("Main.", statement) && continue
-                push!(statements, statement)
+                push!(statements_step1, statement)
             end
         end
+        close(statements_step1)
     end
 
-    mktemp() do precompile_file, precompile_file_h
+    @async mktemp() do precompile_file, precompile_file_h
         # Collect statements from running a REPL process and replaying our REPL script
         pts, ptm = open_fake_pty()
         blackhole = Sys.isunix() ? "/dev/null" : "nul"
@@ -359,8 +361,9 @@ function generate_precompile_statements()
         for statement in split(read(precompile_file, String), '\n')
             # Main should be completely clean
             occursin("Main.", statement) && continue
-            push!(statements, statement)
+            push!(statements_step2, statement)
         end
+        close(statements_step2)
     end
 
     # Create a staging area where all the loaded packages are available
@@ -371,9 +374,12 @@ function generate_precompile_statements()
         end
     end
 
+    # Make statements unique
+    statements = Set{String}()
     # Execute the precompile statements
     n_succeeded = 0
-    include_time = @elapsed for statement in statements
+    include_time = @elapsed for sts in [statements_step1, statements_step2], statement in sts
+        Base.in!(statement, statements) && continue
         # println(statement)
         # XXX: skip some that are broken. these are caused by issue #39902
         occursin("Tuple{Artifacts.var\"#@artifact_str\", LineNumberNode, Module, Any, Any}", statement) && continue

From 0f9186aae914b27dc69f37d43c25b33b74f15b55 Mon Sep 17 00:00:00 2001
From: Petr Vana <petvana@centrum.cz>
Date: Fri, 23 Dec 2022 23:42:02 +0100
Subject: [PATCH 02/19] Clean the code

---
 contrib/generate_precompile.jl | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl
index 50da233e3433c..b1396e206969b 100644
--- a/contrib/generate_precompile.jl
+++ b/contrib/generate_precompile.jl
@@ -275,7 +275,6 @@ function generate_precompile_statements()
         run(`$(julia_exepath()) -O0 --sysimage $sysimg --trace-compile=$tmp_proc --startup-file=no -Cnative -e $s`)
         for f in (tmp_prec, tmp_proc)
             for statement in split(read(f, String), '\n')
-                occursin("Main.", statement) && continue
                 push!(statements_step1, statement)
             end
         end
@@ -359,8 +358,6 @@ function generate_precompile_statements()
         write(debug_output, "\n#### FINISHED ####\n")
 
         for statement in split(read(precompile_file, String), '\n')
-            # Main should be completely clean
-            occursin("Main.", statement) && continue
             push!(statements_step2, statement)
         end
         close(statements_step2)
@@ -378,7 +375,9 @@ function generate_precompile_statements()
     statements = Set{String}()
     # Execute the precompile statements
     n_succeeded = 0
-    include_time = @elapsed for sts in [statements_step1, statements_step2], statement in sts
+    for sts in [statements_step1, statements_step2], statement in sts
+        # Main should be completely clean
+        occursin("Main.", statement) && continue
         Base.in!(statement, statements) && continue
         # println(statement)
         # XXX: skip some that are broken. these are caused by issue #39902
@@ -422,13 +421,8 @@ function generate_precompile_statements()
         n_succeeded > 1200 || @warn "Only $n_succeeded precompile statements"
     end
 
-    include_time *= 1e9
-    gen_time = (time_ns() - start_time) - include_time
     tot_time = time_ns() - start_time
-
     println("Precompilation complete. Summary:")
-    print("Generation ── "); Base.time_print(gen_time);     print(" "); show(IOContext(stdout, :compact=>true), gen_time / tot_time * 100);     println("%")
-    print("Execution ─── "); Base.time_print(include_time); print(" "); show(IOContext(stdout, :compact=>true), include_time / tot_time * 100); println("%")
     print("Total ─────── "); Base.time_print(tot_time);     println()
 
     return

From 6979ff3302056d0e340b149b3488ff11711591b4 Mon Sep 17 00:00:00 2001
From: Petr Vana <petvana@centrum.cz>
Date: Tue, 27 Dec 2022 13:37:44 +0100
Subject: [PATCH 03/19] Improve printing, add errormonitor, and enable serial
 run for debug

---
 contrib/generate_precompile.jl | 46 ++++++++++++++++++++++++++++------
 1 file changed, 38 insertions(+), 8 deletions(-)

diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl
index b1396e206969b..e69e96e1753c0 100644
--- a/contrib/generate_precompile.jl
+++ b/contrib/generate_precompile.jl
@@ -239,6 +239,9 @@ const PKG_PROMPT = "pkg> "
 const SHELL_PROMPT = "shell> "
 const HELP_PROMPT = "help?> "
 
+# You can disable parallel precompiles generation by setting `false`
+const PARALLEL_PRECOMPILATION = true
+
 function generate_precompile_statements()
     start_time = time_ns()
     debug_output = devnull # or stdout
@@ -247,14 +250,37 @@ function generate_precompile_statements()
     # Extract the precompile statements from the precompile file
     statements_step1 = Channel{String}(Inf)
     statements_step2 = Channel{String}(Inf)
+    # Make statements unique
+    statements = Set{String}()
+    # Variables for statistics
+    n_step0 = n_step1 = n_step2 = 0
+    n_succeeded = 0
+    step1 = step2 = nothing
+    repl_state_global = nothing
 
     # From hardcoded statements
     for statement in split(hardcoded_precompile_statements::String, '\n')
         push!(statements_step1, statement)
+        n_step0 += 1
+    end
+
+    # Printing the current state
+    function print_state(;repl_state = nothing)
+        if !isnothing(repl_state)
+            repl_state_global = repl_state
+        end
+        step0_status = "F,$n_step0"
+        step1_status = (isnothing(step1) ? "W" : isopen(statements_step1) ? "R" : "F") * ",$n_step1"
+        step2_status = (isnothing(step2) ? "W" : isopen(statements_step2) ? "R" : "F") * ",$n_step2"
+        repl_status = isnothing(repl_state_global) ? "" : repl_state_global
+        ex_status = "$n_succeeded/$(length(statements))"
+        print("\rCollect(manual($step0_status), normal($step1_status), REPL $repl_status($step2_status)) => Execute $ex_status")
     end
+    println("Precompile statements (Waiting, Running, Finished)")
+    print_state()
 
     # Collect statements from running the script
-    @async mktempdir() do prec_path
+    step1 = @async mktempdir() do prec_path
         # Also precompile a package here
         pkgname = "__PackagePrecompilationStatementModule"
         mkpath(joinpath(prec_path, pkgname, "src"))
@@ -276,12 +302,16 @@ function generate_precompile_statements()
         for f in (tmp_prec, tmp_proc)
             for statement in split(read(f, String), '\n')
                 push!(statements_step1, statement)
+                n_step1 += 1
             end
         end
         close(statements_step1)
+        print_state()
     end
+    errormonitor(step1)
+    !PARALLEL_PRECOMPILATION && wait(step1)
 
-    @async mktemp() do precompile_file, precompile_file_h
+    step2 = @async mktemp() do precompile_file, precompile_file_h
         # Collect statements from running a REPL process and replaying our REPL script
         pts, ptm = open_fake_pty()
         blackhole = Sys.isunix() ? "/dev/null" : "nul"
@@ -330,7 +360,7 @@ function generate_precompile_statements()
             for l in precompile_lines
                 sleep(0.1)
                 curr += 1
-                print("\rGenerating REPL precompile statements... $curr/$(length(precompile_lines))")
+                print_state(repl_state = "$curr/$(length(precompile_lines))")
                 # consume any other output
                 bytesavailable(output_copy) > 0 && readavailable(output_copy)
                 # push our input
@@ -349,7 +379,6 @@ function generate_precompile_statements()
                     sleep(0.1)
                 end
             end
-            println()
         end
         write(ptm, "exit()\n")
         wait(tee)
@@ -359,9 +388,13 @@ function generate_precompile_statements()
 
         for statement in split(read(precompile_file, String), '\n')
             push!(statements_step2, statement)
+            n_step2 += 1
         end
         close(statements_step2)
+        print_state()
     end
+    errormonitor(step2)
+    !PARALLEL_PRECOMPILATION && wait(step2)
 
     # Create a staging area where all the loaded packages are available
     PrecompileStagingArea = Module()
@@ -371,10 +404,7 @@ function generate_precompile_statements()
         end
     end
 
-    # Make statements unique
-    statements = Set{String}()
     # Execute the precompile statements
-    n_succeeded = 0
     for sts in [statements_step1, statements_step2], statement in sts
         # Main should be completely clean
         occursin("Main.", statement) && continue
@@ -408,7 +438,7 @@ function generate_precompile_statements()
             ps = Core.eval(PrecompileStagingArea, ps)
             precompile(ps...)
             n_succeeded += 1
-            print("\rExecuting precompile statements... $n_succeeded/$(length(statements))")
+            print_state()
         catch ex
             # See #28808
             @warn "Failed to precompile expression" form=statement exception=ex _module=nothing _file=nothing _line=0

From 1fb6ec8fce46b7f36771fb31ef5fc5d44b9af090 Mon Sep 17 00:00:00 2001
From: Petr Vana <petvana@centrum.cz>
Date: Tue, 27 Dec 2022 16:51:56 +0100
Subject: [PATCH 04/19] Fix printing

---
 contrib/generate_precompile.jl | 44 ++++++++++++++++++----------------
 1 file changed, 23 insertions(+), 21 deletions(-)

diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl
index e69e96e1753c0..ef971a17381b0 100644
--- a/contrib/generate_precompile.jl
+++ b/contrib/generate_precompile.jl
@@ -250,37 +250,33 @@ function generate_precompile_statements()
     # Extract the precompile statements from the precompile file
     statements_step1 = Channel{String}(Inf)
     statements_step2 = Channel{String}(Inf)
-    # Make statements unique
-    statements = Set{String}()
-    # Variables for statistics
-    n_step0 = n_step1 = n_step2 = 0
-    n_succeeded = 0
-    step1 = step2 = nothing
-    repl_state_global = nothing
 
     # From hardcoded statements
     for statement in split(hardcoded_precompile_statements::String, '\n')
         push!(statements_step1, statement)
-        n_step0 += 1
     end
 
     # Printing the current state
-    function print_state(;repl_state = nothing)
-        if !isnothing(repl_state)
-            repl_state_global = repl_state
+    print_lk = ReentrantLock()
+    status = Dict{String, String}(
+        "step1" => "W",
+        "step2" => "W",
+        "repl" => "0/0",
+        "execute" => "0/0",
+    )
+    function print_state(args::Pair{String,String}...)
+        lock(print_lk) do
+            isempty(args) || push!(status, args...)
+            t1, t2, t3, t4 = (get(status, x, "") for x in ["step1", "repl", "step2", "execute"])
+            print("\rCollect (normal($t1), REPL $t2 ($t3)) => Execute $t4")
         end
-        step0_status = "F,$n_step0"
-        step1_status = (isnothing(step1) ? "W" : isopen(statements_step1) ? "R" : "F") * ",$n_step1"
-        step2_status = (isnothing(step2) ? "W" : isopen(statements_step2) ? "R" : "F") * ",$n_step2"
-        repl_status = isnothing(repl_state_global) ? "" : repl_state_global
-        ex_status = "$n_succeeded/$(length(statements))"
-        print("\rCollect(manual($step0_status), normal($step1_status), REPL $repl_status($step2_status)) => Execute $ex_status")
     end
     println("Precompile statements (Waiting, Running, Finished)")
     print_state()
 
     # Collect statements from running the script
     step1 = @async mktempdir() do prec_path
+        print_state("step1" => "R")
         # Also precompile a package here
         pkgname = "__PackagePrecompilationStatementModule"
         mkpath(joinpath(prec_path, pkgname, "src"))
@@ -299,6 +295,7 @@ function generate_precompile_statements()
             $precompile_script
             """
         run(`$(julia_exepath()) -O0 --sysimage $sysimg --trace-compile=$tmp_proc --startup-file=no -Cnative -e $s`)
+        n_step1 = 0
         for f in (tmp_prec, tmp_proc)
             for statement in split(read(f, String), '\n')
                 push!(statements_step1, statement)
@@ -306,12 +303,13 @@ function generate_precompile_statements()
             end
         end
         close(statements_step1)
-        print_state()
+        print_state("step1" => "F,$n_step1")
     end
     errormonitor(step1)
     !PARALLEL_PRECOMPILATION && wait(step1)
 
     step2 = @async mktemp() do precompile_file, precompile_file_h
+        print_state("step2" => "R")
         # Collect statements from running a REPL process and replaying our REPL script
         pts, ptm = open_fake_pty()
         blackhole = Sys.isunix() ? "/dev/null" : "nul"
@@ -360,7 +358,7 @@ function generate_precompile_statements()
             for l in precompile_lines
                 sleep(0.1)
                 curr += 1
-                print_state(repl_state = "$curr/$(length(precompile_lines))")
+                print_state("repl" => "$curr/$(length(precompile_lines))")
                 # consume any other output
                 bytesavailable(output_copy) > 0 && readavailable(output_copy)
                 # push our input
@@ -386,12 +384,13 @@ function generate_precompile_statements()
         close(ptm)
         write(debug_output, "\n#### FINISHED ####\n")
 
+        n_step2 = 0
         for statement in split(read(precompile_file, String), '\n')
             push!(statements_step2, statement)
             n_step2 += 1
         end
         close(statements_step2)
-        print_state()
+        print_state("step2" => "F,$n_step2")
     end
     errormonitor(step2)
     !PARALLEL_PRECOMPILATION && wait(step2)
@@ -404,6 +403,9 @@ function generate_precompile_statements()
         end
     end
 
+    n_succeeded = 0
+    # Make statements unique
+    statements = Set{String}()
     # Execute the precompile statements
     for sts in [statements_step1, statements_step2], statement in sts
         # Main should be completely clean
@@ -438,7 +440,7 @@ function generate_precompile_statements()
             ps = Core.eval(PrecompileStagingArea, ps)
             precompile(ps...)
             n_succeeded += 1
-            print_state()
+            print_state("execute" => "$n_succeeded/$(length(statements))")
         catch ex
             # See #28808
             @warn "Failed to precompile expression" form=statement exception=ex _module=nothing _file=nothing _line=0

From 1794d64eaf9b19e1037dc646aa30fbc3fa873bbd Mon Sep 17 00:00:00 2001
From: Petr Vana <petvana@centrum.cz>
Date: Tue, 27 Dec 2022 19:43:09 +0100
Subject: [PATCH 05/19] Make printing thread-safe

---
 contrib/generate_precompile.jl | 34 +++++++++++++++++++---------------
 1 file changed, 19 insertions(+), 15 deletions(-)

diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl
index ef971a17381b0..7628a517479a1 100644
--- a/contrib/generate_precompile.jl
+++ b/contrib/generate_precompile.jl
@@ -242,21 +242,9 @@ const HELP_PROMPT = "help?> "
 # You can disable parallel precompiles generation by setting `false`
 const PARALLEL_PRECOMPILATION = true
 
-function generate_precompile_statements()
-    start_time = time_ns()
-    debug_output = devnull # or stdout
-    sysimg = Base.unsafe_string(Base.JLOptions().image_file)
-
-    # Extract the precompile statements from the precompile file
-    statements_step1 = Channel{String}(Inf)
-    statements_step2 = Channel{String}(Inf)
-
-    # From hardcoded statements
-    for statement in split(hardcoded_precompile_statements::String, '\n')
-        push!(statements_step1, statement)
-    end
-
-    # Printing the current state
+# Printing the current state
+let
+    global print_state
     print_lk = ReentrantLock()
     status = Dict{String, String}(
         "step1" => "W",
@@ -271,6 +259,22 @@ function generate_precompile_statements()
             print("\rCollect (normal($t1), REPL $t2 ($t3)) => Execute $t4")
         end
     end
+end
+
+function generate_precompile_statements()
+    start_time = time_ns()
+    debug_output = devnull # or stdout
+    sysimg = Base.unsafe_string(Base.JLOptions().image_file)
+
+    # Extract the precompile statements from the precompile file
+    statements_step1 = Channel{String}(Inf)
+    statements_step2 = Channel{String}(Inf)
+
+    # From hardcoded statements
+    for statement in split(hardcoded_precompile_statements::String, '\n')
+        push!(statements_step1, statement)
+    end
+
     println("Precompile statements (Waiting, Running, Finished)")
     print_state()
 

From a8151920f26905d875f0fe9b56cafebd42a584f5 Mon Sep 17 00:00:00 2001
From: Petr Vana <petvana@centrum.cz>
Date: Tue, 27 Dec 2022 20:07:41 +0100
Subject: [PATCH 06/19] Wait for both steps

---
 contrib/generate_precompile.jl | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl
index 7628a517479a1..cddb17d92ba35 100644
--- a/contrib/generate_precompile.jl
+++ b/contrib/generate_precompile.jl
@@ -457,6 +457,9 @@ function generate_precompile_statements()
         n_succeeded > 1200 || @warn "Only $n_succeeded precompile statements"
     end
 
+    PARALLEL_PRECOMPILATION && wait(step1)
+    PARALLEL_PRECOMPILATION && wait(step2)
+
     tot_time = time_ns() - start_time
     println("Precompilation complete. Summary:")
     print("Total ─────── "); Base.time_print(tot_time);     println()

From 79bee809b75182faae2c8f37146fccd4c7ffd6cc Mon Sep 17 00:00:00 2001
From: Petr Vana <petvana@centrum.cz>
Date: Tue, 27 Dec 2022 23:39:17 +0100
Subject: [PATCH 07/19] Disable errormonitor temporarily

---
 contrib/generate_precompile.jl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl
index cddb17d92ba35..62c23bfd83471 100644
--- a/contrib/generate_precompile.jl
+++ b/contrib/generate_precompile.jl
@@ -309,7 +309,7 @@ function generate_precompile_statements()
         close(statements_step1)
         print_state("step1" => "F,$n_step1")
     end
-    errormonitor(step1)
+    #errormonitor(step1)
     !PARALLEL_PRECOMPILATION && wait(step1)
 
     step2 = @async mktemp() do precompile_file, precompile_file_h
@@ -396,7 +396,7 @@ function generate_precompile_statements()
         close(statements_step2)
         print_state("step2" => "F,$n_step2")
     end
-    errormonitor(step2)
+    #errormonitor(step2)
     !PARALLEL_PRECOMPILATION && wait(step2)
 
     # Create a staging area where all the loaded packages are available

From 68d4ee32b01aa824bfdd10174b8b841be311a1eb Mon Sep 17 00:00:00 2001
From: Petr Vana <petvana@centrum.cz>
Date: Wed, 28 Dec 2022 09:29:58 +0100
Subject: [PATCH 08/19] Use istaskfailed instead of errormonitor

---
 contrib/generate_precompile.jl | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl
index 62c23bfd83471..645d3abf6360d 100644
--- a/contrib/generate_precompile.jl
+++ b/contrib/generate_precompile.jl
@@ -256,7 +256,7 @@ let
         lock(print_lk) do
             isempty(args) || push!(status, args...)
             t1, t2, t3, t4 = (get(status, x, "") for x in ["step1", "repl", "step2", "execute"])
-            print("\rCollect (normal($t1), REPL $t2 ($t3)) => Execute $t4")
+            print("\rCollect (normal ($t1), REPL $t2 ($t3)) => Execute $t4")
         end
     end
 end
@@ -396,7 +396,6 @@ function generate_precompile_statements()
         close(statements_step2)
         print_state("step2" => "F,$n_step2")
     end
-    #errormonitor(step2)
     !PARALLEL_PRECOMPILATION && wait(step2)
 
     # Create a staging area where all the loaded packages are available
@@ -457,8 +456,10 @@ function generate_precompile_statements()
         n_succeeded > 1200 || @warn "Only $n_succeeded precompile statements"
     end
 
-    PARALLEL_PRECOMPILATION && wait(step1)
-    PARALLEL_PRECOMPILATION && wait(step2)
+    wait(step1)
+    istaskfailed(step1) && throw("Step 1 of collecting precompiles failed.")
+    wait(step2)
+    istaskfailed(step2) && throw("Step 2 of collecting precompiles failed.")
 
     tot_time = time_ns() - start_time
     println("Precompilation complete. Summary:")

From 03b9cc63c0dac0195448793c6907d45dc124d9e8 Mon Sep 17 00:00:00 2001
From: Petr Vana <petvana@centrum.cz>
Date: Wed, 28 Dec 2022 11:00:56 +0100
Subject: [PATCH 09/19] Use fetch instead of errormonitor

---
 contrib/generate_precompile.jl | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl
index 645d3abf6360d..fcd6777ccc245 100644
--- a/contrib/generate_precompile.jl
+++ b/contrib/generate_precompile.jl
@@ -308,8 +308,8 @@ function generate_precompile_statements()
         end
         close(statements_step1)
         print_state("step1" => "F,$n_step1")
+        return :ok
     end
-    #errormonitor(step1)
     !PARALLEL_PRECOMPILATION && wait(step1)
 
     step2 = @async mktemp() do precompile_file, precompile_file_h
@@ -395,6 +395,7 @@ function generate_precompile_statements()
         end
         close(statements_step2)
         print_state("step2" => "F,$n_step2")
+        return :ok
     end
     !PARALLEL_PRECOMPILATION && wait(step2)
 
@@ -456,10 +457,8 @@ function generate_precompile_statements()
         n_succeeded > 1200 || @warn "Only $n_succeeded precompile statements"
     end
 
-    wait(step1)
-    istaskfailed(step1) && throw("Step 1 of collecting precompiles failed.")
-    wait(step2)
-    istaskfailed(step2) && throw("Step 2 of collecting precompiles failed.")
+    fetch(step1) == :ok || throw("Step 1 of collecting precompiles failed.")
+    fetch(step2) == :ok || throw("Step 2 of collecting precompiles failed.")
 
     tot_time = time_ns() - start_time
     println("Precompilation complete. Summary:")

From f49f537afae0ac016817fcf785df0f360f0222eb Mon Sep 17 00:00:00 2001
From: Petr Vana <petvana@centrum.cz>
Date: Thu, 29 Dec 2022 13:08:52 +0100
Subject: [PATCH 10/19] Fancy printing

---
 contrib/generate_precompile.jl | 53 +++++++++++++++++++++++++++++-----
 1 file changed, 45 insertions(+), 8 deletions(-)

diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl
index fcd6777ccc245..994a81dec1113 100644
--- a/contrib/generate_precompile.jl
+++ b/contrib/generate_precompile.jl
@@ -242,6 +242,9 @@ const HELP_PROMPT = "help?> "
 # You can disable parallel precompiles generation by setting `false`
 const PARALLEL_PRECOMPILATION = true
 
+# TODO use can_fancyprint(io) from Pkg
+const fancyprint = true
+
 # Printing the current state
 let
     global print_state
@@ -250,13 +253,30 @@ let
         "step1" => "W",
         "step2" => "W",
         "repl" => "0/0",
-        "execute" => "0/0",
+        "step3" => "W",
+        "clock" => "◐",
     )
+    function print_status(key::String)
+        txt = status[key]
+        if startswith(txt, "W") # Waiting
+            printstyled("?", color=Base.warn_color()); print(txt[2:end])
+        elseif startswith(txt, "R") # Running
+            printstyled(status["clock"], color=:magenta); print(txt[2:end])
+        elseif startswith(txt, "F") # Finished
+            printstyled("✓", color=:green); print(txt[2:end])
+        else
+            print(txt)
+        end
+    end
     function print_state(args::Pair{String,String}...)
         lock(print_lk) do
             isempty(args) || push!(status, args...)
-            t1, t2, t3, t4 = (get(status, x, "") for x in ["step1", "repl", "step2", "execute"])
-            print("\rCollect (normal ($t1), REPL $t2 ($t3)) => Execute $t4")
+            print("\rCollect (Basic: ")
+            print_status("step1")
+            print(", REPL ", status["repl"], ": ")
+            print_status("step2")
+            print(") => Execute ")
+            print_status("step3")
         end
     end
 end
@@ -275,8 +295,20 @@ function generate_precompile_statements()
         push!(statements_step1, statement)
     end
 
-    println("Precompile statements (Waiting, Running, Finished)")
+    println("Collect and execute precompile statements")
     print_state()
+    clock = @async begin
+        t = Timer(0; interval=1/10)
+        anim_chars = ["◐","◓","◑","◒"]
+        current = 1
+        if fancyprint
+            while isopen(statements_step2) || !isempty(statements_step2)
+                print_state("clock" => anim_chars[current])
+                wait(t)
+                current = current == 4 ? 1 : current + 1
+            end
+        end
+    end
 
     # Collect statements from running the script
     step1 = @async mktempdir() do prec_path
@@ -301,13 +333,14 @@ function generate_precompile_statements()
         run(`$(julia_exepath()) -O0 --sysimage $sysimg --trace-compile=$tmp_proc --startup-file=no -Cnative -e $s`)
         n_step1 = 0
         for f in (tmp_prec, tmp_proc)
+            isfile(f) || continue
             for statement in split(read(f, String), '\n')
                 push!(statements_step1, statement)
                 n_step1 += 1
             end
         end
         close(statements_step1)
-        print_state("step1" => "F,$n_step1")
+        print_state("step1" => "F$n_step1")
         return :ok
     end
     !PARALLEL_PRECOMPILATION && wait(step1)
@@ -394,7 +427,7 @@ function generate_precompile_statements()
             n_step2 += 1
         end
         close(statements_step2)
-        print_state("step2" => "F,$n_step2")
+        print_state("step2" => "F$n_step2")
         return :ok
     end
     !PARALLEL_PRECOMPILATION && wait(step2)
@@ -444,17 +477,21 @@ function generate_precompile_statements()
             ps = Core.eval(PrecompileStagingArea, ps)
             precompile(ps...)
             n_succeeded += 1
-            print_state("execute" => "$n_succeeded/$(length(statements))")
+            failed = length(statements) - n_succeeded
+            print_state("step3" => "R$n_succeeded ($failed failed)")
         catch ex
             # See #28808
             @warn "Failed to precompile expression" form=statement exception=ex _module=nothing _file=nothing _line=0
         end
     end
+    wait(clock) # Stop asynchronous printing
+    failed = length(statements) - n_succeeded
+    print_state("step3" => "F$n_succeeded ($failed failed)")
     println()
     if have_repl
         # Seems like a reasonable number right now, adjust as needed
         # comment out if debugging script
-        n_succeeded > 1200 || @warn "Only $n_succeeded precompile statements"
+        n_succeeded > 1500 || @warn "Only $n_succeeded precompile statements"
     end
 
     fetch(step1) == :ok || throw("Step 1 of collecting precompiles failed.")

From 5083d8e9cadf438b77c2381b3d1153e5a8760346 Mon Sep 17 00:00:00 2001
From: Petr Vana <petvana@centrum.cz>
Date: Thu, 29 Dec 2022 13:22:42 +0100
Subject: [PATCH 11/19] Make clock spinning

---
 contrib/generate_precompile.jl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl
index 994a81dec1113..baed8884d5626 100644
--- a/contrib/generate_precompile.jl
+++ b/contrib/generate_precompile.jl
@@ -478,6 +478,7 @@ function generate_precompile_statements()
             precompile(ps...)
             n_succeeded += 1
             failed = length(statements) - n_succeeded
+            yield() # Make clock spinning
             print_state("step3" => "R$n_succeeded ($failed failed)")
         catch ex
             # See #28808

From c0f763517f0924cb0d6f7f6fdc6b78ff8484360d Mon Sep 17 00:00:00 2001
From: Petr Vana <petvana@centrum.cz>
Date: Thu, 29 Dec 2022 13:30:27 +0100
Subject: [PATCH 12/19] Fix spacing

---
 contrib/generate_precompile.jl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl
index baed8884d5626..811764ed8dbba 100644
--- a/contrib/generate_precompile.jl
+++ b/contrib/generate_precompile.jl
@@ -259,11 +259,11 @@ let
     function print_status(key::String)
         txt = status[key]
         if startswith(txt, "W") # Waiting
-            printstyled("?", color=Base.warn_color()); print(txt[2:end])
+            printstyled("? ", color=Base.warn_color()); print(txt[2:end])
         elseif startswith(txt, "R") # Running
-            printstyled(status["clock"], color=:magenta); print(txt[2:end])
+            printstyled(status["clock"], " ", color=:magenta); print(txt[2:end])
         elseif startswith(txt, "F") # Finished
-            printstyled("✓", color=:green); print(txt[2:end])
+            printstyled("✓ ", color=:green); print(txt[2:end])
         else
             print(txt)
         end

From 5c1b99a5c33476443160e32463a916fb8f502543 Mon Sep 17 00:00:00 2001
From: Petr Vana <petvana@centrum.cz>
Date: Thu, 29 Dec 2022 13:41:49 +0100
Subject: [PATCH 13/19] Improve wording based on the review

---
 contrib/generate_precompile.jl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl
index 811764ed8dbba..4aace61592427 100644
--- a/contrib/generate_precompile.jl
+++ b/contrib/generate_precompile.jl
@@ -271,7 +271,7 @@ let
     function print_state(args::Pair{String,String}...)
         lock(print_lk) do
             isempty(args) || push!(status, args...)
-            print("\rCollect (Basic: ")
+            print("\r└ Collect (Basic: ")
             print_status("step1")
             print(", REPL ", status["repl"], ": ")
             print_status("step2")
@@ -295,7 +295,7 @@ function generate_precompile_statements()
         push!(statements_step1, statement)
     end
 
-    println("Collect and execute precompile statements")
+    println("Collecting and executing precompile statements")
     print_state()
     clock = @async begin
         t = Timer(0; interval=1/10)

From f0daa0d0e4f0470ddd322840cff9441b18b6ab71 Mon Sep 17 00:00:00 2001
From: Petr Vana <petvana@centrum.cz>
Date: Thu, 29 Dec 2022 14:39:13 +0100
Subject: [PATCH 14/19] Detect if stdout is TTY or CI

---
 contrib/generate_precompile.jl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl
index 4aace61592427..2f8251642abea 100644
--- a/contrib/generate_precompile.jl
+++ b/contrib/generate_precompile.jl
@@ -242,8 +242,8 @@ const HELP_PROMPT = "help?> "
 # You can disable parallel precompiles generation by setting `false`
 const PARALLEL_PRECOMPILATION = true
 
-# TODO use can_fancyprint(io) from Pkg
-const fancyprint = true
+# You can disable fancy printing
+const fancyprint = (io isa Base.TTY) && (get(ENV, "CI", nothing) != "true")
 
 # Printing the current state
 let

From 86ae9e9583ccc9efed03e53310e313c108314228 Mon Sep 17 00:00:00 2001
From: Petr Vana <petvana@centrum.cz>
Date: Thu, 29 Dec 2022 14:50:54 +0100
Subject: [PATCH 15/19] Fix to stdout

---
 contrib/generate_precompile.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl
index 2f8251642abea..85b51e1b39ade 100644
--- a/contrib/generate_precompile.jl
+++ b/contrib/generate_precompile.jl
@@ -243,7 +243,7 @@ const HELP_PROMPT = "help?> "
 const PARALLEL_PRECOMPILATION = true
 
 # You can disable fancy printing
-const fancyprint = (io isa Base.TTY) && (get(ENV, "CI", nothing) != "true")
+const fancyprint = (stdout isa Base.TTY) && (get(ENV, "CI", nothing) != "true")
 
 # Printing the current state
 let

From 338b5627d007e4163897597a7096af8581bb5147 Mon Sep 17 00:00:00 2001
From: Petr Vana <petvana@centrum.cz>
Date: Thu, 29 Dec 2022 15:10:35 +0100
Subject: [PATCH 16/19] White spinner + disable cursor

---
 contrib/generate_precompile.jl | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl
index 85b51e1b39ade..6ec4c282e6168 100644
--- a/contrib/generate_precompile.jl
+++ b/contrib/generate_precompile.jl
@@ -261,7 +261,7 @@ let
         if startswith(txt, "W") # Waiting
             printstyled("? ", color=Base.warn_color()); print(txt[2:end])
         elseif startswith(txt, "R") # Running
-            printstyled(status["clock"], " ", color=:magenta); print(txt[2:end])
+            print(status["clock"], " ", txt[2:end])
         elseif startswith(txt, "F") # Finished
             printstyled("✓ ", color=:green); print(txt[2:end])
         else
@@ -296,6 +296,11 @@ function generate_precompile_statements()
     end
 
     println("Collecting and executing precompile statements")
+
+    ansi_enablecursor = "\e[?25h"
+    ansi_disablecursor = "\e[?25l"
+
+    fancyprint && print(ansi_disablecursor)
     print_state()
     clock = @async begin
         t = Timer(0; interval=1/10)
@@ -488,6 +493,7 @@ function generate_precompile_statements()
     wait(clock) # Stop asynchronous printing
     failed = length(statements) - n_succeeded
     print_state("step3" => "F$n_succeeded ($failed failed)")
+    fancyprint && print(ansi_enablecursor)
     println()
     if have_repl
         # Seems like a reasonable number right now, adjust as needed

From f1ec0cd4199552397c8f678133c95fc9a17f2a96 Mon Sep 17 00:00:00 2001
From: Petr Vana <petvana@centrum.cz>
Date: Thu, 29 Dec 2022 16:08:33 +0100
Subject: [PATCH 17/19] Use try-catch not to loose cursor

---
 contrib/generate_precompile.jl | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl
index 6ec4c282e6168..c728e62c61bdf 100644
--- a/contrib/generate_precompile.jl
+++ b/contrib/generate_precompile.jl
@@ -281,7 +281,10 @@ let
     end
 end
 
-function generate_precompile_statements()
+ansi_enablecursor = "\e[?25h"
+ansi_disablecursor = "\e[?25l"
+
+generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printed
     start_time = time_ns()
     debug_output = devnull # or stdout
     sysimg = Base.unsafe_string(Base.JLOptions().image_file)
@@ -296,10 +299,6 @@ function generate_precompile_statements()
     end
 
     println("Collecting and executing precompile statements")
-
-    ansi_enablecursor = "\e[?25h"
-    ansi_disablecursor = "\e[?25l"
-
     fancyprint && print(ansi_disablecursor)
     print_state()
     clock = @async begin
@@ -493,7 +492,6 @@ function generate_precompile_statements()
     wait(clock) # Stop asynchronous printing
     failed = length(statements) - n_succeeded
     print_state("step3" => "F$n_succeeded ($failed failed)")
-    fancyprint && print(ansi_enablecursor)
     println()
     if have_repl
         # Seems like a reasonable number right now, adjust as needed
@@ -507,7 +505,10 @@ function generate_precompile_statements()
     tot_time = time_ns() - start_time
     println("Precompilation complete. Summary:")
     print("Total ─────── "); Base.time_print(tot_time);     println()
-
+catch e
+    rethrow()
+finally
+    fancyprint && print(ansi_enablecursor)
     return
 end
 

From 8c5e490fa35f96c883cf293c5bd9e64f3119734f Mon Sep 17 00:00:00 2001
From: Petr Vana <petvana@centrum.cz>
Date: Thu, 29 Dec 2022 16:30:27 +0100
Subject: [PATCH 18/19] Remove redundant chatch

---
 contrib/generate_precompile.jl | 2 --
 1 file changed, 2 deletions(-)

diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl
index c728e62c61bdf..de79fac756420 100644
--- a/contrib/generate_precompile.jl
+++ b/contrib/generate_precompile.jl
@@ -505,8 +505,6 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe
     tot_time = time_ns() - start_time
     println("Precompilation complete. Summary:")
     print("Total ─────── "); Base.time_print(tot_time);     println()
-catch e
-    rethrow()
 finally
     fancyprint && print(ansi_enablecursor)
     return

From a1ee9af134685c54e7b4fab1d18c475b6ec284d7 Mon Sep 17 00:00:00 2001
From: Ian Butterworth <i.r.butterworth@gmail.com>
Date: Thu, 29 Dec 2022 12:27:58 -0500
Subject: [PATCH 19/19] don't report skipped comments as failures (and only
 show failed when > 0)

---
 contrib/generate_precompile.jl | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl
index de79fac756420..89bb1d3bb1c06 100644
--- a/contrib/generate_precompile.jl
+++ b/contrib/generate_precompile.jl
@@ -465,7 +465,12 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe
         occursin(", Core.Compiler.AbstractInterpreter, ", statement) && continue
         try
             ps = Meta.parse(statement)
-            isexpr(ps, :call) || continue
+            if !isexpr(ps, :call)
+                # these are typically comments
+                @debug "skipping statement because it does not parse as an expression" statement
+                delete!(statements, statement)
+                continue
+            end
             popfirst!(ps.args) # precompile(...)
             ps.head = :tuple
             l = ps.args[end]
@@ -483,7 +488,7 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe
             n_succeeded += 1
             failed = length(statements) - n_succeeded
             yield() # Make clock spinning
-            print_state("step3" => "R$n_succeeded ($failed failed)")
+            print_state("step3" => string("R$n_succeeded", failed > 0 ? " ($failed failed)" : ""))
         catch ex
             # See #28808
             @warn "Failed to precompile expression" form=statement exception=ex _module=nothing _file=nothing _line=0
@@ -491,7 +496,7 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe
     end
     wait(clock) # Stop asynchronous printing
     failed = length(statements) - n_succeeded
-    print_state("step3" => "F$n_succeeded ($failed failed)")
+    print_state("step3" => string("F$n_succeeded", failed > 0 ? " ($failed failed)" : ""))
     println()
     if have_repl
         # Seems like a reasonable number right now, adjust as needed