From 9602214f73fca6a52e60de8f3a86ba5eaec3aa12 Mon Sep 17 00:00:00 2001 From: kkeundotnet Date: Tue, 20 Dec 2022 13:24:43 +0000 Subject: [PATCH] Run garbage collection once when the main process is waiting for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In https://github.com/ocaml/opam/pull/5376, @dra27 suggested running `Gc.compact` when the main process is waiting for the children processes for the first time. > what I was thinking here was that possibly at the point where opamProcess first is going to wait > for the completion of running jobs, we could add a Gc.compact? In my local running on `opam install ppxlib`, "GC compact" ran in the middle of parallel processing of actions. ``` The following actions will be performed: === install 1 package ∗ ppxlib 0.28.0 00:07.216 XSYS Adding to env { LC_ALL=C } 00:09.099 STATE depexts loaded in 1.883s 00:09.100 SOLUTION parallel_apply 00:09.100 SOLUTION Regroup shared source packages: {} <><> Processing actions <><><><><><><><><><><><><><><><><><><><><><><><><><> 🐫 00:09.106 PARALLEL Iterate over 3 task(s) with 11 process(es) 00:09.106 PARALLEL Starting job 444950918 (worker -/11 -/1 1/3): ⬇ ppxlib.0.28.0 00:09.106 SOLUTION Fetching sources for ppxlib.0.28.0 00:09.106 ACTION download_package: ppxlib.0.28.0 00:09.106 SYSTEM rmdir /Users/scho/.opam/4.14.0/.opam-switch/sources/ppxlib.0.28.0 00:09.109 SYSTEM mkdir /var/folders/gx/6809fgsd3nndyyf69y3h12_h0000gn/T/opam-14220-67d907 00:09.110 PARALLEL Next task in job 444950918: /usr/bin/tar xfj /Users/scho/.opam/download-cache/sha256/d8/d87ae5f9a081206308ca964809b50a66aeb8e83d254801e8b9675448b60cf377 -C /var/folders/gx/6809fgsd3nndyyf69y3h12_h0000gn/T/opam-14220-67d907 00:09.619 PARALLEL GC compact (heap 490 MB -> 328 MB) 00:09.619 PARALLEL Collected task for job 444950918 (ret:0) 00:10.158 SYSTEM rmdir /var/folders/gx/6809fgsd3nndyyf69y3h12_h0000gn/T/opam-14220-67d907 ⬇ retrieved ppxlib.0.28.0 (cached) 00:10.321 PARALLEL Job 444950918 finished ``` Similar to #5376, this PR enabled my 1GB-RAM machine to install `ppxlib` or `js_of_ocaml` without OOM. --- master_changes.md | 2 ++ src/core/opamParallel.ml | 20 ++++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/master_changes.md b/master_changes.md index 33c82c40394..1f9d084ea93 100644 --- a/master_changes.md +++ b/master_changes.md @@ -97,6 +97,7 @@ users) ## Shell ## Internal + * Run `Gc.compact` in OpamParallel, when the main process is waiting for the children processes for the first time [#5396 @kkeundotnet] ## Internal: Windows @@ -127,3 +128,4 @@ users) ## opam-format ## opam-core + * `OpamParallel.*.{map,reduce,iter}`: Run `Gc.compact` when the main process is waiting for the children processes for the first time [#5396 @kkeundotnet] diff --git a/src/core/opamParallel.ml b/src/core/opamParallel.ml index 49abe705d71..658286b45b8 100644 --- a/src/core/opamParallel.ml +++ b/src/core/opamParallel.ml @@ -58,6 +58,16 @@ module type SIG = sig exception Cyclic of G.V.t list list end +let gc_compact () = + let get_heap () = + let {Gc.heap_words; _} = Gc.quick_stat () in + heap_words * Sys.word_size / 8 / 1024 / 1024 + in + let before = get_heap () in + Gc.compact (); + let after = get_heap () in + log "GC compact (heap %d MB -> %d MB)" before after + module Make (G : G) = struct module G = G @@ -92,6 +102,8 @@ module Make (G : G) = struct default :: defined in + let gc_compacted = ref false in + if G.has_cycle g then ( let sccs = G.scc_list g in let sccs = List.filter (function _::_::_ -> true | _ -> false) sccs in @@ -264,8 +276,12 @@ module Make (G : G) = struct (get_slots nslots n) in run_seq_command nslots ready n cmd - else + else ( (* Wait for a process to end *) + if not !gc_compacted then ( + gc_compact (); + gc_compacted := true + ); let processes = M.fold (fun n (p,x,_) acc -> (p,(n,x)) :: acc) running [] in @@ -285,7 +301,7 @@ module Make (G : G) = struct OpamProcess.cleanup result; fail n e in OpamProcess.cleanup result; - run_seq_command nslots ready n next + run_seq_command nslots ready n next ) in let roots = G.fold_vertex