Move pre-existing output directory

If a output_dir already exists, move it to `output_dir_YYYYMMDD_HHMM`

examples/hybrid/driver.jl

@@ -1,16 +1,32 @@
 # When Julia 1.10+ is used interactively, stacktraces contain reduced type information to make them shorter.
-# On the other hand, the full type information is printed when julia is not run interactively. 
+# On the other hand, the full type information is printed when julia is not run interactively.
 # Given that ClimaCore objects are heavily parametrized, non-abbreviated stacktraces are hard to read,
 # so we force abbreviated stacktraces even in non-interactive runs.
 # (See also Base.type_limited_string_from_context())
 redirect_stderr(IOContext(stderr, :stacktrace_types_limited => Ref(false)))
 import ClimaAtmos as CA
+import ClimaComms
 import Random
+import Base.Filesystem: rm, mv
+import Dates
 Random.seed!(1234)
 
 if !(@isdefined config)
     config = CA.AtmosConfig()
 end
+output_dir = CA.get_output_dir(config)
+if ispath(output_dir)
+    if ClimaComms.iamroot(config.comms_ctx)
+        formatted_date = Dates.format(Dates.now(), "yyyymmdd_HHMM")
+        move_to = joinpath(
+            dirname(output_dir),
+            "$(basename(output_dir))_$formatted_date",
+        )
+        @warn "output_dir $output_dir found. Moving it to $move_to"
+        mv(output_dir, move_to)
+    end
+    ClimaComms.barrier(config.comms_ctx)
+end
 simulation = CA.get_simulation(config)
 (; integrator) = simulation
 sol_res = CA.solve_atmos!(simulation)
@@ -24,7 +40,6 @@ import ClimaAtmos.InitialConditions as ICs
 using Statistics: mean
 import ClimaAtmos.Parameters as CAP
 import Thermodynamics as TD
-import ClimaComms
 using SciMLBase
 using PrettyTables
 import DiffEqCallbacks as DECB
@@ -34,7 +49,6 @@ using ClimaTimeSteppers
 import JSON
 using Test
 import Tar
-import Base.Filesystem: rm
 import OrderedCollections
 using ClimaCoreTempestRemap
 using ClimaCorePlots, Plots

src/solver/type_getters.jl

@@ -517,6 +517,18 @@ function get_callbacks(parsed_args, sim_info, atmos, params, comms_ctx)
     return callbacks
 end
 
+function get_output_dir(config::AtmosConfig)
+    (; parsed_args) = config
+    job_id = if isnothing(parsed_args["job_id"])
+        job_id_from_config(parsed_args)
+    else
+        parsed_args["job_id"]
+    end
+    default_output = haskey(ENV, "CI") ? job_id : joinpath("output", job_id)
+    out_dir = parsed_args["output_dir"]
+    return isnothing(out_dir) ? default_output : out_dir
+end
+
 function get_sim_info(config::AtmosConfig)
     (; parsed_args) = config
     FT = eltype(config)
@@ -526,9 +538,8 @@ function get_sim_info(config::AtmosConfig)
     else
         parsed_args["job_id"]
     end
-    default_output = haskey(ENV, "CI") ? job_id : joinpath("output", job_id)
-    out_dir = parsed_args["output_dir"]
-    output_dir = isnothing(out_dir) ? default_output : out_dir
+
+    output_dir = get_output_dir(config)
     mkpath(output_dir)
 
     sim = (;