Skip to content

Commit

Permalink
logging changes to stop clobber of simultaneous runs
Browse files Browse the repository at this point in the history
  • Loading branch information
@a2ray
a2ray committed Nov 13, 2024
1 parent a5b1eb5 commit 20d80cc
Show file tree
Hide file tree
Showing 6 changed files with 75 additions and 58 deletions.
19 changes: 10 additions & 9 deletions src/AEMnoNuisanceGradientInversionTools.jl
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
module AEMnoNuisanceGradientInversionTools
using Distributed, Dates, Printf, PyPlot, DelimitedFiles, StatsBase
using Distributed, Dates, Printf, PyPlot, DelimitedFiles, StatsBase, Random
using ..AbstractOperator, ..CommonToAll, ..GP, ..SoundingDistributor
import ..AbstractOperator.makeoperator
import ..AbstractOperator.Sounding
Expand Down Expand Up @@ -238,24 +238,25 @@ function loopacrossAEMsoundings(soundings::Array{S, 1}, aem_in, σstart, σ0;
zipsaveprefix = "",
minimprovfrac = nothing,
verbose = false,
minimprovkickinstep = round(Int, nstepsmax/2),
minimprovkickinstep = round(Int, nstepsmax/2),
batchstr = randstring(15),
) where S<:Sounding

@assert nsequentialiters != -1
nparallelsoundings = nworkers()
nsoundings = length(soundings)
zall = zboundarytocenter(aem_in.z[aem_in.nfixed+1:end]) # needed for sounding compression in write

writetogloballog("will require $nsequentialiters iterations of $nparallelsoundings soundings", iomode="w")
writetogloballog("starting sequential parallel iterations at $(Dates.now())")
fname = batchstr*"_"*zipsaveprefix*"_global.log"
writetogloballog("will require $nsequentialiters iterations of $nparallelsoundings soundings"; iomode="w", fname)
writetogloballog("starting sequential parallel iterations at $(Dates.now())"; fname)
for iter = 1:nsequentialiters
ss = getss_deterministic(iter, nsequentialiters, nparallelsoundings, nsoundings)
writetogloballog("soundings in loop $iter of $nsequentialiters $ss")
writetogloballog("soundings in loop $iter of $nsequentialiters $ss"; fname)
pids = workers()
t2 = time()
@sync for (i, s) in enumerate(ss)
aem = makeoperator(aem_in, soundings[s])
fname = soundings[s].sounding_string*"_gradientinv.dat"
fname_ = soundings[s].sounding_string*"_gradientinv.dat"
σstart_, σ0_ = map(x->x*ones(length(aem.ρ)-1), [σstart, σ0])
@async remotecall_fetch(gradientinv, pids[i], σstart_, σ0_, aem;
regtype = regtype ,
Expand All @@ -279,7 +280,7 @@ function loopacrossAEMsoundings(soundings::Array{S, 1}, aem_in, σstart, σ0;
κ = κ ,
breakonknown = breakonknown ,
dobo = dobo ,
fname = fname ,
fname = fname_ ,
minimprovfrac, verbose,
minimprovkickinstep)

Expand All @@ -289,7 +290,7 @@ function loopacrossAEMsoundings(soundings::Array{S, 1}, aem_in, σstart, σ0;
compresssoundings && compress(soundings[ss[1]:ss[end]], zall,
isfirstparalleliteration = isfirstparalleliteration, prefix=zipsaveprefix)
dt = time() - t2 # seconds
writetogloballog("done $iter out of $nsequentialiters at $(Dates.now()) in $dt sec")
writetogloballog("done $iter out of $nsequentialiters at $(Dates.now()) in $dt sec"; fname)
end
end

Expand Down
15 changes: 8 additions & 7 deletions src/AEMnoNuisanceMcMCInversionTools.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -322,6 +322,7 @@ end

# Driver code for McMC inversion with no nuisances
function loopacrossAEMsoundings(soundings::Array{S, 1}, aem_in::Operator1D, opt_in::Options;
batchstr = randstring(15),
Tmax = -1,
nsamples = -1,
nchainsatone = 1,
Expand All @@ -335,12 +336,12 @@ function loopacrossAEMsoundings(soundings::Array{S, 1}, aem_in::Operator1D, opt_
@assert Tmax != -1

nsoundings = length(soundings)
nsequentialiters, nparallelsoundings = splittasks(soundings; nchainspersounding, ppn)

writetogloballog("starting sequential parallel iterations at $(Dates.now())")
fname = batchstr*"_global.log"
nsequentialiters, nparallelsoundings = splittasks(soundings; nchainspersounding, ppn, fname)
writetogloballog("starting sequential parallel iterations at $(Dates.now())"; fname)
for iter = 1:nsequentialiters
ss = getss(iter, nsequentialiters, nparallelsoundings, nsoundings)
writetogloballog("soundings in loop $iter of $nsequentialiters $ss")
writetogloballog("soundings in loop $iter of $nsequentialiters $ss"; fname)
t2 = time()
@sync for (i, s) in Iterators.reverse(enumerate(ss))
pids = getpids(i, nchainspersounding)
Expand All @@ -350,12 +351,12 @@ function loopacrossAEMsoundings(soundings::Array{S, 1}, aem_in::Operator1D, opt_
opt.fdataname = soundings[s].sounding_string*"_"

@async remotecall_fetch(main, pids[1], opt, aem, collect(pids[2:end]);
Tmax, nsamples, nchainsatone, nominaltime)
Tmax, nsamples, nchainsatone, nominaltime, batchstr)

end # @sync
dt = time() - t2 #seconds
catlocallogs(nparallelsoundings, nchainspersounding)
writetogloballog("done $iter out of $nsequentialiters at $(Dates.now()) in $dt sec")
catlocallogs(nparallelsoundings, nchainspersounding, batchstr)
writetogloballog("done $iter out of $nsequentialiters at $(Dates.now()) in $dt sec"; fname)
GC.gc()
end
end
Expand Down
19 changes: 10 additions & 9 deletions src/AEMwithNuisanceGradientInversionTools.jl
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
module AEMwithNuisanceGradientInversionTools
using Distributed, Dates, Printf, PyPlot, DelimitedFiles, StatsBase
using Distributed, Dates, Printf, PyPlot, DelimitedFiles, StatsBase, Random
using ..AbstractOperator, ..CommonToAll, ..GP, ..SoundingDistributor
import ..AbstractOperator.makeoperator
import ..AbstractOperator.setnuboundsandstartforgradinv
Expand Down Expand Up @@ -243,25 +243,26 @@ function loopacrossAEMsoundings(soundings::Array{S, 1}, aem_in, σstart, σ0, nu
usebox = true,
reducenuto = 0.2,
debuglevel = 0,
breaknuonknown = false,
breaknuonknown = false,
batchstr = randstring(15),
) where S<:Sounding

@assert nsequentialiters != -1
nparallelsoundings = nworkers()
nsoundings = length(soundings)
zall = zboundarytocenter(aem_in.z[aem_in.nfixed+1:end]) # needed for sounding compression in write

writetogloballog("will require $nsequentialiters iterations of $nparallelsoundings soundings", iomode="w")
writetogloballog("starting sequential parallel iterations at $(Dates.now())")
fname = batchstr*"_"*zipsaveprefix*"_global.log"
writetogloballog("will require $nsequentialiters iterations of $nparallelsoundings soundings"; iomode="w", fname)
writetogloballog("starting sequential parallel iterations at $(Dates.now())"; fname)
for iter = 1:nsequentialiters
ss = getss_deterministic(iter, nsequentialiters, nparallelsoundings, nsoundings)
writetogloballog("soundings in loop $iter of $nsequentialiters $ss")
writetogloballog("soundings in loop $iter of $nsequentialiters $ss"; fname)
pids = workers()
t2 = time()
@sync for (i, s) in enumerate(ss)
aem = makeoperator(aem_in, soundings[s])
nubounds, nustart = setnuboundsandstartforgradinv(aem, nuboundsΔ)
fname = soundings[s].sounding_string*"_gradientinv.dat"
fname_ = soundings[s].sounding_string*"_gradientinv.dat"
σstart_, σ0_ = map(x->x*ones(length(aem.ρ)-1), [σstart, σ0])
@async remotecall_fetch(gradientinv, pids[i], σstart_, σ0_, nustart, aem;
regtype = regtype ,
Expand All @@ -275,7 +276,7 @@ function loopacrossAEMsoundings(soundings::Array{S, 1}, aem_in, σstart, σ0, nu
β² = β² ,
regularizeupdate = regularizeupdate,
knownvalue = knownvalue ,
fname = fname ,
fname = fname_ ,
ntriesnu, nubounds, boxiters, usebox, reducenuto, debuglevel, breaknuonknown,
minimprovfrac, verbose, minimprovkickinstep)

Expand All @@ -285,7 +286,7 @@ function loopacrossAEMsoundings(soundings::Array{S, 1}, aem_in, σstart, σ0, nu
compresssoundings && compress(soundings[ss[1]:ss[end]], zall, size(nuboundsΔ, 1),
isfirstparalleliteration = isfirstparalleliteration, prefix=zipsaveprefix)
dt = time() - t2 # seconds
writetogloballog("done $iter out of $nsequentialiters at $(Dates.now()) in $dt sec")
writetogloballog("done $iter out of $nsequentialiters at $(Dates.now()) in $dt sec"; fname)
end
end

Expand Down
15 changes: 8 additions & 7 deletions src/AEMwithNuisanceMcMCInversionTools.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,7 @@ end

# Driver code for McMC inversion with nuisances
function loopacrossAEMsoundings(soundings::Array{S, 1}, aem_in::Operator1D, opt_in::Options, optn_in::OptionsNuisance;
batchstr = randstring(15),
Tmax = -1,
nsamples = -1,
nchainsatone = 1,
Expand All @@ -171,12 +172,12 @@ function loopacrossAEMsoundings(soundings::Array{S, 1}, aem_in::Operator1D, opt_
@assert Tmax != -1

nsoundings = length(soundings)
nsequentialiters, nparallelsoundings = splittasks(soundings; nchainspersounding, ppn)

writetogloballog("starting sequential parallel iterations at $(Dates.now())")
fname = batchstr*"_global.log"
nsequentialiters, nparallelsoundings = splittasks(soundings; nchainspersounding, ppn, fname)
writetogloballog("starting sequential parallel iterations at $(Dates.now())"; fname)
for iter = 1:nsequentialiters
ss = getss(iter, nsequentialiters, nparallelsoundings, nsoundings)
writetogloballog("soundings in loop $iter of $nsequentialiters $ss")
writetogloballog("soundings in loop $iter of $nsequentialiters $ss"; fname)
t2 = time()
@sync for (i, s) in Iterators.reverse(enumerate(ss))
pids = getpids(i, nchainspersounding)
Expand All @@ -187,12 +188,12 @@ function loopacrossAEMsoundings(soundings::Array{S, 1}, aem_in::Operator1D, opt_
optn = getoptnfromexisting(optn_in, opt, soundings[s])

@async remotecall_fetch(main, pids[1], opt, optn, aem, collect(pids[2:end]);
Tmax, nsamples, nchainsatone, nominaltime)
Tmax, nsamples, nchainsatone, nominaltime, batchstr)

end # @sync
dt = time() - t2 #seconds
catlocallogs(nparallelsoundings, nchainspersounding)
writetogloballog("done $iter out of $nsequentialiters at $(Dates.now()) in $dt sec")
catlocallogs(nparallelsoundings, nchainspersounding, batchstr)
writetogloballog("done $iter out of $nsequentialiters at $(Dates.now()) in $dt sec"; fname)
GC.gc()
end
end
Expand Down
Loading

0 comments on commit 20d80cc

Please sign in to comment.