Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Em log and data softening #124

Merged
merged 6 commits into from
Oct 10, 2022
Merged

Em log and data softening #124

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a1739e2
minor fixes
MhDang Apr 11, 2022
ab194f0
prune lower bound flag
MhDang May 2, 2022
afd7572
bound
MhDang May 3, 2022
a260d20
Merge branch 'master' into em-log
MhDang Oct 10, 2022
06a3294
data softening for binary dataset
MhDang Oct 10, 2022
0c9fe8a
un-commit flow flag
MhDang Oct 10, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion src/nodes/indicator_dist.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,11 @@ bits(d::Indicator, _ = nothing) = d
unbits(d::Indicator, _ = nothing) = d

loglikelihood(d::Indicator, value, _ = nothing) =
(d.value == value) ? zero(Float32) : -Inf32
if value isa AbstractFloat && d isa Literal
(d.value) ? log(value) : log1p(-value)
else
(d.value == value) ? zero(Float32) : -Inf32
end

init_params(d::Indicator, _) = d

Expand Down
25 changes: 18 additions & 7 deletions src/parameters/em.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,10 @@ function full_batch_em(bpc::CuBitsProbCircuit, raw_data::CuArray, num_epochs;
marginals, flows, node_aggr, edge_aggr,
mine, maxe, debug)
push!(log_likelihoods, log_likelihood)
call(callbacks, epoch, log_likelihood)
done = call(callbacks, epoch, log_likelihood)
if !isnothing(done) && done[end] == true
break
end
end

cleanup_memory((data, raw_data), (flows, flows_mem),
Expand Down Expand Up @@ -467,10 +470,13 @@ function mini_batch_em(bpc::CuBitsProbCircuit, raw_data::CuArray, num_epochs;
end
log_likelihood = sum(log_likelihoods_epoch) / batch_size / num_batches
push!(log_likelihoods, log_likelihood)
call(callbacks, epoch, log_likelihood)
done = call(callbacks, epoch, log_likelihood)

param_inertia += Δparam_inertia
flow_memory += Δflow_memory
if !isnothing(done) && done[end] == true
break
end
end

cleanup_memory((data, raw_data), (flows, flows_mem),
Expand All @@ -490,10 +496,11 @@ end

function call(callbacks::CALLBACKList, epoch, log_likelihood)
if callbacks.list[1].verbose
for x in callbacks.list
done = map(callbacks.list) do x
call(x, epoch, log_likelihood)
end
println()
done
end
end

Expand Down Expand Up @@ -543,16 +550,20 @@ call(caller::FullBatchLog, epoch, log_likelihood) = begin
caller.verbose && print("Full-batch EM epoch $epoch; train LL $log_likelihood")
end
call(caller::LikelihoodsLog, epoch, log_likelihood) = begin
valid_ll, test_ll = nothing, nothing
if epoch % caller.iter == 0 && (!isnothing(caller.valid_x) || !isnothing(caller.test_x))
if !isnothing(caller.valid_x)
print("; valid LL ", loglikelihood(caller.bpc, caller.valid_x;
batch_size=caller.batch_size,mars_mem=caller.mars_mem))
valid_ll = loglikelihood(caller.bpc, caller.valid_x;
batch_size=caller.batch_size,mars_mem=caller.mars_mem)
print("; valid LL ", valid_ll)
end
if !isnothing(caller.test_x)
print("; test LL ", loglikelihood(caller.bpc, caller.test_x;
batch_size=caller.batch_size,mars_mem=caller.mars_mem))
test_ll = loglikelihood(caller.bpc, caller.test_x;
batch_size=caller.batch_size,mars_mem=caller.mars_mem)
print("; test LL ", test_ll)
end
end
valid_ll, test_ll
end

cleanup(caller::CALLBACK) = nothing
Expand Down