Skip to content

Commit

Permalink
fixes GC
Browse files Browse the repository at this point in the history
Signed-off-by: Michael Hausenblas <[email protected]>
  • Loading branch information
mhausenblas committed Feb 22, 2020
1 parent 147e50b commit 6fba208
Showing 1 changed file with 8 additions and 24 deletions.
32 changes: 8 additions & 24 deletions gc.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,38 +7,22 @@ import (

var (
// how often we check for orphans:
gcPause = 10 * time.Second
// how long a pod of a terminating dproc can run
// before we consider it an orphan (in seconds):
maxOrphanRuntimeSec = 600.0
gcPause = 30 * time.Second
)

func gcDProcs() {
for {
poNstart, err := kubectl(false, "get", "po",
"--selector=dproctype="+string(DProcTerminating), "-o=custom-columns=:metadata.name,:status.startTime", "--field-selector=status.phase=Running",
"--no-headers")
orphandeploys, err := kubectl(false, "get", "deploy",
"--selector=dproctype="+string(DProcTerminating), "-o=custom-columns=:metadata.name", "--no-headers")
if err != nil {
debug(err.Error())
}
debug(poNstart)
if poNstart != "" {
for _, pns := range strings.Split(poNstart, "\n") {
poname, start := strings.Split(pns, " ")[0], strings.Split(pns, " ")[1]
debug("GC: looking at candidate pod " + poname + " with start timestamp " + start + "\n")
layout := "2006-01-02T15:04:05Z"
st, err := time.Parse(layout, start)
debug(orphandeploys)
if orphandeploys != "" {
for _, d := range strings.Split(orphandeploys, "\n") {
_, err := kubectl(false, "delete", "deploy", d)
if err != nil {
debug("GC: couldn't parse start time of pod " + poname)
}
now := time.Now()
diff := now.Sub(st)
if diff.Seconds() > maxOrphanRuntimeSec {
debug("GC: found orphaned pod " + poname + " with a start time of " + st.String())
_, err = kubectl(false, "delete", "pod", poname)
if err != nil {
warn("GC: couldn't remove orphaned pod " + poname)
}
warn("GC: couldn't reap orphaned deployment " + d)
}
}
}
Expand Down

0 comments on commit 6fba208

Please sign in to comment.