diff --git a/renderer/renderer.go b/renderer/renderer.go
index e5a83065c..59931c19e 100644
--- a/renderer/renderer.go
+++ b/renderer/renderer.go
@@ -165,8 +165,12 @@ func AtomicWrite(path string, createDestDirs bool, contents []byte, perms os.Fil
 	// If we got this far, it means we are about to save the file. Copy the
 	// current file so we have a backup. Note that os.Link preserves the Mode.
 	if backup {
-		if err := os.Link(path, path+".bak"); err != nil {
+		bak, old := path+".bak", path+".old.bak"
+		os.Rename(bak, old) // ignore error
+		if err := os.Link(path, bak); err != nil {
 			log.Printf("[WARN] (runner) could not backup %q: %v", path, err)
+		} else {
+			os.Remove(old) // ignore error
 		}
 	}
 
diff --git a/renderer/renderer_test.go b/renderer/renderer_test.go
index eed31d987..85f142ede 100644
--- a/renderer/renderer_test.go
+++ b/renderer/renderer_test.go
@@ -160,4 +160,41 @@ func TestAtomicWrite(t *testing.T) {
 			}
 		}
 	})
+
+	t.Run("backup_backup", func(t *testing.T) {
+		outDir, err := ioutil.TempDir("", "")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer os.RemoveAll(outDir)
+		outFile, err := ioutil.TempFile(outDir, "")
+		if err != nil {
+			t.Fatal(err)
+		}
+		if _, err := outFile.Write([]byte("first")); err != nil {
+			t.Fatal(err)
+		}
+
+		contains := func(filename, content string) {
+			f, err := ioutil.ReadFile(filename + ".bak")
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !bytes.Equal(f, []byte(content)) {
+				t.Fatalf("expected %q to be %q", f, []byte(content))
+			}
+		}
+
+		err = AtomicWrite(outFile.Name(), true, []byte("second"), 0644, true)
+		if err != nil {
+			t.Fatal(err)
+		}
+		contains(outFile.Name(), "first")
+
+		err = AtomicWrite(outFile.Name(), true, []byte("third"), 0644, true)
+		if err != nil {
+			t.Fatal(err)
+		}
+		contains(outFile.Name(), "second")
+	})
 }