write lockfiles for in flight services

we need a mechanism for the bosh drain lifecycle to know if there are
terraform processes in flight. the lockfiles do just that. they filesystem
approach was favoured over a in memory map to allow an easier interface with
bosh. assuming the csb process crashes while shutting down, recovering the
SIs that were in flight is tricky because the in memory state got lost when
the crash happened.

additionally having the files use the GUIDs for their names allows us to log
the ecact SIs that were not succesfully finished via the drain script.

internal/storage/storage.go

@@ -1,17 +1,31 @@
 // Package storage implements a Database Access Object (DAO)
 package storage
 
-import "gorm.io/gorm"
+import (
+	"os"
+
+	"gorm.io/gorm"
+)
 
 type Storage struct {
-	db        *gorm.DB
-	encryptor Encryptor
+	db          *gorm.DB
+	encryptor   Encryptor
+	lockFileDir string
 }
 
 func New(db *gorm.DB, encryptor Encryptor) *Storage {
+	// the VM based HA deployment requires a drain mechanism. LockFiles are a simple solution.
+	// but not every environment will opt for using VM based deployments. So detect if the lockfile
+	// director is present.
+
+	dirDefault := os.Getenv("CSB_LOCKFILE_DIR")
+	if _, err := os.Stat(dirDefault); err != nil {
+		dirDefault, _ = os.MkdirTemp("/tmp/", "lockfiles")
+	}
 	return &Storage{
-		db:        db,
-		encryptor: encryptor,
+		db:          db,
+		encryptor:   encryptor,
+		lockFileDir: dirDefault,
 	}
 }
 

internal/storage/terraform_deployment.go

@@ -2,6 +2,8 @@ package storage
 
 import (
 	"fmt"
+	"os"
+	"strings"
 	"time"
 
 	"github.com/hashicorp/go-version"
@@ -156,3 +158,29 @@ func (s *Storage) loadTerraformDeploymentIfExists(id string, receiver any) error
 
 	return s.db.Where("id = ?", id).First(receiver).Error
 }
+
+func (s *Storage) LockFilesExist() bool {
+	entries, _ := os.ReadDir(s.lockFileDir)
+	return len(entries) != 0
+}
+
+func (s *Storage) WriteLockFile(deploymentID string) error {
+	return os.WriteFile(fmt.Sprintf("%s/%s", s.lockFileDir, sanitizeFileName(deploymentID)), []byte{}, 0o644)
+}
+
+func (s *Storage) RemoveLockFile(deploymentID string) error {
+	return os.Remove(fmt.Sprintf("%s/%s", s.lockFileDir, sanitizeFileName(deploymentID)))
+}
+
+func (s *Storage) GetLockedDeploymentIds() ([]string, error) {
+	entries, err := os.ReadDir(s.lockFileDir)
+	var names []string
+	for _, entry := range entries {
+		names = append(names, strings.ReplaceAll(entry.Name(), "_", ":"))
+	}
+	return names, err
+}
+
+func sanitizeFileName(name string) string {
+	return strings.ReplaceAll(name, ":", "_")
+}

internal/storage/terraform_deployment_test.go

@@ -206,10 +206,54 @@ var _ = Describe("TerraformDeployments", func() {
 			Expect(store.DeleteTerraformDeployment("not-there")).NotTo(HaveOccurred())
 		})
 	})
+
+	Describe("LockFileExists", func() {
+		It("reports correct status", func() {
+			Expect(store.WriteLockFile("1234")).To(Succeed())
+			Expect(store.WriteLockFile("5678")).To(Succeed())
+
+			Expect(store.LockFilesExist()).To(BeTrue())
+
+			Expect(store.RemoveLockFile("1234")).To(Succeed())
+
+			Expect(store.LockFilesExist()).To(BeTrue())
+
+			Expect(store.RemoveLockFile("5678")).To(Succeed())
+
+			Expect(store.LockFilesExist()).To(BeFalse())
+		})
+	})
+
+	Describe("GetLockedDeploymentIds", func() {
+		It("returns correct names", func() {
+			names, err := store.GetLockedDeploymentIds()
+			Expect(err).NotTo(HaveOccurred())
+			Expect(names).To(BeEmpty())
+
+			Expect(store.WriteLockFile("tf:1234:")).To(Succeed())
+			Expect(store.WriteLockFile("tf:5678:9123")).To(Succeed())
+
+			names, err = store.GetLockedDeploymentIds()
+			Expect(err).NotTo(HaveOccurred())
+			Expect(names).To(ContainElements("tf:1234:", "tf:5678:9123"))
+
+			Expect(store.RemoveLockFile("tf:1234:")).To(Succeed())
+
+			names, err = store.GetLockedDeploymentIds()
+			Expect(err).NotTo(HaveOccurred())
+			Expect(names).To(ContainElements("tf:5678:9123"))
+			Expect(names).ToNot(ContainElements("tf:1234:"))
+
+			Expect(store.RemoveLockFile("tf:5678:9123")).To(Succeed())
+
+			names, err = store.GetLockedDeploymentIds()
+			Expect(err).NotTo(HaveOccurred())
+			Expect(names).To(BeEmpty())
+		})
+	})
 })
 
 func addFakeTerraformDeployments() {
-
 	Expect(db.Create(&models.TerraformDeployment{
 		ID:                   "fake-id-1",
 		Workspace:            fakeWorkspace("fake-1", "1.2.3"),