feat: local and remote cache options (#399)

- Adds ability to perform cleanup of items added to the remote cache upon build completion in addition to the existing cancelled or halted. To enable, set `remote_cache_cleanup = true`. Defaults to `false`. - Adds the ability to specify the datastore for the placement of the remote cache. To enable, set `remote_cache_datastore = "myCacheDatastoreName"`. If not set, the datastore of the virtual machine is used. - Adds the ability to specify the path on the datastore for the placement of the remote cache. To enable, set `remote_cache_path = "foo/bar"` If not set, the default path is `packer_cache` at the root of the datastore. - Adds the ability to overwrite items in the remote cache. To enable, set `remote_cache_overwrite = true`. - Adds the ability to overwrite items in the local cache. To enable, set `local_cache_overwrite = true`. If `local_cache_overwrite = true`, `remote_cache_overwrite` is implicitly be set to `true`. This is to ensure consistency between the local and remote cache. Signed-off-by: Ryan Johnson <[email protected]>
hashicorp · Apr 17, 2024 · de0c305 · de0c305
1 parent 79fbec4
commit de0c305
Show file tree

Hide file tree

Showing 8 changed files with 231 additions and 65 deletions.
diff --git a/.web-docs/components/builder/vsphere-iso/README.md b/.web-docs/components/builder/vsphere-iso/README.md
@@ -33,23 +33,42 @@ necessary for this build to succeed and can be found further down the page.
 
 <!-- Code generated from the comments of the Config struct in builder/vsphere/iso/config.go; DO NOT EDIT MANUALLY -->
 
-- `create_snapshot` (bool) - Specifies to create a snapshot of the virtual machine to use as a base for linked clones.
+- `create_snapshot` (bool) - Create a snapshot of the virtual machine to use as a base for linked clones.
   Defaults to `false`.
 
-- `snapshot_name` (string) - Specifies the name of the snapshot when `create_snapshot` is `true`.
+- `snapshot_name` (string) - The name of the snapshot when `create_snapshot` is `true`.
   Defaults to `Created By Packer`.
 
-- `convert_to_template` (bool) - Specifies to convert the cloned virtual machine to a template after the build is complete.
+- `convert_to_template` (bool) - Convert the virtual machine to a template after the build is complete.
   Defaults to `false`.
-  If set to `true`, the virtual machine can not be imported to a content library.
+  If set to `true`, the virtual machine can not be imported into a content library.
 
-- `export` (\*common.ExportConfig) - Specifies the configuration for exporting the virtual machine to an OVF.
+- `export` (\*common.ExportConfig) - The configuration for exporting the virtual machine to an OVF.
   The virtual machine is not exported if [export configuration](#export-configuration) is not specified.
 
-- `content_library_destination` (\*common.ContentLibraryDestinationConfig) - Specifies the configuration for importing a VM template or OVF template to a content library.
+- `content_library_destination` (\*common.ContentLibraryDestinationConfig) - Import the virtual machine as a VM template or OVF template to a content library.
   The template will not be imported if no [content library import configuration](#content-library-import-configuration) is specified.
   If set, `convert_to_template` must be set to `false`.
 
+- `local_cache_overwrite` (bool) - Overwrite files in the local cache if they already exist.
+  Defaults to `false`.
+
+- `remote_cache_cleanup` (bool) - Cleanup items added to the remote cache after the build is complete.
+  Defaults to `false`.
+
+  -> **Note:** If the local cache overwrite flag is set to `true`, `RemoteCacheOverwrite` will
+  implicitly be set to `true`. This is to ensure consistency between the local and remote
+  cache.
+
+- `remote_cache_overwrite` (bool) - Overwrite files in the remote cache if they already exist.
+  Defaults to `false`.
+
+- `remote_cache_datastore` (string) - The remote cache datastore to use for the build.
+  If not set, the datastore of the virtual machine is used.
+
+- `remote_cache_path` (string) - The directory path on the remote cache datastore to use for the build.
+  If not set, the default path is `packer_cache/`.
+
 <!-- End of code generated from the comments of the Config struct in builder/vsphere/iso/config.go; -->
 
 

diff --git a/builder/vsphere/common/step_download.go b/builder/vsphere/common/step_download.go
@@ -6,65 +6,119 @@ package common
 import (
 	"context"
 	"fmt"
+	"log"
 	"net/url"
+	"os"
+	"path/filepath"
 
 	"github.com/hashicorp/packer-plugin-sdk/multistep"
 	packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
 	"github.com/hashicorp/packer-plugin-vsphere/builder/vsphere/driver"
 )
 
-// Defining this interface ensures that we use the common step download, or the
-// mock created to test this wrapper
+// Defining this interface ensures that we use the common StepDownload, or the mock created to
+// test this wrapper.
 type DownloadStep interface {
 	Run(context.Context, multistep.StateBag) multistep.StepAction
 	Cleanup(multistep.StateBag)
 	UseSourceToFindCacheTarget(source string) (*url.URL, string, error)
 }
 
-// vSphere has a specialized need -- before we waste time downloading an iso,
-// we need to check whether that iso already exists on the remote datastore.
-// if it does, we skip the download. This wrapping-step still uses the common
-// StepDownload but only if the image isn't already present on the datastore.
+// Before downloading and uploading an ISO file to the remote cache, check if the ISO file
+// already exists on the datastore.
+//
+// If it exists, we check if the overwrite flag is set to true. If it is, we delete the file
+// and download it again. If it is not set or false, the download is skipped.
+//
+// This wrapping-step still uses the common StepDownload, but only if the ISO file does not
+// already exist on the datastore.
 type StepDownload struct {
 	DownloadStep DownloadStep
-	// These keys are vSphere-specific and used to check the remote datastore.
-	Url       []string
-	ResultKey string
-	Datastore string
-	Host      string
+	// These keys are vSphere-specific and are used to check the remote datastore.
+	Url                  []string
+	ResultKey            string
+	Datastore            string
+	Host                 string
+	LocalCacheOverwrite  bool
+	RemoteCacheOverwrite bool
+	RemoteCacheDatastore string
+	RemoteCachePath      string
 }
 
 func (s *StepDownload) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
 	driver := state.Get("driver").(driver.Driver)
 	ui := state.Get("ui").(packersdk.Ui)
 
-	// Check whether iso is present on remote datastore.
-	ds, err := driver.FindDatastore(s.Datastore, s.Host)
+	// Set the remote cache datastore. If not set, use the default datastore for the build.
+	remoteCacheDatastore := s.Datastore
+	if s.RemoteCacheDatastore != "" {
+		remoteCacheDatastore = s.RemoteCacheDatastore
+	}
+
+	// Find the datastore to use for the remote cache.
+	ds, err := driver.FindDatastore(remoteCacheDatastore, s.Host)
 	if err != nil {
-		state.Put("error", fmt.Errorf("datastore doesn't exist: %v", err))
+		state.Put("error", fmt.Errorf("error finding the datastore: %v", err))
 		return multistep.ActionHalt
 	}
 
-	// loop over URLs to see if any are already present. If they are, store that
-	// one instate and continue
+	// Set the remote cache path. If not set, use the default cache path.
+	remoteCachePath := s.RemoteCachePath
+	if remoteCachePath == "" {
+		remoteCachePath = DefaultRemoteCachePath
+	}
+
+	// Loop over the URLs to see if any are already present.
+	// If they are, store in state and continue.
 	for _, source := range s.Url {
 		_, targetPath, err := s.DownloadStep.UseSourceToFindCacheTarget(source)
 		if err != nil {
-			state.Put("error", fmt.Errorf("Error getting target path: %s", err))
+			state.Put("error", fmt.Errorf("error returning target path: %s", err))
 			return multistep.ActionHalt
 		}
-		_, remotePath, _, _ := GetRemoteDirectoryAndPath(targetPath, ds)
+
+		filename := filepath.Base(targetPath)
+
+		// Check if the ISO file is already present in the local cache.
+		if _, err := os.Stat(targetPath); !os.IsNotExist(err) {
+			// If the local cache overwrite flag is set to true, delete the file and download the
+			// ISO file again.
+			if s.LocalCacheOverwrite {
+				ui.Say(fmt.Sprintf("Overwriting %s in local cache...", filename))
+				// Delete the file from the local cache.
+				if err := os.Remove(targetPath); err != nil {
+					state.Put("error", fmt.Errorf("error overwriting file in local cache: %w", err))
+					return multistep.ActionHalt
+				}
+			}
+		}
+
+		_, remotePath, remoteDirectory, _ := GetRemoteDirectoryAndPath(filename, ds, remoteCachePath)
 
 		if exists := ds.FileExists(remotePath); exists {
-			ui.Say(fmt.Sprintf("File %s already uploaded; continuing", targetPath))
-			state.Put(s.ResultKey, targetPath)
-			state.Put("SourceImageURL", source)
-			return multistep.ActionContinue
+			// If the remote cache overwrite flag is set to true, delete the file and download the
+			// ISO file again.
+			if s.RemoteCacheOverwrite {
+				if s.LocalCacheOverwrite {
+					log.Println("The local cache overwrite flag is set to true. Files will also be overwritten in the remote cache datastore to ensure consistency.")
+				}
+				ui.Say(fmt.Sprintf("Overwriting %s in remote cache %s...", filename, remoteDirectory))
+				// Delete the file from the remote cache datastore.
+				if err := ds.Delete(remotePath); err != nil {
+					state.Put("error", fmt.Errorf("error overwriting file in remote cache: %w", err))
+					return multistep.ActionHalt
+				}
+			} else {
+				// Skip the download step if the file exists in the local cache.
+				ui.Say(fmt.Sprintf("Skipping download, %s already exists in the local cache...", filename))
+				state.Put(s.ResultKey, targetPath)
+				state.Put("SourceImageURL", source)
+				return multistep.ActionContinue
+			}
 		}
 	}
 
-	// ISO is not present on datastore, so we need to download, then upload it.
-	// Pass through to the common download step.
+	// The ISO file is not present on the remote cache datastore. Continue with the download step.
 	return s.DownloadStep.Run(ctx, state)
 }
 

diff --git a/builder/vsphere/common/step_remote_upload.go b/builder/vsphere/common/step_remote_upload.go
@@ -6,18 +6,23 @@ package common
 import (
 	"context"
 	"fmt"
-	"log"
 	"path/filepath"
 
 	"github.com/hashicorp/packer-plugin-sdk/multistep"
 	packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
 	"github.com/hashicorp/packer-plugin-vsphere/builder/vsphere/driver"
 )
 
+const DefaultRemoteCachePath = "packer_cache"
+
 type StepRemoteUpload struct {
 	Datastore                  string
 	Host                       string
 	SetHostForDatastoreUploads bool
+	RemoteCacheCleanup         bool
+	RemoteCacheOverwrite       bool
+	RemoteCacheDatastore       string
+	RemoteCachePath            string
 	UploadedCustomCD           bool
 }
 
@@ -45,40 +50,71 @@ func (s *StepRemoteUpload) Run(_ context.Context, state multistep.StateBag) mult
 		state.Put("cd_path", fullRemotePath)
 	}
 
+	if s.RemoteCacheCleanup {
+		state.Put("remote_cache_cleanup", s.RemoteCacheCleanup)
+	}
+
 	return multistep.ActionContinue
 }
 
-func GetRemoteDirectoryAndPath(path string, ds driver.Datastore) (string, string, string, string) {
+func GetRemoteDirectoryAndPath(path string, ds driver.Datastore, remoteCachePath string) (string, string, string, string) {
 	filename := filepath.Base(path)
-	remotePath := fmt.Sprintf("packer_cache/%s", filename)
-	remoteDirectory := fmt.Sprintf("[%s] packer_cache/", ds.Name())
+	remotePath := fmt.Sprintf("%s/%s", remoteCachePath, filename)
+	remoteDirectory := fmt.Sprintf("[%s] %s", ds.Name(), remoteCachePath)
 	fullRemotePath := fmt.Sprintf("%s/%s", remoteDirectory, filename)
 
 	return filename, remotePath, remoteDirectory, fullRemotePath
-
 }
+
 func (s *StepRemoteUpload) uploadFile(path string, d driver.Driver, ui packersdk.Ui) (string, error) {
-	ds, err := d.FindDatastore(s.Datastore, s.Host)
+
+	// Set the remote cache datastore. If not set, use the default datastore for the build.
+	remoteCacheDatastore := s.Datastore
+	if s.RemoteCacheDatastore != "" {
+		remoteCacheDatastore = s.RemoteCacheDatastore
+	}
+
+	// Find the datastore to use for the remote cache.
+	ds, err := d.FindDatastore(remoteCacheDatastore, s.Host)
 	if err != nil {
-		return "", fmt.Errorf("datastore doesn't exist: %v", err)
+		return "", fmt.Errorf("error finding the remote cache datastore: %v", err)
 	}
 
-	filename, remotePath, remoteDirectory, fullRemotePath := GetRemoteDirectoryAndPath(path, ds)
+	// Set the remote cache path. If not set, use the default cache path.
+	remoteCachePath := s.RemoteCachePath
+	if remoteCachePath == "" {
+		remoteCachePath = DefaultRemoteCachePath
+	}
 
-	if exists := ds.FileExists(remotePath); exists == true {
-		ui.Say(fmt.Sprintf("File %s already exists; skipping upload.", fullRemotePath))
-		return fullRemotePath, nil
+	filename, remotePath, remoteDirectory, fullRemotePath := GetRemoteDirectoryAndPath(path, ds, remoteCachePath)
+
+	if exists := ds.FileExists(remotePath); exists {
+		// If the remote cache overwrite flag is set to true, delete the file and download the
+		// ISO file again.
+		if s.RemoteCacheOverwrite {
+			ui.Say(fmt.Sprintf("Overwriting %s in remote cache %s...", filename, remoteDirectory))
+			// Delete the file from the remote cache datastore.
+			if err := ds.Delete(remotePath); err != nil {
+				return "", fmt.Errorf("error overwriting file in remote cache: %w", err)
+			}
+		} else {
+			// Skip the download step if the remote cache overwrite flag is not set.
+			ui.Say(fmt.Sprintf("Skipping upload, %s already exists in remote cache...", fullRemotePath))
+			return fullRemotePath, nil
+		}
 	}
 
-	ui.Say(fmt.Sprintf("Uploading %s to %s", filename, remotePath))
+	ui.Say(fmt.Sprintf("Uploading %s to %s...", filename, remoteDirectory))
 
+	// Check if the remote cache directory exists. If not, create it.
 	if exists := ds.DirExists(remotePath); exists == false {
-		log.Printf("Remote directory doesn't exist; creating...")
+		ui.Say(fmt.Sprintf("Remote cache directory does not exist; creating %s...", remoteDirectory))
 		if err := ds.MakeDirectory(remoteDirectory); err != nil {
 			return "", err
 		}
 	}
 
+	// Upload the file to the remote cache datastore.
 	if err := ds.UploadFile(path, remotePath, s.Host, s.SetHostForDatastoreUploads); err != nil {
 		return "", err
 	}
@@ -88,7 +124,9 @@ func (s *StepRemoteUpload) uploadFile(path string, d driver.Driver, ui packersdk
 func (s *StepRemoteUpload) Cleanup(state multistep.StateBag) {
 	_, cancelled := state.GetOk(multistep.StateCancelled)
 	_, halted := state.GetOk(multistep.StateHalted)
-	if !cancelled && !halted {
+	_, remoteCacheCleanup := state.GetOk("remote_cache_cleanup")
+
+	if !cancelled && !halted && !remoteCacheCleanup {
 		return
 	}
 
@@ -103,18 +141,17 @@ func (s *StepRemoteUpload) Cleanup(state multistep.StateBag) {
 
 	ui := state.Get("ui").(packersdk.Ui)
 	d := state.Get("driver").(*driver.VCenterDriver)
-	ui.Say("Deleting cd_files image from remote datastore ...")
+	ui.Say(fmt.Sprintf("Removing %s...", UploadedCDPath))
 
 	ds, err := d.FindDatastore(s.Datastore, s.Host)
 	if err != nil {
-		log.Printf("Error finding datastore to delete custom CD; please delete manually: %s", err)
+		ui.Say(fmt.Sprintf("Unable to find the remote cache datastore. Please remove the item manually: %s", err))
 		return
 	}
 
 	err = ds.Delete(UploadedCDPath.(string))
 	if err != nil {
-		log.Printf("Error deleting custom CD from remote datastore; please delete manually: %s", err)
+		ui.Say(fmt.Sprintf("Unable to remove item from the remote cache. Please remove the item manually: %s", err))
 		return
-
 	}
 }
diff --git a/builder/vsphere/common/step_remote_upload_test.go b/builder/vsphere/common/step_remote_upload_test.go
@@ -48,7 +48,7 @@ func TestStepRemoteUpload_Run(t *testing.T) {
 	if !ok {
 		t.Fatalf("state should contain iso_remote_path")
 	}
-	expectedRemovePath := fmt.Sprintf("[%s] packer_cache//path", driverMock.DatastoreMock.Name())
+	expectedRemovePath := fmt.Sprintf("[%s] packer_cache/path", driverMock.DatastoreMock.Name())
 	if remotePath != expectedRemovePath {
 		t.Fatalf("iso_remote_path expected to be %s but was %s", expectedRemovePath, remotePath)
 	}

diff --git a/builder/vsphere/iso/builder.go b/builder/vsphere/iso/builder.go
@@ -52,10 +52,14 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook)
 				TargetPath:  b.config.TargetPath,
 				Url:         b.config.ISOUrls,
 			},
-			Url:       b.config.ISOUrls,
-			ResultKey: "iso_path",
-			Datastore: b.config.Datastore,
-			Host:      b.config.Host,
+			Url:                  b.config.ISOUrls,
+			ResultKey:            "iso_path",
+			Datastore:            b.config.Datastore,
+			Host:                 b.config.Host,
+			LocalCacheOverwrite:  b.config.LocalCacheOverwrite,
+			RemoteCacheOverwrite: b.config.RemoteCacheOverwrite || b.config.LocalCacheOverwrite,
+			RemoteCacheDatastore: b.config.RemoteCacheDatastore,
+			RemoteCachePath:      b.config.RemoteCachePath,
 		},
 		&commonsteps.StepCreateCD{
 			Files:   b.config.CDConfig.CDFiles,
@@ -66,6 +70,10 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook)
 			Datastore:                  b.config.Datastore,
 			Host:                       b.config.Host,
 			SetHostForDatastoreUploads: b.config.SetHostForDatastoreUploads,
+			RemoteCacheCleanup:         b.config.RemoteCacheCleanup,
+			RemoteCacheOverwrite:       b.config.RemoteCacheOverwrite,
+			RemoteCacheDatastore:       b.config.RemoteCacheDatastore,
+			RemoteCachePath:            b.config.RemoteCachePath,
 		},
 		&StepCreateVM{
 			Config:   &b.config.CreateConfig,