diff --git a/.web-docs/components/builder/vsphere-iso/README.md b/.web-docs/components/builder/vsphere-iso/README.md index 8569708e..98def1df 100644 --- a/.web-docs/components/builder/vsphere-iso/README.md +++ b/.web-docs/components/builder/vsphere-iso/README.md @@ -33,23 +33,42 @@ necessary for this build to succeed and can be found further down the page. -- `create_snapshot` (bool) - Specifies to create a snapshot of the virtual machine to use as a base for linked clones. +- `create_snapshot` (bool) - Create a snapshot of the virtual machine to use as a base for linked clones. Defaults to `false`. -- `snapshot_name` (string) - Specifies the name of the snapshot when `create_snapshot` is `true`. +- `snapshot_name` (string) - The name of the snapshot when `create_snapshot` is `true`. Defaults to `Created By Packer`. -- `convert_to_template` (bool) - Specifies to convert the cloned virtual machine to a template after the build is complete. +- `convert_to_template` (bool) - Convert the virtual machine to a template after the build is complete. Defaults to `false`. - If set to `true`, the virtual machine can not be imported to a content library. + If set to `true`, the virtual machine can not be imported into a content library. -- `export` (\*common.ExportConfig) - Specifies the configuration for exporting the virtual machine to an OVF. +- `export` (\*common.ExportConfig) - The configuration for exporting the virtual machine to an OVF. The virtual machine is not exported if [export configuration](#export-configuration) is not specified. -- `content_library_destination` (\*common.ContentLibraryDestinationConfig) - Specifies the configuration for importing a VM template or OVF template to a content library. +- `content_library_destination` (\*common.ContentLibraryDestinationConfig) - Import the virtual machine as a VM template or OVF template to a content library. The template will not be imported if no [content library import configuration](#content-library-import-configuration) is specified. If set, `convert_to_template` must be set to `false`. +- `local_cache_overwrite` (bool) - Overwrite files in the local cache if they already exist. + Defaults to `false`. + +- `remote_cache_cleanup` (bool) - Cleanup items added to the remote cache after the build is complete. + Defaults to `false`. + + -> **Note:** If the local cache overwrite flag is set to `true`, `RemoteCacheOverwrite` will + implicitly be set to `true`. This is to ensure consistency between the local and remote + cache. + +- `remote_cache_overwrite` (bool) - Overwrite files in the remote cache if they already exist. + Defaults to `false`. + +- `remote_cache_datastore` (string) - The remote cache datastore to use for the build. + If not set, the datastore of the virtual machine is used. + +- `remote_cache_path` (string) - The directory path on the remote cache datastore to use for the build. + If not set, the default path is `packer_cache/`. + diff --git a/builder/vsphere/common/step_download.go b/builder/vsphere/common/step_download.go index 28ff0dde..a4cb248f 100644 --- a/builder/vsphere/common/step_download.go +++ b/builder/vsphere/common/step_download.go @@ -6,65 +6,119 @@ package common import ( "context" "fmt" + "log" "net/url" + "os" + "path/filepath" "github.com/hashicorp/packer-plugin-sdk/multistep" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" "github.com/hashicorp/packer-plugin-vsphere/builder/vsphere/driver" ) -// Defining this interface ensures that we use the common step download, or the -// mock created to test this wrapper +// Defining this interface ensures that we use the common StepDownload, or the mock created to +// test this wrapper. type DownloadStep interface { Run(context.Context, multistep.StateBag) multistep.StepAction Cleanup(multistep.StateBag) UseSourceToFindCacheTarget(source string) (*url.URL, string, error) } -// vSphere has a specialized need -- before we waste time downloading an iso, -// we need to check whether that iso already exists on the remote datastore. -// if it does, we skip the download. This wrapping-step still uses the common -// StepDownload but only if the image isn't already present on the datastore. +// Before downloading and uploading an ISO file to the remote cache, check if the ISO file +// already exists on the datastore. +// +// If it exists, we check if the overwrite flag is set to true. If it is, we delete the file +// and download it again. If it is not set or false, the download is skipped. +// +// This wrapping-step still uses the common StepDownload, but only if the ISO file does not +// already exist on the datastore. type StepDownload struct { DownloadStep DownloadStep - // These keys are vSphere-specific and used to check the remote datastore. - Url []string - ResultKey string - Datastore string - Host string + // These keys are vSphere-specific and are used to check the remote datastore. + Url []string + ResultKey string + Datastore string + Host string + LocalCacheOverwrite bool + RemoteCacheOverwrite bool + RemoteCacheDatastore string + RemoteCachePath string } func (s *StepDownload) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction { driver := state.Get("driver").(driver.Driver) ui := state.Get("ui").(packersdk.Ui) - // Check whether iso is present on remote datastore. - ds, err := driver.FindDatastore(s.Datastore, s.Host) + // Set the remote cache datastore. If not set, use the default datastore for the build. + remoteCacheDatastore := s.Datastore + if s.RemoteCacheDatastore != "" { + remoteCacheDatastore = s.RemoteCacheDatastore + } + + // Find the datastore to use for the remote cache. + ds, err := driver.FindDatastore(remoteCacheDatastore, s.Host) if err != nil { - state.Put("error", fmt.Errorf("datastore doesn't exist: %v", err)) + state.Put("error", fmt.Errorf("error finding the datastore: %v", err)) return multistep.ActionHalt } - // loop over URLs to see if any are already present. If they are, store that - // one instate and continue + // Set the remote cache path. If not set, use the default cache path. + remoteCachePath := s.RemoteCachePath + if remoteCachePath == "" { + remoteCachePath = DefaultRemoteCachePath + } + + // Loop over the URLs to see if any are already present. + // If they are, store in state and continue. for _, source := range s.Url { _, targetPath, err := s.DownloadStep.UseSourceToFindCacheTarget(source) if err != nil { - state.Put("error", fmt.Errorf("Error getting target path: %s", err)) + state.Put("error", fmt.Errorf("error returning target path: %s", err)) return multistep.ActionHalt } - _, remotePath, _, _ := GetRemoteDirectoryAndPath(targetPath, ds) + + filename := filepath.Base(targetPath) + + // Check if the ISO file is already present in the local cache. + if _, err := os.Stat(targetPath); !os.IsNotExist(err) { + // If the local cache overwrite flag is set to true, delete the file and download the + // ISO file again. + if s.LocalCacheOverwrite { + ui.Say(fmt.Sprintf("Overwriting %s in local cache...", filename)) + // Delete the file from the local cache. + if err := os.Remove(targetPath); err != nil { + state.Put("error", fmt.Errorf("error overwriting file in local cache: %w", err)) + return multistep.ActionHalt + } + } + } + + _, remotePath, remoteDirectory, _ := GetRemoteDirectoryAndPath(filename, ds, remoteCachePath) if exists := ds.FileExists(remotePath); exists { - ui.Say(fmt.Sprintf("File %s already uploaded; continuing", targetPath)) - state.Put(s.ResultKey, targetPath) - state.Put("SourceImageURL", source) - return multistep.ActionContinue + // If the remote cache overwrite flag is set to true, delete the file and download the + // ISO file again. + if s.RemoteCacheOverwrite { + if s.LocalCacheOverwrite { + log.Println("The local cache overwrite flag is set to true. Files will also be overwritten in the remote cache datastore to ensure consistency.") + } + ui.Say(fmt.Sprintf("Overwriting %s in remote cache %s...", filename, remoteDirectory)) + // Delete the file from the remote cache datastore. + if err := ds.Delete(remotePath); err != nil { + state.Put("error", fmt.Errorf("error overwriting file in remote cache: %w", err)) + return multistep.ActionHalt + } + } else { + // Skip the download step if the file exists in the local cache. + ui.Say(fmt.Sprintf("Skipping download, %s already exists in the local cache...", filename)) + state.Put(s.ResultKey, targetPath) + state.Put("SourceImageURL", source) + return multistep.ActionContinue + } } } - // ISO is not present on datastore, so we need to download, then upload it. - // Pass through to the common download step. + // The ISO file is not present on the remote cache datastore. Continue with the download step. return s.DownloadStep.Run(ctx, state) } diff --git a/builder/vsphere/common/step_remote_upload.go b/builder/vsphere/common/step_remote_upload.go index 5dd2d679..c5e0bbe5 100644 --- a/builder/vsphere/common/step_remote_upload.go +++ b/builder/vsphere/common/step_remote_upload.go @@ -6,7 +6,6 @@ package common import ( "context" "fmt" - "log" "path/filepath" "github.com/hashicorp/packer-plugin-sdk/multistep" @@ -14,10 +13,16 @@ import ( "github.com/hashicorp/packer-plugin-vsphere/builder/vsphere/driver" ) +const DefaultRemoteCachePath = "packer_cache" + type StepRemoteUpload struct { Datastore string Host string SetHostForDatastoreUploads bool + RemoteCacheCleanup bool + RemoteCacheOverwrite bool + RemoteCacheDatastore string + RemoteCachePath string UploadedCustomCD bool } @@ -45,40 +50,71 @@ func (s *StepRemoteUpload) Run(_ context.Context, state multistep.StateBag) mult state.Put("cd_path", fullRemotePath) } + if s.RemoteCacheCleanup { + state.Put("remote_cache_cleanup", s.RemoteCacheCleanup) + } + return multistep.ActionContinue } -func GetRemoteDirectoryAndPath(path string, ds driver.Datastore) (string, string, string, string) { +func GetRemoteDirectoryAndPath(path string, ds driver.Datastore, remoteCachePath string) (string, string, string, string) { filename := filepath.Base(path) - remotePath := fmt.Sprintf("packer_cache/%s", filename) - remoteDirectory := fmt.Sprintf("[%s] packer_cache/", ds.Name()) + remotePath := fmt.Sprintf("%s/%s", remoteCachePath, filename) + remoteDirectory := fmt.Sprintf("[%s] %s", ds.Name(), remoteCachePath) fullRemotePath := fmt.Sprintf("%s/%s", remoteDirectory, filename) return filename, remotePath, remoteDirectory, fullRemotePath - } + func (s *StepRemoteUpload) uploadFile(path string, d driver.Driver, ui packersdk.Ui) (string, error) { - ds, err := d.FindDatastore(s.Datastore, s.Host) + + // Set the remote cache datastore. If not set, use the default datastore for the build. + remoteCacheDatastore := s.Datastore + if s.RemoteCacheDatastore != "" { + remoteCacheDatastore = s.RemoteCacheDatastore + } + + // Find the datastore to use for the remote cache. + ds, err := d.FindDatastore(remoteCacheDatastore, s.Host) if err != nil { - return "", fmt.Errorf("datastore doesn't exist: %v", err) + return "", fmt.Errorf("error finding the remote cache datastore: %v", err) } - filename, remotePath, remoteDirectory, fullRemotePath := GetRemoteDirectoryAndPath(path, ds) + // Set the remote cache path. If not set, use the default cache path. + remoteCachePath := s.RemoteCachePath + if remoteCachePath == "" { + remoteCachePath = DefaultRemoteCachePath + } - if exists := ds.FileExists(remotePath); exists == true { - ui.Say(fmt.Sprintf("File %s already exists; skipping upload.", fullRemotePath)) - return fullRemotePath, nil + filename, remotePath, remoteDirectory, fullRemotePath := GetRemoteDirectoryAndPath(path, ds, remoteCachePath) + + if exists := ds.FileExists(remotePath); exists { + // If the remote cache overwrite flag is set to true, delete the file and download the + // ISO file again. + if s.RemoteCacheOverwrite { + ui.Say(fmt.Sprintf("Overwriting %s in remote cache %s...", filename, remoteDirectory)) + // Delete the file from the remote cache datastore. + if err := ds.Delete(remotePath); err != nil { + return "", fmt.Errorf("error overwriting file in remote cache: %w", err) + } + } else { + // Skip the download step if the remote cache overwrite flag is not set. + ui.Say(fmt.Sprintf("Skipping upload, %s already exists in remote cache...", fullRemotePath)) + return fullRemotePath, nil + } } - ui.Say(fmt.Sprintf("Uploading %s to %s", filename, remotePath)) + ui.Say(fmt.Sprintf("Uploading %s to %s...", filename, remoteDirectory)) + // Check if the remote cache directory exists. If not, create it. if exists := ds.DirExists(remotePath); exists == false { - log.Printf("Remote directory doesn't exist; creating...") + ui.Say(fmt.Sprintf("Remote cache directory does not exist; creating %s...", remoteDirectory)) if err := ds.MakeDirectory(remoteDirectory); err != nil { return "", err } } + // Upload the file to the remote cache datastore. if err := ds.UploadFile(path, remotePath, s.Host, s.SetHostForDatastoreUploads); err != nil { return "", err } @@ -88,7 +124,9 @@ func (s *StepRemoteUpload) uploadFile(path string, d driver.Driver, ui packersdk func (s *StepRemoteUpload) Cleanup(state multistep.StateBag) { _, cancelled := state.GetOk(multistep.StateCancelled) _, halted := state.GetOk(multistep.StateHalted) - if !cancelled && !halted { + _, remoteCacheCleanup := state.GetOk("remote_cache_cleanup") + + if !cancelled && !halted && !remoteCacheCleanup { return } @@ -103,18 +141,17 @@ func (s *StepRemoteUpload) Cleanup(state multistep.StateBag) { ui := state.Get("ui").(packersdk.Ui) d := state.Get("driver").(*driver.VCenterDriver) - ui.Say("Deleting cd_files image from remote datastore ...") + ui.Say(fmt.Sprintf("Removing %s...", UploadedCDPath)) ds, err := d.FindDatastore(s.Datastore, s.Host) if err != nil { - log.Printf("Error finding datastore to delete custom CD; please delete manually: %s", err) + ui.Say(fmt.Sprintf("Unable to find the remote cache datastore. Please remove the item manually: %s", err)) return } err = ds.Delete(UploadedCDPath.(string)) if err != nil { - log.Printf("Error deleting custom CD from remote datastore; please delete manually: %s", err) + ui.Say(fmt.Sprintf("Unable to remove item from the remote cache. Please remove the item manually: %s", err)) return - } } diff --git a/builder/vsphere/common/step_remote_upload_test.go b/builder/vsphere/common/step_remote_upload_test.go index 4c021ffa..56a91f41 100644 --- a/builder/vsphere/common/step_remote_upload_test.go +++ b/builder/vsphere/common/step_remote_upload_test.go @@ -48,7 +48,7 @@ func TestStepRemoteUpload_Run(t *testing.T) { if !ok { t.Fatalf("state should contain iso_remote_path") } - expectedRemovePath := fmt.Sprintf("[%s] packer_cache//path", driverMock.DatastoreMock.Name()) + expectedRemovePath := fmt.Sprintf("[%s] packer_cache/path", driverMock.DatastoreMock.Name()) if remotePath != expectedRemovePath { t.Fatalf("iso_remote_path expected to be %s but was %s", expectedRemovePath, remotePath) } diff --git a/builder/vsphere/iso/builder.go b/builder/vsphere/iso/builder.go index 023aa0b5..01925b1a 100644 --- a/builder/vsphere/iso/builder.go +++ b/builder/vsphere/iso/builder.go @@ -52,10 +52,14 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook) TargetPath: b.config.TargetPath, Url: b.config.ISOUrls, }, - Url: b.config.ISOUrls, - ResultKey: "iso_path", - Datastore: b.config.Datastore, - Host: b.config.Host, + Url: b.config.ISOUrls, + ResultKey: "iso_path", + Datastore: b.config.Datastore, + Host: b.config.Host, + LocalCacheOverwrite: b.config.LocalCacheOverwrite, + RemoteCacheOverwrite: b.config.RemoteCacheOverwrite || b.config.LocalCacheOverwrite, + RemoteCacheDatastore: b.config.RemoteCacheDatastore, + RemoteCachePath: b.config.RemoteCachePath, }, &commonsteps.StepCreateCD{ Files: b.config.CDConfig.CDFiles, @@ -66,6 +70,10 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook) Datastore: b.config.Datastore, Host: b.config.Host, SetHostForDatastoreUploads: b.config.SetHostForDatastoreUploads, + RemoteCacheCleanup: b.config.RemoteCacheCleanup, + RemoteCacheOverwrite: b.config.RemoteCacheOverwrite, + RemoteCacheDatastore: b.config.RemoteCacheDatastore, + RemoteCachePath: b.config.RemoteCachePath, }, &StepCreateVM{ Config: &b.config.CreateConfig, diff --git a/builder/vsphere/iso/config.go b/builder/vsphere/iso/config.go index 33093819..47fda9f5 100644 --- a/builder/vsphere/iso/config.go +++ b/builder/vsphere/iso/config.go @@ -39,23 +39,42 @@ type Config struct { common.ShutdownConfig `mapstructure:",squash"` - // Specifies to create a snapshot of the virtual machine to use as a base for linked clones. + // Create a snapshot of the virtual machine to use as a base for linked clones. // Defaults to `false`. CreateSnapshot bool `mapstructure:"create_snapshot"` - // Specifies the name of the snapshot when `create_snapshot` is `true`. + // The name of the snapshot when `create_snapshot` is `true`. // Defaults to `Created By Packer`. SnapshotName string `mapstructure:"snapshot_name"` - // Specifies to convert the cloned virtual machine to a template after the build is complete. + // Convert the virtual machine to a template after the build is complete. // Defaults to `false`. - // If set to `true`, the virtual machine can not be imported to a content library. + // If set to `true`, the virtual machine can not be imported into a content library. ConvertToTemplate bool `mapstructure:"convert_to_template"` - // Specifies the configuration for exporting the virtual machine to an OVF. + // The configuration for exporting the virtual machine to an OVF. // The virtual machine is not exported if [export configuration](#export-configuration) is not specified. Export *common.ExportConfig `mapstructure:"export"` - // Specifies the configuration for importing a VM template or OVF template to a content library. + // Import the virtual machine as a VM template or OVF template to a content library. // The template will not be imported if no [content library import configuration](#content-library-import-configuration) is specified. // If set, `convert_to_template` must be set to `false`. ContentLibraryDestinationConfig *common.ContentLibraryDestinationConfig `mapstructure:"content_library_destination"` + // Overwrite files in the local cache if they already exist. + // Defaults to `false`. + LocalCacheOverwrite bool `mapstructure:"local_cache_overwrite"` + // Cleanup items added to the remote cache after the build is complete. + // Defaults to `false`. + // + // -> **Note:** If the local cache overwrite flag is set to `true`, `RemoteCacheOverwrite` will + // implicitly be set to `true`. This is to ensure consistency between the local and remote + // cache. + RemoteCacheCleanup bool `mapstructure:"remote_cache_cleanup"` + // Overwrite files in the remote cache if they already exist. + // Defaults to `false`. + RemoteCacheOverwrite bool `mapstructure:"remote_cache_overwrite"` + // The remote cache datastore to use for the build. + // If not set, the datastore of the virtual machine is used. + RemoteCacheDatastore string `mapstructure:"remote_cache_datastore"` + // The directory path on the remote cache datastore to use for the build. + // If not set, the default path is `packer_cache/`. + RemoteCachePath string `mapstructure:"remote_cache_path"` ctx interpolate.Context } diff --git a/builder/vsphere/iso/config.hcl2spec.go b/builder/vsphere/iso/config.hcl2spec.go index 1c27b356..c75ca9dd 100644 --- a/builder/vsphere/iso/config.hcl2spec.go +++ b/builder/vsphere/iso/config.hcl2spec.go @@ -149,6 +149,11 @@ type FlatConfig struct { ConvertToTemplate *bool `mapstructure:"convert_to_template" cty:"convert_to_template" hcl:"convert_to_template"` Export *common.FlatExportConfig `mapstructure:"export" cty:"export" hcl:"export"` ContentLibraryDestinationConfig *common.FlatContentLibraryDestinationConfig `mapstructure:"content_library_destination" cty:"content_library_destination" hcl:"content_library_destination"` + LocalCacheOverwrite *bool `mapstructure:"local_cache_overwrite" cty:"local_cache_overwrite" hcl:"local_cache_overwrite"` + RemoteCacheCleanup *bool `mapstructure:"remote_cache_cleanup" cty:"remote_cache_cleanup" hcl:"remote_cache_cleanup"` + RemoteCacheOverwrite *bool `mapstructure:"remote_cache_overwrite" cty:"remote_cache_overwrite" hcl:"remote_cache_overwrite"` + RemoteCacheDatastore *string `mapstructure:"remote_cache_datastore" cty:"remote_cache_datastore" hcl:"remote_cache_datastore"` + RemoteCachePath *string `mapstructure:"remote_cache_path" cty:"remote_cache_path" hcl:"remote_cache_path"` } // FlatMapstructure returns a new FlatConfig. @@ -301,6 +306,11 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec { "convert_to_template": &hcldec.AttrSpec{Name: "convert_to_template", Type: cty.Bool, Required: false}, "export": &hcldec.BlockSpec{TypeName: "export", Nested: hcldec.ObjectSpec((*common.FlatExportConfig)(nil).HCL2Spec())}, "content_library_destination": &hcldec.BlockSpec{TypeName: "content_library_destination", Nested: hcldec.ObjectSpec((*common.FlatContentLibraryDestinationConfig)(nil).HCL2Spec())}, + "local_cache_overwrite": &hcldec.AttrSpec{Name: "local_cache_overwrite", Type: cty.Bool, Required: false}, + "remote_cache_cleanup": &hcldec.AttrSpec{Name: "remote_cache_cleanup", Type: cty.Bool, Required: false}, + "remote_cache_overwrite": &hcldec.AttrSpec{Name: "remote_cache_overwrite", Type: cty.Bool, Required: false}, + "remote_cache_datastore": &hcldec.AttrSpec{Name: "remote_cache_datastore", Type: cty.String, Required: false}, + "remote_cache_path": &hcldec.AttrSpec{Name: "remote_cache_path", Type: cty.String, Required: false}, } return s } diff --git a/docs-partials/builder/vsphere/iso/Config-not-required.mdx b/docs-partials/builder/vsphere/iso/Config-not-required.mdx index 4380821b..1ca96a12 100644 --- a/docs-partials/builder/vsphere/iso/Config-not-required.mdx +++ b/docs-partials/builder/vsphere/iso/Config-not-required.mdx @@ -1,20 +1,39 @@ -- `create_snapshot` (bool) - Specifies to create a snapshot of the virtual machine to use as a base for linked clones. +- `create_snapshot` (bool) - Create a snapshot of the virtual machine to use as a base for linked clones. Defaults to `false`. -- `snapshot_name` (string) - Specifies the name of the snapshot when `create_snapshot` is `true`. +- `snapshot_name` (string) - The name of the snapshot when `create_snapshot` is `true`. Defaults to `Created By Packer`. -- `convert_to_template` (bool) - Specifies to convert the cloned virtual machine to a template after the build is complete. +- `convert_to_template` (bool) - Convert the virtual machine to a template after the build is complete. Defaults to `false`. - If set to `true`, the virtual machine can not be imported to a content library. + If set to `true`, the virtual machine can not be imported into a content library. -- `export` (\*common.ExportConfig) - Specifies the configuration for exporting the virtual machine to an OVF. +- `export` (\*common.ExportConfig) - The configuration for exporting the virtual machine to an OVF. The virtual machine is not exported if [export configuration](#export-configuration) is not specified. -- `content_library_destination` (\*common.ContentLibraryDestinationConfig) - Specifies the configuration for importing a VM template or OVF template to a content library. +- `content_library_destination` (\*common.ContentLibraryDestinationConfig) - Import the virtual machine as a VM template or OVF template to a content library. The template will not be imported if no [content library import configuration](#content-library-import-configuration) is specified. If set, `convert_to_template` must be set to `false`. +- `local_cache_overwrite` (bool) - Overwrite files in the local cache if they already exist. + Defaults to `false`. + +- `remote_cache_cleanup` (bool) - Cleanup items added to the remote cache after the build is complete. + Defaults to `false`. + + -> **Note:** If the local cache overwrite flag is set to `true`, `RemoteCacheOverwrite` will + implicitly be set to `true`. This is to ensure consistency between the local and remote + cache. + +- `remote_cache_overwrite` (bool) - Overwrite files in the remote cache if they already exist. + Defaults to `false`. + +- `remote_cache_datastore` (string) - The remote cache datastore to use for the build. + If not set, the datastore of the virtual machine is used. + +- `remote_cache_path` (string) - The directory path on the remote cache datastore to use for the build. + If not set, the default path is `packer_cache/`. +