From c6f6449e2e77b8eb897a23276e8d27f3343ba1bf Mon Sep 17 00:00:00 2001 From: staebler Date: Wed, 12 Dec 2018 15:24:00 -0500 Subject: [PATCH] assets: move asset store impl to own package Tests are being created that use the asset store implementation to validate restoring assets from the state file and disk. The tests are much easier when they have access to the private fields of the asset store. Rather than exposing more public fields and methods on the store, the tests will be placed in the same package as the store. Unfortunately, this creates cyclic imports between the base asset package and the various packages containing the targeted assets. To rectify this, the concrete asset store implementation has been moved to a new pkg/asset/store package. Changes for https://jira.coreos.com/browse/CORS-940 --- cmd/openshift-install/create.go | 3 +- cmd/openshift-install/destroy.go | 4 +- pkg/asset/asset.go | 4 +- pkg/asset/filefetcher.go | 48 --- pkg/asset/store.go | 350 --------------------- pkg/asset/store/data | 1 + pkg/asset/store/filefetcher.go | 51 ++++ pkg/asset/{ => store}/filefetcher_test.go | 16 +- pkg/asset/store/store.go | 357 ++++++++++++++++++++++ pkg/asset/{ => store}/store_test.go | 64 ++-- 10 files changed, 457 insertions(+), 441 deletions(-) create mode 120000 pkg/asset/store/data create mode 100644 pkg/asset/store/filefetcher.go rename pkg/asset/{ => store}/filefetcher_test.go (94%) create mode 100644 pkg/asset/store/store.go rename pkg/asset/{ => store}/store_test.go (82%) diff --git a/cmd/openshift-install/create.go b/cmd/openshift-install/create.go index c58946fed2c..c5349128525 100644 --- a/cmd/openshift-install/create.go +++ b/cmd/openshift-install/create.go @@ -32,6 +32,7 @@ import ( "github.com/openshift/installer/pkg/asset/installconfig" "github.com/openshift/installer/pkg/asset/kubeconfig" "github.com/openshift/installer/pkg/asset/manifests" + assetstore "github.com/openshift/installer/pkg/asset/store" "github.com/openshift/installer/pkg/asset/templates" "github.com/openshift/installer/pkg/asset/tls" destroybootstrap "github.com/openshift/installer/pkg/destroy/bootstrap" @@ -154,7 +155,7 @@ func newCreateCmd() *cobra.Command { func runTargetCmd(targets ...asset.WritableAsset) func(cmd *cobra.Command, args []string) { runner := func(directory string) error { - assetStore, err := asset.NewStore(directory) + assetStore, err := assetstore.NewStore(directory) if err != nil { return errors.Wrapf(err, "failed to create asset store") } diff --git a/cmd/openshift-install/destroy.go b/cmd/openshift-install/destroy.go index 002a6a04519..747c106850b 100644 --- a/cmd/openshift-install/destroy.go +++ b/cmd/openshift-install/destroy.go @@ -5,7 +5,7 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/openshift/installer/pkg/asset" + assetstore "github.com/openshift/installer/pkg/asset/store" "github.com/openshift/installer/pkg/destroy" "github.com/openshift/installer/pkg/destroy/bootstrap" _ "github.com/openshift/installer/pkg/destroy/libvirt" @@ -52,7 +52,7 @@ func runDestroyCmd(directory string) error { return errors.Wrap(err, "Failed to destroy cluster") } - store, err := asset.NewStore(directory) + store, err := assetstore.NewStore(directory) if err != nil { return errors.Wrapf(err, "failed to create asset store") } diff --git a/pkg/asset/asset.go b/pkg/asset/asset.go index b08652b0d2a..6fb4e05b8fe 100644 --- a/pkg/asset/asset.go +++ b/pkg/asset/asset.go @@ -58,9 +58,9 @@ func PersistToFile(asset WritableAsset, directory string) error { return nil } -// deleteAssetFromDisk removes all the files for asset from disk. +// DeleteAssetFromDisk removes all the files for asset from disk. // this is function is not safe for calling concurrently on the same directory. -func deleteAssetFromDisk(asset WritableAsset, directory string) error { +func DeleteAssetFromDisk(asset WritableAsset, directory string) error { logrus.Debugf("Purging asset %q from disk", asset.Name()) for _, f := range asset.Files() { path := filepath.Join(directory, f.Filename) diff --git a/pkg/asset/filefetcher.go b/pkg/asset/filefetcher.go index 2520caffffa..5aece890ea3 100644 --- a/pkg/asset/filefetcher.go +++ b/pkg/asset/filefetcher.go @@ -1,11 +1,5 @@ package asset -import ( - "io/ioutil" - "path/filepath" - "sort" -) - //go:generate mockgen -source=./filefetcher.go -destination=./mock/filefetcher_generated.go -package=mock // FileFetcher fetches the asset files from disk. @@ -15,45 +9,3 @@ type FileFetcher interface { // FetchByPattern returns the files whose name match the given glob. FetchByPattern(pattern string) ([]*File, error) } - -type fileFetcher struct { - directory string -} - -// FetchByName returns the file with the given name. -func (f *fileFetcher) FetchByName(name string) (*File, error) { - data, err := ioutil.ReadFile(filepath.Join(f.directory, name)) - if err != nil { - return nil, err - } - return &File{Filename: name, Data: data}, nil -} - -// FetchByPattern returns the files whose name match the given regexp. -func (f *fileFetcher) FetchByPattern(pattern string) (files []*File, err error) { - matches, err := filepath.Glob(filepath.Join(f.directory, pattern)) - if err != nil { - return nil, err - } - - files = make([]*File, 0, len(matches)) - for _, path := range matches { - data, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - - filename, err := filepath.Rel(f.directory, path) - if err != nil { - return nil, err - } - - files = append(files, &File{ - Filename: filename, - Data: data, - }) - } - - sort.Slice(files, func(i, j int) bool { return files[i].Filename < files[j].Filename }) - return files, nil -} diff --git a/pkg/asset/store.go b/pkg/asset/store.go index 9557aefd640..867ec3f521b 100644 --- a/pkg/asset/store.go +++ b/pkg/asset/store.go @@ -1,20 +1,5 @@ package asset -import ( - "encoding/json" - "io/ioutil" - "os" - "path/filepath" - "reflect" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" -) - -const ( - stateFileName = ".openshift_install_state.json" -) - // Store is a store for the states of assets. type Store interface { // Fetch retrieves the state of the given asset, generating it and its @@ -29,338 +14,3 @@ type Store interface { // state file DestroyState() error } - -// assetSource indicates from where the asset was fetched -type assetSource int - -const ( - // unsourced indicates that the asset has not been fetched - unfetched assetSource = iota - // generatedSource indicates that the asset was generated - generatedSource - // onDiskSource indicates that the asset was fetched from disk - onDiskSource - // stateFileSource indicates that the asset was fetched from the state file - stateFileSource -) - -type assetState struct { - // asset is the asset. - // If the asset has not been fetched, then this will be nil. - asset Asset - // source is the source from which the asset was fetched - source assetSource - // anyParentsDirty is true if any of the parents of the asset are dirty - anyParentsDirty bool - // presentOnDisk is true if the asset in on-disk. This is set whether the - // asset is sourced from on-disk or not. It is used in purging consumed assets. - presentOnDisk bool -} - -// StoreImpl is the implementation of Store. -type StoreImpl struct { - directory string - assets map[reflect.Type]*assetState - stateFileAssets map[string]json.RawMessage - fileFetcher FileFetcher -} - -// NewStore returns an asset store that implements the Store interface. -func NewStore(dir string) (Store, error) { - store := &StoreImpl{ - directory: dir, - fileFetcher: &fileFetcher{directory: dir}, - assets: map[reflect.Type]*assetState{}, - } - - if err := store.loadStateFile(); err != nil { - return nil, err - } - return store, nil -} - -// Fetch retrieves the state of the given asset, generating it and its -// dependencies if necessary. -func (s *StoreImpl) Fetch(asset Asset) error { - if err := s.fetch(asset, ""); err != nil { - return err - } - if err := s.saveStateFile(); err != nil { - return errors.Wrapf(err, "failed to save state") - } - if wa, ok := asset.(WritableAsset); ok { - return errors.Wrapf(s.purge(wa), "failed to purge asset") - } - return nil -} - -// Destroy removes the asset from all its internal state and also from -// disk if possible. -func (s *StoreImpl) Destroy(asset Asset) error { - if sa, ok := s.assets[reflect.TypeOf(asset)]; ok { - reflect.ValueOf(asset).Elem().Set(reflect.ValueOf(sa.asset).Elem()) - } else if s.isAssetInState(asset) { - if err := s.loadAssetFromState(asset); err != nil { - return err - } - } else { - // nothing to do - return nil - } - - if wa, ok := asset.(WritableAsset); ok { - if err := deleteAssetFromDisk(wa, s.directory); err != nil { - return err - } - } - - delete(s.assets, reflect.TypeOf(asset)) - delete(s.stateFileAssets, reflect.TypeOf(asset).String()) - return s.saveStateFile() -} - -// DestroyState removes the state file from disk -func (s *StoreImpl) DestroyState() error { - s.stateFileAssets = nil - path := filepath.Join(s.directory, stateFileName) - err := os.Remove(path) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - return nil -} - -// loadStateFile retrieves the state from the state file present in the given directory -// and returns the assets map -func (s *StoreImpl) loadStateFile() error { - path := filepath.Join(s.directory, stateFileName) - assets := map[string]json.RawMessage{} - data, err := ioutil.ReadFile(path) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - err = json.Unmarshal(data, &assets) - if err != nil { - return errors.Wrapf(err, "failed to unmarshal state file %q", path) - } - s.stateFileAssets = assets - return nil -} - -// loadAssetFromState renders the asset object arguments from the state file contents. -func (s *StoreImpl) loadAssetFromState(asset Asset) error { - bytes, ok := s.stateFileAssets[reflect.TypeOf(asset).String()] - if !ok { - return errors.Errorf("asset %q is not found in the state file", asset.Name()) - } - return json.Unmarshal(bytes, asset) -} - -// isAssetInState tests whether the asset is in the state file. -func (s *StoreImpl) isAssetInState(asset Asset) bool { - _, ok := s.stateFileAssets[reflect.TypeOf(asset).String()] - return ok -} - -// saveStateFile dumps the entire state map into a file -func (s *StoreImpl) saveStateFile() error { - if s.stateFileAssets == nil { - s.stateFileAssets = map[string]json.RawMessage{} - } - for k, v := range s.assets { - if v.source == unfetched { - continue - } - data, err := json.MarshalIndent(v.asset, "", " ") - if err != nil { - return err - } - s.stateFileAssets[k.String()] = json.RawMessage(data) - } - data, err := json.MarshalIndent(s.stateFileAssets, "", " ") - if err != nil { - return err - } - - path := filepath.Join(s.directory, stateFileName) - if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { - return err - } - if err := ioutil.WriteFile(path, data, 0644); err != nil { - return err - } - return nil -} - -// fetch populates the given asset, generating it and its dependencies if -// necessary, and returns whether or not the asset had to be regenerated and -// any errors. -func (s *StoreImpl) fetch(asset Asset, indent string) error { - logrus.Debugf("%sFetching %q...", indent, asset.Name()) - - assetState, ok := s.assets[reflect.TypeOf(asset)] - if !ok { - if _, err := s.load(asset, ""); err != nil { - return err - } - assetState = s.assets[reflect.TypeOf(asset)] - } - - // Return immediately if the asset has been fetched before, - // this is because we are doing a depth-first-search, it's guaranteed - // that we always fetch the parent before children, so we don't need - // to worry about invalidating anything in the cache. - if assetState.source != unfetched { - logrus.Debugf("%sReusing previously-fetched %q", indent, asset.Name()) - reflect.ValueOf(asset).Elem().Set(reflect.ValueOf(assetState.asset).Elem()) - return nil - } - - // Re-generate the asset - dependencies := asset.Dependencies() - parents := make(Parents, len(dependencies)) - for _, d := range dependencies { - if err := s.fetch(d, increaseIndent(indent)); err != nil { - return errors.Wrapf(err, "failed to fetch dependency of %q", asset.Name()) - } - parents.Add(d) - } - logrus.Debugf("%sGenerating %q...", indent, asset.Name()) - if err := asset.Generate(parents); err != nil { - return errors.Wrapf(err, "failed to generate asset %q", asset.Name()) - } - assetState.asset = asset - assetState.source = generatedSource - return nil -} - -// load loads the asset and all of its ancestors from on-disk and the state file. -func (s *StoreImpl) load(asset Asset, indent string) (*assetState, error) { - logrus.Debugf("%sLoading %q...", indent, asset.Name()) - - // Stop descent if the asset has already been loaded. - if state, ok := s.assets[reflect.TypeOf(asset)]; ok { - return state, nil - } - - // Load dependencies from on-disk. - anyParentsDirty := false - for _, d := range asset.Dependencies() { - state, err := s.load(d, increaseIndent(indent)) - if err != nil { - return nil, err - } - if state.anyParentsDirty || state.source == onDiskSource { - anyParentsDirty = true - } - } - - // Try to load from on-disk. - var ( - onDiskAsset WritableAsset - foundOnDisk bool - ) - if _, isWritable := asset.(WritableAsset); isWritable { - onDiskAsset = reflect.New(reflect.TypeOf(asset).Elem()).Interface().(WritableAsset) - var err error - foundOnDisk, err = onDiskAsset.Load(s.fileFetcher) - if err != nil { - return nil, errors.Wrapf(err, "failed to load asset %q", asset.Name()) - } - } - - // Try to load from state file. - var ( - stateFileAsset Asset - foundInStateFile bool - onDiskMatchesStateFile bool - ) - // Do not need to bother with loading from state file if any of the parents - // are dirty because the asset must be re-generated in this case. - if !anyParentsDirty { - foundInStateFile = s.isAssetInState(asset) - if foundInStateFile { - stateFileAsset = reflect.New(reflect.TypeOf(asset).Elem()).Interface().(Asset) - if err := s.loadAssetFromState(stateFileAsset); err != nil { - return nil, errors.Wrapf(err, "failed to load asset %q from state file", asset.Name()) - } - } - - if foundOnDisk && foundInStateFile { - logrus.Debugf("%sLoading %q from both state file and target directory", indent, asset.Name()) - - // If the on-disk asset is the same as the one in the state file, there - // is no need to consider the one on disk and to mark the asset dirty. - onDiskMatchesStateFile = reflect.DeepEqual(onDiskAsset, stateFileAsset) - if onDiskMatchesStateFile { - logrus.Debugf("%sOn-disk %q matches asset in state file", indent, asset.Name()) - } - } - } - - var ( - assetToStore Asset - source assetSource - ) - switch { - // A parent is dirty. The asset must be re-generated. - case anyParentsDirty: - if foundOnDisk { - logrus.Warningf("%sDiscarding the %q that was provided in the target directory because its dependencies are dirty and it needs to be regenerated", indent, asset.Name()) - } - source = unfetched - // The asset is on disk and that differs from what is in the source file. - // The asset is sourced from on disk. - case foundOnDisk && !onDiskMatchesStateFile: - logrus.Debugf("%sUsing %q loaded from target directory", indent, asset.Name()) - assetToStore = onDiskAsset - source = onDiskSource - // The asset is in the state file. The asset is sourced from state file. - case foundInStateFile: - logrus.Debugf("%sUsing %q loaded from state file", indent, asset.Name()) - assetToStore = stateFileAsset - source = stateFileSource - // There is no existing source for the asset. The asset will be generated. - default: - source = unfetched - } - - state := &assetState{ - asset: assetToStore, - source: source, - anyParentsDirty: anyParentsDirty, - presentOnDisk: foundOnDisk, - } - s.assets[reflect.TypeOf(asset)] = state - return state, nil -} - -// purge deletes the on-disk assets that are consumed already. -// E.g., install-config.yaml will be deleted after fetching 'manifests'. -// The target asset is excluded. -func (s *StoreImpl) purge(excluded WritableAsset) error { - for _, assetState := range s.assets { - if !assetState.presentOnDisk { - continue - } - if reflect.TypeOf(assetState.asset) == reflect.TypeOf(excluded) { - continue - } - logrus.Infof("Consuming %q from target directory", assetState.asset.Name()) - if err := deleteAssetFromDisk(assetState.asset.(WritableAsset), s.directory); err != nil { - return err - } - assetState.presentOnDisk = false - } - return nil -} - -func increaseIndent(indent string) string { - return indent + " " -} diff --git a/pkg/asset/store/data b/pkg/asset/store/data new file mode 120000 index 00000000000..45606f08b05 --- /dev/null +++ b/pkg/asset/store/data @@ -0,0 +1 @@ +../../../data/data \ No newline at end of file diff --git a/pkg/asset/store/filefetcher.go b/pkg/asset/store/filefetcher.go new file mode 100644 index 00000000000..03703ab1039 --- /dev/null +++ b/pkg/asset/store/filefetcher.go @@ -0,0 +1,51 @@ +package store + +import ( + "io/ioutil" + "path/filepath" + "sort" + + "github.com/openshift/installer/pkg/asset" +) + +type fileFetcher struct { + directory string +} + +// FetchByName returns the file with the given name. +func (f *fileFetcher) FetchByName(name string) (*asset.File, error) { + data, err := ioutil.ReadFile(filepath.Join(f.directory, name)) + if err != nil { + return nil, err + } + return &asset.File{Filename: name, Data: data}, nil +} + +// FetchByPattern returns the files whose name match the given regexp. +func (f *fileFetcher) FetchByPattern(pattern string) (files []*asset.File, err error) { + matches, err := filepath.Glob(filepath.Join(f.directory, pattern)) + if err != nil { + return nil, err + } + + files = make([]*asset.File, 0, len(matches)) + for _, path := range matches { + data, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + + filename, err := filepath.Rel(f.directory, path) + if err != nil { + return nil, err + } + + files = append(files, &asset.File{ + Filename: filename, + Data: data, + }) + } + + sort.Slice(files, func(i, j int) bool { return files[i].Filename < files[j].Filename }) + return files, nil +} diff --git a/pkg/asset/filefetcher_test.go b/pkg/asset/store/filefetcher_test.go similarity index 94% rename from pkg/asset/filefetcher_test.go rename to pkg/asset/store/filefetcher_test.go index 43cb52b6c26..42af7928fba 100644 --- a/pkg/asset/filefetcher_test.go +++ b/pkg/asset/store/filefetcher_test.go @@ -1,4 +1,4 @@ -package asset +package store import ( "io/ioutil" @@ -7,6 +7,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + + "github.com/openshift/installer/pkg/asset" ) func TestFetchByName(t *testing.T) { @@ -14,7 +16,7 @@ func TestFetchByName(t *testing.T) { name string files map[string][]byte input string - expectFile *File + expectFile *asset.File }{ { name: "input doesn't match", @@ -26,7 +28,7 @@ func TestFetchByName(t *testing.T) { name: "with contents", files: map[string][]byte{"foo.bar": []byte("some data")}, input: "foo.bar", - expectFile: &File{ + expectFile: &asset.File{ Filename: "foo.bar", Data: []byte("some data"), }, @@ -35,7 +37,7 @@ func TestFetchByName(t *testing.T) { name: "match one file", files: map[string][]byte{"foo.bar": []byte("some data")}, input: "foo.bar", - expectFile: &File{ + expectFile: &asset.File{ Filename: "foo.bar", Data: []byte("some data"), }, @@ -110,11 +112,11 @@ func TestFetchByPattern(t *testing.T) { } tests := []struct { input string - expectFiles []*File + expectFiles []*asset.File }{ { input: "master-[0-9]*.ign", - expectFiles: []*File{ + expectFiles: []*asset.File{ { Filename: "master-0.ign", Data: []byte("some data 0"), @@ -147,7 +149,7 @@ func TestFetchByPattern(t *testing.T) { }, { input: filepath.Join("manifests", "*"), - expectFiles: []*File{ + expectFiles: []*asset.File{ { Filename: "manifests/0", Data: []byte("some data 11"), diff --git a/pkg/asset/store/store.go b/pkg/asset/store/store.go new file mode 100644 index 00000000000..ee966687b47 --- /dev/null +++ b/pkg/asset/store/store.go @@ -0,0 +1,357 @@ +package store + +import ( + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + "reflect" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + "github.com/openshift/installer/pkg/asset" +) + +const ( + stateFileName = ".openshift_install_state.json" +) + +// assetSource indicates from where the asset was fetched +type assetSource int + +const ( + // unsourced indicates that the asset has not been fetched + unfetched assetSource = iota + // generatedSource indicates that the asset was generated + generatedSource + // onDiskSource indicates that the asset was fetched from disk + onDiskSource + // stateFileSource indicates that the asset was fetched from the state file + stateFileSource +) + +type assetState struct { + // asset is the asset. + // If the asset has not been fetched, then this will be nil. + asset asset.Asset + // source is the source from which the asset was fetched + source assetSource + // anyParentsDirty is true if any of the parents of the asset are dirty + anyParentsDirty bool + // presentOnDisk is true if the asset in on-disk. This is set whether the + // asset is sourced from on-disk or not. It is used in purging consumed assets. + presentOnDisk bool +} + +// storeImpl is the implementation of Store. +type storeImpl struct { + directory string + assets map[reflect.Type]*assetState + stateFileAssets map[string]json.RawMessage + fileFetcher asset.FileFetcher +} + +// NewStore returns an asset store that implements the asset.Store interface. +func NewStore(dir string) (asset.Store, error) { + return newStore(dir) +} + +func newStore(dir string) (*storeImpl, error) { + store := &storeImpl{ + directory: dir, + fileFetcher: &fileFetcher{directory: dir}, + assets: map[reflect.Type]*assetState{}, + } + + if err := store.loadStateFile(); err != nil { + return nil, err + } + return store, nil +} + +// Fetch retrieves the state of the given asset, generating it and its +// dependencies if necessary. +func (s *storeImpl) Fetch(a asset.Asset) error { + if err := s.fetch(a, ""); err != nil { + return err + } + if err := s.saveStateFile(); err != nil { + return errors.Wrapf(err, "failed to save state") + } + if wa, ok := a.(asset.WritableAsset); ok { + return errors.Wrapf(s.purge(wa), "failed to purge asset") + } + return nil +} + +// Destroy removes the asset from all its internal state and also from +// disk if possible. +func (s *storeImpl) Destroy(a asset.Asset) error { + if sa, ok := s.assets[reflect.TypeOf(a)]; ok { + reflect.ValueOf(a).Elem().Set(reflect.ValueOf(sa.asset).Elem()) + } else if s.isAssetInState(a) { + if err := s.loadAssetFromState(a); err != nil { + return err + } + } else { + // nothing to do + return nil + } + + if wa, ok := a.(asset.WritableAsset); ok { + if err := asset.DeleteAssetFromDisk(wa, s.directory); err != nil { + return err + } + } + + delete(s.assets, reflect.TypeOf(a)) + delete(s.stateFileAssets, reflect.TypeOf(a).String()) + return s.saveStateFile() +} + +// DestroyState removes the state file from disk +func (s *storeImpl) DestroyState() error { + s.stateFileAssets = nil + path := filepath.Join(s.directory, stateFileName) + err := os.Remove(path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + return nil +} + +// loadStateFile retrieves the state from the state file present in the given directory +// and returns the assets map +func (s *storeImpl) loadStateFile() error { + path := filepath.Join(s.directory, stateFileName) + assets := map[string]json.RawMessage{} + data, err := ioutil.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + err = json.Unmarshal(data, &assets) + if err != nil { + return errors.Wrapf(err, "failed to unmarshal state file %q", path) + } + s.stateFileAssets = assets + return nil +} + +// loadAssetFromState renders the asset object arguments from the state file contents. +func (s *storeImpl) loadAssetFromState(a asset.Asset) error { + bytes, ok := s.stateFileAssets[reflect.TypeOf(a).String()] + if !ok { + return errors.Errorf("asset %q is not found in the state file", a.Name()) + } + return json.Unmarshal(bytes, a) +} + +// isAssetInState tests whether the asset is in the state file. +func (s *storeImpl) isAssetInState(a asset.Asset) bool { + _, ok := s.stateFileAssets[reflect.TypeOf(a).String()] + return ok +} + +// saveStateFile dumps the entire state map into a file +func (s *storeImpl) saveStateFile() error { + if s.stateFileAssets == nil { + s.stateFileAssets = map[string]json.RawMessage{} + } + for k, v := range s.assets { + if v.source == unfetched { + continue + } + data, err := json.MarshalIndent(v.asset, "", " ") + if err != nil { + return err + } + s.stateFileAssets[k.String()] = json.RawMessage(data) + } + data, err := json.MarshalIndent(s.stateFileAssets, "", " ") + if err != nil { + return err + } + + path := filepath.Join(s.directory, stateFileName) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return err + } + if err := ioutil.WriteFile(path, data, 0644); err != nil { + return err + } + return nil +} + +// fetch populates the given asset, generating it and its dependencies if +// necessary, and returns whether or not the asset had to be regenerated and +// any errors. +func (s *storeImpl) fetch(a asset.Asset, indent string) error { + logrus.Debugf("%sFetching %q...", indent, a.Name()) + + assetState, ok := s.assets[reflect.TypeOf(a)] + if !ok { + if _, err := s.load(a, ""); err != nil { + return err + } + assetState = s.assets[reflect.TypeOf(a)] + } + + // Return immediately if the asset has been fetched before, + // this is because we are doing a depth-first-search, it's guaranteed + // that we always fetch the parent before children, so we don't need + // to worry about invalidating anything in the cache. + if assetState.source != unfetched { + logrus.Debugf("%sReusing previously-fetched %q", indent, a.Name()) + reflect.ValueOf(a).Elem().Set(reflect.ValueOf(assetState.asset).Elem()) + return nil + } + + // Re-generate the asset + dependencies := a.Dependencies() + parents := make(asset.Parents, len(dependencies)) + for _, d := range dependencies { + if err := s.fetch(d, increaseIndent(indent)); err != nil { + return errors.Wrapf(err, "failed to fetch dependency of %q", a.Name()) + } + parents.Add(d) + } + logrus.Debugf("%sGenerating %q...", indent, a.Name()) + if err := a.Generate(parents); err != nil { + return errors.Wrapf(err, "failed to generate asset %q", a.Name()) + } + assetState.asset = a + assetState.source = generatedSource + return nil +} + +// load loads the asset and all of its ancestors from on-disk and the state file. +func (s *storeImpl) load(a asset.Asset, indent string) (*assetState, error) { + logrus.Debugf("%sLoading %q...", indent, a.Name()) + + // Stop descent if the asset has already been loaded. + if state, ok := s.assets[reflect.TypeOf(a)]; ok { + return state, nil + } + + // Load dependencies from on-disk. + anyParentsDirty := false + for _, d := range a.Dependencies() { + state, err := s.load(d, increaseIndent(indent)) + if err != nil { + return nil, err + } + if state.anyParentsDirty || state.source == onDiskSource { + anyParentsDirty = true + } + } + + // Try to load from on-disk. + var ( + onDiskAsset asset.WritableAsset + foundOnDisk bool + ) + if _, isWritable := a.(asset.WritableAsset); isWritable { + onDiskAsset = reflect.New(reflect.TypeOf(a).Elem()).Interface().(asset.WritableAsset) + var err error + foundOnDisk, err = onDiskAsset.Load(s.fileFetcher) + if err != nil { + return nil, errors.Wrapf(err, "failed to load asset %q", a.Name()) + } + } + + // Try to load from state file. + var ( + stateFileAsset asset.Asset + foundInStateFile bool + onDiskMatchesStateFile bool + ) + // Do not need to bother with loading from state file if any of the parents + // are dirty because the asset must be re-generated in this case. + if !anyParentsDirty { + foundInStateFile = s.isAssetInState(a) + if foundInStateFile { + stateFileAsset = reflect.New(reflect.TypeOf(a).Elem()).Interface().(asset.Asset) + if err := s.loadAssetFromState(stateFileAsset); err != nil { + return nil, errors.Wrapf(err, "failed to load asset %q from state file", a.Name()) + } + } + + if foundOnDisk && foundInStateFile { + logrus.Debugf("%sLoading %q from both state file and target directory", indent, a.Name()) + + // If the on-disk asset is the same as the one in the state file, there + // is no need to consider the one on disk and to mark the asset dirty. + onDiskMatchesStateFile = reflect.DeepEqual(onDiskAsset, stateFileAsset) + if onDiskMatchesStateFile { + logrus.Debugf("%sOn-disk %q matches asset in state file", indent, a.Name()) + } + } + } + + var ( + assetToStore asset.Asset + source assetSource + ) + switch { + // A parent is dirty. The asset must be re-generated. + case anyParentsDirty: + if foundOnDisk { + logrus.Warningf("%sDiscarding the %q that was provided in the target directory because its dependencies are dirty and it needs to be regenerated", indent, a.Name()) + } + source = unfetched + // The asset is on disk and that differs from what is in the source file. + // The asset is sourced from on disk. + case foundOnDisk && !onDiskMatchesStateFile: + logrus.Debugf("%sUsing %q loaded from target directory", indent, a.Name()) + assetToStore = onDiskAsset + source = onDiskSource + // The asset is in the state file. The asset is sourced from state file. + case foundInStateFile: + logrus.Debugf("%sUsing %q loaded from state file", indent, a.Name()) + assetToStore = stateFileAsset + source = stateFileSource + // There is no existing source for the asset. The asset will be generated. + default: + source = unfetched + } + + state := &assetState{ + asset: assetToStore, + source: source, + anyParentsDirty: anyParentsDirty, + presentOnDisk: foundOnDisk, + } + s.assets[reflect.TypeOf(a)] = state + return state, nil +} + +// purge deletes the on-disk assets that are consumed already. +// E.g., install-config.yaml will be deleted after fetching 'manifests'. +// The target asset is excluded. +func (s *storeImpl) purge(excluded asset.WritableAsset) error { + for _, assetState := range s.assets { + if !assetState.presentOnDisk { + continue + } + if reflect.TypeOf(assetState.asset) == reflect.TypeOf(excluded) { + continue + } + logrus.Infof("Consuming %q from target directory", assetState.asset.Name()) + if err := asset.DeleteAssetFromDisk(assetState.asset.(asset.WritableAsset), s.directory); err != nil { + return err + } + assetState.presentOnDisk = false + } + return nil +} + +func increaseIndent(indent string) string { + return indent + " " +} diff --git a/pkg/asset/store_test.go b/pkg/asset/store/store_test.go similarity index 82% rename from pkg/asset/store_test.go rename to pkg/asset/store/store_test.go index 8c13d44b2d8..7b99c1f33bd 100644 --- a/pkg/asset/store_test.go +++ b/pkg/asset/store/store_test.go @@ -1,4 +1,4 @@ -package asset +package store import ( "io/ioutil" @@ -7,6 +7,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + + "github.com/openshift/installer/pkg/asset" ) var ( @@ -14,30 +16,30 @@ var ( // asset store creates new assets by type, so the tests cannot store behavior // state in the assets themselves. generationLog []string - dependencies map[reflect.Type][]Asset + dependencies map[reflect.Type][]asset.Asset onDiskAssets map[reflect.Type]bool ) func clearAssetBehaviors() { generationLog = []string{} - dependencies = map[reflect.Type][]Asset{} + dependencies = map[reflect.Type][]asset.Asset{} onDiskAssets = map[reflect.Type]bool{} } -func dependenciesTestStoreAsset(a Asset) []Asset { +func dependenciesTestStoreAsset(a asset.Asset) []asset.Asset { return dependencies[reflect.TypeOf(a)] } -func generateTestStoreAsset(a Asset) error { +func generateTestStoreAsset(a asset.Asset) error { generationLog = append(generationLog, a.Name()) return nil } -func fileTestStoreAsset(a Asset) []*File { - return []*File{{Filename: a.Name()}} +func fileTestStoreAsset(a asset.Asset) []*asset.File { + return []*asset.File{{Filename: a.Name()}} } -func loadTestStoreAsset(a Asset) (bool, error) { +func loadTestStoreAsset(a asset.Asset) (bool, error) { return onDiskAssets[reflect.TypeOf(a)], nil } @@ -47,19 +49,19 @@ func (a *testStoreAssetA) Name() string { return "a" } -func (a *testStoreAssetA) Dependencies() []Asset { +func (a *testStoreAssetA) Dependencies() []asset.Asset { return dependenciesTestStoreAsset(a) } -func (a *testStoreAssetA) Generate(Parents) error { +func (a *testStoreAssetA) Generate(asset.Parents) error { return generateTestStoreAsset(a) } -func (a *testStoreAssetA) Files() []*File { +func (a *testStoreAssetA) Files() []*asset.File { return fileTestStoreAsset(a) } -func (a *testStoreAssetA) Load(FileFetcher) (bool, error) { +func (a *testStoreAssetA) Load(asset.FileFetcher) (bool, error) { return loadTestStoreAsset(a) } @@ -69,19 +71,19 @@ func (a *testStoreAssetB) Name() string { return "b" } -func (a *testStoreAssetB) Dependencies() []Asset { +func (a *testStoreAssetB) Dependencies() []asset.Asset { return dependenciesTestStoreAsset(a) } -func (a *testStoreAssetB) Generate(Parents) error { +func (a *testStoreAssetB) Generate(asset.Parents) error { return generateTestStoreAsset(a) } -func (a *testStoreAssetB) Files() []*File { +func (a *testStoreAssetB) Files() []*asset.File { return fileTestStoreAsset(a) } -func (a *testStoreAssetB) Load(FileFetcher) (bool, error) { +func (a *testStoreAssetB) Load(asset.FileFetcher) (bool, error) { return loadTestStoreAsset(a) } @@ -91,19 +93,19 @@ func (a *testStoreAssetC) Name() string { return "c" } -func (a *testStoreAssetC) Dependencies() []Asset { +func (a *testStoreAssetC) Dependencies() []asset.Asset { return dependenciesTestStoreAsset(a) } -func (a *testStoreAssetC) Generate(Parents) error { +func (a *testStoreAssetC) Generate(asset.Parents) error { return generateTestStoreAsset(a) } -func (a *testStoreAssetC) Files() []*File { +func (a *testStoreAssetC) Files() []*asset.File { return fileTestStoreAsset(a) } -func (a *testStoreAssetC) Load(FileFetcher) (bool, error) { +func (a *testStoreAssetC) Load(asset.FileFetcher) (bool, error) { return loadTestStoreAsset(a) } @@ -113,23 +115,23 @@ func (a *testStoreAssetD) Name() string { return "d" } -func (a *testStoreAssetD) Dependencies() []Asset { +func (a *testStoreAssetD) Dependencies() []asset.Asset { return dependenciesTestStoreAsset(a) } -func (a *testStoreAssetD) Generate(Parents) error { +func (a *testStoreAssetD) Generate(asset.Parents) error { return generateTestStoreAsset(a) } -func (a *testStoreAssetD) Files() []*File { +func (a *testStoreAssetD) Files() []*asset.File { return fileTestStoreAsset(a) } -func (a *testStoreAssetD) Load(FileFetcher) (bool, error) { +func (a *testStoreAssetD) Load(asset.FileFetcher) (bool, error) { return loadTestStoreAsset(a) } -func newTestStoreAsset(name string) Asset { +func newTestStoreAsset(name string) asset.Asset { switch name { case "a": return &testStoreAssetA{} @@ -262,16 +264,16 @@ func TestStoreFetch(t *testing.T) { t.Fatalf("failed to create temporary directory: %v", err) } defer os.RemoveAll(dir) - store := &StoreImpl{ + store := &storeImpl{ directory: dir, assets: map[reflect.Type]*assetState{}, } - assets := make(map[string]Asset, len(tc.assets)) + assets := make(map[string]asset.Asset, len(tc.assets)) for name := range tc.assets { assets[name] = newTestStoreAsset(name) } for name, deps := range tc.assets { - dependenciesOfAsset := make([]Asset, len(deps)) + dependenciesOfAsset := make([]asset.Asset, len(deps)) for i, d := range deps { dependenciesOfAsset[i] = assets[d] } @@ -361,15 +363,15 @@ func TestStoreFetchOnDiskAssets(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { clearAssetBehaviors() - store := &StoreImpl{ + store := &storeImpl{ assets: map[reflect.Type]*assetState{}, } - assets := make(map[string]Asset, len(tc.assets)) + assets := make(map[string]asset.Asset, len(tc.assets)) for name := range tc.assets { assets[name] = newTestStoreAsset(name) } for name, deps := range tc.assets { - dependenciesOfAsset := make([]Asset, len(deps)) + dependenciesOfAsset := make([]asset.Asset, len(deps)) for i, d := range deps { dependenciesOfAsset[i] = assets[d] }