From 8abf94f2afa1892587417dafd491fa8f77669d83 Mon Sep 17 00:00:00 2001 From: Thorsten de Buhr Date: Fri, 23 Jun 2023 09:43:39 +0200 Subject: [PATCH] [cpackget] update-index: added option "-a" to download all missing PDSC files that are listed in index.pidx (#186) Command "update-index": - added "--all-pdsc-files", "-a" - downloads all PDSC files that are listed in index.pidx but are not present in .Web - if "sparse" is not set, the files that have already been in .Web are updated, if necessary (newly downloaded files are not checked) --------- Co-authored-by: Sourabh Mehta <73165318+soumeh01@users.noreply.github.com> --- cmd/commands/index.go | 2 +- cmd/commands/init.go | 2 +- cmd/commands/root.go | 2 +- cmd/commands/update_index.go | 5 +- cmd/installer/root.go | 216 ++++++++++++++++++++--------------- cmd/installer/root_test.go | 50 ++++++-- 6 files changed, 168 insertions(+), 109 deletions(-) diff --git a/cmd/commands/index.go b/cmd/commands/index.go index 6574f97..c3631ad 100644 --- a/cmd/commands/index.go +++ b/cmd/commands/index.go @@ -26,7 +26,7 @@ var IndexCmd = &cobra.Command{ log.Infof("Updating index %v", args) indexPath := args[0] installer.UnlockPackRoot() - err := installer.UpdatePublicIndex(indexPath, overwrite, true, false, viper.GetInt("concurrent-downloads"), viper.GetInt("timeout")) + err := installer.UpdatePublicIndex(indexPath, overwrite, true, false, false, viper.GetInt("concurrent-downloads"), viper.GetInt("timeout")) installer.LockPackRoot() return err }, diff --git a/cmd/commands/init.go b/cmd/commands/init.go index 6882b71..941da10 100644 --- a/cmd/commands/init.go +++ b/cmd/commands/init.go @@ -38,7 +38,7 @@ The index-url is mandatory. Ex "cpackget init --pack-root path/to/mypackroot htt } installer.UnlockPackRoot() - err = installer.UpdatePublicIndex(indexPath, true, true, initCmdFlags.downloadPdscFiles, viper.GetInt("concurrent-downloads"), viper.GetInt("timeout")) + err = installer.UpdatePublicIndex(indexPath, true, true, initCmdFlags.downloadPdscFiles, false, viper.GetInt("concurrent-downloads"), viper.GetInt("timeout")) installer.LockPackRoot() return err }, diff --git a/cmd/commands/root.go b/cmd/commands/root.go index 27be126..d3269b0 100644 --- a/cmd/commands/root.go +++ b/cmd/commands/root.go @@ -82,7 +82,7 @@ func configureInstaller(cmd *cobra.Command, args []string) error { // Exclude index updating commands to not double update if cmd.Name() != "init" && cmd.Name() != "index" && cmd.Name() != "update-index" { installer.UnlockPackRoot() - err = installer.UpdatePublicIndex(defaultPublicIndex, true, true, false, 0, 0) + err = installer.UpdatePublicIndex(defaultPublicIndex, true, true, false, false, 0, 0) if err != nil { return err } diff --git a/cmd/commands/update_index.go b/cmd/commands/update_index.go index 266174d..1cc59ac 100644 --- a/cmd/commands/update_index.go +++ b/cmd/commands/update_index.go @@ -15,6 +15,8 @@ import ( var updateIndexCmdFlags struct { // sparse indicates whether the update should update all installed pack's pdscs (false) or simply update the index (true) sparse bool + // downloadPdscFiles forces all pdsc files from the public index to be downloaded + downloadUpdatePdscFiles bool } var UpdateIndexCmd = &cobra.Command{ @@ -26,7 +28,7 @@ var UpdateIndexCmd = &cobra.Command{ RunE: func(cmd *cobra.Command, args []string) error { log.Infof("Updating public index") installer.UnlockPackRoot() - err := installer.UpdatePublicIndex("", true, updateIndexCmdFlags.sparse, false, viper.GetInt("concurrent-downloads"), viper.GetInt("timeout")) + err := installer.UpdatePublicIndex("", true, updateIndexCmdFlags.sparse, false, updateIndexCmdFlags.downloadUpdatePdscFiles, viper.GetInt("concurrent-downloads"), viper.GetInt("timeout")) installer.LockPackRoot() return err }, @@ -46,4 +48,5 @@ By default it will also check if all PDSC files under .Web/ need update as well. func init() { UpdateIndexCmd.Flags().BoolVarP(&updateIndexCmdFlags.sparse, "sparse", "s", false, "avoid updating the pdsc files within .Web/ folder") + UpdateIndexCmd.Flags().BoolVarP(&updateIndexCmdFlags.downloadUpdatePdscFiles, "all-pdsc-files", "a", false, "updates/downloads all the latest .pdsc files from the public index") } diff --git a/cmd/installer/root.go b/cmd/installer/root.go index a1cf9e2..14b06cb 100644 --- a/cmd/installer/root.go +++ b/cmd/installer/root.go @@ -285,8 +285,110 @@ func RemovePdsc(pdscPath string) error { return Installation.touchPackIdx() } +// Workaround wrapper function to still log errors +func massDownloadPdscFiles(pdscTag xml.PdscTag, skipInstalledPdscFiles bool, wg *sync.WaitGroup, timeout int) { + if err := Installation.downloadPdscFile(pdscTag, skipInstalledPdscFiles, wg, timeout); err != nil { + log.Error(err) + } +} + +func DownloadPDSCFiles(skipInstalledPdscFiles bool, concurrency int, timeout int) error { + var wg sync.WaitGroup + log.Info("Downloading all PDSC files available on the public index") + if err := Installation.PublicIndexXML.Read(); err != nil { + return err + } + + pdscTags := Installation.PublicIndexXML.ListPdscTags() + if len(pdscTags) == 0 { + log.Info("(no packs in public index)") + return nil + } + + queue := concurrency + for _, pdscTag := range pdscTags { + if concurrency == 0 || len(pdscTags) <= concurrency { + if err := Installation.downloadPdscFile(pdscTag, skipInstalledPdscFiles, nil, timeout); err != nil { + log.Error(err) + } + } else { + // Don't queue more downloads than specified + if queue == 0 { + if err := Installation.downloadPdscFile(pdscTag, skipInstalledPdscFiles, nil, timeout); err != nil { + log.Error(err) + } + wg.Add(concurrency) + queue = concurrency + } else { + wg.Add(1) + go massDownloadPdscFiles(pdscTag, skipInstalledPdscFiles, &wg, timeout) + queue-- + } + } + } + return nil +} + +func UpdateInstalledPDSCFiles(pidxXML *xml.PidxXML, concurrency int, timeout int) error { + var wg sync.WaitGroup + log.Info("Updating PDSC files of installed packs referenced in index.pidx") + pdscFiles, err := utils.ListDir(Installation.WebDir, ".pdsc$") + if err != nil { + return err + } + + queue := concurrency + for _, pdscFile := range pdscFiles { + log.Debugf("Checking if \"%s\" needs updating", pdscFile) + pdscXML := xml.NewPdscXML(pdscFile) + err := pdscXML.Read() + if err != nil { + log.Errorf("%s: %v", pdscFile, err) + continue + } + + searchTag := xml.PdscTag{ + Vendor: pdscXML.Vendor, + Name: pdscXML.Name, + } + + // Warn the user if the pack is no longer present in index.pidx + tags := pidxXML.FindPdscTags(searchTag) + if len(tags) == 0 { + log.Warnf("The pack %s::%s is no longer present in the updated index.pidx, deleting PDSC file \"%v\"", pdscXML.Vendor, pdscXML.Name, pdscFile) + utils.UnsetReadOnly(pdscFile) + os.Remove(pdscFile) + continue + } + + versionInIndex := tags[0].Version + latestVersion := pdscXML.LatestVersion() + if versionInIndex != latestVersion { + log.Infof("%s::%s can be upgraded from \"%s\" to \"%s\"", pdscXML.Vendor, pdscXML.Name, latestVersion, versionInIndex) + if concurrency == 0 || len(pdscFiles) <= concurrency { + if err := Installation.downloadPdscFile(tags[0], false, nil, timeout); err != nil { + log.Error(err) + } + } else { + if queue == 0 { + if err := Installation.downloadPdscFile(tags[0], false, nil, timeout); err != nil { + log.Error(err) + } + wg.Add(concurrency) + queue = concurrency + } else { + wg.Add(1) + go massDownloadPdscFiles(tags[0], false, &wg, timeout) + queue-- + } + } + } + } + return nil +} + // UpdatePublicIndex receives a index path and place it under .Web/index.pidx. -func UpdatePublicIndex(indexPath string, overwrite bool, sparse bool, downloadPdsc bool, concurrency int, timeout int) error { +func UpdatePublicIndex(indexPath string, overwrite bool, sparse bool, downloadPdsc bool, downloadRemainingPdscFiles bool, concurrency int, timeout int) error { // TODO: Remove overwrite when cpackget v1 gets released if !overwrite { return errs.ErrCannotOverwritePublicIndex @@ -337,102 +439,24 @@ func UpdatePublicIndex(indexPath string, overwrite bool, sparse bool, downloadPd } utils.SetReadOnly(Installation.PublicIndex) - // Workaround wrapper function to still log errors - // and not make the linter angry - massDownloadPdscFiles := func(pdscTag xml.PdscTag, wg *sync.WaitGroup, timeout int) { - if err := Installation.downloadPdscFile(pdscTag, wg, timeout); err != nil { - log.Error(err) - } - } if downloadPdsc { - var wg sync.WaitGroup - log.Info("Downloading all PDSC files available on the public index") - if err := Installation.PublicIndexXML.Read(); err != nil { + err = DownloadPDSCFiles(false, concurrency, timeout) + if err != nil { return err } - - pdscTags := Installation.PublicIndexXML.ListPdscTags() - if len(pdscTags) == 0 { - log.Info("(no packs in public index)") - return nil - } - - queue := concurrency - for _, pdscTag := range pdscTags { - if concurrency == 0 || len(pdscTags) <= concurrency { - if err := Installation.downloadPdscFile(pdscTag, nil, timeout); err != nil { - log.Error(err) - } - } else { - // Don't queue more downloads than specified - if queue == 0 { - if err := Installation.downloadPdscFile(pdscTag, nil, timeout); err != nil { - log.Error(err) - } - wg.Add(concurrency) - queue = concurrency - } else { - wg.Add(1) - go massDownloadPdscFiles(pdscTag, &wg, timeout) - queue-- - } - } - } } + if !sparse { - var wg sync.WaitGroup - log.Info("Updating PDSC files of installed packs referenced in index.pidx") - pdscFiles, err := utils.ListDir(Installation.WebDir, ".pdsc$") + err = UpdateInstalledPDSCFiles(pidxXML, concurrency, timeout) if err != nil { return err } + } - queue := concurrency - for _, pdscFile := range pdscFiles { - log.Debugf("Checking if \"%s\" needs updating", pdscFile) - pdscXML := xml.NewPdscXML(pdscFile) - err := pdscXML.Read() - if err != nil { - log.Errorf("%s: %v", pdscFile, err) - continue - } - - searchTag := xml.PdscTag{ - Vendor: pdscXML.Vendor, - Name: pdscXML.Name, - } - - // Warn the user if the pack is no longer present in index.pidx - tags := pidxXML.FindPdscTags(searchTag) - if len(tags) == 0 { - log.Warnf("The pack %s::%s is no longer present in the updated index.pidx, deleting PDSC file \"%v\"", pdscXML.Vendor, pdscXML.Name, pdscFile) - utils.UnsetReadOnly(pdscFile) - os.Remove(pdscFile) - continue - } - - versionInIndex := tags[0].Version - latestVersion := pdscXML.LatestVersion() - if versionInIndex != latestVersion { - log.Infof("%s::%s can be upgraded from \"%s\" to \"%s\"", pdscXML.Vendor, pdscXML.Name, latestVersion, versionInIndex) - if concurrency == 0 || len(pdscFiles) <= concurrency { - if err := Installation.downloadPdscFile(tags[0], nil, timeout); err != nil { - log.Error(err) - } - } else { - if queue == 0 { - if err := Installation.downloadPdscFile(tags[0], nil, timeout); err != nil { - log.Error(err) - } - wg.Add(concurrency) - queue = concurrency - } else { - wg.Add(1) - go massDownloadPdscFiles(tags[0], &wg, timeout) - queue-- - } - } - } + if downloadRemainingPdscFiles { + err = DownloadPDSCFiles(true, concurrency, timeout) + if err != nil { + return err } } @@ -1042,17 +1066,26 @@ func (p *PacksInstallationType) packIsPublic(pack *PackType, timeout int) (bool, // Sometimes a pidx file might have multiple pdsc tags for same key // which is not the case here, so we'll take only the first one pdscTag := pdscTags[0] - return true, p.downloadPdscFile(pdscTag, nil, timeout) + return true, p.downloadPdscFile(pdscTag, false, nil, timeout) } // downloadPdscFile takes in a xml.PdscTag containing URL, Vendor and Name of the pack // so it can be downloaded into .Web/ -func (p *PacksInstallationType) downloadPdscFile(pdscTag xml.PdscTag, wg *sync.WaitGroup, timeout int) error { +func (p *PacksInstallationType) downloadPdscFile(pdscTag xml.PdscTag, skipInstalledPdscFiles bool, wg *sync.WaitGroup, timeout int) error { // Only change use if it's not a concurrent download if wg != nil { defer wg.Done() } + basePdscFile := fmt.Sprintf("%s.%s.pdsc", pdscTag.Vendor, pdscTag.Name) + pdscFilePath := filepath.Join(p.WebDir, basePdscFile) + + if skipInstalledPdscFiles { + if utils.FileExists(pdscFilePath) { + return nil + } + } + pdscURL := pdscTag.URL // switch to keil.com cache for PDSC file @@ -1061,9 +1094,6 @@ func (p *PacksInstallationType) downloadPdscFile(pdscTag xml.PdscTag, wg *sync.W pdscURL = KeilDefaultPackRoot } - basePdscFile := fmt.Sprintf("%s.%s.pdsc", pdscTag.Vendor, pdscTag.Name) - pdscFilePath := filepath.Join(p.WebDir, basePdscFile) - log.Debugf("Downloading %s from \"%s\"", basePdscFile, pdscURL) pdscFileURL, err := url.Parse(pdscURL) diff --git a/cmd/installer/root_test.go b/cmd/installer/root_test.go index f5d42cb..1b49753 100644 --- a/cmd/installer/root_test.go +++ b/cmd/installer/root_test.go @@ -10,6 +10,7 @@ import ( "net/http" "net/http/httptest" "os" + "path" "path/filepath" "strings" "testing" @@ -382,6 +383,7 @@ func TestUpdatePublicIndex(t *testing.T) { var Overwrite = true var Sparse = true var DownloadPdsc = false + var DownloadRemainingPdscFiles = true var Concurrency = 0 // Re-enable this test when a flag --enforce-security is implemented @@ -416,7 +418,7 @@ func TestUpdatePublicIndex(t *testing.T) { indexPath := server.URL() + "this-file-does-not-exist" - err := installer.UpdatePublicIndex(indexPath, Overwrite, Sparse, DownloadPdsc, Concurrency, Timeout) + err := installer.UpdatePublicIndex(indexPath, Overwrite, Sparse, DownloadPdsc, !DownloadRemainingPdscFiles, Concurrency, Timeout) assert.NotNil(err) assert.Equal(errs.ErrBadRequest, err) @@ -435,7 +437,7 @@ func TestUpdatePublicIndex(t *testing.T) { indexServer.AddRoute("index.pidx", indexContent) indexPath := indexServer.URL() + "index.pidx" - err = installer.UpdatePublicIndex(indexPath, Overwrite, Sparse, DownloadPdsc, Concurrency, Timeout) + err = installer.UpdatePublicIndex(indexPath, Overwrite, Sparse, DownloadPdsc, !DownloadRemainingPdscFiles, Concurrency, Timeout) assert.NotNil(err) assert.Equal(err.Error(), "XML syntax error on line 3: unexpected EOF") @@ -453,7 +455,7 @@ func TestUpdatePublicIndex(t *testing.T) { indexServer.AddRoute("index.pidx", indexContent) indexPath := indexServer.URL() + "index.pidx" - err = installer.UpdatePublicIndex(indexPath, Overwrite, Sparse, DownloadPdsc, Concurrency, Timeout) + err = installer.UpdatePublicIndex(indexPath, Overwrite, Sparse, DownloadPdsc, !DownloadRemainingPdscFiles, Concurrency, Timeout) assert.Nil(err) @@ -465,6 +467,30 @@ func TestUpdatePublicIndex(t *testing.T) { assert.Equal(copied, indexContent) }) + t.Run("test add remaining PDSC from index.pidx", func(t *testing.T) { + localTestingDir := "test-add-remaining-PDSC-from-index" + assert.Nil(installer.SetPackRoot(localTestingDir, CreatePackRoot)) + installer.UnlockPackRoot() + defer os.RemoveAll(localTestingDir) + + indexContent, err := os.ReadFile(samplePublicIndexLocalhostPdsc) + assert.Nil(err) + indexServer := NewServer() + // The psd URL needs to be updated as it's not known beforehand + updatedIndex := []byte(strings.Replace(string(indexContent), "https://127.0.0.1", indexServer.URL(), -1)) + indexServer.AddRoute("index.pidx", updatedIndex) + indexPath := indexServer.URL() + "index.pidx" + + pdscContent, err := os.ReadFile(publicLocalPack123Pdsc) + assert.Nil(err) + indexServer.AddRoute("TheVendor.PublicLocalPack.pdsc", pdscContent) + + err = installer.UpdatePublicIndex(indexPath, Overwrite, Sparse, DownloadPdsc, DownloadRemainingPdscFiles, Concurrency, Timeout) + assert.Nil(err) + + assert.True(utils.FileExists(path.Join(localTestingDir, ".Web", "TheVendor.PublicLocalPack.pdsc"))) + }) + t.Run("test update-index delete pdsc when not in index.pidx", func(t *testing.T) { localTestingDir := "test-update-index-delete-pdsc-when-not-in-index" assert.Nil(installer.SetPackRoot(localTestingDir, CreatePackRoot)) @@ -477,7 +503,7 @@ func TestUpdatePublicIndex(t *testing.T) { indexServer.AddRoute("index.pidx", indexContent) indexPath := indexServer.URL() + "index.pidx" - err = installer.UpdatePublicIndex(indexPath, Overwrite, Sparse, DownloadPdsc, Concurrency, Timeout) + err = installer.UpdatePublicIndex(indexPath, Overwrite, Sparse, DownloadPdsc, !DownloadRemainingPdscFiles, Concurrency, Timeout) assert.Nil(err) publicIndex := installer.Installation.PublicIndex @@ -491,7 +517,7 @@ func TestUpdatePublicIndex(t *testing.T) { err = utils.CopyFile(pdscPackNotInIndex, filepath.Join(localTestingDir, ".Web", "TheVendor.PackNotInIndex.pdsc")) assert.Nil(err) - err = installer.UpdatePublicIndex(indexPath, Overwrite, false, DownloadPdsc, Concurrency, Timeout) + err = installer.UpdatePublicIndex(indexPath, Overwrite, false, DownloadPdsc, !DownloadRemainingPdscFiles, Concurrency, Timeout) assert.Nil(err) assert.False(utils.FileExists(filepath.Join(localTestingDir, ".Web", "TheVendor.PackNotInIndex.pdsc"))) @@ -506,7 +532,7 @@ func TestUpdatePublicIndex(t *testing.T) { indexContent, err := os.ReadFile(samplePublicIndex) assert.Nil(err) - assert.Nil(installer.UpdatePublicIndex(samplePublicIndex, Overwrite, Sparse, DownloadPdsc, Concurrency, Timeout)) + assert.Nil(installer.UpdatePublicIndex(samplePublicIndex, Overwrite, Sparse, DownloadPdsc, !DownloadRemainingPdscFiles, Concurrency, Timeout)) assert.True(utils.FileExists(installer.Installation.PublicIndex)) @@ -530,7 +556,7 @@ func TestUpdatePublicIndex(t *testing.T) { indexServer.AddRoute("index.pidx", indexContent) indexPath := indexServer.URL() + "index.pidx" - err = installer.UpdatePublicIndex(indexPath, !Overwrite, Sparse, DownloadPdsc, Concurrency, Timeout) + err = installer.UpdatePublicIndex(indexPath, !Overwrite, Sparse, DownloadPdsc, !DownloadRemainingPdscFiles, Concurrency, Timeout) assert.NotNil(err) assert.Equal(errs.ErrCannotOverwritePublicIndex, err) @@ -550,7 +576,7 @@ func TestUpdatePublicIndex(t *testing.T) { indexServer.AddRoute("index.pidx", indexContent) indexPath := indexServer.URL() + "index.pidx" - err = installer.UpdatePublicIndex(indexPath, Overwrite, Sparse, DownloadPdsc, Concurrency, Timeout) + err = installer.UpdatePublicIndex(indexPath, Overwrite, Sparse, DownloadPdsc, !DownloadRemainingPdscFiles, Concurrency, Timeout) assert.Nil(err) assert.True(utils.FileExists(installer.Installation.PublicIndex)) @@ -579,7 +605,7 @@ func TestUpdatePublicIndex(t *testing.T) { assert.Nil(err) indexServer.AddRoute("TheVendor.PublicLocalPack.pdsc", pdscContent) - err = installer.UpdatePublicIndex(indexPath, Overwrite, Sparse, !DownloadPdsc, Concurrency, Timeout) + err = installer.UpdatePublicIndex(indexPath, Overwrite, Sparse, !DownloadPdsc, !DownloadRemainingPdscFiles, Concurrency, Timeout) assert.Nil(err) assert.True(utils.FileExists(installer.Installation.PublicIndex)) @@ -612,7 +638,7 @@ func TestUpdatePublicIndex(t *testing.T) { indexServer.AddRoute(publicConcurrentLocalPdscBase+fmt.Sprint(i)+".pdsc", pdscContent) } - err = installer.UpdatePublicIndex(indexPath, Overwrite, Sparse, !DownloadPdsc, 5, Timeout) + err = installer.UpdatePublicIndex(indexPath, Overwrite, Sparse, !DownloadPdsc, !DownloadRemainingPdscFiles, 5, Timeout) assert.Nil(err) assert.True(utils.FileExists(installer.Installation.PublicIndex)) @@ -683,7 +709,7 @@ func TestUpdatePublicIndex(t *testing.T) { indexServer.AddRoute(filepath.Base(publicLocalPack124Pdsc), pdscContent) - assert.Nil(installer.UpdatePublicIndex("", Overwrite, !Sparse, DownloadPdsc, Concurrency, Timeout)) + assert.Nil(installer.UpdatePublicIndex("", Overwrite, !Sparse, DownloadPdsc, !DownloadRemainingPdscFiles, Concurrency, Timeout)) // Make sure index.pidx exists and it is updated assert.FileExists(installer.Installation.PublicIndex) @@ -751,7 +777,7 @@ func TestUpdatePublicIndex(t *testing.T) { indexServer.AddRoute(pdsc, pdscContent) } - assert.Nil(installer.UpdatePublicIndex("", Overwrite, !Sparse, DownloadPdsc, 5, Timeout)) + assert.Nil(installer.UpdatePublicIndex("", Overwrite, !Sparse, DownloadPdsc, !DownloadRemainingPdscFiles, 5, Timeout)) // Make sure index.pidx exists and it is updated assert.FileExists(installer.Installation.PublicIndex)