Skip to content

Commit

Permalink
refactor: use a progress bar
Browse files Browse the repository at this point in the history
  • Loading branch information
daniel-palmer-gu committed Feb 3, 2024
1 parent 154e103 commit d306aa7
Show file tree
Hide file tree
Showing 3 changed files with 37 additions and 16 deletions.
8 changes: 8 additions & 0 deletions src/pkg/message/progress.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,3 +96,11 @@ func (p *ProgressBar) Errorf(err error, format string, a ...any) {
p.Stop()
WarnErrf(err, format, a...)
}

// GetCurrent returns the current total
func (p *ProgressBar) GetCurrent() int {
if p.progress != nil {
return p.progress.Current
}
return -1
}
8 changes: 4 additions & 4 deletions src/pkg/packager/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -335,22 +335,22 @@ func (p *Packager) archivePackage(destinationTarball string) error {
if err != nil {
return fmt.Errorf("unable to read the package archive: %w", err)
}
spinner.Successf("Package saved to %q", destinationTarball)

// Convert Megabytes to bytes.
chunkSize := p.cfg.CreateOpts.MaxPackageSizeMB * 1000 * 1000

// If a chunk size was specified and the package is larger than the chunk size, split it into chunks.
if p.cfg.CreateOpts.MaxPackageSizeMB > 0 && fi.Size() > int64(chunkSize) {
if fi.Size() / int64(chunkSize) > 999 {
if fi.Size()/int64(chunkSize) > 999 {
return fmt.Errorf("unable to split the package archive into multiple files: must be less than 1,000 files")
}
spinner.Updatef("Package is larger than %dMB, splitting into multiple files", p.cfg.CreateOpts.MaxPackageSizeMB)
_, _, err := utils.SplitFile(destinationTarball, chunkSize)
message.Notef("Package is larger than %dMB, splitting into multiple files", p.cfg.CreateOpts.MaxPackageSizeMB)
err := utils.SplitFile(destinationTarball, chunkSize)
if err != nil {
return fmt.Errorf("unable to split the package archive into multiple files: %w", err)
}
}
spinner.Successf("Package saved to %q", destinationTarball)
return nil
}

Expand Down
37 changes: 25 additions & 12 deletions src/pkg/utils/io.go
Original file line number Diff line number Diff line change
Expand Up @@ -333,7 +333,9 @@ func ReadFileByChunks(path string, chunkSizeBytes int) (chunks [][]byte, sha256s
// - fileNames: list of file paths srcFile was split across
// - sha256sum: sha256sum of the srcFile before splitting
// - err: any errors encountered
func SplitFile(srcFile string, chunkSizeBytes int) (fileNames []string, sha256sum string, err error) {
func SplitFile(srcFile string, chunkSizeBytes int) (err error) {
var fileNames []string
var sha256sum string
hash := sha256.New()

// Set buffer size to some multiple of 4096 KiB for modern file system cluster sizes
Expand All @@ -347,22 +349,27 @@ func SplitFile(srcFile string, chunkSizeBytes int) (fileNames []string, sha256su
// get file size
fi, err := os.Stat(srcFile)
if err != nil {
return fileNames, "", err
return err
}
fileSize := fi.Size()

// start progress bar
title := fmt.Sprintf("[0/%d] MB bytes written", fileSize/1000/1000)
progressBar := message.NewProgressBar(fileSize, title)
defer progressBar.Stop()

// open file
file, err := os.Open(srcFile)
defer file.Close()
if err != nil {
return fileNames, "", err
return err
}

// create file path starting from part 001
path := fmt.Sprintf("%s.part001", srcFile)
chunkFile, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return fileNames, "", err
return err
}
fileNames = append(fileNames, path)
defer chunkFile.Close()
Expand All @@ -378,7 +385,7 @@ func SplitFile(srcFile string, chunkSizeBytes int) (fileNames []string, sha256su
// At end of file, break out of loop
break
}
return fileNames, "", err
return err
}

// Pass data to hash
Expand All @@ -389,34 +396,40 @@ func SplitFile(srcFile string, chunkSizeBytes int) (fileNames []string, sha256su
// write the remaining chunk size to file
_, err := chunkFile.Write(buf[0:chunkBytesRemaining])
if err != nil {
return fileNames, "", err
return err
}

// create new file
path = fmt.Sprintf("%s.part%03d", srcFile, len(fileNames)+1)
chunkFile, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return fileNames, "", err
return err
}
fileNames = append(fileNames, path)
defer chunkFile.Close()

// write to new file where we left off
_, err = chunkFile.Write(buf[chunkBytesRemaining:bytesRead])
if err != nil {
return fileNames, "", err
return err
}

// set chunkBytesRemaining considering how many bytes are already written to new file
chunkBytesRemaining = chunkSizeBytes - (bufferSize - chunkBytesRemaining)
} else {
_, err := chunkFile.Write(buf[0:bytesRead])
if err != nil {
return fileNames, "", err
return err
}
chunkBytesRemaining = chunkBytesRemaining - bytesRead
}

// update progress bar
progressBar.Add(bufferSize)
title := fmt.Sprintf("[%d/%d] MB bytes written", progressBar.GetCurrent()/1000/1000, fileSize/1000/1000)
progressBar.UpdateTitle(title)
}
progressBar.Successf("Package split across %d files", len(fileNames))
file.Close()
_ = os.RemoveAll(srcFile)

Expand All @@ -430,17 +443,17 @@ func SplitFile(srcFile string, chunkSizeBytes int) (fileNames []string, sha256su
Sha256Sum: sha256sum,
})
if err != nil {
return fileNames, "", fmt.Errorf("unable to marshal the split package data: %w", err)
return fmt.Errorf("unable to marshal the split package data: %w", err)
}

// write header file
path = fmt.Sprintf("%s.part000", srcFile)
if err := os.WriteFile(path, jsonData, 0644); err != nil {
return fileNames, sha256sum, fmt.Errorf("unable to write the file %s: %w", path, err)
return fmt.Errorf("unable to write the file %s: %w", path, err)
}
fileNames = append(fileNames, path)

return fileNames, sha256sum, nil
return nil
}

// IsTextFile returns true if the given file is a text file.
Expand Down

0 comments on commit d306aa7

Please sign in to comment.