Skip to content

Commit

Permalink
Open parts files one at a time
Browse files Browse the repository at this point in the history
When completing a file-based upload, open the parts files one at a time
and write them to the upload, closing each file before opening the next
one.

This is preferrable to opening them all at once and closing all files at
the end, because it consumes less file descriptors.

Updates #10660
  • Loading branch information
zmb3 committed Mar 3, 2022
1 parent 29d6ed2 commit 221b9ce
Showing 1 changed file with 15 additions and 19 deletions.
34 changes: 15 additions & 19 deletions lib/events/filesessions/filestream.go
Original file line number Diff line number Diff line change
Expand Up @@ -124,30 +124,26 @@ func (h *Handler) CompleteUpload(ctx context.Context, upload events.StreamUpload
}
}()

files := make([]*os.File, 0, len(parts))
readers := make([]io.Reader, 0, len(parts))

defer func() {
for i := 0; i < len(files); i++ {
if err := files[i].Close(); err != nil {
h.WithError(err).Errorf("Failed to close file %q.", files[i].Name())
}
writePartToFile := func(path string) error {
file, err := os.Open(path)
if err != nil {
return err
}
}()
defer func() {
if err := file.Close(); err != nil {
h.WithError(err).Errorf("failed to close file %q", path)
}
}()

_, err = io.Copy(f, file)
return err
}

for _, part := range parts {
partPath := h.partPath(upload, part.Number)
file, err := os.Open(partPath)
if err != nil {
return trace.ConvertSystemError(err)
if err := writePartToFile(partPath); err != nil {
return trace.Wrap(err)
}
files = append(files, file)
readers = append(readers, file)
}

_, err = io.Copy(f, io.MultiReader(readers...))
if err != nil {
return trace.Wrap(err)
}

err = h.Config.OnBeforeComplete(ctx, upload)
Expand Down

0 comments on commit 221b9ce

Please sign in to comment.