Skip to content

Commit

Permalink
Open parts files one at a time
Browse files Browse the repository at this point in the history
When completing a file-based upload, open the parts files one at a time
and write them to the upload, closing each file before opening the next
one.

This is preferrable to opening them all at once and closing all files at
the end, because it consumes less file descriptors.

Updates #10660
  • Loading branch information
zmb3 committed Mar 8, 2022
1 parent 9a6369b commit d4b7512
Showing 1 changed file with 15 additions and 19 deletions.
34 changes: 15 additions & 19 deletions lib/events/filesessions/filestream.go
Original file line number Diff line number Diff line change
Expand Up @@ -124,30 +124,26 @@ func (h *Handler) CompleteUpload(ctx context.Context, upload events.StreamUpload
}
}()

files := make([]*os.File, 0, len(parts))
readers := make([]io.Reader, 0, len(parts))

defer func() {
for i := 0; i < len(files); i++ {
if err := files[i].Close(); err != nil {
h.WithError(err).Errorf("Failed to close file %q.", files[i].Name())
}
writePartToFile := func(path string) error {
file, err := os.Open(path)
if err != nil {
return err
}
}()
defer func() {
if err := file.Close(); err != nil {
h.WithError(err).Errorf("failed to close file %q", path)
}
}()

_, err = io.Copy(f, file)
return err
}

for _, part := range parts {
partPath := h.partPath(upload, part.Number)
file, err := os.Open(partPath)
if err != nil {
return trace.Wrap(err, "failed to open part file for upload")
if err := writePartToFile(partPath); err != nil {
return trace.Wrap(err)
}
files = append(files, file)
readers = append(readers, file)
}

_, err = io.Copy(f, io.MultiReader(readers...))
if err != nil {
return trace.Wrap(err)
}

err = h.Config.OnBeforeComplete(ctx, upload)
Expand Down

0 comments on commit d4b7512

Please sign in to comment.