Skip to content

Commit

Permalink
[FEAT] Read transferred bytes per upload
Browse files Browse the repository at this point in the history
Signed-off-by: rita.canavarro <[email protected]>
  • Loading branch information
rita.canavarro committed Aug 7, 2023
1 parent ad6113e commit 7ac6c07
Show file tree
Hide file tree
Showing 2 changed files with 39 additions and 2 deletions.
22 changes: 20 additions & 2 deletions objstore.go
Original file line number Diff line number Diff line change
Expand Up @@ -457,11 +457,11 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu
bkt.opsDuration.WithLabelValues(op)
bkt.opsFetchedBytes.WithLabelValues(op)
}
// fetched bytes only relevant for get and getrange
// fetched bytes only relevant for get, getrange and upload
for _, op := range []string{
OpGet,
OpGetRange,
// TODO: Add uploads
OpUpload,
} {
bkt.opsTransferredBytes.WithLabelValues(op)
}
Expand Down Expand Up @@ -602,8 +602,26 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err
}
return err
}

b.lastSuccessfulUploadTime.WithLabelValues(b.bkt.Name()).SetToCurrentTime()
b.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds())

rc := nopCloserWithObjectSize{
Reader: r,
}

trc := newTimingReadCloser(
rc,
op,
b.opsDuration,
b.opsFailures,
b.isOpFailureExpected,
b.opsFetchedBytes,
b.opsTransferredBytes,
)

trc.Close()

return nil
}

Expand Down
19 changes: 19 additions & 0 deletions objstore_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,25 @@ func TestDownloadUploadDirConcurrency(t *testing.T) {
objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get_range",le="+Inf"} 0
objstore_bucket_operation_transferred_bytes_sum{bucket="",operation="get_range"} 0
objstore_bucket_operation_transferred_bytes_count{bucket="",operation="get_range"} 0
objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="32768"} 3
objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="65536"} 3
objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="131072"} 3
objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="262144"} 3
objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="524288"} 3
objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="1.048576e+06"} 3
objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="2.097152e+06"} 3
objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="4.194304e+06"} 3
objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="8.388608e+06"} 3
objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="1.6777216e+07"} 3
objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="3.3554432e+07"} 3
objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="6.7108864e+07"} 3
objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="1.34217728e+08"} 3
objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="2.68435456e+08"} 3
objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="5.36870912e+08"} 3
objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="1.073741824e+09"} 3
objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="+Inf"} 3
objstore_bucket_operation_transferred_bytes_sum{bucket="",operation="upload"} 0
objstore_bucket_operation_transferred_bytes_count{bucket="",operation="upload"} 3
`), `objstore_bucket_operation_transferred_bytes`))

testutil.Ok(t, UploadDir(context.Background(), log.NewNopLogger(), m, tempDir, "/dir-copy", WithUploadConcurrency(10)))
Expand Down

0 comments on commit 7ac6c07

Please sign in to comment.