Skip to content

Commit

Permalink
[content-service] Log filesystem usage error
Browse files Browse the repository at this point in the history
  • Loading branch information
Furisto authored and roboquat committed Dec 5, 2022
1 parent 0298079 commit d0ccc7f
Show file tree
Hide file tree
Showing 23 changed files with 84 additions and 153 deletions.
14 changes: 7 additions & 7 deletions components/content-service/pkg/initializer/download.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,9 @@ func (ws *fileDownloadInitializer) Run(ctx context.Context, mappings []archive.I
span, ctx := opentracing.StartSpanFromContext(ctx, "FileDownloadInitializer.Run")
defer tracing.FinishSpan(span, &err)
start := time.Now()
initialSize, diskErr := getDiskUsage()
if diskErr != nil {
log.WithError(err).Error("could not get disk usage")
initialSize, fsErr := getFsUsage()
if fsErr != nil {
log.WithError(fsErr).Error("could not get disk usage")
}

for _, info := range ws.FilesInfos {
Expand All @@ -62,10 +62,10 @@ func (ws *fileDownloadInitializer) Run(ctx context.Context, mappings []archive.I
}
}

if diskErr == nil {
currentSize, diskErr := getDiskUsage()
if diskErr != nil {
log.WithError(err).Error("could not get disk usage")
if fsErr == nil {
currentSize, fsErr := getFsUsage()
if fsErr != nil {
log.WithError(fsErr).Error("could not get disk usage")
}

metrics = csapi.InitializerMetrics{csapi.InitializerMetric{
Expand Down
14 changes: 7 additions & 7 deletions components/content-service/pkg/initializer/git.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,9 @@ func (ws *GitInitializer) Run(ctx context.Context, mappings []archive.IDMapping)
span.SetTag("isGitWS", isGitWS)
defer tracing.FinishSpan(span, &err)
start := time.Now()
initialSize, diskErr := getDiskUsage()
if diskErr != nil {
log.WithError(err).Error("could not get disk usage")
initialSize, fsErr := getFsUsage()
if fsErr != nil {
log.WithError(fsErr).Error("could not get disk usage")
}

src = csapi.WorkspaceInitFromOther
Expand Down Expand Up @@ -177,10 +177,10 @@ func (ws *GitInitializer) Run(ctx context.Context, mappings []archive.IDMapping)

log.WithField("stage", "init").WithField("location", ws.Location).Debug("Git operations complete")

if diskErr == nil {
currentSize, diskErr := getDiskUsage()
if diskErr != nil {
log.WithError(err).Error("could not get disk usage")
if fsErr == nil {
currentSize, fsErr := getFsUsage()
if fsErr != nil {
log.WithError(fsErr).Error("could not get disk usage")
}

stats = csapi.InitializerMetrics{csapi.InitializerMetric{
Expand Down
37 changes: 21 additions & 16 deletions components/content-service/pkg/initializer/initializer.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,9 @@ func (e CompositeInitializer) Run(ctx context.Context, mappings []archive.IDMapp
span, ctx := opentracing.StartSpanFromContext(ctx, "CompositeInitializer.Run")
defer tracing.FinishSpan(span, &err)
start := time.Now()
initialSize, diskErr := getDiskUsage()
if diskErr != nil {
log.WithError(err).Error("could not get disk usage")
initialSize, fsErr := getFsUsage()
if fsErr != nil {
log.WithError(fsErr).Error("could not get disk usage")
}

total := []csapi.InitializerMetric{}
Expand All @@ -81,10 +81,10 @@ func (e CompositeInitializer) Run(ctx context.Context, mappings []archive.IDMapp
total = append(total, stats...)
}

if diskErr == nil {
currentSize, diskErr := getDiskUsage()
if diskErr != nil {
log.WithError(err).Error("could not get disk usage")
if fsErr == nil {
currentSize, fsErr := getFsUsage()
if fsErr != nil {
log.WithError(fsErr).Error("could not get disk usage")
}

total = append(total, csapi.InitializerMetric{
Expand Down Expand Up @@ -215,9 +215,9 @@ func (bi *fromBackupInitializer) Run(ctx context.Context, mappings []archive.IDM
}

start := time.Now()
initialSize, diskErr := getDiskUsage()
if diskErr != nil {
log.WithError(err).Error("could not get disk usage")
initialSize, fsErr := getFsUsage()
if fsErr != nil {
log.WithError(fsErr).Error("could not get disk usage")
}

hasBackup, err := bi.RemoteStorage.Download(ctx, bi.Location, storage.DefaultBackup, mappings)
Expand All @@ -231,10 +231,10 @@ func (bi *fromBackupInitializer) Run(ctx context.Context, mappings []archive.IDM
return src, nil, xerrors.Errorf("cannot restore backup: %w", err)
}

if diskErr == nil {
currentSize, diskErr := getDiskUsage()
if diskErr != nil {
log.WithError(err).Error("could not get disk usage")
if fsErr == nil {
currentSize, fsErr := getFsUsage()
if fsErr != nil {
log.WithError(fsErr).Error("could not get disk usage")
}

stats = csapi.InitializerMetrics{csapi.InitializerMetric{
Expand Down Expand Up @@ -574,9 +574,14 @@ func PlaceWorkspaceReadyFile(ctx context.Context, wspath string, initsrc csapi.W
return nil
}

func getDiskUsage() (uint64, error) {
func getFsUsage() (uint64, error) {
var stat syscall.Statfs_t
err := syscall.Statfs("/workspace", &stat)

err := syscall.Statfs("/dst", &stat)
if os.IsNotExist(err) {
err = syscall.Statfs("/workspace", &stat)
}

if err != nil {
return 0, err
}
Expand Down
12 changes: 6 additions & 6 deletions components/content-service/pkg/initializer/prebuild.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,9 @@ func (p *PrebuildInitializer) Run(ctx context.Context, mappings []archive.IDMapp
span, ctx := opentracing.StartSpanFromContext(ctx, "PrebuildInitializer")
defer tracing.FinishSpan(span, &err)
startTime := time.Now()
initialSize, diskErr := getDiskUsage()
if diskErr != nil {
log.WithError(err).Error("could not get disk usage")
initialSize, fsErr := getFsUsage()
if fsErr != nil {
log.WithError(fsErr).Error("could not get disk usage")
}

var spandata []tracelog.Field
Expand Down Expand Up @@ -109,10 +109,10 @@ func (p *PrebuildInitializer) Run(ctx context.Context, mappings []archive.IDMapp
}
log.Debug("Initialized workspace with prebuilt snapshot")

if diskErr == nil {
currentSize, err := getDiskUsage()
if fsErr == nil {
currentSize, fsErr := getFsUsage()
if err != nil {
log.WithError(err).Error("could not get disk usage")
log.WithError(fsErr).Error("could not get disk usage")
}

stats = append(stats, csapi.InitializerMetric{
Expand Down
14 changes: 7 additions & 7 deletions components/content-service/pkg/initializer/snapshot.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,9 @@ func (s *SnapshotInitializer) Run(ctx context.Context, mappings []archive.IDMapp
span.SetTag("snapshot", s.Snapshot)
defer tracing.FinishSpan(span, &err)
start := time.Now()
initialSize, diskErr := getDiskUsage()
if diskErr != nil {
log.WithError(err).Error("could not get disk usage")
initialSize, fsErr := getFsUsage()
if fsErr != nil {
log.WithError(fsErr).Error("could not get disk usage")
}

src = csapi.WorkspaceInitFromBackup
Expand All @@ -53,10 +53,10 @@ func (s *SnapshotInitializer) Run(ctx context.Context, mappings []archive.IDMapp
return src, nil, xerrors.Errorf("did not find snapshot %s", s.Snapshot)
}

if diskErr == nil {
currentSize, diskErr := getDiskUsage()
if diskErr == nil {
log.WithError(err).Error("could not get disk usage")
if fsErr == nil {
currentSize, fsErr := getFsUsage()
if fsErr == nil {
log.WithError(fsErr).Error("could not get disk usage")
}

stats = csapi.InitializerMetrics{csapi.InitializerMetric{
Expand Down
2 changes: 1 addition & 1 deletion components/supervisor/pkg/metrics/reporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ func NewGrpcMetricsReporter(gitpodHost string) *GrpcMetricsReporter {
"grpc_server_started_total": true,
"grpc_server_handling_seconds": true,
"supervisor_ide_ready_duration_total": true,
"initializer_bytes_second": true,
"supervisor_initializer_bytes_second": true,
},
values: make(map[string]float64),
addCounter: func(name string, labels map[string]string, value uint64) {
Expand Down
4 changes: 2 additions & 2 deletions components/ws-daemon/pkg/content/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ import (
type metrics struct {
BackupWaitingTimeHist prometheus.Histogram
BackupWaitingTimeoutCounter prometheus.Counter
InitializerHistogram prometheus.HistogramVec
InitializerHistogram *prometheus.HistogramVec
}

// WorkspaceService implements the InitService and WorkspaceService
Expand Down Expand Up @@ -130,7 +130,7 @@ func NewWorkspaceService(ctx context.Context, cfg Config, kubernetesNamespace st
metrics: &metrics{
BackupWaitingTimeHist: waitingTimeHist,
BackupWaitingTimeoutCounter: waitingTimeoutCounter,
InitializerHistogram: *initializerHistogram,
InitializerHistogram: initializerHistogram,
},
// we permit five concurrent backups at any given time, hence the five in the channel
backupWorkspaceLimiter: make(chan struct{}, 5),
Expand Down
9 changes: 2 additions & 7 deletions install/installer/cmd/testdata/render/aws-setup/output.golden

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 2 additions & 7 deletions install/installer/cmd/testdata/render/gcp-setup/output.golden

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 2 additions & 7 deletions install/installer/cmd/testdata/render/kind-ide/output.golden

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit d0ccc7f

Please sign in to comment.