Skip to content

Commit

Permalink
Change UploadFile to UploadUsageReport
Browse files Browse the repository at this point in the history
The method will upload to the usage-records bucket so should not take
arbitrary inputs, only usage reports.

Do the encoding and gzipping of the report in the method rather than
the caller.
  • Loading branch information
Andrew Farries authored and roboquat committed Jul 25, 2022
1 parent fb941f0 commit 8ce9022
Show file tree
Hide file tree
Showing 3 changed files with 22 additions and 21 deletions.
22 changes: 18 additions & 4 deletions components/usage/pkg/contentservice/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,22 @@
package contentservice

import (
"bytes"
"compress/gzip"
"context"
"encoding/json"
"fmt"
"io"
"net/http"

"github.com/gitpod-io/gitpod/common-go/log"
"github.com/gitpod-io/gitpod/content-service/api"
"github.com/gitpod-io/gitpod/usage/pkg/db"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)

type Interface interface {
UploadFile(ctx context.Context, filename string, body io.Reader) error
UploadUsageReport(ctx context.Context, filename string, report map[db.AttributionID][]db.WorkspaceInstanceForUsage) error
}

type Client struct {
Expand All @@ -28,13 +31,24 @@ func New(url string) *Client {
return &Client{url: url}
}

func (c *Client) UploadFile(ctx context.Context, filename string, body io.Reader) error {
func (c *Client) UploadUsageReport(ctx context.Context, filename string, report map[db.AttributionID][]db.WorkspaceInstanceForUsage) error {
url, err := c.getSignedUploadUrl(ctx, filename)
if err != nil {
return fmt.Errorf("failed to obtain signed upload URL: %w", err)
}

req, err := http.NewRequest(http.MethodPut, url, body)
reportBytes := &bytes.Buffer{}
gz := gzip.NewWriter(reportBytes)
err = json.NewEncoder(gz).Encode(report)
if err != nil {
return fmt.Errorf("failed to marshal report to JSON: %w", err)
}
err = gz.Close()
if err != nil {
return fmt.Errorf("failed to compress usage report: %w", err)
}

req, err := http.NewRequest(http.MethodPut, url, reportBytes)
if err != nil {
return fmt.Errorf("failed to construct http request: %w", err)
}
Expand Down
5 changes: 3 additions & 2 deletions components/usage/pkg/contentservice/noop.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,12 @@ package contentservice

import (
"context"
"io"

"github.com/gitpod-io/gitpod/usage/pkg/db"
)

type NoOpClient struct{}

func (c *NoOpClient) UploadFile(ctx context.Context, filename string, body io.Reader) error {
func (c *NoOpClient) UploadUsageReport(ctx context.Context, filename string, report map[db.AttributionID][]db.WorkspaceInstanceForUsage) error {
return nil
}
16 changes: 1 addition & 15 deletions components/usage/pkg/controller/reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,8 @@
package controller

import (
"bytes"
"compress/gzip"
"context"
"database/sql"
"encoding/json"
"fmt"
"math"
"time"
Expand Down Expand Up @@ -75,24 +72,13 @@ func (u *UsageReconciler) Reconcile() (err error) {
}
log.WithField("usage_reconcile_status", status).Info("Reconcile completed.")

reportBytes := &bytes.Buffer{}
gz := gzip.NewWriter(reportBytes)
err = json.NewEncoder(gz).Encode(report)
if err != nil {
return fmt.Errorf("failed to marshal report to JSON: %w", err)
}
err = gz.Close()
if err != nil {
return fmt.Errorf("failed to compress usage report: %w", err)
}

err = db.CreateUsageRecords(ctx, u.conn, usageReportToUsageRecords(report, u.pricer, u.nowFunc().UTC()))
if err != nil {
return fmt.Errorf("failed to write usage records to database: %s", err)
}

filename := fmt.Sprintf("%s.gz", now.Format(time.RFC3339))
err = u.contentService.UploadFile(ctx, filename, reportBytes)
err = u.contentService.UploadUsageReport(ctx, filename, report)
if err != nil {
return fmt.Errorf("failed to upload usage report: %w", err)
}
Expand Down

0 comments on commit 8ce9022

Please sign in to comment.