Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[usage] Join workspace instances with workspaces to get project and type #11312

Merged
merged 4 commits into from
Jul 12, 2022
Merged

[usage] Join workspace instances with workspaces to get project and type #11312

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a04a7ba
[usage] Join workspace instances with workspaces to get project and type
easyCZ Jul 12, 2022
fccb8c7
Fix up
easyCZ Jul 12, 2022
0ffb7bb
fix up
easyCZ Jul 12, 2022
7f8eb57
fix up
easyCZ Jul 12, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions components/usage/pkg/controller/reconciler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ func (u *UsageReconciler) ReconcileTimeRange(ctx context.Context, from, to time.
return status, instancesByAttributionID, nil
}

type UsageReport map[db.AttributionID][]db.WorkspaceInstance
type UsageReport map[db.AttributionID][]db.WorkspaceInstanceForUsage

func (u UsageReport) CreditSummaryForTeams(pricer *WorkspacePricer, maxStopTime time.Time) map[string]int64 {
creditsPerTeamID := map[string]int64{}
Expand Down Expand Up @@ -149,7 +149,7 @@ type invalidWorkspaceInstance struct {
workspaceInstanceID uuid.UUID
}

func (u *UsageReconciler) loadWorkspaceInstances(ctx context.Context, from, to time.Time) ([]db.WorkspaceInstance, []invalidWorkspaceInstance, error) {
func (u *UsageReconciler) loadWorkspaceInstances(ctx context.Context, from, to time.Time) ([]db.WorkspaceInstanceForUsage, []invalidWorkspaceInstance, error) {
log.Infof("Gathering usage data from %s to %s", from, to)
instances, err := db.ListWorkspaceInstancesInRange(ctx, u.conn, from, to)
if err != nil {
Expand All @@ -162,7 +162,7 @@ func (u *UsageReconciler) loadWorkspaceInstances(ctx context.Context, from, to t
return trimmed, invalid, nil
}

func validateInstances(instances []db.WorkspaceInstance) (valid []db.WorkspaceInstance, invalid []invalidWorkspaceInstance) {
func validateInstances(instances []db.WorkspaceInstanceForUsage) (valid []db.WorkspaceInstanceForUsage, invalid []invalidWorkspaceInstance) {
for _, i := range instances {
// i is a pointer to the current element, we need to assign it to ensure we're copying the value, not the current pointer.
instance := i
Expand Down Expand Up @@ -196,8 +196,8 @@ func validateInstances(instances []db.WorkspaceInstance) (valid []db.WorkspaceIn
}

// trimStartStopTime ensures that start time or stop time of an instance is never outside of specified start or stop time range.
func trimStartStopTime(instances []db.WorkspaceInstance, maximumStart, minimumStop time.Time) []db.WorkspaceInstance {
var updated []db.WorkspaceInstance
func trimStartStopTime(instances []db.WorkspaceInstanceForUsage, maximumStart, minimumStop time.Time) []db.WorkspaceInstanceForUsage {
var updated []db.WorkspaceInstanceForUsage

for _, instance := range instances {
if instance.CreationTime.Time().Before(maximumStart) {
Expand All @@ -213,11 +213,11 @@ func trimStartStopTime(instances []db.WorkspaceInstance, maximumStart, minimumSt
return updated
}

func groupInstancesByAttributionID(instances []db.WorkspaceInstance) map[db.AttributionID][]db.WorkspaceInstance {
result := map[db.AttributionID][]db.WorkspaceInstance{}
func groupInstancesByAttributionID(instances []db.WorkspaceInstanceForUsage) map[db.AttributionID][]db.WorkspaceInstanceForUsage {
result := map[db.AttributionID][]db.WorkspaceInstanceForUsage{}
for _, instance := range instances {
if _, ok := result[instance.UsageAttributionID]; !ok {
result[instance.UsageAttributionID] = []db.WorkspaceInstance{}
result[instance.UsageAttributionID] = []db.WorkspaceInstanceForUsage{}
}

result[instance.UsageAttributionID] = append(result[instance.UsageAttributionID], instance)
Expand Down
47 changes: 26 additions & 21 deletions components/usage/pkg/controller/reconciler_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,34 +79,38 @@ func TestUsageReport_CreditSummaryForTeams(t *testing.T) {
}{
{
Name: "no instances in report, no summary",
Report: map[db.AttributionID][]db.WorkspaceInstance{},
Report: map[db.AttributionID][]db.WorkspaceInstanceForUsage{},
Expected: map[string]int64{},
},
{
Name: "skips user attributions",
Report: map[db.AttributionID][]db.WorkspaceInstance{
Report: map[db.AttributionID][]db.WorkspaceInstanceForUsage{
db.NewUserAttributionID(uuid.New().String()): {
dbtest.NewWorkspaceInstance(t, db.WorkspaceInstance{}),
db.WorkspaceInstanceForUsage{
UsageAttributionID: db.NewUserAttributionID(uuid.New().String()),
},
},
},
Expected: map[string]int64{},
},
{
Name: "two workspace instances",
Report: map[db.AttributionID][]db.WorkspaceInstance{
Report: map[db.AttributionID][]db.WorkspaceInstanceForUsage{
teamAttributionID: {
dbtest.NewWorkspaceInstance(t, db.WorkspaceInstance{
db.WorkspaceInstanceForUsage{
// has 1 day and 23 hours of usage
WorkspaceClass: defaultWorkspaceClass,
CreationTime: db.NewVarcharTime(time.Date(2022, 05, 30, 00, 00, 00, 00, time.UTC)),
StoppedTime: db.NewVarcharTime(time.Date(2022, 06, 1, 1, 0, 0, 0, time.UTC)),
}),
dbtest.NewWorkspaceInstance(t, db.WorkspaceInstance{
UsageAttributionID: teamAttributionID,
WorkspaceClass: defaultWorkspaceClass,
CreationTime: db.NewVarcharTime(time.Date(2022, 05, 30, 00, 00, 00, 00, time.UTC)),
StoppedTime: db.NewVarcharTime(time.Date(2022, 06, 1, 1, 0, 0, 0, time.UTC)),
},
db.WorkspaceInstanceForUsage{
// has 1 hour of usage
WorkspaceClass: defaultWorkspaceClass,
CreationTime: db.NewVarcharTime(time.Date(2022, 05, 30, 00, 00, 00, 00, time.UTC)),
StoppedTime: db.NewVarcharTime(time.Date(2022, 05, 30, 1, 0, 0, 0, time.UTC)),
}),
UsageAttributionID: teamAttributionID,
WorkspaceClass: defaultWorkspaceClass,
CreationTime: db.NewVarcharTime(time.Date(2022, 05, 30, 00, 00, 00, 00, time.UTC)),
StoppedTime: db.NewVarcharTime(time.Date(2022, 05, 30, 1, 0, 0, 0, time.UTC)),
},
},
},
Expected: map[string]int64{
Expand All @@ -116,14 +120,15 @@ func TestUsageReport_CreditSummaryForTeams(t *testing.T) {
},
{
Name: "unknown workspace class uses default",
Report: map[db.AttributionID][]db.WorkspaceInstance{
Report: map[db.AttributionID][]db.WorkspaceInstanceForUsage{
teamAttributionID: {
dbtest.NewWorkspaceInstance(t, db.WorkspaceInstance{
// has 1 hour of usage
WorkspaceClass: "yolo-workspace-class",
CreationTime: db.NewVarcharTime(time.Date(2022, 05, 30, 00, 00, 00, 00, time.UTC)),
StoppedTime: db.NewVarcharTime(time.Date(2022, 05, 30, 1, 0, 0, 0, time.UTC)),
}),
// has 1 hour of usage
db.WorkspaceInstanceForUsage{
WorkspaceClass: "yolo-workspace-class",
UsageAttributionID: teamAttributionID,
CreationTime: db.NewVarcharTime(time.Date(2022, 05, 30, 00, 00, 00, 00, time.UTC)),
StoppedTime: db.NewVarcharTime(time.Date(2022, 05, 30, 1, 0, 0, 0, time.UTC)),
},
},
},
Expected: map[string]int64{
Expand Down
59 changes: 37 additions & 22 deletions components/usage/pkg/db/workspace_instance.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,21 +45,6 @@ type WorkspaceInstance struct {
_ bool `gorm:"column:deleted;type:tinyint;default:0;" json:"deleted"`
}

// WorkspaceRuntimeSeconds computes how long this WorkspaceInstance has been running.
// If the instance is still running (no stop time set), maxStopTime is used to to compute the duration - this is an upper bound on stop
func (i *WorkspaceInstance) WorkspaceRuntimeSeconds(maxStopTime time.Time) int64 {
start := i.CreationTime.Time()
stop := maxStopTime

if i.StoppedTime.IsSet() {
if i.StoppedTime.Time().Before(maxStopTime) {
stop = i.StoppedTime.Time()
}
}

return int64(stop.Sub(start).Round(time.Second).Seconds())
}

// TableName sets the insert table name for this struct type
func (i *WorkspaceInstance) TableName() string {
return "d_b_workspace_instance"
Expand All @@ -71,16 +56,20 @@ func (i *WorkspaceInstance) TableName() string {
// - running
// - instances which only just terminated after the start period
// - instances which only just started in the period specified
func ListWorkspaceInstancesInRange(ctx context.Context, conn *gorm.DB, from, to time.Time) ([]WorkspaceInstance, error) {
var instances []WorkspaceInstance
var instancesInBatch []WorkspaceInstance
func ListWorkspaceInstancesInRange(ctx context.Context, conn *gorm.DB, from, to time.Time) ([]WorkspaceInstanceForUsage, error) {
var instances []WorkspaceInstanceForUsage
var instancesInBatch []WorkspaceInstanceForUsage

tx := conn.WithContext(ctx).
Table(fmt.Sprintf("%s as wsi", (&WorkspaceInstance{}).TableName())).
Select("wsi.id as id, ws.projectId as projectId, ws.type as workspaceType, wsi.workspaceClass as workspaceClass, wsi.usageAttributionId as usageAttributionId, wsi.stoppedTime as stoppedTime, wsi.creationTime as creationTime").
Joins(fmt.Sprintf("LEFT JOIN %s AS ws ON wsi.workspaceId = ws.id", (&Workspace{}).TableName())).
Where(
conn.Where("stoppedTime >= ?", TimeToISO8601(from)).Or("stoppedTime = ?", ""),
conn.Where("wsi.stoppedTime >= ?", TimeToISO8601(from)).Or("wsi.stoppedTime = ?", ""),
).
Where("creationTime < ?", TimeToISO8601(to)).
Where("startedTime != ?", "").
Where("usageAttributionId != ?", "").
Where("wsi.creationTime < ?", TimeToISO8601(to)).
Where("wsi.startedTime != ?", "").
Where("wsi.usageAttributionId != ?", "").
Comment on lines +64 to +72
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need to test this query against larger data sets to see how it performs, or happy to deploy to prod as is?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can give it a go against prod data

FindInBatches(&instancesInBatch, 1000, func(_ *gorm.DB, _ int) error {
instances = append(instances, instancesInBatch...)
return nil
Expand Down Expand Up @@ -125,3 +114,29 @@ func (a AttributionID) Values() (entity string, identifier string) {
const (
WorkspaceClass_Default = "default"
)

type WorkspaceInstanceForUsage struct {
ID uuid.UUID `gorm:"column:id;type:char;size:36;" json:"id"`
ProjectID sql.NullString `gorm:"column:projectId;type:char;size:36;" json:"projectId"`
WorkspaceClass string `gorm:"column:workspaceClass;type:varchar;size:255;" json:"workspaceClass"`
Type WorkspaceType `gorm:"column:workspaceType;type:char;size:16;default:regular;" json:"workspaceType"`
UsageAttributionID AttributionID `gorm:"column:usageAttributionId;type:varchar;size:60;" json:"usageAttributionId"`

CreationTime VarcharTime `gorm:"column:creationTime;type:varchar;size:255;" json:"creationTime"`
StoppedTime VarcharTime `gorm:"column:stoppedTime;type:varchar;size:255;" json:"stoppedTime"`
}

// WorkspaceRuntimeSeconds computes how long this WorkspaceInstance has been running.
// If the instance is still running (no stop time set), maxStopTime is used to to compute the duration - this is an upper bound on stop
func (i *WorkspaceInstanceForUsage) WorkspaceRuntimeSeconds(maxStopTime time.Time) int64 {
start := i.CreationTime.Time()
stop := maxStopTime

if i.StoppedTime.IsSet() {
if i.StoppedTime.Time().Before(maxStopTime) {
stop = i.StoppedTime.Time()
}
}

return int64(stop.Sub(start).Round(time.Second).Seconds())
}
21 changes: 11 additions & 10 deletions components/usage/pkg/db/workspace_instance_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,66 +18,67 @@ import (
func TestListWorkspaceInstancesInRange(t *testing.T) {
conn := dbtest.ConnectForTests(t)

workspaceID := "gitpodio-gitpod-gyjr82jkfnd"
workspace := dbtest.CreateWorkspaces(t, conn, dbtest.NewWorkspace(t, db.Workspace{}))[0]

valid := []db.WorkspaceInstance{
// In the middle of May
dbtest.NewWorkspaceInstance(t, db.WorkspaceInstance{
WorkspaceID: workspaceID,
WorkspaceID: workspace.ID,
Comment on lines +21 to +26
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was this change necessary or just tidying up?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To ensure the joining works correctly, we needed to create an actual Workspace record in the DB (line 21). That Workspace gets a random ID each time, so it's references here.

Aside, getting a random ID is a good thing, as it allows tests against the DB to run concurrently without interfering with each other.

CreationTime: db.NewVarcharTime(time.Date(2022, 05, 15, 12, 00, 00, 00, time.UTC)),
StartedTime: db.NewVarcharTime(time.Date(2022, 05, 15, 12, 00, 00, 00, time.UTC)),
StoppedTime: db.NewVarcharTime(time.Date(2022, 05, 15, 13, 00, 00, 00, time.UTC)),
}),
// Start of May
dbtest.NewWorkspaceInstance(t, db.WorkspaceInstance{
ID: uuid.New(),
WorkspaceID: workspaceID,
WorkspaceID: workspace.ID,
CreationTime: db.NewVarcharTime(time.Date(2022, 05, 1, 0, 00, 00, 00, time.UTC)),
StartedTime: db.NewVarcharTime(time.Date(2022, 05, 1, 0, 00, 00, 00, time.UTC)),
StoppedTime: db.NewVarcharTime(time.Date(2022, 05, 1, 1, 00, 00, 00, time.UTC)),
}),
// End of May
dbtest.NewWorkspaceInstance(t, db.WorkspaceInstance{
ID: uuid.New(),
WorkspaceID: workspaceID,
WorkspaceID: workspace.ID,
CreationTime: db.NewVarcharTime(time.Date(2022, 05, 31, 23, 00, 00, 00, time.UTC)),
StartedTime: db.NewVarcharTime(time.Date(2022, 05, 31, 23, 00, 00, 00, time.UTC)),
StoppedTime: db.NewVarcharTime(time.Date(2022, 05, 31, 23, 59, 59, 999999, time.UTC)),
}),
// Started in April, but continued into May
dbtest.NewWorkspaceInstance(t, db.WorkspaceInstance{
ID: uuid.New(),
WorkspaceID: workspaceID,
WorkspaceID: workspace.ID,
CreationTime: db.NewVarcharTime(time.Date(2022, 04, 30, 23, 00, 00, 00, time.UTC)),
StartedTime: db.NewVarcharTime(time.Date(2022, 04, 30, 23, 00, 00, 00, time.UTC)),
StoppedTime: db.NewVarcharTime(time.Date(2022, 05, 1, 0, 0, 0, 0, time.UTC)),
}),
// Started in May, but continued into June
dbtest.NewWorkspaceInstance(t, db.WorkspaceInstance{
ID: uuid.New(),
WorkspaceID: workspaceID,
WorkspaceID: workspace.ID,
CreationTime: db.NewVarcharTime(time.Date(2022, 05, 31, 23, 00, 00, 00, time.UTC)),
StartedTime: db.NewVarcharTime(time.Date(2022, 05, 31, 23, 00, 00, 00, time.UTC)),
StoppedTime: db.NewVarcharTime(time.Date(2022, 06, 1, 1, 0, 0, 0, time.UTC)),
}),
// Started in April, but continued into June (ran for all of May)
dbtest.NewWorkspaceInstance(t, db.WorkspaceInstance{
ID: uuid.New(),
WorkspaceID: workspaceID,
WorkspaceID: workspace.ID,
CreationTime: db.NewVarcharTime(time.Date(2022, 04, 31, 23, 00, 00, 00, time.UTC)),
StartedTime: db.NewVarcharTime(time.Date(2022, 04, 31, 23, 00, 00, 00, time.UTC)),
StoppedTime: db.NewVarcharTime(time.Date(2022, 06, 1, 1, 0, 0, 0, time.UTC)),
}),
// Stopped in May, no creation time, should be retrieved but this is a poor data quality record.
dbtest.NewWorkspaceInstance(t, db.WorkspaceInstance{
ID: uuid.New(),
WorkspaceID: workspaceID,
WorkspaceID: workspace.ID,
StartedTime: db.NewVarcharTime(time.Date(2022, 05, 1, 1, 0, 0, 0, time.UTC)),
StoppedTime: db.NewVarcharTime(time.Date(2022, 05, 1, 1, 0, 0, 0, time.UTC)),
}),
// Started in April, no stop time, still running
dbtest.NewWorkspaceInstance(t, db.WorkspaceInstance{
ID: uuid.New(),
WorkspaceID: workspaceID,
WorkspaceID: workspace.ID,
CreationTime: db.NewVarcharTime(time.Date(2022, 04, 31, 23, 00, 00, 00, time.UTC)),
StartedTime: db.NewVarcharTime(time.Date(2022, 04, 31, 23, 00, 00, 00, time.UTC)),
}),
Expand All @@ -86,7 +87,7 @@ func TestListWorkspaceInstancesInRange(t *testing.T) {
// Start of June
dbtest.NewWorkspaceInstance(t, db.WorkspaceInstance{
ID: uuid.New(),
WorkspaceID: workspaceID,
WorkspaceID: workspace.ID,
CreationTime: db.NewVarcharTime(time.Date(2022, 06, 1, 00, 00, 00, 00, time.UTC)),
StartedTime: db.NewVarcharTime(time.Date(2022, 06, 1, 00, 00, 00, 00, time.UTC)),
StoppedTime: db.NewVarcharTime(time.Date(2022, 06, 1, 1, 0, 0, 0, time.UTC)),
Expand Down