Skip to content

Commit

Permalink
Merge pull request #1458 from merico-dev/db-migration
Browse files Browse the repository at this point in the history
feat: db migration
  • Loading branch information
klesh authored Apr 12, 2022
2 parents cb4bac0 + 218810f commit 3d239f0
Show file tree
Hide file tree
Showing 174 changed files with 2,483 additions and 353 deletions.
12 changes: 12 additions & 0 deletions migration/interface.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
package migration

import (
"context"
"gorm.io/gorm"
)

type Script interface {
Up(ctx context.Context, db *gorm.DB) error
Version() uint64
Name() string
}
98 changes: 98 additions & 0 deletions migration/migrator.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
package migration

import (
"context"
"fmt"
"gorm.io/gorm"
"sort"
"sync"
)

var m = migrator{scripts: make(map[string]scriptWithComment)}

type scriptWithComment struct {
Script
comment string
}
type migrator struct {
sync.Mutex
db *gorm.DB
scripts map[string]scriptWithComment
}

func Init(db *gorm.DB) {
m.db = db
}

func (m *migrator) register(scripts []Script, comment string) {
m.Lock()
defer m.Unlock()
for _, script := range scripts {
m.scripts[fmt.Sprintf("%s:%d", script.Name(), script.Version())] = scriptWithComment{
Script: script,
comment: comment,
}
}
}

func (m *migrator) bookKeep(script scriptWithComment) error {
record := &MigrationHistory{
ScriptVersion: script.Version(),
ScriptName: script.Name(),
Comment: script.comment,
}
return m.db.Create(record).Error
}

func (m *migrator) execute(ctx context.Context) error {
versions, err := m.getExecuted()
if err != nil {
return err
}
for key := range versions {
delete(m.scripts, key)
}
var scriptSlice []scriptWithComment
for _, script := range m.scripts {
scriptSlice = append(scriptSlice, script)
}
sort.Slice(scriptSlice, func(i, j int) bool {
return scriptSlice[i].Version() < scriptSlice[j].Version()
})
for _, script := range scriptSlice {
err = script.Up(ctx, m.db)
if err != nil {
return err
}
err = m.bookKeep(script)
if err != nil {
return err
}
}
return nil
}
func (m *migrator) getExecuted() (map[string]struct{}, error) {
var err error
versions := make(map[string]struct{})
err = m.db.Migrator().AutoMigrate(&MigrationHistory{})
if err != nil {
return nil, err
}
var records []MigrationHistory
err = m.db.Find(&records).Error
if err != nil {
return nil, err
}
for _, record := range records {
versions[fmt.Sprintf("%s:%d", record.ScriptName, record.ScriptVersion)] = struct{}{}
}
return versions, nil
}

func Register(scripts []Script, comment string) {
m.register(scripts, comment)
}

func Execute(ctx context.Context) error {
return m.execute(ctx)
}
20 changes: 20 additions & 0 deletions migration/models.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package migration

import (
"time"
)

const (
tableName = "_devlake_migration_history"
)

type MigrationHistory struct {
CreatedAt time.Time
ScriptVersion uint64 `gorm:"primarykey"`
ScriptName string `gorm:"primarykey;type:varchar(255)"`
Comment string
}

func (MigrationHistory) TableName() string {
return tableName
}
4 changes: 4 additions & 0 deletions models/blueprint.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,7 @@ type InputBlueprint struct {
}

type EditBlueprint InputBlueprint

func (Blueprint) TableName() string {
return "_devlake_blueprints"
}
2 changes: 1 addition & 1 deletion models/common/base.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ type NoPKModel struct {
type RawDataOrigin struct {
// can be used for flushing outdated records from table
RawDataParams string `gorm:"column:_raw_data_params;type:varchar(255);index"`
RawDataTable string `gorm:"column:_raw_data_table"`
RawDataTable string `gorm:"column:_raw_data_table;type:varchar(255)"`
// can be used for debugging
RawDataId uint64 `gorm:"column:_raw_data_id"`
// we can store record index into this field, which is helpful for debugging
Expand Down
4 changes: 2 additions & 2 deletions models/domainlayer/code/note.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ import (
type Note struct {
domainlayer.DomainEntity
PrId string `gorm:"index;comment:References the pull request for this note;type:varchar(100)"`
Type string
Author string
Type string `gorm:"type:char(100)"`
Author string `gorm:"type:char(255)"`
Body string
Resolvable bool `gorm:"comment:Is or is not a review comment"`
System bool `gorm:"comment:Is or is not auto-generated vs. human generated"`
Expand Down
19 changes: 10 additions & 9 deletions models/domainlayer/code/pull_request.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@ import (

type PullRequest struct {
domainlayer.DomainEntity
RepoId string `gorm:"index"`
Status string `gorm:"comment:open/closed or other"`
BaseRepoId string `gorm:"index"`
HeadRepoId string `gorm:"index"`
Status string `gorm:"type:varchar(100);comment:open/closed or other"`
Title string
Description string
Url string `gorm:"type:char(255)"`
Expand All @@ -19,12 +20,12 @@ type PullRequest struct {
Key int
CreatedDate time.Time
MergedDate *time.Time
ClosedAt *time.Time
Type string
Component string
ClosedDate *time.Time
Type string `gorm:"type:char(100)"`
Component string `gorm:"type:char(100)"`
MergeCommitSha string `gorm:"type:char(40)"`
HeadRef string
BaseRef string
BaseCommitSha string
HeadCommitSha string
HeadRef string `gorm:"type:char(255)"`
BaseRef string `gorm:"type:char(255)"`
BaseCommitSha string `gorm:"type:char(40)"`
HeadCommitSha string `gorm:"type:char(40)"`
}
2 changes: 1 addition & 1 deletion models/domainlayer/code/pull_request_commit.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package code
import "github.com/merico-dev/lake/models/common"

type PullRequestCommit struct {
CommitSha string `gorm:"primaryKey"`
CommitSha string `gorm:"primaryKey;type:char(40)"`
PullRequestId string `json:"id" gorm:"primaryKey;type:varchar(255);comment:This key is generated based on details from the original plugin"` // format: <Plugin>:<Entity>:<PK0>:<PK1>
common.NoPKModel
}
2 changes: 1 addition & 1 deletion models/domainlayer/code/pull_request_labels.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,6 @@ import (

type PullRequestLabel struct {
PullRequestId string `json:"id" gorm:"primaryKey;type:varchar(255);comment:This key is generated based on details from the original plugin"` // format: <Plugin>:<Entity>:<PK0>:<PK1>
LabelName string `gorm:"primaryKey"`
LabelName string `gorm:"primaryKey;type:varchar(255)"`
common.NoPKModel
}
3 changes: 2 additions & 1 deletion models/domainlayer/code/ref.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
package code

import (
"github.com/merico-dev/lake/models/domainlayer"
"time"

"github.com/merico-dev/lake/models/domainlayer"
)

type Ref struct {
Expand Down
8 changes: 4 additions & 4 deletions models/domainlayer/code/refs_pr_cherry_pick.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@ package code
import "github.com/merico-dev/lake/models/common"

type RefsPrCherrypick struct {
RepoName string
RepoName string`gorm:"type:char(255)"`
ParentPrKey int
CherrypickBaseBranches string
CherrypickPrKeys string
ParentPrUrl string
CherrypickBaseBranches string`gorm:"type:char(255)"`
CherrypickPrKeys string`gorm:"type:char(255)"`
ParentPrUrl string`gorm:"type:char(255)"`
ParentPrId string `json:"parent_pr_id" gorm:"primaryKey;type:varchar(255);comment:This key is generated based on details from the original plugin"` // format: <Plugin>:<Entity>:<PK0>:<PK1>
common.NoPKModel
}
6 changes: 3 additions & 3 deletions models/domainlayer/devops/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ import (
type Build struct {
domainlayer.DomainEntity
JobId string `gorm:"index"`
Name string
CommitSha string
Name string `gorm:"type:char(255)"`
CommitSha string `gorm:"type:char(40)"`
DurationSec uint64
Status string
Status string `gorm:"type:char(100)"`
StartedDate time.Time
}
2 changes: 1 addition & 1 deletion models/domainlayer/devops/job.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,6 @@ import (
)

type Job struct {
Name string
Name string `gorm:"type:char(255)"`
domainlayer.DomainEntity
}
8 changes: 4 additions & 4 deletions models/domainlayer/ticket/board.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,14 @@ import (

type Board struct {
domainlayer.DomainEntity
Name string
Name string `gorm:"type:char(255)"`
Description string
Url string
Url string `gorm:"type:char(255)"`
CreatedDate *time.Time
}

type BoardSprint struct {
common.NoPKModel
BoardId string `gorm:"primaryKey"`
SprintId string `gorm:"primaryKey"`
BoardId string `gorm:"primaryKey;type:varchar(255)"`
SprintId string `gorm:"primaryKey;type:varchar(255)"`
}
8 changes: 4 additions & 4 deletions models/domainlayer/ticket/changelog.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@ type Changelog struct {

// collected fields
IssueId string `gorm:"index"`
AuthorId string
AuthorName string
FieldId string
FieldName string
AuthorId string `gorm:"type:char(255)"`
AuthorName string `gorm:"type:char(255)"`
FieldId string `gorm:"type:char(255)"`
FieldName string `gorm:"type:char(255)"`
From string
To string
CreatedDate time.Time
Expand Down
27 changes: 14 additions & 13 deletions models/domainlayer/ticket/issue.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,28 +8,29 @@ import (

type Issue struct {
domainlayer.DomainEntity
Url string
Key string
Url string `gorm:"type:char(255)"`
Key string `gorm:"type:char(255)"`
Title string
Summary string
EpicKey string
Type string
Status string
Description string
EpicKey string `gorm:"type:char(255)"`
Type string `gorm:"type:char(100)"`
Status string `gorm:"type:char(100)"`
OriginalStatus string `gorm:"type:char(100)"`
StoryPoint uint
ResolutionDate *time.Time
CreatedDate *time.Time
UpdatedDate *time.Time
LeadTimeMinutes uint
ParentIssueId string
Priority string
ParentIssueId string `gorm:"type:char(255)"`
Priority string `gorm:"type:char(255)"`
OriginalEstimateMinutes int64
TimeSpentMinutes int64
TimeRemainingMinutes int64
CreatorId string
AssigneeId string
AssigneeName string
Severity string
Component string
CreatorId string `gorm:"type:char(255)"`
AssigneeId string `gorm:"type:char(255)"`
AssigneeName string `gorm:"type:char(255)"`
Severity string `gorm:"type:char(255)"`
Component string `gorm:"type:char(255)"`
}

const (
Expand Down
12 changes: 6 additions & 6 deletions models/domainlayer/ticket/issue_history.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ import (

type IssueStatusHistory struct {
common.NoPKModel
IssueId string `gorm:"primaryKey"`
OriginalStatus string `gorm:"primaryKey"`
IssueId string `gorm:"primaryKey;type:varchar(255)"`
OriginalStatus string `gorm:"primaryKey;type:varchar(255)"`
StartDate time.Time `gorm:"primaryKey"`
EndDate *time.Time
}
Expand All @@ -20,8 +20,8 @@ func (IssueStatusHistory) TableName() string {

type IssueAssigneeHistory struct {
common.NoPKModel
IssueId string `gorm:"primaryKey"`
Assignee string `gorm:"primaryKey"`
IssueId string `gorm:"primaryKey;type:varchar(255)"`
Assignee string `gorm:"primaryKey;type:varchar(255)"`
StartDate time.Time `gorm:"primaryKey"`
EndDate *time.Time
}
Expand All @@ -32,8 +32,8 @@ func (IssueAssigneeHistory) TableName() string {

type IssueSprintsHistory struct {
common.NoPKModel
IssueId string `gorm:"primaryKey"`
SprintId string `gorm:"primaryKey"`
IssueId string `gorm:"primaryKey;type:varchar(255)"`
SprintId string `gorm:"primaryKey;type:varchar(255)"`
StartDate time.Time `gorm:"primaryKey"`
EndDate *time.Time
}
Expand Down
2 changes: 1 addition & 1 deletion models/domainlayer/ticket/issue_label.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,6 @@ import "github.com/merico-dev/lake/models/common"

type IssueLabel struct {
IssueId string `json:"id" gorm:"primaryKey;type:varchar(255);comment:This key is generated based on details from the original plugin"` // format: <Plugin>:<Entity>:<PK0>:<PK1>
LabelName string `gorm:"primaryKey"`
LabelName string `gorm:"primaryKey;type:varchar(255)"`
common.NoPKModel
}
Loading

0 comments on commit 3d239f0

Please sign in to comment.