diff --git a/.gitignore b/.gitignore index 00b92cb33..acb4088f2 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ cli/cli cli/migrate .coverage .godoc.pid -vendor/ \ No newline at end of file +vendor/ +.vscode/ diff --git a/database/mssql/examples/migrations/1085649617_create_users_table.down.sql b/database/mssql/examples/migrations/1085649617_create_users_table.down.sql new file mode 100644 index 000000000..c99ddcdc8 --- /dev/null +++ b/database/mssql/examples/migrations/1085649617_create_users_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS users; diff --git a/database/mssql/examples/migrations/1085649617_create_users_table.up.sql b/database/mssql/examples/migrations/1085649617_create_users_table.up.sql new file mode 100644 index 000000000..92897dcab --- /dev/null +++ b/database/mssql/examples/migrations/1085649617_create_users_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE users ( + user_id integer unique, + name varchar(40), + email varchar(40) +); diff --git a/database/mssql/examples/migrations/1185749658_add_city_to_users.down.sql b/database/mssql/examples/migrations/1185749658_add_city_to_users.down.sql new file mode 100644 index 000000000..940c60712 --- /dev/null +++ b/database/mssql/examples/migrations/1185749658_add_city_to_users.down.sql @@ -0,0 +1 @@ +ALTER TABLE users DROP COLUMN IF EXISTS city; diff --git a/database/mssql/examples/migrations/1185749658_add_city_to_users.up.sql b/database/mssql/examples/migrations/1185749658_add_city_to_users.up.sql new file mode 100644 index 000000000..2add820be --- /dev/null +++ b/database/mssql/examples/migrations/1185749658_add_city_to_users.up.sql @@ -0,0 +1,3 @@ +ALTER TABLE users ADD city varchar(100); + + diff --git a/database/mssql/examples/migrations/1285849751_add_index_on_user_emails.down.sql b/database/mssql/examples/migrations/1285849751_add_index_on_user_emails.down.sql new file mode 100644 index 000000000..3e87dd229 --- /dev/null +++ b/database/mssql/examples/migrations/1285849751_add_index_on_user_emails.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS users_email_index; diff --git a/database/mssql/examples/migrations/1285849751_add_index_on_user_emails.up.sql b/database/mssql/examples/migrations/1285849751_add_index_on_user_emails.up.sql new file mode 100644 index 000000000..03a04639c --- /dev/null +++ b/database/mssql/examples/migrations/1285849751_add_index_on_user_emails.up.sql @@ -0,0 +1,3 @@ +CREATE UNIQUE INDEX users_email_index ON users (email); + +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/mssql/examples/migrations/1385949617_create_books_table.down.sql b/database/mssql/examples/migrations/1385949617_create_books_table.down.sql new file mode 100644 index 000000000..1a0b1a214 --- /dev/null +++ b/database/mssql/examples/migrations/1385949617_create_books_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS books; diff --git a/database/mssql/examples/migrations/1385949617_create_books_table.up.sql b/database/mssql/examples/migrations/1385949617_create_books_table.up.sql new file mode 100644 index 000000000..f1503b518 --- /dev/null +++ b/database/mssql/examples/migrations/1385949617_create_books_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE books ( + user_id integer, + name varchar(40), + author varchar(40) +); diff --git a/database/mssql/examples/migrations/1485949617_create_movies_table.down.sql b/database/mssql/examples/migrations/1485949617_create_movies_table.down.sql new file mode 100644 index 000000000..3a5187689 --- /dev/null +++ b/database/mssql/examples/migrations/1485949617_create_movies_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS movies; diff --git a/database/mssql/examples/migrations/1485949617_create_movies_table.up.sql b/database/mssql/examples/migrations/1485949617_create_movies_table.up.sql new file mode 100644 index 000000000..f0ef5943b --- /dev/null +++ b/database/mssql/examples/migrations/1485949617_create_movies_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE movies ( + user_id integer, + name varchar(40), + director varchar(40) +); diff --git a/database/mssql/examples/migrations/1585849751_just_a_comment.up.sql b/database/mssql/examples/migrations/1585849751_just_a_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/mssql/examples/migrations/1585849751_just_a_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/mssql/examples/migrations/1685849751_another_comment.up.sql b/database/mssql/examples/migrations/1685849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/mssql/examples/migrations/1685849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/mssql/examples/migrations/1785849751_another_comment.up.sql b/database/mssql/examples/migrations/1785849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/mssql/examples/migrations/1785849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/mssql/examples/migrations/1885849751_another_comment.up.sql b/database/mssql/examples/migrations/1885849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/mssql/examples/migrations/1885849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/mssql/mssql.go b/database/mssql/mssql.go index 387e9a6ca..ff06d70e1 100644 --- a/database/mssql/mssql.go +++ b/database/mssql/mssql.go @@ -8,9 +8,10 @@ import ( "io/ioutil" nurl "net/url" - _ "github.com/denisenkom/go-mssqldb" // mssql support + mssql "github.com/denisenkom/go-mssqldb" // mssql support "github.com/golang-migrate/migrate/v4" "github.com/golang-migrate/migrate/v4/database" + "github.com/hashicorp/go-multierror" ) func init() { @@ -29,6 +30,13 @@ var ( ErrDatabaseDirty = fmt.Errorf("database is dirty") ) +var lockErrorMap = map[int]string{ + -1: "The lock request timed out.", + -2: "The lock request was canceled.", + -3: "The lock request was chosen as a deadlock victim.", + -999: "Parameter validation or other call error.", +} + // Config for database type Config struct { MigrationsTable string @@ -47,8 +55,7 @@ type MSSQL struct { config *Config } -// WithInstance returns a database instance from an already create database connection -// TODO: WithInstance double check that docs are correct for this function +// WithInstance returns a database instance from an already created database connection func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { if config == nil { return nil, ErrNilConfig @@ -105,10 +112,6 @@ func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { return ss, nil } -func dbConnectionString(host, port string) string { - return fmt.Sprintf("postgres://postgres@%s:%s/postgres?sslmode=disable", host, port) -} - // Open a connection to the database func (ss *MSSQL) Open(url string) (database.Driver, error) { purl, err := nurl.Parse(url) @@ -122,14 +125,12 @@ func (ss *MSSQL) Open(url string) (database.Driver, error) { } migrationsTable := purl.Query().Get("x-migrations-table") - if len(migrationsTable) == 0 { - migrationsTable = DefaultMigrationsTable - } px, err := WithInstance(db, &Config{ DatabaseName: purl.Path, MigrationsTable: migrationsTable, }) + if err != nil { return nil, err } @@ -158,22 +159,20 @@ func (ss *MSSQL) Lock() error { return err } - // start a transaction - query := "BEGIN TRANSACTION" - if _, err := ss.conn.ExecContext(context.Background(), query); err != nil { - return &database.Error{OrigErr: err, Err: "get lock transaction", Query: []byte(query)} - } - // This will either obtain the lock immediately and return true, // or return false if the lock cannot be acquired immediately. // MS Docs: sp_getapplock: https://docs.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-getapplock-transact-sql?view=sql-server-2017 - query = `EXEC sp_getapplock @Resource = ?, @LockMode = 'Update'` - if _, err := ss.conn.ExecContext(context.Background(), query, aid); err != nil { + query := `EXEC sp_getapplock @Resource = ?, @LockMode = 'Update', @LockOwner = 'Session', @LockTimeout = 0` + + var status mssql.ReturnStatus + if _, err = ss.conn.ExecContext(context.Background(), query, aid, &status); err == nil && status > -1 { + ss.isLocked = true + return nil + } else if err != nil { return &database.Error{OrigErr: err, Err: "try lock failed", Query: []byte(query)} + } else { + return &database.Error{Err: fmt.Sprintf("try lock failed with error %v", lockErrorMap[int(status)]), Query: []byte(query)} } - - ss.isLocked = true - return nil } // Unlock froms the migration lock from the database @@ -188,18 +187,12 @@ func (ss *MSSQL) Unlock() error { } // MS Docs: sp_releaseapplock: https://docs.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-releaseapplock-transact-sql?view=sql-server-2017 - query := `EXEC sp_releaseapplock @Resource = ?` + query := `EXEC sp_releaseapplock @Resource = ?, @LockOwner = 'Session'` if _, err := ss.conn.ExecContext(context.Background(), query, aid); err != nil { return &database.Error{OrigErr: err, Query: []byte(query)} } ss.isLocked = false - // end lock transaction - query = "COMMIT" - if _, err := ss.conn.ExecContext(context.Background(), query); err != nil { - return &database.Error{OrigErr: err, Err: "commit lock transaction", Query: []byte(query)} - } - return nil } @@ -213,78 +206,25 @@ func (ss *MSSQL) Run(migration io.Reader) error { // run migration query := string(migr[:]) if _, err := ss.conn.ExecContext(context.Background(), query); err != nil { - // // FIXME: check for mssql error here - // if pgErr, ok := err.(*pq.Error); ok { - // var line uint - // var col uint - // var lineColOK bool - // if pgErr.Position != "" { - // if pos, err := strconv.ParseUint(pgErr.Position, 10, 64); err == nil { - // line, col, lineColOK = computeLineFromPos(query, int(pos)) - // } - // } - // message := fmt.Sprintf("migration failed: %s", pgErr.Message) - // if lineColOK { - // message = fmt.Sprintf("%s (column %d)", message, col) - // } - // if pgErr.Detail != "" { - // message = fmt.Sprintf("%s, %s", message, pgErr.Detail) - // } - // return database.Error{OrigErr: err, Err: message, Query: migr, Line: line} - // } return database.Error{OrigErr: err, Err: "migration failed", Query: migr} } return nil } -// func computeLineFromPos(s string, pos int) (line uint, col uint, ok bool) { -// // replace crlf with lf -// s = strings.Replace(s, "\r\n", "\n", -1) -// // pg docs: pos uses index 1 for the first character, and positions are measured in characters not bytes -// runes := []rune(s) -// if pos > len(runes) { -// return 0, 0, false -// } -// sel := runes[:pos] -// line = uint(runesCount(sel, newLine) + 1) -// col = uint(pos - 1 - runesLastIndex(sel, newLine)) -// return line, col, true -// } - -const newLine = '\n' - -func runesCount(input []rune, target rune) int { - var count int - for _, r := range input { - if r == target { - count++ - } - } - return count -} - -func runesLastIndex(input []rune, target rune) int { - for i := len(input) - 1; i >= 0; i-- { - if input[i] == target { - return i - } - } - return -1 -} - // SetVersion for the current database func (ss *MSSQL) SetVersion(version int, dirty bool) error { - tx := ss.db - // tx, err := ss.conn.BeginTx(context.Background(), &sql.TxOptions{}) - // if err != nil { - // return &database.Error{OrigErr: err, Err: "transaction start failed"} - // } + tx, err := ss.conn.BeginTx(context.Background(), &sql.TxOptions{}) + if err != nil { + return &database.Error{OrigErr: err, Err: "transaction start failed"} + } query := `TRUNCATE TABLE "` + ss.config.MigrationsTable + `"` if _, err := tx.Exec(query); err != nil { - // tx.Rollback() + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } return &database.Error{OrigErr: err, Query: []byte(query)} } @@ -295,14 +235,16 @@ func (ss *MSSQL) SetVersion(version int, dirty bool) error { } query = `INSERT INTO "` + ss.config.MigrationsTable + `" (version, dirty) VALUES ($1, $2)` if _, err := tx.Exec(query, version, dirtyBit); err != nil { - // tx.Rollback() + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } return &database.Error{OrigErr: err, Query: []byte(query)} } } - // if err := tx.Commit(); err != nil { - // return &database.Error{OrigErr: err, Err: "transaction commit failed"} - // } + if err := tx.Commit(); err != nil { + return &database.Error{OrigErr: err, Err: "transaction commit failed"} + } return nil } @@ -317,11 +259,6 @@ func (ss *MSSQL) Version() (version int, dirty bool, err error) { case err != nil: // FIXME: convert to MSSQL error - // if e, ok := err.(*pq.Error); ok { - // if e.Code.Name() == "undefined_table" { - // return database.NilVersion, false, nil - // } - // } return 0, false, &database.Error{OrigErr: err, Query: []byte(query)} default: @@ -361,50 +298,24 @@ func (ss *MSSQL) Drop() error { return &database.Error{OrigErr: err, Query: []byte(query)} } - if err := ss.ensureVersionTable(); err != nil { - return err - } - return nil - - // // select all tables in current schema - // query := `SELECT t.name FROM sys.tables AS t INNER JOIN sys.schemas AS s ON t.[schema_id] = s.[schema_id] WHERE s.name = N'dbo';` - // tables, err := ss.conn.QueryContext(context.Background(), query) - // if err != nil { - // return &database.Error{OrigErr: err, Query: []byte(query)} - // } - // defer tables.Close() - - // // delete one table after another - // tableNames := make([]string, 0) - // for tables.Next() { - // var tableName string - // if err := tables.Scan(&tableName); err != nil { - // return err - // } - // if len(tableName) > 0 { - // tableNames = append(tableNames, tableName) - // } - // } - - // if len(tableNames) > 0 { - // // delete one by one ... - // for _, t := range tableNames { - // query = `DROP TABLE IF EXISTS "` + t + `"` - // if _, err := ss.conn.ExecContext(context.Background(), query); err != nil { - // return &database.Error{OrigErr: err, Query: []byte(query)} - // } - // } - - // if err := ss.ensureVersionTable(); err != nil { - // return err - // } - // } - - // return nil } func (ss *MSSQL) ensureVersionTable() (err error) { + if err = ss.Lock(); err != nil { + return err + } + + defer func() { + if e := ss.Unlock(); e != nil { + if err == nil { + err = e + } else { + err = multierror.Append(err, e) + } + } + }() + query := `IF NOT EXISTS (SELECT * FROM sysobjects diff --git a/database/mssql/mssql_test.go b/database/mssql/mssql_test.go new file mode 100755 index 000000000..d350b9554 --- /dev/null +++ b/database/mssql/mssql_test.go @@ -0,0 +1,157 @@ +package mssql + +import ( + "context" + "database/sql" + sqldriver "database/sql/driver" + "fmt" + "testing" + + "github.com/dhui/dktest" + "github.com/golang-migrate/migrate/v4" + + dt "github.com/golang-migrate/migrate/v4/database/testing" + "github.com/golang-migrate/migrate/v4/dktesting" + + _ "github.com/golang-migrate/migrate/v4/source/file" +) + +const defaultPort = 1433 +const saPassword = "Root1234" + +var ( + opts = dktest.Options{ + Env: map[string]string{"ACCEPT_EULA": "Y", "SA_PASSWORD": saPassword, "MSSQL_PID": "Express"}, + PortRequired: true, ReadyFunc: isReady, + } + // Supported versions: https://www.mysql.com/support/supportedplatforms/database.html + specs = []dktesting.ContainerSpec{ + {ImageName: "mcr.microsoft.com/mssql/server:2017-latest-ubuntu", Options: opts}, + {ImageName: "mcr.microsoft.com/mssql/server:2019-latest", Options: opts}, + } +) + +func isReady(ctx context.Context, c dktest.ContainerInfo) bool { + ip, port, err := c.Port(defaultPort) + if err != nil { + return false + } + uri := fmt.Sprintf("sqlserver://sa:%v@%v:%v?database=master", saPassword, ip, port) + db, err := sql.Open("sqlserver", uri) + if err != nil { + return false + } + defer db.Close() + if err = db.PingContext(ctx); err != nil { + switch err { + case sqldriver.ErrBadConn: + return false + default: + fmt.Println(err) + } + return false + } + + return true +} + +func Test(t *testing.T) { + // mysql.SetLogger(mysql.Logger(log.New(ioutil.Discard, "", log.Ltime))) + + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(defaultPort) + if err != nil { + t.Fatal(err) + } + + addr := fmt.Sprintf("sqlserver://sa:%v@%v:%v?master", saPassword, ip, port) + p := &MSSQL{} + d, err := p.Open(addr) + if err != nil { + t.Fatalf("%v", err) + } + defer d.Close() + dt.Test(t, d, []byte("SELECT 1")) + + // check ensureVersionTable + if err := d.(*MSSQL).ensureVersionTable(); err != nil { + t.Fatal(err) + } + // check again + if err := d.(*MSSQL).ensureVersionTable(); err != nil { + t.Fatal(err) + } + }) +} + +func TestMigrate(t *testing.T) { + // mysql.SetLogger(mysql.Logger(log.New(ioutil.Discard, "", log.Ltime))) + + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(defaultPort) + if err != nil { + t.Fatal(err) + } + + addr := fmt.Sprintf("sqlserver://sa:%v@%v:%v?master", saPassword, ip, port) + p := &MSSQL{} + d, err := p.Open(addr) + if err != nil { + t.Fatalf("%v", err) + } + defer d.Close() + + m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "public", d) + if err != nil { + t.Fatalf("%v", err) + } + dt.TestMigrate(t, m, []byte("SELECT 1")) + + // check ensureVersionTable + if err := d.(*MSSQL).ensureVersionTable(); err != nil { + t.Fatal(err) + } + // check again + if err := d.(*MSSQL).ensureVersionTable(); err != nil { + t.Fatal(err) + } + }) +} + +func TestLockWorks(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(defaultPort) + if err != nil { + t.Fatal(err) + } + + addr := fmt.Sprintf("sqlserver://sa:%v@%v:%v?master", saPassword, ip, port) + p := &MSSQL{} + d, err := p.Open(addr) + if err != nil { + t.Fatalf("%v", err) + } + dt.Test(t, d, []byte("SELECT 1")) + + ms := d.(*MSSQL) + + err = ms.Lock() + if err != nil { + t.Fatal(err) + } + err = ms.Unlock() + if err != nil { + t.Fatal(err) + } + + // make sure the 2nd lock works (RELEASE_LOCK is very finicky) + err = ms.Lock() + if err != nil { + t.Fatal(err) + } + err = ms.Unlock() + if err != nil { + t.Fatal(err) + } + }) +} diff --git a/go.sum b/go.sum index c6a7a9c57..9b6b62c7d 100644 --- a/go.sum +++ b/go.sum @@ -56,7 +56,7 @@ github.com/dhui/dktest v0.3.0/go.mod h1:cyzIUfGsBEbZ6BT7tnXqAShHSXCZhSNmFl70sZ7c github.com/docker/distribution v2.7.0+incompatible h1:neUDAlf3wX6Ml4HdqTrbcOHXtfRN0TFIwt6YFL7N9RU= github.com/docker/distribution v2.7.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v0.7.3-0.20190103212154-2b7e084dc98b/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf h1:ZjoOm2c/ckflZVEbHBOwuf4FoUajM5Jx9dDTVBHK9vc= +github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf h1:2v/98rHzs3v6X0AHtoCH9u+e56SdnpogB1Z2fFe1KqQ= github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=