feat: migration box now supports autocommit DDL changes

go.mod

@@ -25,10 +25,10 @@ require (
 	github.com/go-sql-driver/mysql v1.8.1
 	github.com/gobuffalo/fizz v1.14.4
 	github.com/gobuffalo/httptest v1.5.2
-	github.com/gobuffalo/pop/v6 v6.0.8
+	github.com/gobuffalo/pop/v6 v6.1.1
 	github.com/gobwas/glob v0.2.3
 	github.com/goccy/go-yaml v1.9.6
-	github.com/gofrs/uuid v4.3.0+incompatible
+	github.com/gofrs/uuid v4.3.1+incompatible
 	github.com/golang-jwt/jwt/v5 v5.2.1
 	github.com/golang/mock v1.6.0
 	github.com/google/go-jsonnet v0.20.0
@@ -138,12 +138,12 @@ require (
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/go-pdf/fpdf v0.6.0 // indirect
 	github.com/gobuffalo/envy v1.10.2 // indirect
-	github.com/gobuffalo/flect v0.3.0 // indirect
+	github.com/gobuffalo/flect v1.0.0 // indirect
 	github.com/gobuffalo/github_flavored_markdown v1.1.3 // indirect
 	github.com/gobuffalo/helpers v0.6.7 // indirect
 	github.com/gobuffalo/here v0.6.7 // indirect
 	github.com/gobuffalo/nulls v0.4.2 // indirect
-	github.com/gobuffalo/plush/v4 v4.1.16 // indirect
+	github.com/gobuffalo/plush/v4 v4.1.18 // indirect
 	github.com/gobuffalo/tags/v3 v3.1.4 // indirect
 	github.com/gobuffalo/validate/v3 v3.3.3 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect

go.sum

@@ -236,6 +236,7 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+
 github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
 github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
@@ -253,8 +254,9 @@ github.com/gobuffalo/fizz v1.14.4/go.mod h1:9/2fGNXNeIFOXEEgTPJwiK63e44RjG+Nc4hf
 github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs=
 github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
 github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
-github.com/gobuffalo/flect v0.3.0 h1:erfPWM+K1rFNIQeRPdeEXxo8yFr/PO17lhRnS8FUrtk=
 github.com/gobuffalo/flect v0.3.0/go.mod h1:5pf3aGnsvqvCj50AVni7mJJF8ICxGZ8HomberC3pXLE=
+github.com/gobuffalo/flect v1.0.0 h1:eBFmskjXZgAOagiTXJH25Nt5sdFwNRcb8DKZsIsAUQI=
+github.com/gobuffalo/flect v1.0.0/go.mod h1:l9V6xSb4BlXwsxEMj3FVEub2nkdQjWhPvD8XTTlHPQc=
 github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk=
 github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28=
 github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo=
@@ -284,10 +286,11 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe
 github.com/gobuffalo/packd v1.0.2/go.mod h1:sUc61tDqGMXON80zpKGp92lDb86Km28jfvX7IAyxFT8=
 github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
 github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
-github.com/gobuffalo/plush/v4 v4.1.16 h1:Y6jVVTLdg1BxRXDIbTJz+J8QRzEAtv5ZwYpGdIFR7VU=
 github.com/gobuffalo/plush/v4 v4.1.16/go.mod h1:6t7swVsarJ8qSLw1qyAH/KbrcSTwdun2ASEQkOznakg=
-github.com/gobuffalo/pop/v6 v6.0.8 h1:9+5ShHYh3x9NDFCITfm/gtKDDRSgOwiY7kA0Hf7N9aQ=
-github.com/gobuffalo/pop/v6 v6.0.8/go.mod h1:f4JQ4Zvkffcevz+t+XAwBLStD7IQs19DiIGIDFYw1eA=
+github.com/gobuffalo/plush/v4 v4.1.18 h1:bnPjdMTEUQHqj9TNX2Ck3mxEXYZa+0nrFMNM07kpX9g=
+github.com/gobuffalo/plush/v4 v4.1.18/go.mod h1:xi2tJIhFI4UdzIL8sxZtzGYOd2xbBpcFbLZlIPGGZhU=
+github.com/gobuffalo/pop/v6 v6.1.1 h1:eUDBaZcb0gYrmFnKwpuTEUA7t5ZHqNfvS4POqJYXDZY=
+github.com/gobuffalo/pop/v6 v6.1.1/go.mod h1:1n7jAmI1i7fxuXPZjZb0VBPQDbksRtCoFnrDV5IsvaI=
 github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
 github.com/gobuffalo/tags/v3 v3.1.4 h1:X/ydLLPhgXV4h04Hp2xlbI2oc5MDaa7eub6zw8oHjsM=
 github.com/gobuffalo/tags/v3 v3.1.4/go.mod h1:ArRNo3ErlHO8BtdA0REaZxijuWnWzF6PUXngmMXd2I0=
@@ -303,8 +306,8 @@ github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
 github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
-github.com/gofrs/uuid v4.3.0+incompatible h1:CaSVZxm5B+7o45rtab4jC2G37WGYX1zQfuU2i6DSvnc=
-github.com/gofrs/uuid v4.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gofrs/uuid v4.3.1+incompatible h1:0/KbAdpx3UXAx1kEOWHJeOkpbgRFGHVgv+CFIY7dBJI=
+github.com/gofrs/uuid v4.3.1+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
@@ -416,6 +419,7 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf h1:FtEj8sfIcaaBfAKrE1Cwb61YDtYq9JxChK1c7AKce7s=
@@ -745,7 +749,7 @@ github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/
 github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
 github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
-github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
+github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
 github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
 github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=

popx/match.go

@@ -0,0 +1,64 @@
+// Copyright © 2024 Ory Corp
+// SPDX-License-Identifier: Apache-2.0
+
+package popx
+
+import (
+	"fmt"
+	"regexp"
+
+	"github.com/gobuffalo/pop/v6"
+)
+
+var mrx = regexp.MustCompile(`^(\d+)_([^.]+)(\.[a-z0-9]+)?(|\.autocommit)\.(up|down)\.(sql|fizz)$`)
+
+// Match holds the information parsed from a migration filename.
+type Match struct {
+	Version    string
+	Name       string
+	DBType     string
+	Direction  string
+	Type       string
+	Autocommit bool
+}
+
+// ParseMigrationFilename parses a migration filename.
+func ParseMigrationFilename(filename string) (*Match, error) {
+	matches := mrx.FindAllStringSubmatch(filename, -1)
+	if len(matches) == 0 {
+		return nil, nil
+	}
+	m := matches[0]
+
+	var dbType string
+	if m[3] == "" {
+		dbType = "all"
+	} else {
+		dbType = pop.CanonicalDialect(m[3][1:])
+		if !pop.DialectSupported(dbType) {
+			return nil, fmt.Errorf("unsupported dialect %s", dbType)
+		}
+	}
+
+	if m[6] == "fizz" && dbType != "all" {
+		return nil, fmt.Errorf("invalid database type %q, expected \"all\" because fizz is database type independent", dbType)
+	}
+
+	autocommit := false
+	if m[4] == ".autocommit" {
+		autocommit = true
+	} else if m[4] != "" {
+		return nil, fmt.Errorf("invalid autocommit flag %q", m[4])
+	}
+
+	match := &Match{
+		Version:    m[1],
+		Name:       m[2],
+		DBType:     dbType,
+		Autocommit: autocommit,
+		Direction:  m[5],
+		Type:       m[6],
+	}
+
+	return match, nil
+}

popx/match_test.go

@@ -0,0 +1,112 @@
+// Copyright © 2024 Ory Corp
+// SPDX-License-Identifier: Apache-2.0
+
+package popx
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func Test_ParseMigrationFilenameFizzDown(t *testing.T) {
+	r := require.New(t)
+
+	m, err := ParseMigrationFilename("20190611004000_create_providers.down.fizz")
+	r.NoError(err)
+	r.NotNil(m)
+	r.Equal(m.Version, "20190611004000")
+	r.Equal(m.Name, "create_providers")
+	r.Equal(m.DBType, "all")
+	r.Equal(m.Direction, "down")
+	r.Equal(m.Type, "fizz")
+	r.Equal(m.Autocommit, false)
+}
+
+func Test_ParseMigrationFilenameFizzUp(t *testing.T) {
+	r := require.New(t)
+
+	m, err := ParseMigrationFilename("20190611004000_create_providers.up.fizz")
+	r.NoError(err)
+	r.NotNil(m)
+	r.Equal(m.Version, "20190611004000")
+	r.Equal(m.Name, "create_providers")
+	r.Equal(m.DBType, "all")
+	r.Equal(m.Direction, "up")
+	r.Equal(m.Type, "fizz")
+	r.Equal(m.Autocommit, false)
+}
+
+func Test_ParseMigrationFilenameFizzUpPostgres(t *testing.T) {
+	r := require.New(t)
+
+	m, err := ParseMigrationFilename("20190611004000_create_providers.pg.up.fizz")
+	r.NotNil(err)
+	r.Equal(err.Error(), "invalid database type \"postgres\", expected \"all\" because fizz is database type independent")
+	r.Nil(m)
+}
+
+func Test_ParseMigrationFilenameFizzDownPostgres(t *testing.T) {
+	r := require.New(t)
+
+	m, err := ParseMigrationFilename("20190611004000_create_providers.pg.down.fizz")
+	r.NotNil(err)
+	r.Equal(err.Error(), "invalid database type \"postgres\", expected \"all\" because fizz is database type independent")
+	r.Nil(m)
+}
+
+func Test_ParseMigrationFilenameSQLUp(t *testing.T) {
+	r := require.New(t)
+
+	m, err := ParseMigrationFilename("20190611004000_create_providers.up.sql")
+	r.NoError(err)
+	r.NotNil(m)
+	r.Equal(m.Version, "20190611004000")
+	r.Equal(m.Name, "create_providers")
+	r.Equal(m.DBType, "all")
+	r.Equal(m.Direction, "up")
+	r.Equal(m.Type, "sql")
+	r.Equal(m.Autocommit, false)
+}
+
+func Test_ParseMigrationFilenameSQLUpPostgres(t *testing.T) {
+	r := require.New(t)
+
+	m, err := ParseMigrationFilename("20190611004000_create_providers.pg.up.sql")
+	r.NoError(err)
+	r.NotNil(m)
+	r.Equal(m.Version, "20190611004000")
+	r.Equal(m.Name, "create_providers")
+	r.Equal(m.DBType, "postgres")
+	r.Equal(m.Direction, "up")
+	r.Equal(m.Type, "sql")
+	r.Equal(m.Autocommit, false)
+}
+
+func Test_ParseMigrationFilenameSQLUpAutocommit(t *testing.T) {
+	r := require.New(t)
+
+	m, err := ParseMigrationFilename("20190611004000_create_providers.autocommit.up.sql")
+	r.NoError(err)
+	r.NotNil(m)
+	r.Equal(m.Version, "20190611004000")
+	r.Equal(m.Name, "create_providers")
+	r.Equal(m.DBType, "all")
+	r.Equal(m.Direction, "up")
+	r.Equal(m.Type, "sql")
+	r.Equal(m.Autocommit, true)
+}
+
+func Test_ParseMigrationFilenameSQLDownAutocommit(t *testing.T) {
+	r := require.New(t)
+
+	m, err := ParseMigrationFilename("20190611004000_create_providers.mysql.autocommit.down.sql")
+	r.NoError(err)
+	r.NotNil(m)
+	r.Equal(m.Version, "20190611004000")
+	r.Equal(m.Name, "create_providers")
+	r.Equal(m.DBType, "mysql")
+	r.Equal(m.Direction, "down")
+	r.Equal(m.Type, "sql")
+	r.Equal(m.Autocommit, true)
+}

popx/migration_box.go

@@ -146,7 +146,7 @@ func NewMigrationBox(dir fs.FS, m *Migrator, opts ...MigrationBoxOption) (*Migra
 		mb = o(mb)
 	}
 
-	runner := func(b []byte) func(Migration, *pop.Connection, *pop.Tx) error {
+	txRunner := func(b []byte) func(Migration, *pop.Connection, *pop.Tx) error {
 		return func(mf Migration, c *pop.Connection, tx *pop.Tx) error {
 			content, err := mb.migrationContent(mf, c, b, true)
 			if err != nil {
@@ -163,7 +163,24 @@ func NewMigrationBox(dir fs.FS, m *Migrator, opts ...MigrationBoxOption) (*Migra
 		}
 	}
 
-	err := mb.findMigrations(runner)
+	autoCommitRunner := func(b []byte) func(Migration, *pop.Connection) error {
+		return func(mf Migration, c *pop.Connection) error {
+			content, err := mb.migrationContent(mf, c, b, true)
+			if err != nil {
+				return errors.Wrapf(err, "error processing %s", mf.Path)
+			}
+			if isMigrationEmpty(content) {
+				m.l.WithField("migration", mf.Path).Trace("This is usually ok - ignoring migration because content is empty. This is ok!")
+				return nil
+			}
+			if _, err = c.RawQuery(content).ExecWithCount(); err != nil {
+				return errors.Wrapf(err, "error executing %s, sql: %s", mf.Path, content)
+			}
+			return nil
+		}
+	}
+
+	err := mb.findMigrations(txRunner, autoCommitRunner)
 	if err != nil {
 		return mb, err
 	}
@@ -178,7 +195,10 @@ func NewMigrationBox(dir fs.FS, m *Migrator, opts ...MigrationBoxOption) (*Migra
 	return mb, nil
 }
 
-func (fm *MigrationBox) findMigrations(runner func([]byte) func(mf Migration, c *pop.Connection, tx *pop.Tx) error) error {
+func (fm *MigrationBox) findMigrations(
+	runner func([]byte) func(mf Migration, c *pop.Connection, tx *pop.Tx) error,
+	runnerNoTx func([]byte) func(mf Migration, c *pop.Connection) error,
+) error {
 	return fs.WalkDir(fm.Dir, ".", func(p string, info fs.DirEntry, err error) error {
 		if err != nil {
 			return errors.WithStack(err)
@@ -188,7 +208,7 @@ func (fm *MigrationBox) findMigrations(runner func([]byte) func(mf Migration, c
 			return nil
 		}
 
-		match, err := pop.ParseMigrationFilename(info.Name())
+		match, err := ParseMigrationFilename(info.Name())
 		if err != nil {
 			if strings.HasPrefix(err.Error(), "unsupported dialect") {
 				fm.l.Tracef("This is usually ok - ignoring migration file %s because dialect is not supported: %s", info.Name(), err.Error())
@@ -219,8 +239,14 @@ func (fm *MigrationBox) findMigrations(runner func([]byte) func(mf Migration, c
 			DBType:    match.DBType,
 			Direction: match.Direction,
 			Type:      match.Type,
-			Runner:    runner(content),
 		}
+
+		if match.Autocommit {
+			mf.RunnerNoTx = runnerNoTx(content)
+		} else {
+			mf.Runner = runner(content)
+		}
+
 		fm.Migrations[mf.Direction] = append(fm.Migrations[mf.Direction], mf)
 		mod := sort.Interface(fm.Migrations[mf.Direction])
 		if mf.Direction == "down" {

popx/migration_box_testdata_test.go

@@ -22,6 +22,9 @@ var testData embed.FS
 //go:embed stub/migrations/testdata_migrations/*
 var empty embed.FS
 
+//go:embed stub/migrations/notx/*
+var notx embed.FS
+
 //go:embed stub/migrations/check/valid/*
 var checkValidFS embed.FS
 
@@ -54,6 +57,25 @@ func TestMigrationBoxWithTestdata(t *testing.T) {
 	assert.Equal(t, "testdata", data.Data)
 }
 
+func TestMigrationBoxWithoutTransaction(t *testing.T) {
+	c, err := pop.NewConnection(&pop.ConnectionDetails{
+		URL: "sqlite://file::memory:?_fk=true",
+	})
+	require.NoError(t, err)
+	require.NoError(t, c.Open())
+
+	mb, err := popx.NewMigrationBox(
+		notx,
+		popx.NewMigrator(c, logrusx.New("", ""), nil, 0),
+	)
+
+	require.NoError(t, err)
+	assert.Len(t, mb.Migrations["up"], 1)
+	assert.Len(t, mb.Migrations["down"], 1)
+
+	require.NoError(t, mb.Up(context.Background()), "should not fail even though we are creating a transaction in the migration")
+}
+
 func TestMigrationBox_CheckNoErr(t *testing.T) {
 	c, err := pop.NewConnection(&pop.ConnectionDetails{
 		URL: "sqlite://file::memory:?_fk=true",

popx/stub/migrations/notx/20241031_notx.autocommit.down.sql

@@ -0,0 +1 @@
+BEGIN;ROLLBACK;

popx/stub/migrations/notx/20241031_notx.autocommit.up.sql

@@ -0,0 +1 @@
+BEGIN;ROLLBACK;