fix: namespace migrator (#417)

closes #404

cmd/namespace/migrate_up.go

@@ -1,8 +1,10 @@
 package namespace
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
+	"strings"
 
 	"github.com/ory/x/cmdx"
 	"github.com/ory/x/flagx"
@@ -38,22 +40,22 @@ func NewMigrateUpCmd() *cobra.Command {
 				return cmdx.FailSilently(cmd)
 			}
 
-			status, err := reg.NamespaceMigrator().NamespaceStatus(ctx, n.ID)
-			if err != nil {
+			status := &bytes.Buffer{}
+			if err := reg.NamespaceMigrator().NamespaceStatus(ctx, status, n); err != nil {
 				if !errors.Is(err, persistence.ErrNamespaceUnknown) {
 					_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not get status for namespace \"%s\": %+v\n", n.Name, err)
 					return cmdx.FailSilently(cmd)
 				}
 			} else {
-				if status.CurrentVersion == status.NextVersion {
+				_, _ = cmd.OutOrStdout().Write(status.Bytes())
+
+				if !strings.Contains(status.String(), "Pending") {
 					_, _ = fmt.Fprintln(cmd.OutOrStdout(), "The namespace is already migrated up to the most recent version, there is noting to do.")
 					return nil
 				}
 
-				cmdx.PrintRow(cmd, status)
-
 				if !flagx.MustGetBool(cmd, YesFlag) {
-					if !cmdx.AskForConfirmation("Are you sure that you want to apply this migration? Make sure to check the CHANGELOG and UPGRADE for breaking changes beforehand.", cmd.InOrStdin(), cmd.OutOrStdout()) {
+					if !cmdx.AskForConfirmation("Are you sure that you want to apply this migration? Make sure to check the CHANGELOG.md and UPGRADE.md for breaking changes beforehand.", cmd.InOrStdin(), cmd.OutOrStdout()) {
 						_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Migration of namespace \"%s\" aborted.\n", n.Name)
 						return nil
 					}
@@ -65,14 +67,11 @@ func NewMigrateUpCmd() *cobra.Command {
 				return cmdx.FailSilently(cmd)
 			}
 
-			status, err = reg.NamespaceMigrator().NamespaceStatus(ctx, n.ID)
-			if err != nil {
+			if err := reg.NamespaceMigrator().NamespaceStatus(ctx, cmd.OutOrStdout(), n); err != nil {
 				_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not get status for namespace \"%s\": %+v\n", n.Name, err)
 				return cmdx.FailSilently(cmd)
 			}
 
-			cmdx.PrintRow(cmd, status)
-
 			return nil
 		},
 	}

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/HdrHistogram/hdrhistogram-go v1.0.1 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect
 	github.com/bufbuild/buf v0.31.1
-	github.com/cenkalti/backoff v2.2.1+incompatible
+	github.com/cenkalti/backoff/v3 v3.0.0
 	github.com/containerd/continuity v0.0.0-20200228182428-0f16d7a0959c // indirect
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-openapi/errors v0.19.4
@@ -28,7 +28,7 @@ require (
 	github.com/ory/graceful v0.1.1
 	github.com/ory/herodot v0.9.1
 	github.com/ory/jsonschema/v3 v3.0.1
-	github.com/ory/x v0.0.178
+	github.com/ory/x v0.0.179
 	github.com/pelletier/go-toml v1.8.0
 	github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
 	github.com/pkg/errors v0.9.1

go.sum

@@ -91,6 +91,8 @@ github.com/cockroachdb/cockroach-go v0.0.0-20190925194419-606b3d062051/go.mod h1
 github.com/cockroachdb/cockroach-go v0.0.0-20200312223839-f565e4789405/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/codegangsta/negroni v1.0.0/go.mod h1:v0y3T5G7Y1UlFfyxFn/QLRU4a2EuNau2iZY63YTKWo0=
+github.com/containerd/containerd v1.4.3 h1:ijQT13JedHSHrQGWFcGEwzcNKrAGIiZ+jSD5QQG07SY=
+github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
 github.com/containerd/continuity v0.0.0-20181203112020-004b46473808/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/containerd/continuity v0.0.0-20200107194136-26c1120b8d41/go.mod h1:Dq467ZllaHgAtVp4p1xUQWBrFXR9s/wyoTpG8zOJGkY=
@@ -124,6 +126,10 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
+github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/docker v17.12.0-ce-rc1.0.20201201034508-7d75c1d40d88+incompatible h1:rsPfdypSNWulLrsXo3WiBdlNQpokgBqfWLjEa/aXiBc=
+github.com/docker/docker v17.12.0-ce-rc1.0.20201201034508-7d75c1d40d88+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk=
@@ -537,6 +543,7 @@ github.com/gofrs/uuid/v3 v3.1.2/go.mod h1:xPwMqoocQ1L5G6pXX5BcE7N5jlzn2o19oqAKxw
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
 github.com/golang/gddo v0.0.0-20180828051604-96d2a289f41e/go.mod h1:xEhNfoBDX1hzLm2Nf80qUvZ2sVwoMZ8d6IE2SrsQfh4=
@@ -890,6 +897,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
 github.com/monoculum/formam v0.0.0-20180901015400-4e68be1d79ba/go.mod h1:RKgILGEJq24YyJ2ban8EO0RUVSJlF1pGsEvoLEACr/Q=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
+github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/moul/http2curl v0.0.0-20170919181001-9ac6cf4d929b/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q=
@@ -967,8 +976,8 @@ github.com/ory/x v0.0.93/go.mod h1:lfcTaGXpTZs7IEQAW00r9EtTCOxD//SiP5uWtNiz31g=
 github.com/ory/x v0.0.110/go.mod h1:DJfkE3GdakhshNhw4zlKoRaL/ozg/lcTahA9OCih2BE=
 github.com/ory/x v0.0.127/go.mod h1:FwUujfFuCj5d+xgLn4fGMYPnzriR5bdAIulFXMtnK0M=
 github.com/ory/x v0.0.128/go.mod h1:ykx1XOsl9taQtoW2yNvuxl/feEfTfrZTcbY1U7841tI=
-github.com/ory/x v0.0.178 h1:pdX2PJLxci+qT2U0lFEjyF7Mu7rTbYocoKimQQ9F1ZA=
-github.com/ory/x v0.0.178/go.mod h1:G+X1V3YTzMiQAMy3tE58cKou+4dcnkjx/jRyawStiPo=
+github.com/ory/x v0.0.179 h1:ewzGC1n2uWEmGRDVzYYzHsbdHrxNtkCwGjbkpZeeNKI=
+github.com/ory/x v0.0.179/go.mod h1:SGETCUk1DgQC30bb7y4hjhkKGQ1x0YOsldrmGmy6MNc=
 github.com/parnurzeal/gorequest v0.2.15/go.mod h1:3Kh2QUMJoqw3icWAecsyzkpY7UzRfDhbRdTjtNwNiUE=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g=
@@ -1438,6 +1447,8 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d h1:nc5K6ox/4lTFbMVSL9WRR81ix
 golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y=
 golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a h1:i47hUS795cOydZI4AwJQCKXOr4BvxzvikwDoDtHhP2Y=
+golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To=

internal/check/engine_test.go

@@ -20,6 +20,11 @@ import (
 
 func newDepsProvider(t *testing.T, namespaces []*namespace.Namespace, pageOpts ...x.PaginationOptionSetter) *relationtuple.ManagerWrapper {
 	reg := driver.NewMemoryTestRegistry(t, namespaces)
+	t.Cleanup(func() {
+		for _, n := range namespaces {
+			require.NoError(t, reg.NamespaceMigrator().MigrateNamespaceDown(context.Background(), n, 0))
+		}
+	})
 	return relationtuple.NewManagerWrapper(t, reg, pageOpts...)
 }
 

internal/driver/config/provider.go

@@ -33,7 +33,7 @@ const (
 
 	KeyNamespaces = "namespaces"
 
-	DSNMemory = "sqlite://:memory:?_fk=true"
+	DSNMemory = "sqlite://file::memory:?_fk=true&cache=shared"
 )
 
 type (

internal/driver/registry_default.go

@@ -1,20 +1,15 @@
 package driver
 
 import (
+	"bytes"
 	"context"
-	"time"
+	"strings"
 
 	"github.com/julienschmidt/httprouter"
 	"google.golang.org/grpc"
 
-	"github.com/cenkalti/backoff"
-	"github.com/ory/x/sqlcon"
-
 	"github.com/ory/keto/internal/driver/config"
 
-	"github.com/pkg/errors"
-
-	"github.com/gobuffalo/pop/v5"
 	"github.com/ory/herodot"
 	"github.com/ory/x/healthx"
 	"github.com/ory/x/logrusx"
@@ -38,13 +33,12 @@ var (
 
 type (
 	RegistryDefault struct {
-		p    persistence.Persister
-		l    *logrusx.Logger
-		w    herodot.Writer
-		ce   *check.Engine
-		ee   *expand.Engine
-		conn *pop.Connection
-		c    *config.Provider
+		p  persistence.Persister
+		l  *logrusx.Logger
+		w  herodot.Writer
+		ce *check.Engine
+		ee *expand.Engine
+		c  *config.Provider
 
 		healthH  *healthx.Handler
 		handlers []Handler
@@ -130,40 +124,12 @@ func (r *RegistryDefault) Migrator() persistence.Migrator {
 }
 
 func (r *RegistryDefault) Init(ctx context.Context) error {
-	bc := backoff.NewExponentialBackOff()
-	bc.MaxElapsedTime = time.Minute * 5
-	bc.Reset()
-
-	if err := backoff.Retry(func() error {
-		pool, idlePool, connMaxLifetime, cleanedDSN := sqlcon.ParseConnectionOptions(r.l, r.c.DSN())
-		c, err := pop.NewConnection(&pop.ConnectionDetails{
-			URL:             sqlcon.FinalizeDSN(r.l, cleanedDSN),
-			IdlePool:        idlePool,
-			ConnMaxLifetime: connMaxLifetime,
-			Pool:            pool,
-		})
-		if err != nil {
-			r.Logger().WithError(err).Warnf("Unable to connect to database, retrying.")
-			return errors.WithStack(err)
-		}
-
-		r.conn = c
-		if err := c.Open(); err != nil {
-			r.Logger().WithError(err).Warnf("Unable to open the database connection, retrying.")
-			return errors.WithStack(err)
-		}
-
-		return nil
-	}, bc); err != nil {
-		return err
-	}
-
 	nm, err := r.c.NamespaceManager()
 	if err != nil {
 		return err
 	}
 
-	r.p, err = sql.NewPersister(r.conn, r.Logger(), nm)
+	r.p, err = sql.NewPersister(r.c.DSN(), r.Logger(), nm)
 	if err != nil {
 		return err
 	}
@@ -179,9 +145,11 @@ func (r *RegistryDefault) Init(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
+	nStatus := &bytes.Buffer{}
 	for _, n := range namespaceConfigs {
-		s, err := r.NamespaceMigrator().NamespaceStatus(ctx, n.ID)
-		if err != nil {
+		if err := r.NamespaceMigrator().NamespaceStatus(ctx, nStatus, n); err != nil {
+			return err
+		} else if strings.Contains(nStatus.String(), "Pending") {
 			if r.c.DSN() == config.DSNMemory {
 				// auto migrate when DSN is memory
 				if err := r.NamespaceMigrator().MigrateNamespaceUp(ctx, n); err != nil {
@@ -193,10 +161,6 @@ func (r *RegistryDefault) Init(ctx context.Context) error {
 			r.l.Warnf("Namespace %s is defined in the config but not yet migrated. It is ignored until you explicitly migrate it.", n.Name)
 			continue
 		}
-
-		if s.CurrentVersion != s.NextVersion {
-			r.l.Warnf("Namespace %s is not migrated to the latest version, it will be ignored until you explicitly migrate it.", n.Name)
-		}
 	}
 
 	return nil

internal/driver/registry_factory.go

@@ -52,6 +52,7 @@ func NewMemoryTestRegistry(t *testing.T, namespaces []*namespace.Namespace) Regi
 	c, err := config.New(ctx, nil, l)
 	require.NoError(t, err)
 	require.NoError(t, c.Set(config.KeyDSN, config.DSNMemory))
+	require.NoError(t, c.Set("log.level", "debug"))
 	require.NoError(t, c.Set(config.KeyNamespaces, namespaces))
 
 	r := &RegistryDefault{

internal/e2e/full_suit_test.go

@@ -47,9 +47,10 @@ func Test(t *testing.T) {
 		//})
 
 		for _, n := range nn {
-			s, err := r.NamespaceMigrator().NamespaceStatus(ctx, n.ID)
-			require.NoError(t, err)
-			assert.Equal(t, s.NextVersion, s.CurrentVersion)
+			out := bytes.Buffer{}
+			require.NoError(t, r.NamespaceMigrator().NamespaceStatus(ctx, &out, n))
+			assert.Contains(t, out.String(), "Applied")
+			assert.NotContains(t, out.String(), "Pending")
 
 			// TODO
 			//t.Cleanup(func() {
@@ -58,7 +59,7 @@ func Test(t *testing.T) {
 		}
 	}
 
-	for _, dsn := range GetDSNs(t) {
+	for _, dsn := range dsns {
 		t.Run(fmt.Sprintf("dsn=%s", dsn.Name), func(t *testing.T) {
 			nspaces := []*namespace.Namespace{{
 				Name: "dreams",

internal/e2e/helpers.go

@@ -109,6 +109,7 @@ func GetDSNs(t testing.TB) []*DsnT {
 			},
 		)
 	}
+	t.Cleanup(dockertest.KillAllTestDatabases)
 
 	return dsns
 }

internal/expand/engine_test.go

@@ -20,8 +20,13 @@ import (
 )
 
 func newTestEngine(t *testing.T, namespaces []*namespace.Namespace, paginationOpts ...x.PaginationOptionSetter) (*relationtuple.ManagerWrapper, *expand.Engine) {
-	reg := relationtuple.NewManagerWrapper(t, driver.NewMemoryTestRegistry(t, namespaces), paginationOpts...)
-
+	innerReg := driver.NewMemoryTestRegistry(t, namespaces)
+	reg := relationtuple.NewManagerWrapper(t, innerReg, paginationOpts...)
+	t.Cleanup(func() {
+		for _, n := range namespaces {
+			require.NoError(t, innerReg.NamespaceMigrator().MigrateNamespaceDown(context.Background(), n, 0))
+		}
+	})
 	e := expand.NewEngine(reg)
 	return reg, e
 }

internal/namespace/definitons.go

@@ -3,9 +3,7 @@ package namespace
 import (
 	"context"
 	"encoding/json"
-	"fmt"
-
-	"github.com/ory/x/cmdx"
+	"io"
 )
 
 type (
@@ -14,14 +12,10 @@ type (
 		Name   string          `json:"name" db:"-" toml:"name"`
 		Config json.RawMessage `json:"config,omitempty" db:"-" toml:"config,omitempty"`
 	}
-	Status struct {
-		CurrentVersion int `json:"current_version" db:"-"`
-		NextVersion    int `json:"next_version" db:"-"`
-	}
 	Migrator interface {
 		MigrateNamespaceUp(ctx context.Context, n *Namespace) error
 		MigrateNamespaceDown(ctx context.Context, n *Namespace, steps int) error
-		NamespaceStatus(ctx context.Context, id int) (*Status, error)
+		NamespaceStatus(ctx context.Context, w io.Writer, n *Namespace) error
 	}
 	Manager interface {
 		GetNamespace(ctx context.Context, name string) (*Namespace, error)
@@ -34,25 +28,3 @@ type (
 		NamespaceMigrator() Migrator
 	}
 )
-
-var (
-	_ cmdx.TableRow = &Status{}
-)
-
-func (s *Status) Header() []string {
-	return []string{
-		"CURRENT VERSION",
-		"NEXT VERSION",
-	}
-}
-
-func (s *Status) Columns() []string {
-	return []string{
-		fmt.Sprintf("%d", s.CurrentVersion),
-		fmt.Sprintf("%d", s.NextVersion),
-	}
-}
-
-func (s *Status) Interface() interface{} {
-	return s
-}