diff --git a/cmd/namespace/migrate_up.go b/cmd/namespace/migrate_up.go index 4527563f5..2294eb2ef 100644 --- a/cmd/namespace/migrate_up.go +++ b/cmd/namespace/migrate_up.go @@ -1,8 +1,10 @@ package namespace import ( + "bytes" "errors" "fmt" + "strings" "github.com/ory/x/cmdx" "github.com/ory/x/flagx" @@ -38,22 +40,22 @@ func NewMigrateUpCmd() *cobra.Command { return cmdx.FailSilently(cmd) } - status, err := reg.NamespaceMigrator().NamespaceStatus(ctx, n.ID) - if err != nil { + status := &bytes.Buffer{} + if err := reg.NamespaceMigrator().NamespaceStatus(ctx, status, n); err != nil { if !errors.Is(err, persistence.ErrNamespaceUnknown) { _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not get status for namespace \"%s\": %+v\n", n.Name, err) return cmdx.FailSilently(cmd) } } else { - if status.CurrentVersion == status.NextVersion { + _, _ = cmd.OutOrStdout().Write(status.Bytes()) + + if !strings.Contains(status.String(), "Pending") { _, _ = fmt.Fprintln(cmd.OutOrStdout(), "The namespace is already migrated up to the most recent version, there is noting to do.") return nil } - cmdx.PrintRow(cmd, status) - if !flagx.MustGetBool(cmd, YesFlag) { - if !cmdx.AskForConfirmation("Are you sure that you want to apply this migration? Make sure to check the CHANGELOG and UPGRADE for breaking changes beforehand.", cmd.InOrStdin(), cmd.OutOrStdout()) { + if !cmdx.AskForConfirmation("Are you sure that you want to apply this migration? Make sure to check the CHANGELOG.md and UPGRADE.md for breaking changes beforehand.", cmd.InOrStdin(), cmd.OutOrStdout()) { _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Migration of namespace \"%s\" aborted.\n", n.Name) return nil } @@ -65,14 +67,11 @@ func NewMigrateUpCmd() *cobra.Command { return cmdx.FailSilently(cmd) } - status, err = reg.NamespaceMigrator().NamespaceStatus(ctx, n.ID) - if err != nil { + if err := reg.NamespaceMigrator().NamespaceStatus(ctx, cmd.OutOrStdout(), n); err != nil { _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not get status for namespace \"%s\": %+v\n", n.Name, err) return cmdx.FailSilently(cmd) } - cmdx.PrintRow(cmd, status) - return nil }, } diff --git a/go.mod b/go.mod index a47c86b4e..77c9be617 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/HdrHistogram/hdrhistogram-go v1.0.1 // indirect github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect github.com/bufbuild/buf v0.31.1 - github.com/cenkalti/backoff v2.2.1+incompatible + github.com/cenkalti/backoff/v3 v3.0.0 github.com/containerd/continuity v0.0.0-20200228182428-0f16d7a0959c // indirect github.com/ghodss/yaml v1.0.0 github.com/go-openapi/errors v0.19.4 @@ -28,7 +28,7 @@ require ( github.com/ory/graceful v0.1.1 github.com/ory/herodot v0.9.1 github.com/ory/jsonschema/v3 v3.0.1 - github.com/ory/x v0.0.178 + github.com/ory/x v0.0.179 github.com/pelletier/go-toml v1.8.0 github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 689796c6c..8f01828bb 100644 --- a/go.sum +++ b/go.sum @@ -91,6 +91,8 @@ github.com/cockroachdb/cockroach-go v0.0.0-20190925194419-606b3d062051/go.mod h1 github.com/cockroachdb/cockroach-go v0.0.0-20200312223839-f565e4789405/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/negroni v1.0.0/go.mod h1:v0y3T5G7Y1UlFfyxFn/QLRU4a2EuNau2iZY63YTKWo0= +github.com/containerd/containerd v1.4.3 h1:ijQT13JedHSHrQGWFcGEwzcNKrAGIiZ+jSD5QQG07SY= +github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/continuity v0.0.0-20181203112020-004b46473808/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20200107194136-26c1120b8d41/go.mod h1:Dq467ZllaHgAtVp4p1xUQWBrFXR9s/wyoTpG8zOJGkY= @@ -124,6 +126,10 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= +github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v17.12.0-ce-rc1.0.20201201034508-7d75c1d40d88+incompatible h1:rsPfdypSNWulLrsXo3WiBdlNQpokgBqfWLjEa/aXiBc= +github.com/docker/docker v17.12.0-ce-rc1.0.20201201034508-7d75c1d40d88+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk= @@ -537,6 +543,7 @@ github.com/gofrs/uuid/v3 v3.1.2/go.mod h1:xPwMqoocQ1L5G6pXX5BcE7N5jlzn2o19oqAKxw github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/gddo v0.0.0-20180828051604-96d2a289f41e/go.mod h1:xEhNfoBDX1hzLm2Nf80qUvZ2sVwoMZ8d6IE2SrsQfh4= @@ -890,6 +897,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/monoculum/formam v0.0.0-20180901015400-4e68be1d79ba/go.mod h1:RKgILGEJq24YyJ2ban8EO0RUVSJlF1pGsEvoLEACr/Q= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/moul/http2curl v0.0.0-20170919181001-9ac6cf4d929b/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= @@ -967,8 +976,8 @@ github.com/ory/x v0.0.93/go.mod h1:lfcTaGXpTZs7IEQAW00r9EtTCOxD//SiP5uWtNiz31g= github.com/ory/x v0.0.110/go.mod h1:DJfkE3GdakhshNhw4zlKoRaL/ozg/lcTahA9OCih2BE= github.com/ory/x v0.0.127/go.mod h1:FwUujfFuCj5d+xgLn4fGMYPnzriR5bdAIulFXMtnK0M= github.com/ory/x v0.0.128/go.mod h1:ykx1XOsl9taQtoW2yNvuxl/feEfTfrZTcbY1U7841tI= -github.com/ory/x v0.0.178 h1:pdX2PJLxci+qT2U0lFEjyF7Mu7rTbYocoKimQQ9F1ZA= -github.com/ory/x v0.0.178/go.mod h1:G+X1V3YTzMiQAMy3tE58cKou+4dcnkjx/jRyawStiPo= +github.com/ory/x v0.0.179 h1:ewzGC1n2uWEmGRDVzYYzHsbdHrxNtkCwGjbkpZeeNKI= +github.com/ory/x v0.0.179/go.mod h1:SGETCUk1DgQC30bb7y4hjhkKGQ1x0YOsldrmGmy6MNc= github.com/parnurzeal/gorequest v0.2.15/go.mod h1:3Kh2QUMJoqw3icWAecsyzkpY7UzRfDhbRdTjtNwNiUE= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= @@ -1438,6 +1447,8 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d h1:nc5K6ox/4lTFbMVSL9WRR81ix golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a h1:i47hUS795cOydZI4AwJQCKXOr4BvxzvikwDoDtHhP2Y= +golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To= diff --git a/internal/check/engine_test.go b/internal/check/engine_test.go index 46e092c6e..70acb55ed 100644 --- a/internal/check/engine_test.go +++ b/internal/check/engine_test.go @@ -20,6 +20,11 @@ import ( func newDepsProvider(t *testing.T, namespaces []*namespace.Namespace, pageOpts ...x.PaginationOptionSetter) *relationtuple.ManagerWrapper { reg := driver.NewMemoryTestRegistry(t, namespaces) + t.Cleanup(func() { + for _, n := range namespaces { + require.NoError(t, reg.NamespaceMigrator().MigrateNamespaceDown(context.Background(), n, 0)) + } + }) return relationtuple.NewManagerWrapper(t, reg, pageOpts...) } diff --git a/internal/driver/config/provider.go b/internal/driver/config/provider.go index 27f99bbed..fd4bc571d 100644 --- a/internal/driver/config/provider.go +++ b/internal/driver/config/provider.go @@ -33,7 +33,7 @@ const ( KeyNamespaces = "namespaces" - DSNMemory = "sqlite://:memory:?_fk=true" + DSNMemory = "sqlite://file::memory:?_fk=true&cache=shared" ) type ( diff --git a/internal/driver/registry_default.go b/internal/driver/registry_default.go index 2c7e05a9c..1ab9405ee 100644 --- a/internal/driver/registry_default.go +++ b/internal/driver/registry_default.go @@ -1,20 +1,15 @@ package driver import ( + "bytes" "context" - "time" + "strings" "github.com/julienschmidt/httprouter" "google.golang.org/grpc" - "github.com/cenkalti/backoff" - "github.com/ory/x/sqlcon" - "github.com/ory/keto/internal/driver/config" - "github.com/pkg/errors" - - "github.com/gobuffalo/pop/v5" "github.com/ory/herodot" "github.com/ory/x/healthx" "github.com/ory/x/logrusx" @@ -38,13 +33,12 @@ var ( type ( RegistryDefault struct { - p persistence.Persister - l *logrusx.Logger - w herodot.Writer - ce *check.Engine - ee *expand.Engine - conn *pop.Connection - c *config.Provider + p persistence.Persister + l *logrusx.Logger + w herodot.Writer + ce *check.Engine + ee *expand.Engine + c *config.Provider healthH *healthx.Handler handlers []Handler @@ -130,40 +124,12 @@ func (r *RegistryDefault) Migrator() persistence.Migrator { } func (r *RegistryDefault) Init(ctx context.Context) error { - bc := backoff.NewExponentialBackOff() - bc.MaxElapsedTime = time.Minute * 5 - bc.Reset() - - if err := backoff.Retry(func() error { - pool, idlePool, connMaxLifetime, cleanedDSN := sqlcon.ParseConnectionOptions(r.l, r.c.DSN()) - c, err := pop.NewConnection(&pop.ConnectionDetails{ - URL: sqlcon.FinalizeDSN(r.l, cleanedDSN), - IdlePool: idlePool, - ConnMaxLifetime: connMaxLifetime, - Pool: pool, - }) - if err != nil { - r.Logger().WithError(err).Warnf("Unable to connect to database, retrying.") - return errors.WithStack(err) - } - - r.conn = c - if err := c.Open(); err != nil { - r.Logger().WithError(err).Warnf("Unable to open the database connection, retrying.") - return errors.WithStack(err) - } - - return nil - }, bc); err != nil { - return err - } - nm, err := r.c.NamespaceManager() if err != nil { return err } - r.p, err = sql.NewPersister(r.conn, r.Logger(), nm) + r.p, err = sql.NewPersister(r.c.DSN(), r.Logger(), nm) if err != nil { return err } @@ -179,9 +145,11 @@ func (r *RegistryDefault) Init(ctx context.Context) error { if err != nil { return err } + nStatus := &bytes.Buffer{} for _, n := range namespaceConfigs { - s, err := r.NamespaceMigrator().NamespaceStatus(ctx, n.ID) - if err != nil { + if err := r.NamespaceMigrator().NamespaceStatus(ctx, nStatus, n); err != nil { + return err + } else if strings.Contains(nStatus.String(), "Pending") { if r.c.DSN() == config.DSNMemory { // auto migrate when DSN is memory if err := r.NamespaceMigrator().MigrateNamespaceUp(ctx, n); err != nil { @@ -193,10 +161,6 @@ func (r *RegistryDefault) Init(ctx context.Context) error { r.l.Warnf("Namespace %s is defined in the config but not yet migrated. It is ignored until you explicitly migrate it.", n.Name) continue } - - if s.CurrentVersion != s.NextVersion { - r.l.Warnf("Namespace %s is not migrated to the latest version, it will be ignored until you explicitly migrate it.", n.Name) - } } return nil diff --git a/internal/driver/registry_factory.go b/internal/driver/registry_factory.go index 621a10af2..c1993b009 100644 --- a/internal/driver/registry_factory.go +++ b/internal/driver/registry_factory.go @@ -52,6 +52,7 @@ func NewMemoryTestRegistry(t *testing.T, namespaces []*namespace.Namespace) Regi c, err := config.New(ctx, nil, l) require.NoError(t, err) require.NoError(t, c.Set(config.KeyDSN, config.DSNMemory)) + require.NoError(t, c.Set("log.level", "debug")) require.NoError(t, c.Set(config.KeyNamespaces, namespaces)) r := &RegistryDefault{ diff --git a/internal/e2e/full_suit_test.go b/internal/e2e/full_suit_test.go index 1a33a17aa..577320003 100644 --- a/internal/e2e/full_suit_test.go +++ b/internal/e2e/full_suit_test.go @@ -47,9 +47,10 @@ func Test(t *testing.T) { //}) for _, n := range nn { - s, err := r.NamespaceMigrator().NamespaceStatus(ctx, n.ID) - require.NoError(t, err) - assert.Equal(t, s.NextVersion, s.CurrentVersion) + out := bytes.Buffer{} + require.NoError(t, r.NamespaceMigrator().NamespaceStatus(ctx, &out, n)) + assert.Contains(t, out.String(), "Applied") + assert.NotContains(t, out.String(), "Pending") // TODO //t.Cleanup(func() { @@ -58,7 +59,7 @@ func Test(t *testing.T) { } } - for _, dsn := range GetDSNs(t) { + for _, dsn := range dsns { t.Run(fmt.Sprintf("dsn=%s", dsn.Name), func(t *testing.T) { nspaces := []*namespace.Namespace{{ Name: "dreams", diff --git a/internal/e2e/helpers.go b/internal/e2e/helpers.go index c437c1d6c..1fe26e78a 100644 --- a/internal/e2e/helpers.go +++ b/internal/e2e/helpers.go @@ -109,6 +109,7 @@ func GetDSNs(t testing.TB) []*DsnT { }, ) } + t.Cleanup(dockertest.KillAllTestDatabases) return dsns } diff --git a/internal/expand/engine_test.go b/internal/expand/engine_test.go index b13d6b293..7fd83f837 100644 --- a/internal/expand/engine_test.go +++ b/internal/expand/engine_test.go @@ -20,8 +20,13 @@ import ( ) func newTestEngine(t *testing.T, namespaces []*namespace.Namespace, paginationOpts ...x.PaginationOptionSetter) (*relationtuple.ManagerWrapper, *expand.Engine) { - reg := relationtuple.NewManagerWrapper(t, driver.NewMemoryTestRegistry(t, namespaces), paginationOpts...) - + innerReg := driver.NewMemoryTestRegistry(t, namespaces) + reg := relationtuple.NewManagerWrapper(t, innerReg, paginationOpts...) + t.Cleanup(func() { + for _, n := range namespaces { + require.NoError(t, innerReg.NamespaceMigrator().MigrateNamespaceDown(context.Background(), n, 0)) + } + }) e := expand.NewEngine(reg) return reg, e } diff --git a/internal/namespace/definitons.go b/internal/namespace/definitons.go index 04e261391..47dd5d60d 100644 --- a/internal/namespace/definitons.go +++ b/internal/namespace/definitons.go @@ -3,9 +3,7 @@ package namespace import ( "context" "encoding/json" - "fmt" - - "github.com/ory/x/cmdx" + "io" ) type ( @@ -14,14 +12,10 @@ type ( Name string `json:"name" db:"-" toml:"name"` Config json.RawMessage `json:"config,omitempty" db:"-" toml:"config,omitempty"` } - Status struct { - CurrentVersion int `json:"current_version" db:"-"` - NextVersion int `json:"next_version" db:"-"` - } Migrator interface { MigrateNamespaceUp(ctx context.Context, n *Namespace) error MigrateNamespaceDown(ctx context.Context, n *Namespace, steps int) error - NamespaceStatus(ctx context.Context, id int) (*Status, error) + NamespaceStatus(ctx context.Context, w io.Writer, n *Namespace) error } Manager interface { GetNamespace(ctx context.Context, name string) (*Namespace, error) @@ -34,25 +28,3 @@ type ( NamespaceMigrator() Migrator } ) - -var ( - _ cmdx.TableRow = &Status{} -) - -func (s *Status) Header() []string { - return []string{ - "CURRENT VERSION", - "NEXT VERSION", - } -} - -func (s *Status) Columns() []string { - return []string{ - fmt.Sprintf("%d", s.CurrentVersion), - fmt.Sprintf("%d", s.NextVersion), - } -} - -func (s *Status) Interface() interface{} { - return s -} diff --git a/internal/persistence/sql/namespace.go b/internal/persistence/sql/namespace.go index 7c2ce836b..0e74c3585 100644 --- a/internal/persistence/sql/namespace.go +++ b/internal/persistence/sql/namespace.go @@ -2,111 +2,62 @@ package sql import ( "context" - "database/sql" "fmt" + "io" - "github.com/pkg/errors" - - "github.com/gobuffalo/pop/v5" + "github.com/ory/x/pkgerx" "github.com/ory/keto/internal/namespace" - "github.com/ory/keto/internal/persistence" -) - -type ( - namespaceRow struct { - ID int `db:"id"` - Version int `db:"schema_version"` - } -) - -const ( - namespaceCreateStatement = ` -CREATE TABLE %[1]s -( - shard_id varchar(64), - object varchar(64), - relation varchar(64), - subject varchar(256), /* can be or */ - commit_time timestamp, - - PRIMARY KEY (shard_id, object, relation, subject, commit_time) -); - -CREATE INDEX %[1]s_object_idx ON %[1]s (object); - -CREATE INDEX %[1]s_user_set_idx ON %[1]s (object, relation); -` - namespaceDropStatement = ` -DROP INDEX %[1]s_user_set_idx; - -DROP INDEX %[1]s_object_idx; - -DROP TABLE %[1]s; -` - - mostRecentSchemaVersion = 1 ) func tableFromNamespace(n *namespace.Namespace) string { return fmt.Sprintf("keto_%0.10d_relation_tuples", n.ID) } -func createStmt(n *namespace.Namespace) string { - return fmt.Sprintf(namespaceCreateStatement, tableFromNamespace(n)) +func migrationTableFromNamespace(n *namespace.Namespace) string { + return fmt.Sprintf("keto_namespace_%0.10d_migrations", n.ID) } -func dropStmt(n *namespace.Namespace) string { - return fmt.Sprintf(namespaceDropStatement, tableFromNamespace(n)) -} +func (p *Persister) namespaceMigrationBox(n *namespace.Namespace) (*pkgerx.MigrationBox, error) { + c, err := p.newConnection(map[string]string{ + "migration_table_name": migrationTableFromNamespace(n), + }) + if err != nil { + return nil, err + } -func (p *Persister) MigrateNamespaceUp(ctx context.Context, n *namespace.Namespace) error { - return p.transaction(ctx, func(ctx context.Context, c *pop.Connection) error { - // TODO this is only creating new namespaces and not applying migrations - nr := namespaceRow{ - ID: n.ID, - Version: mostRecentSchemaVersion, - } + return pkgerx.NewMigrationBox(namespaceMigrations, c, p.l, pkgerx.WithTemplateValues(map[string]interface{}{ + "tableName": tableFromNamespace(n), + })) +} - // first create the table because of cockroach limitations, see https://github.com/cockroachdb/cockroach/issues/54477 - if err := c.RawQuery(createStmt(n)).Exec(); err != nil { - return errors.WithStack(err) - } +func (p *Persister) MigrateNamespaceUp(_ context.Context, n *namespace.Namespace) error { + mb, err := p.namespaceMigrationBox(n) + if err != nil { + return err + } - return errors.WithStack(c.RawQuery(fmt.Sprintf("INSERT INTO %s (id, schema_version) VALUES (?, ?)", nr.TableName()), nr.ID, nr.Version).Exec()) - }) + return mb.Up() } -func (p *Persister) MigrateNamespaceDown(ctx context.Context, n *namespace.Namespace, _ int) error { - return p.transaction(ctx, func(ctx context.Context, c *pop.Connection) error { - if err := c.RawQuery(dropStmt(n)).Exec(); err != nil { - return errors.WithStack(err) - } +func (p *Persister) MigrateNamespaceDown(_ context.Context, n *namespace.Namespace, steps int) error { + mb, err := p.namespaceMigrationBox(n) + if err != nil { + return err + } - return errors.WithStack(c.RawQuery(fmt.Sprintf("DELETE FROM %s WHERE id = ?", (&namespaceRow{}).TableName()), n.ID).Exec()) - }) + return mb.Down(steps) } func (p *Persister) NamespaceFromName(ctx context.Context, name string) (*namespace.Namespace, error) { return p.namespaces.GetNamespace(ctx, name) } -func (p *Persister) NamespaceStatus(ctx context.Context, id int) (*namespace.Status, error) { - var n namespaceRow - if err := p.connection(ctx).Find(&n, id); err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, persistence.ErrNamespaceUnknown - } - - return nil, err +func (p *Persister) NamespaceStatus(_ context.Context, w io.Writer, n *namespace.Namespace) error { + mb, err := p.namespaceMigrationBox(n) + if err != nil { + return err } - return &namespace.Status{ - CurrentVersion: n.Version, - NextVersion: mostRecentSchemaVersion, - }, nil -} - -func (n *namespaceRow) TableName() string { - return "keto_namespace" + return mb.Status(w) } diff --git a/internal/persistence/sql/namespace_migrations/20210128140414_namespace.down.sql b/internal/persistence/sql/namespace_migrations/20210128140414_namespace.down.sql new file mode 100644 index 000000000..cc7e55c39 --- /dev/null +++ b/internal/persistence/sql/namespace_migrations/20210128140414_namespace.down.sql @@ -0,0 +1,5 @@ +DROP INDEX {{ identifier .Parameters.tableName }}_user_set_idx; + +DROP INDEX {{ identifier .Parameters.tableName }}_object_idx; + +DROP TABLE {{ identifier .Parameters.tableName }}; diff --git a/internal/persistence/sql/namespace_migrations/20210128140414_namespace.mysql.down.sql b/internal/persistence/sql/namespace_migrations/20210128140414_namespace.mysql.down.sql new file mode 100644 index 000000000..62b69a636 --- /dev/null +++ b/internal/persistence/sql/namespace_migrations/20210128140414_namespace.mysql.down.sql @@ -0,0 +1,5 @@ +DROP INDEX {{ identifier .Parameters.tableName }}_user_set_idx ON {{ identifier .Parameters.tableName }}; + +DROP INDEX {{ identifier .Parameters.tableName }}_object_idx ON {{ identifier .Parameters.tableName }}; + +DROP TABLE {{ identifier .Parameters.tableName }}; diff --git a/internal/persistence/sql/namespace_migrations/20210128140414_namespace.up.sql b/internal/persistence/sql/namespace_migrations/20210128140414_namespace.up.sql new file mode 100644 index 000000000..b9642b792 --- /dev/null +++ b/internal/persistence/sql/namespace_migrations/20210128140414_namespace.up.sql @@ -0,0 +1,14 @@ +CREATE TABLE {{ identifier .Parameters.tableName }} +( + shard_id varchar(64), + object varchar(64), + relation varchar(64), + subject varchar(256), /* can be or */ + commit_time timestamp, + + PRIMARY KEY (shard_id, object, relation, subject, commit_time) +); + +CREATE INDEX {{ identifier .Parameters.tableName }}_object_idx ON {{ identifier .Parameters.tableName }} (object); + +CREATE INDEX {{ identifier .Parameters.tableName }}_user_set_idx ON {{ identifier .Parameters.tableName }} (object, relation); diff --git a/internal/persistence/sql/namespace_test.go b/internal/persistence/sql/namespace_test.go new file mode 100644 index 000000000..c85198ad8 --- /dev/null +++ b/internal/persistence/sql/namespace_test.go @@ -0,0 +1,16 @@ +package sql + +//import ( +// "github.com/ory/keto/internal/e2e" +// "testing" +//) +// +//func TestNamespaceMigrations(t *testing.T) { +// setup := func(t *testing.T, dsn *e2e.DsnT) *Persister { +// NewPersister() +// } +// +// t.Run("case=migrates up", func(t *testing.T) { +// +// }) +//} diff --git a/internal/persistence/sql/persister.go b/internal/persistence/sql/persister.go index a662bd5a4..40fd46556 100644 --- a/internal/persistence/sql/persister.go +++ b/internal/persistence/sql/persister.go @@ -5,6 +5,10 @@ import ( "fmt" "io" "strconv" + "time" + + "github.com/cenkalti/backoff/v3" + "github.com/ory/x/sqlcon" "github.com/ory/keto/internal/namespace" @@ -24,6 +28,8 @@ type ( conn *pop.Connection mb *pkgerx.MigrationBox namespaces namespace.Manager + l *logrusx.Logger + dsn string } internalPagination struct { Offset int @@ -38,23 +44,70 @@ const ( ) var ( - migrations = pkger.Dir("/internal/persistence/sql/migrations") + migrations = pkger.Dir("/internal/persistence/sql/migrations") + namespaceMigrations = pkger.Dir("/internal/persistence/sql/namespace_migrations") _ persistence.Persister = &Persister{} ) -func NewPersister(c *pop.Connection, l *logrusx.Logger, namespaces namespace.Manager) (*Persister, error) { +func NewPersister(dsn string, l *logrusx.Logger, namespaces namespace.Manager) (*Persister, error) { pop.SetLogger(l.PopLogger) - mb, err := pkgerx.NewMigrationBox(migrations, c, l) + p := &Persister{ + namespaces: namespaces, + l: l, + dsn: dsn, + } + + var err error + p.conn, err = p.newConnection(nil) if err != nil { return nil, err } - return &Persister{ - mb: mb, - conn: c, - namespaces: namespaces, - }, nil + + p.mb, err = pkgerx.NewMigrationBox(migrations, p.conn, l) + if err != nil { + return nil, err + } + return p, nil +} + +func (p *Persister) newConnection(options map[string]string) (c *pop.Connection, err error) { + pool, idlePool, connMaxLifetime, cleanedDSN := sqlcon.ParseConnectionOptions(p.l, p.dsn) + connDetails := &pop.ConnectionDetails{ + URL: sqlcon.FinalizeDSN(p.l, cleanedDSN), + IdlePool: idlePool, + ConnMaxLifetime: connMaxLifetime, + Pool: pool, + Options: options, + } + + bc := backoff.NewExponentialBackOff() + bc.MaxElapsedTime = time.Minute * 5 + bc.Reset() + + if err := backoff.Retry(func() (err error) { + c, err = pop.NewConnection(connDetails) + if err != nil { + p.l.WithError(err).Warnf("Unable to connect to database, retrying.") + return errors.WithStack(err) + } + + if err := c.Open(); err != nil { + p.l.WithError(err).Warnf("Unable to open the database connection, retrying.") + return errors.WithStack(err) + } + + if err := c.Store.(interface{ Ping() error }).Ping(); err != nil { + return errors.WithStack(err) + } + + return nil + }, bc); err != nil { + return nil, errors.WithStack(err) + } + + return c, nil } func (p *Persister) MigrateUp(_ context.Context) error {