From 1b5f41509ebcf640e0e7180e84024800be2b8923 Mon Sep 17 00:00:00 2001 From: zepatrik Date: Mon, 13 Sep 2021 17:39:25 +0200 Subject: [PATCH 01/18] feat: migration to single table SQL schema --- cmd/namespace/migrate_legacy.go | 104 +++++++ cmd/namespace/root.go | 2 +- go.mod | 2 +- go.sum | 8 +- internal/driver/pop_connection.go | 76 +++++ internal/driver/registry.go | 6 + internal/driver/registry_default.go | 64 +---- .../20210128140414_namespace.down.sql | 0 .../20210128140414_namespace.mysql.down.sql | 0 .../20210128140414_namespace.up.sql | 0 .../sql/migrations/single_table.go | 271 ++++++++++++++++++ .../sql/migrations/single_table_test.go | 168 +++++++++++ internal/persistence/sql/persister.go | 6 +- internal/persistence/sql/relationtuples.go | 14 +- scripts/single-table-migration-e2e.sh | 8 + 15 files changed, 654 insertions(+), 75 deletions(-) create mode 100644 cmd/namespace/migrate_legacy.go create mode 100644 internal/driver/pop_connection.go rename internal/persistence/sql/{ => migrations}/namespace_migrations/20210128140414_namespace.down.sql (100%) rename internal/persistence/sql/{ => migrations}/namespace_migrations/20210128140414_namespace.mysql.down.sql (100%) rename internal/persistence/sql/{ => migrations}/namespace_migrations/20210128140414_namespace.up.sql (100%) create mode 100644 internal/persistence/sql/migrations/single_table.go create mode 100644 internal/persistence/sql/migrations/single_table_test.go create mode 100644 scripts/single-table-migration-e2e.sh diff --git a/cmd/namespace/migrate_legacy.go b/cmd/namespace/migrate_legacy.go new file mode 100644 index 000000000..24d6799bf --- /dev/null +++ b/cmd/namespace/migrate_legacy.go @@ -0,0 +1,104 @@ +package namespace + +import ( + "fmt" + + "github.com/ory/x/cmdx" + "github.com/ory/x/flagx" + "github.com/pkg/errors" + "github.com/spf13/cobra" + + "github.com/ory/keto/cmd/migrate" + "github.com/ory/keto/internal/driver" + "github.com/ory/keto/internal/namespace" + "github.com/ory/keto/internal/persistence" + "github.com/ory/keto/internal/persistence/sql/migrations" +) + +func NewMigrateLegacyCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "legacy []", + Short: "Migrate a namespace from a legacy table.", + Long: "Migrate a legacy (v0.6.0) table to the v0.7.0 table.\n" + + "This step only has to be executed once.\n" + + "If no namespace is specified, all legacy namespaces will be migrated.\n" + + "Please ensure that namespace IDs did not change in the config file and you have a backup in case something goes wrong!", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + reg, err := driver.NewDefaultRegistry(cmd.Context(), cmd.Flags(), false) + if errors.Is(err, persistence.ErrNetworkMigrationsMissing) { + _, _ = fmt.Fprintln(cmd.ErrOrStderr(), "Migrations were not applied yet, please apply them first using `keto migrate up`.") + return cmdx.FailSilently(cmd) + } else if err != nil { + return err + } + + migrator := migrations.NewToSingleTableMigrator(reg) + + var nn []*namespace.Namespace + if len(args) == 1 { + nm, err := reg.Config().NamespaceManager() + if err != nil { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "There seems to be a problem with the config: %s\n", err.Error()) + return cmdx.FailSilently(cmd) + } + n, err := nm.GetNamespaceByName(cmd.Context(), args[0]) + if err != nil { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "There seems to be a problem with the config: %s\n", err.Error()) + return cmdx.FailSilently(cmd) + } + + nn = []*namespace.Namespace{n} + + if !flagx.MustGetBool(cmd, migrate.FlagYes) && + !cmdx.AskForConfirmation( + fmt.Sprintf("Are you sure that you want to migrate the namespace '%s'?", args[0]), + cmd.InOrStdin(), cmd.OutOrStdout()) { + _, _ = fmt.Fprintln(cmd.OutOrStdout(), "OK, aborting.") + return nil + } + } else { + nn, err = migrator.LegacyNamespaces(cmd.Context()) + if err != nil { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not get legacy namespaces: %s\n", err.Error()) + return cmdx.FailSilently(cmd) + } + + if len(nn) == 0 { + _, _ = fmt.Fprintln(cmd.OutOrStdout(), "Could not find legacy namespaces, there seems nothing to be done.") + return nil + } + + var names string + for _, n := range nn { + names += " " + n.Name + "\n" + } + if !flagx.MustGetBool(cmd, migrate.FlagYes) && + !cmdx.AskForConfirmation( + fmt.Sprintf("I found the following legacy namespaces:\n%sDo you want to migrate all of them?", names), + cmd.InOrStdin(), cmd.OutOrStdout()) { + _, _ = fmt.Fprintln(cmd.OutOrStdout(), "OK, aborting.") + return nil + } + } + + for _, n := range nn { + if err := migrator.MigrateNamespace(cmd.Context(), n); err != nil { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Encountered error while migrating: %s\nAborting.\n", err.Error()) + return cmdx.FailSilently(cmd) + } + if err := migrator.MigrateDown(cmd.Context(), n); err != nil { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not migrate down: %s\n", err.Error()) + return cmdx.FailSilently(cmd) + } + } + + return nil + }, + } + + migrate.RegisterYesFlag(cmd.Flags()) + registerPackageFlags(cmd.Flags()) + + return cmd +} diff --git a/cmd/namespace/root.go b/cmd/namespace/root.go index 482e4155a..c4d680147 100644 --- a/cmd/namespace/root.go +++ b/cmd/namespace/root.go @@ -25,7 +25,7 @@ func NewMigrateCmd() *cobra.Command { func RegisterCommandsRecursive(parent *cobra.Command) { rootCmd := NewNamespaceCmd() migrateCmd := NewMigrateCmd() - migrateCmd.AddCommand(NewMigrateUpCmd(), NewMigrateDownCmd(), NewMigrateStatusCmd()) + migrateCmd.AddCommand(NewMigrateUpCmd(), NewMigrateDownCmd(), NewMigrateStatusCmd(), NewMigrateLegacyCmd()) rootCmd.AddCommand(migrateCmd, NewValidateCmd()) diff --git a/go.mod b/go.mod index cae316fdf..41a15dafc 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/ory/herodot v0.9.6 github.com/ory/jsonschema/v3 v3.0.3 github.com/ory/keto/proto v0.0.0-00010101000000-000000000000 - github.com/ory/x v0.0.275 + github.com/ory/x v0.0.280 github.com/pelletier/go-toml v1.8.1 github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 1fef39d20..250cb0ca9 100644 --- a/go.sum +++ b/go.sum @@ -660,8 +660,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/knadh/koanf v0.14.1-0.20201201075439-e0853799f9ec/go.mod h1:H5mEFsTeWizwFXHKtsITL5ipsLTuAMQoGuQpp+1JL9U= -github.com/knadh/koanf v1.0.0 h1:tGQ1L53Tp4uPx6agVGBN1U7A4f83ZpH3SwZNG+BkfqI= -github.com/knadh/koanf v1.0.0/go.mod h1:vrMMuhIH0k7EoxiMbVfFlRvJYmxcT2Eha3DH8Tx5+X4= +github.com/knadh/koanf v1.2.2 h1:CydaDM/2mtza/ytVVnj4iQSVPjDq+pSV2vWMFDiQS08= +github.com/knadh/koanf v1.2.2/go.mod h1:xpPTwMhsA/aaQLAilyCCqfpEiY1gpa160AiCuWHJUjY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -865,8 +865,8 @@ github.com/ory/x v0.0.93/go.mod h1:lfcTaGXpTZs7IEQAW00r9EtTCOxD//SiP5uWtNiz31g= github.com/ory/x v0.0.110/go.mod h1:DJfkE3GdakhshNhw4zlKoRaL/ozg/lcTahA9OCih2BE= github.com/ory/x v0.0.127/go.mod h1:FwUujfFuCj5d+xgLn4fGMYPnzriR5bdAIulFXMtnK0M= github.com/ory/x v0.0.205/go.mod h1:A1s4iwmFIppRXZLF3J9GGWeY/HpREVm0Dk5z/787iek= -github.com/ory/x v0.0.275 h1:rFNmrmXUEGforCf+T1MUTk0VzJbMLsyZ31h/vV0II4w= -github.com/ory/x v0.0.275/go.mod h1:6bZ1d2RwvlC3iQOasMCQnliBC1DOLKLc/buxk4ync+Q= +github.com/ory/x v0.0.280 h1:yh/tzyFZtQlK2vI7G9StY25V36FGlRMPUXnwvrlKsCE= +github.com/ory/x v0.0.280/go.mod h1:APpShLyJcVzKw1kTgrHI+j/L9YM+8BRjHlcYObc7C1U= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/parnurzeal/gorequest v0.2.15/go.mod h1:3Kh2QUMJoqw3icWAecsyzkpY7UzRfDhbRdTjtNwNiUE= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= diff --git a/internal/driver/pop_connection.go b/internal/driver/pop_connection.go new file mode 100644 index 000000000..ef48ba532 --- /dev/null +++ b/internal/driver/pop_connection.go @@ -0,0 +1,76 @@ +package driver + +import ( + "time" + + "github.com/cenkalti/backoff/v3" + "github.com/gobuffalo/pop/v5" + "github.com/luna-duclos/instrumentedsql" + "github.com/luna-duclos/instrumentedsql/opentracing" + "github.com/ory/x/sqlcon" + "github.com/pkg/errors" +) + +func (r *RegistryDefault) PopConnectionWithOpts(popOpts ...func(*pop.ConnectionDetails)) (*pop.Connection, error) { + tracer := r.Tracer() + + var opts []instrumentedsql.Opt + if tracer.IsLoaded() { + opts = []instrumentedsql.Opt{ + instrumentedsql.WithTracer(opentracing.NewTracer(true)), + instrumentedsql.WithOmitArgs(), + } + } + pool, idlePool, connMaxLifetime, connMaxIdleTime, cleanedDSN := sqlcon.ParseConnectionOptions(r.Logger(), r.Config().DSN()) + connDetails := &pop.ConnectionDetails{ + URL: sqlcon.FinalizeDSN(r.Logger(), cleanedDSN), + IdlePool: idlePool, + ConnMaxLifetime: connMaxLifetime, + ConnMaxIdleTime: connMaxIdleTime, + Pool: pool, + UseInstrumentedDriver: tracer != nil && tracer.IsLoaded(), + InstrumentedDriverOptions: opts, + } + for _, o := range popOpts { + o(connDetails) + } + + bc := backoff.NewExponentialBackOff() + bc.MaxElapsedTime = time.Minute * 5 + bc.Reset() + + var conn *pop.Connection + if err := backoff.Retry(func() (err error) { + conn, err = pop.NewConnection(connDetails) + if err != nil { + r.Logger().WithError(err).Error("Unable to connect to database, retrying.") + return errors.WithStack(err) + } + + if err := conn.Open(); err != nil { + r.Logger().WithError(err).Error("Unable to open the database connection, retrying.") + return errors.WithStack(err) + } + + if err := conn.Store.(interface{ Ping() error }).Ping(); err != nil { + r.Logger().WithError(err).Error("Unable to ping the database connection, retrying.") + return errors.WithStack(err) + } + + return nil + }, bc); err != nil { + return nil, errors.WithStack(err) + } + + return conn, nil +} + +// PopConnection returns the standard connection that is kept for the whole time. +func (r *RegistryDefault) PopConnection() (*pop.Connection, error) { + if r.conn == nil { + var err error + r.conn, err = r.PopConnectionWithOpts() + return r.conn, err + } + return r.conn, nil +} diff --git a/internal/driver/registry.go b/internal/driver/registry.go index c3e35deea..08fbc1330 100644 --- a/internal/driver/registry.go +++ b/internal/driver/registry.go @@ -4,6 +4,8 @@ import ( "context" "net/http" + "github.com/gobuffalo/pop/v5" + "github.com/ory/keto/internal/driver/config" "github.com/spf13/cobra" @@ -35,6 +37,10 @@ type ( expand.EngineProvider check.EngineProvider persistence.Migrator + persistence.Provider + + PopConnection() (*pop.Connection, error) + PopConnectionWithOpts(f ...func(*pop.ConnectionDetails)) (*pop.Connection, error) HealthHandler() *healthx.Handler Tracer() *tracing.Tracer diff --git a/internal/driver/registry_default.go b/internal/driver/registry_default.go index 7e2f7ed2b..e9ac3d4bf 100644 --- a/internal/driver/registry_default.go +++ b/internal/driver/registry_default.go @@ -4,16 +4,11 @@ import ( "context" "net/http" "sync" - "time" "github.com/ory/x/networkx" - "github.com/cenkalti/backoff/v3" "github.com/gobuffalo/pop/v5" - "github.com/luna-duclos/instrumentedsql" - "github.com/luna-duclos/instrumentedsql/opentracing" "github.com/ory/x/popx" - "github.com/ory/x/sqlcon" "github.com/pkg/errors" "github.com/ory/x/dbal" @@ -153,6 +148,13 @@ func (r *RegistryDefault) RelationTupleManager() relationtuple.Manager { return r.p } +func (r *RegistryDefault) Persister() persistence.Persister { + if r.p == nil { + panic("no persister, but expected to have one") + } + return r.p +} + func (r *RegistryDefault) PermissionEngine() *check.Engine { if r.ce == nil { r.ce = check.NewEngine(r) @@ -201,58 +203,6 @@ func (r *RegistryDefault) MigrateDown(ctx context.Context) error { return mb.Up(ctx) } -func (r *RegistryDefault) PopConnection() (*pop.Connection, error) { - if r.conn == nil { - tracer := r.Tracer() - - var opts []instrumentedsql.Opt - if tracer.IsLoaded() { - opts = []instrumentedsql.Opt{ - instrumentedsql.WithTracer(opentracing.NewTracer(true)), - instrumentedsql.WithOmitArgs(), - } - } - pool, idlePool, connMaxLifetime, connMaxIdleTime, cleanedDSN := sqlcon.ParseConnectionOptions(r.Logger(), r.Config().DSN()) - connDetails := &pop.ConnectionDetails{ - URL: sqlcon.FinalizeDSN(r.Logger(), cleanedDSN), - IdlePool: idlePool, - ConnMaxLifetime: connMaxLifetime, - ConnMaxIdleTime: connMaxIdleTime, - Pool: pool, - UseInstrumentedDriver: tracer != nil && tracer.IsLoaded(), - InstrumentedDriverOptions: opts, - } - - bc := backoff.NewExponentialBackOff() - bc.MaxElapsedTime = time.Minute * 5 - bc.Reset() - - if err := backoff.Retry(func() (err error) { - conn, err := pop.NewConnection(connDetails) - if err != nil { - r.Logger().WithError(err).Error("Unable to connect to database, retrying.") - return errors.WithStack(err) - } - - if err := conn.Open(); err != nil { - r.Logger().WithError(err).Error("Unable to open the database connection, retrying.") - return errors.WithStack(err) - } - - if err := conn.Store.(interface{ Ping() error }).Ping(); err != nil { - r.Logger().WithError(err).Error("Unable to ping the database connection, retrying.") - return errors.WithStack(err) - } - - r.conn = conn - return nil - }, bc); err != nil { - return nil, errors.WithStack(err) - } - } - return r.conn, nil -} - func (r *RegistryDefault) determineNetwork(ctx context.Context) (*networkx.Network, error) { c, err := r.PopConnection() if err != nil { diff --git a/internal/persistence/sql/namespace_migrations/20210128140414_namespace.down.sql b/internal/persistence/sql/migrations/namespace_migrations/20210128140414_namespace.down.sql similarity index 100% rename from internal/persistence/sql/namespace_migrations/20210128140414_namespace.down.sql rename to internal/persistence/sql/migrations/namespace_migrations/20210128140414_namespace.down.sql diff --git a/internal/persistence/sql/namespace_migrations/20210128140414_namespace.mysql.down.sql b/internal/persistence/sql/migrations/namespace_migrations/20210128140414_namespace.mysql.down.sql similarity index 100% rename from internal/persistence/sql/namespace_migrations/20210128140414_namespace.mysql.down.sql rename to internal/persistence/sql/migrations/namespace_migrations/20210128140414_namespace.mysql.down.sql diff --git a/internal/persistence/sql/namespace_migrations/20210128140414_namespace.up.sql b/internal/persistence/sql/migrations/namespace_migrations/20210128140414_namespace.up.sql similarity index 100% rename from internal/persistence/sql/namespace_migrations/20210128140414_namespace.up.sql rename to internal/persistence/sql/migrations/namespace_migrations/20210128140414_namespace.up.sql diff --git a/internal/persistence/sql/migrations/single_table.go b/internal/persistence/sql/migrations/single_table.go new file mode 100644 index 000000000..5c9ce0600 --- /dev/null +++ b/internal/persistence/sql/migrations/single_table.go @@ -0,0 +1,271 @@ +package migrations + +import ( + "context" + "embed" + "fmt" + "time" + + "github.com/gobuffalo/pop/v5" + "github.com/gofrs/uuid" + "github.com/ory/x/popx" + "github.com/ory/x/sqlcon" + "github.com/pkg/errors" + + "github.com/ory/keto/internal/driver/config" + "github.com/ory/keto/internal/namespace" + "github.com/ory/keto/internal/persistence" + "github.com/ory/keto/internal/persistence/sql" + "github.com/ory/keto/internal/relationtuple" + "github.com/ory/keto/internal/x" +) + +// Partially transferred from tree at https://github.com/ory/keto/tree/88cedc35b5bcb79ee54e361e00b9d7f60f27b431 +// Right before https://github.com/ory/keto/pull/638 + +type ( + dependencies interface { + persistence.Provider + x.LoggerProvider + config.Provider + + PopConnection() (*pop.Connection, error) + PopConnectionWithOpts(...func(*pop.ConnectionDetails)) (*pop.Connection, error) + } + toSingleTableMigrator struct { + d dependencies + perPage int + } + + relationTuple struct { + // An ID field is required to make pop happy. The actual ID is a composite primary key. + ID string `db:"shard_id"` + Object string `db:"object"` + Relation string `db:"relation"` + Subject string `db:"subject"` + CommitTime time.Time `db:"commit_time"` + Namespace *namespace.Namespace `db:"-"` + } + relationTuples []*relationTuple + contextKey string +) + +var ( + //go:embed namespace_migrations/*.sql + namespaceMigrations embed.FS +) + +const namespaceCtxKey contextKey = "namespace" + +func tableFromNamespace(n *namespace.Namespace) string { + return fmt.Sprintf("keto_%0.10d_relation_tuples", n.ID) +} + +func namespaceIDFromTable(t string) (nID int32, err error) { + _, err = fmt.Sscanf(t, "keto_%d_relation_tuples", &nID) + return nID, errors.WithStack(err) +} + +func migrationTableFromNamespace(n *namespace.Namespace) string { + return fmt.Sprintf("keto_namespace_%0.10d_migrations", n.ID) +} + +func namespaceTableFromContext(ctx context.Context) string { + n, ok := ctx.Value(namespaceCtxKey).(*namespace.Namespace) + if n == nil || !ok { + panic("namespace context key not set") + } + return tableFromNamespace(n) +} + +func (r *relationTuple) toInternal() (*relationtuple.InternalRelationTuple, error) { + if r == nil { + return nil, nil + } + + sub, err := relationtuple.SubjectFromString(r.Subject) + if err != nil { + return nil, err + } + + return &relationtuple.InternalRelationTuple{ + Relation: r.Relation, + Object: r.Object, + Namespace: r.Namespace.Name, + Subject: sub, + }, nil +} + +func (relationTuples) TableName(ctx context.Context) string { + return namespaceTableFromContext(ctx) +} + +func (relationTuple) TableName(ctx context.Context) string { + return namespaceTableFromContext(ctx) +} + +func NewToSingleTableMigrator(d dependencies) *toSingleTableMigrator { + return &toSingleTableMigrator{ + d: d, + perPage: 100, + } +} + +func (m *toSingleTableMigrator) namespaceMigrationBox(n *namespace.Namespace) (*popx.MigrationBox, error) { + c, err := m.d.PopConnectionWithOpts(func(d *pop.ConnectionDetails) { + d.Options = map[string]string{ + "migration_table_name": migrationTableFromNamespace(n), + } + }) + if err != nil { + return nil, err + } + + return popx.NewMigrationBox( + namespaceMigrations, + popx.NewMigrator(c, m.d.Logger(), nil, 0), + popx.WithTemplateValues(map[string]interface{}{ + "tableName": tableFromNamespace(n), + }), + ) +} + +func (m *toSingleTableMigrator) getOldRelationTuples(ctx context.Context, n *namespace.Namespace, page, perPage int) (relationTuples, bool, error) { + q := m.d.Persister().Connection(ctx). + WithContext(context.WithValue(ctx, namespaceCtxKey, n)). + Order("object, relation, subject, commit_time"). + Paginate(page, perPage) + + var res relationTuples + if err := q.All(&res); err != nil { + return nil, false, sqlcon.HandleError(err) + } + for _, r := range res { + r.Namespace = n + } + return res, q.Paginator.Page < q.Paginator.TotalPages, nil +} + +func (m *toSingleTableMigrator) insertOldRelationTuples(ctx context.Context, n *namespace.Namespace, rs ...*relationtuple.InternalRelationTuple) error { + for _, r := range rs { + if r.Subject == nil { + return errors.New("subject is not allowed to be nil") + } + + m.d.Logger().WithFields(r.ToLoggerFields()).Trace("creating in legacy database") + + if err := m.d.Persister().Connection(context.WithValue(ctx, namespaceCtxKey, n)).Create(&relationTuple{ + ID: "testing only", + Object: r.Object, + Relation: r.Relation, + Subject: r.Subject.String(), + CommitTime: time.Now(), + }); err != nil { + return err + } + } + return nil +} + +func (m *toSingleTableMigrator) MigrateNamespace(ctx context.Context, n *namespace.Namespace) error { + p, ok := m.d.Persister().(*sql.Persister) + if !ok { + panic("got unexpected persister") + } + + for done, page := false, 1; !done; { + if err := p.Transaction(ctx, func(ctx context.Context, _ *pop.Connection) error { + rs, hasNext, err := m.getOldRelationTuples(ctx, n, page, m.perPage) + if err != nil { + return err + } + + for _, r := range rs { + ri, err := r.toInternal() + if err != nil { + m.d.Logger().WithField("relation_tuple", r).WithField("hint", "https://github.com/ory/keto/issues/661").WithError(err).Warn("Skipping relation tuple, it seems to be in a broken state. Please recreate it manually.") + continue + } + rt := &sql.RelationTuple{ + ID: uuid.Must(uuid.NewV4()), + CommitTime: r.CommitTime, + } + if err := rt.FromInternal(ctx, p, ri); err != nil { + return err + } + + m.d.Logger().WithFields(ri.ToLoggerFields()).Debug("creating in new table...") + if err := sqlcon.HandleError( + p.CreateWithNetwork(ctx, rt), + ); err != nil { + return err + } + } + if !hasNext { + done = true + return nil + } + + page++ + return nil + }); err != nil { + return err + } + } + + return nil +} + +func (m *toSingleTableMigrator) LegacyNamespaces(ctx context.Context) ([]*namespace.Namespace, error) { + c, err := m.d.PopConnection() + if err != nil { + return nil, err + } + + var query *pop.Query + switch d := c.Dialect.Name(); d { + case "sqlite3": + query = c.RawQuery("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'keto_%_relation_tuples'") + case "postgres": + query = c.RawQuery("SELECT tablename FROM pg_catalog.pg_tables WHERE tablename LIKE 'keto_%_relation_tuples'") + case "cockroach", "mysql": + query = c.RawQuery("SELECT table_name FROM information_schema.tables WHERE table_name LIKE 'keto_%_relation_tuples'") + default: + panic("got unknown database dialect " + d) + } + + var tableNames []string + if err := sqlcon.HandleError(query.All(&tableNames)); err != nil { + return nil, err + } + + nm, err := m.d.Config().NamespaceManager() + if err != nil { + return nil, err + } + + namespaces := make([]*namespace.Namespace, len(tableNames)) + for i := range tableNames { + cID, err := namespaceIDFromTable(tableNames[i]) + if err != nil { + return nil, err + } + namespaces[i], err = nm.GetNamespaceByConfigID(ctx, cID) + if err != nil { + return nil, err + } + } + + return namespaces, nil +} + +func (m *toSingleTableMigrator) MigrateDown(ctx context.Context, n *namespace.Namespace) error { + mb, err := m.namespaceMigrationBox(n) + if err != nil { + return err + } + if err := mb.Down(ctx, 0); err != nil { + return errors.WithStack(err) + } + return nil +} diff --git a/internal/persistence/sql/migrations/single_table_test.go b/internal/persistence/sql/migrations/single_table_test.go new file mode 100644 index 000000000..3e975988a --- /dev/null +++ b/internal/persistence/sql/migrations/single_table_test.go @@ -0,0 +1,168 @@ +package migrations + +import ( + "context" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/ory/keto/internal/driver" + "github.com/ory/keto/internal/driver/config" + "github.com/ory/keto/internal/namespace" + "github.com/ory/keto/internal/relationtuple" + "github.com/ory/keto/internal/x" + "github.com/ory/keto/internal/x/dbx" +) + +func TestToSingleTableMigrator(t *testing.T) { + const debugOnDisk = false + + for _, dsn := range dbx.GetDSNs(t, debugOnDisk) { + t.Run("db="+dsn.Name, func(t *testing.T) { + r := driver.NewTestRegistry(t, dsn) + ctx := context.Background() + var nn []*namespace.Namespace + m := NewToSingleTableMigrator(r) + + setup := func(t *testing.T) *namespace.Namespace { + n := &namespace.Namespace{ + Name: t.Name(), + ID: int32(len(nn)), + } + + nn = append(nn, n) + + mb, err := m.namespaceMigrationBox(n) + require.NoError(t, err) + require.NoError(t, mb.Up(ctx)) + + t.Cleanup(func() { + if debugOnDisk { + return + } + require.NoError(t, mb.Down(ctx, -1)) + }) + + require.NoError(t, r.Config().Set(config.KeyNamespaces, nn)) + + return n + } + + t.Run("case=simple tuples", func(t *testing.T) { + n := setup(t) + + // insert tuples into the old table + sID := &relationtuple.InternalRelationTuple{ + Namespace: n.Name, + Object: "a", + Relation: "a", + Subject: &relationtuple.SubjectID{ID: "a"}, + } + sSet := &relationtuple.InternalRelationTuple{ + Namespace: n.Name, + Object: "b", + Relation: "b", + Subject: &relationtuple.SubjectSet{ + Namespace: n.Name, + Object: "b", + Relation: "b", + }, + } + require.NoError(t, m.insertOldRelationTuples(ctx, n, sID, sSet)) + + // get the tuple from the old table + oldRts, next, err := m.getOldRelationTuples(ctx, n, 0, 100) + require.NoError(t, err) + assert.False(t, next) + require.Len(t, oldRts, 2) + for i, r := range []*relationtuple.InternalRelationTuple{sID, sSet} { + assert.Equal(t, r.Namespace, oldRts[i].Namespace.Name) + assert.Equal(t, r.Object, oldRts[i].Object) + assert.Equal(t, r.Relation, oldRts[i].Relation) + assert.Equal(t, r.Subject.String(), oldRts[i].Subject) + } + + // migrate to new table + require.NoError(t, m.MigrateNamespace(ctx, n)) + + // get the tuple from the new table + rts, nextToken, err := r.RelationTupleManager().GetRelationTuples(ctx, &relationtuple.RelationQuery{Namespace: n.Name}) + require.NoError(t, err) + assert.Equal(t, "", nextToken) + require.Len(t, rts, 2) + assert.Equal(t, sID, rts[0]) + assert.Equal(t, sSet, rts[1]) + }) + + t.Run("case=paginates", func(t *testing.T) { + n := setup(t) + + defer func(old int) { + m.perPage = old + }(m.perPage) + m.perPage = 1 + + rts := make([]*relationtuple.InternalRelationTuple, 10) + for i := range rts { + rts[i] = &relationtuple.InternalRelationTuple{ + Namespace: n.Name, + Object: strconv.Itoa(i), + Relation: strconv.Itoa(i), + Subject: &relationtuple.SubjectID{ID: strconv.Itoa(i)}, + } + } + + require.NoError(t, m.insertOldRelationTuples(ctx, n, rts...)) + require.NoError(t, m.MigrateNamespace(ctx, n)) + + migrated, nextToken, err := r.RelationTupleManager().GetRelationTuples(ctx, &relationtuple.RelationQuery{Namespace: n.Name}, x.WithSize(len(rts))) + require.NoError(t, err) + assert.Equal(t, "", nextToken) + assert.Equal(t, rts, migrated) + }) + }) + } +} + +func TestToSingleTableMigrator_HasLegacyTable(t *testing.T) { + const debugOnDisk = false + + for _, dsn := range dbx.GetDSNs(t, debugOnDisk) { + t.Run("db="+dsn.Name, func(t *testing.T) { + ctx := context.Background() + reg := driver.NewTestRegistry(t, dsn) + m := NewToSingleTableMigrator(reg) + + nspaces := []*namespace.Namespace{{ + ID: 3, + Name: "foo", + }} + require.NoError(t, reg.Config().Set(config.KeyNamespaces, nspaces)) + + // expect to not report legacy table + legacyNamespaces, err := m.LegacyNamespaces(ctx) + require.NoError(t, err) + assert.Len(t, legacyNamespaces, 0) + + // migrate legacy table up + mb, err := m.namespaceMigrationBox(nspaces[0]) + require.NoError(t, err) + require.NoError(t, mb.Up(ctx)) + + // expect to report legacy table + legacyNamespaces, err = m.LegacyNamespaces(ctx) + require.NoError(t, err) + assert.Equal(t, nspaces, legacyNamespaces) + + // migrate legacy down + require.NoError(t, mb.Down(ctx, -1)) + + // expect to not report legacy + legacyNamespaces, err = m.LegacyNamespaces(ctx) + require.NoError(t, err) + assert.Len(t, legacyNamespaces, 0) + }) + } +} diff --git a/internal/persistence/sql/persister.go b/internal/persistence/sql/persister.go index b44aeeb9d..64fb0678c 100644 --- a/internal/persistence/sql/persister.go +++ b/internal/persistence/sql/persister.go @@ -38,7 +38,6 @@ type ( config.Provider x.LoggerProvider - Tracer() *tracing.Tracer PopConnection() (*pop.Connection, error) } ) @@ -51,9 +50,6 @@ var ( //go:embed migrations/sql/*.sql migrations embed.FS - ////go:embed namespace_migrations/*.sql - //namespaceMigrations embed.FS - _ persistence.Persister = &Persister{} ) @@ -102,7 +98,7 @@ func (p *Persister) QueryWithNetwork(ctx context.Context) *pop.Query { return p.Connection(ctx).Where("nid = ?", p.NetworkID(ctx)) } -func (p *Persister) transaction(ctx context.Context, f func(ctx context.Context, c *pop.Connection) error) error { +func (p *Persister) Transaction(ctx context.Context, f func(ctx context.Context, c *pop.Connection) error) error { return popx.Transaction(ctx, p.conn.WithContext(ctx), f) } diff --git a/internal/persistence/sql/relationtuples.go b/internal/persistence/sql/relationtuples.go index 1e353c2c0..a4091710d 100644 --- a/internal/persistence/sql/relationtuples.go +++ b/internal/persistence/sql/relationtuples.go @@ -112,7 +112,7 @@ func (r *RelationTuple) insertSubject(ctx context.Context, p *Persister, s relat return nil } -func (r *RelationTuple) fromInternal(ctx context.Context, p *Persister, rt *relationtuple.InternalRelationTuple) error { +func (r *RelationTuple) FromInternal(ctx context.Context, p *Persister, rt *relationtuple.InternalRelationTuple) error { n, err := p.GetNamespaceByName(ctx, rt.Namespace) if err != nil { return err @@ -125,7 +125,7 @@ func (r *RelationTuple) fromInternal(ctx context.Context, p *Persister, rt *rela return r.insertSubject(ctx, p, rt.Subject) } -func (p *Persister) insertRelationTuple(ctx context.Context, rel *relationtuple.InternalRelationTuple) error { +func (p *Persister) InsertRelationTuple(ctx context.Context, rel *relationtuple.InternalRelationTuple) error { if rel.Subject == nil { return errors.New("subject is not allowed to be nil") } @@ -136,7 +136,7 @@ func (p *Persister) insertRelationTuple(ctx context.Context, rel *relationtuple. ID: uuid.Must(uuid.NewV4()), CommitTime: time.Now(), } - if err := rt.fromInternal(ctx, p, rel); err != nil { + if err := rt.FromInternal(ctx, p, rel); err != nil { return err } @@ -176,7 +176,7 @@ func (p *Persister) whereSubject(ctx context.Context, q *pop.Query, sub relation } func (p *Persister) DeleteRelationTuples(ctx context.Context, rs ...*relationtuple.InternalRelationTuple) error { - return p.transaction(ctx, func(ctx context.Context, c *pop.Connection) error { + return p.Transaction(ctx, func(ctx context.Context, c *pop.Connection) error { for _, r := range rs { n, err := p.GetNamespaceByName(ctx, r.Namespace) if err != nil { @@ -258,9 +258,9 @@ func (p *Persister) GetRelationTuples(ctx context.Context, query *relationtuple. } func (p *Persister) WriteRelationTuples(ctx context.Context, rs ...*relationtuple.InternalRelationTuple) error { - return p.transaction(ctx, func(ctx context.Context, _ *pop.Connection) error { + return p.Transaction(ctx, func(ctx context.Context, _ *pop.Connection) error { for _, r := range rs { - if err := p.insertRelationTuple(ctx, r); err != nil { + if err := p.InsertRelationTuple(ctx, r); err != nil { return err } } @@ -269,7 +269,7 @@ func (p *Persister) WriteRelationTuples(ctx context.Context, rs ...*relationtupl } func (p *Persister) TransactRelationTuples(ctx context.Context, ins []*relationtuple.InternalRelationTuple, del []*relationtuple.InternalRelationTuple) error { - return p.transaction(ctx, func(ctx context.Context, _ *pop.Connection) error { + return p.Transaction(ctx, func(ctx context.Context, _ *pop.Connection) error { if err := p.WriteRelationTuples(ctx, ins...); err != nil { return err } diff --git a/scripts/single-table-migration-e2e.sh b/scripts/single-table-migration-e2e.sh new file mode 100644 index 000000000..0ea03ca43 --- /dev/null +++ b/scripts/single-table-migration-e2e.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -euxo pipefail + +bash <(curl https://raw.githubusercontent.com/ory/keto/master/install.sh) -b . v0.6.0-alpha.3 + +export DSN=sqlite://./migrate_e2e.sqlite + +./keto migrate up -y -c ./contrib/cat-videos-example/keto.yml From e010d822ed20538253017a2ad38d7e69ce3c709e Mon Sep 17 00:00:00 2001 From: aeneasr <3372410+aeneasr@users.noreply.github.com> Date: Mon, 13 Sep 2021 15:40:39 +0000 Subject: [PATCH 02/18] autogen(docs): generate cli docs --- .../docs/cli/keto-namespace-migrate-legacy.md | 48 +++++++++++++++++++ docs/docs/cli/keto-namespace-migrate.md | 29 +++++++---- docs/docs/cli/keto-namespace.md | 1 + docs/sidebar.json | 2 + 4 files changed, 72 insertions(+), 8 deletions(-) create mode 100644 docs/docs/cli/keto-namespace-migrate-legacy.md diff --git a/docs/docs/cli/keto-namespace-migrate-legacy.md b/docs/docs/cli/keto-namespace-migrate-legacy.md new file mode 100644 index 000000000..7d097acfd --- /dev/null +++ b/docs/docs/cli/keto-namespace-migrate-legacy.md @@ -0,0 +1,48 @@ +--- +id: keto-namespace-migrate-legacy +title: keto namespace migrate legacy +description: + keto namespace migrate legacy Migrate a namespace from a legacy table. +--- + + + +## keto namespace migrate legacy + +Migrate a namespace from a legacy table. + +### Synopsis + +Migrate a legacy (v0.6.0) table to the v0.7.0 table. This step only has to be +executed once. If no namespace is specified, all legacy namespaces will be +migrated. Please ensure that namespace IDs did not change in the config file and +you have a backup in case something goes wrong! + +``` +keto namespace migrate legacy [<namespace-name>] [flags] +``` + +### Options + +``` + -f, --format string Set the output format. One of table, json, and json-pretty. (default "default") + -h, --help help for legacy + -q, --quiet Be quiet with output printing. + --read-remote string Remote URL of the read API endpoint. (default "127.0.0.1:4466") + --write-remote string Remote URL of the write API endpoint. (default "127.0.0.1:4467") + -y, --yes yes to all questions, no user input required +``` + +### Options inherited from parent commands + +``` + -c, --config strings Config files to load, overwriting in the order specified. (default [/home/circleci/keto.yml]) +``` + +### SEE ALSO + +- [keto namespace migrate](keto-namespace-migrate) - Migrate a namespace diff --git a/docs/docs/cli/keto-namespace-migrate.md b/docs/docs/cli/keto-namespace-migrate.md index bcfebd470..f39eb060f 100644 --- a/docs/docs/cli/keto-namespace-migrate.md +++ b/docs/docs/cli/keto-namespace-migrate.md @@ -4,17 +4,30 @@ title: keto namespace migrate description: keto namespace migrate Migrate a namespace --- + + ## keto namespace migrate -Deprecated: this step is not required anymore since -https://github.com/ory/keto/pull/638 +Migrate a namespace + +### Options + +``` + -h, --help help for migrate +``` + +### Options inherited from parent commands + +``` + -c, --config strings Config files to load, overwriting in the order specified. (default [/home/circleci/keto.yml]) +``` ### SEE ALSO - [keto namespace](keto-namespace) - Read and manipulate namespaces -- [keto namespace migrate down](keto-namespace-migrate-down) - Migrate a - namespace down -- [keto namespace migrate status](keto-namespace-migrate-status) - Get the - current namespace migration status -- [keto namespace migrate up](keto-namespace-migrate-up) - Migrate a namespace - up +- [keto namespace migrate legacy](keto-namespace-migrate-legacy) - Migrate a + namespace from a legacy table. diff --git a/docs/docs/cli/keto-namespace.md b/docs/docs/cli/keto-namespace.md index 5f7c6b20b..a47d60528 100644 --- a/docs/docs/cli/keto-namespace.md +++ b/docs/docs/cli/keto-namespace.md @@ -29,5 +29,6 @@ Read and manipulate namespaces ### SEE ALSO - [keto](keto) - Global and consistent permission and authorization server +- [keto namespace migrate](keto-namespace-migrate) - Migrate a namespace - [keto namespace validate](keto-namespace-validate) - Validate namespace definitions diff --git a/docs/sidebar.json b/docs/sidebar.json index 4b64b3508..dbbafc7c5 100644 --- a/docs/sidebar.json +++ b/docs/sidebar.json @@ -50,6 +50,8 @@ "cli/keto-migrate-status", "cli/keto-migrate-up", "cli/keto-namespace", + "cli/keto-namespace-migrate", + "cli/keto-namespace-migrate-legacy", "cli/keto-namespace-validate", "cli/keto-relation-tuple", "cli/keto-relation-tuple-create", From 89484b314d8b5690e64f4147edbe256006d74b80 Mon Sep 17 00:00:00 2001 From: zepatrik Date: Tue, 14 Sep 2021 12:00:36 +0200 Subject: [PATCH 03/18] test: add e2e migration script & CI task --- .../workflows/single-table-migration-e2e.yml | 21 ++++++++++++ internal/persistence/sql/persister.go | 3 -- scripts/single-table-migration-e2e.sh | 32 ++++++++++++++++++- 3 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/single-table-migration-e2e.yml mode change 100644 => 100755 scripts/single-table-migration-e2e.sh diff --git a/.github/workflows/single-table-migration-e2e.yml b/.github/workflows/single-table-migration-e2e.yml new file mode 100644 index 000000000..3d7fbfa7e --- /dev/null +++ b/.github/workflows/single-table-migration-e2e.yml @@ -0,0 +1,21 @@ +name: Run full e2e test of the migration to single table persister (see https://github.com/ory/keto/issues/628) + +on: + workflow_dispatch: + +jobs: + test-migration: + runs-on: ubuntu-latest + name: Test Migration + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-go@v2 + with: + go-version: '1.16' + - name: Run test script + run: ./scripts/single-table-migration-e2e.sh + - uses: actions/upload-artifact@v2 + if: failure() + with: + name: sqlite-db + path: migrate_e2e.sqlite diff --git a/internal/persistence/sql/persister.go b/internal/persistence/sql/persister.go index 64fb0678c..c80119a42 100644 --- a/internal/persistence/sql/persister.go +++ b/internal/persistence/sql/persister.go @@ -54,9 +54,6 @@ var ( ) func NewPersister(reg dependencies, nid uuid.UUID) (*Persister, error) { - //pop.SetLogger(reg.Logger().PopLogger) - - pop.Debug = true conn, err := reg.PopConnection() if err != nil { return nil, err diff --git a/scripts/single-table-migration-e2e.sh b/scripts/single-table-migration-e2e.sh old mode 100644 new mode 100755 index 0ea03ca43..fa534ea60 --- a/scripts/single-table-migration-e2e.sh +++ b/scripts/single-table-migration-e2e.sh @@ -1,8 +1,38 @@ #!/bin/bash set -euxo pipefail +rm migrate_e2e.sqlite || true + bash <(curl https://raw.githubusercontent.com/ory/keto/master/install.sh) -b . v0.6.0-alpha.3 -export DSN=sqlite://./migrate_e2e.sqlite +export DSN="sqlite://./migrate_e2e.sqlite?_fk=true" +export KETO_READ_REMOTE="127.0.0.1:4466" +export KETO_WRITE_REMOTE="127.0.0.1:4467" ./keto migrate up -y -c ./contrib/cat-videos-example/keto.yml +./keto namespace migrate up -y -c ./contrib/cat-videos-example/keto.yml videos + +./keto serve all -c ./contrib/cat-videos-example/keto.yml & +keto_server_pid=$! + +function teardown() { + kill $keto_server_pid || true +} +trap teardown EXIT + +jq '[range(200)] | map({namespace: "videos", object: . | tostring, relation: "view", subject: "user"})' <(echo '{}') \ + | ./keto relation-tuple create -q - + +kill $keto_server_pid + +go build -tags sqlite -o keto_new . + +./keto_new migrate up -y -c ./contrib/cat-videos-example/keto.yml +./keto_new namespace migrate legacy -y -c ./contrib/cat-videos-example/keto.yml videos + +./keto_new serve all -c ./contrib/cat-videos-example/keto.yml & +keto_server_pid=$! + +for i in {1..200..25} ; do + diff <(echo 'Allowed') <(./keto_new check user view videos "$i") +done From c7b1bd31e2e0ac7fda8118370314f093fc1c10b6 Mon Sep 17 00:00:00 2001 From: zepatrik Date: Tue, 14 Sep 2021 12:04:11 +0200 Subject: [PATCH 04/18] ci: trigger e2e on this branch --- .github/workflows/single-table-migration-e2e.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/single-table-migration-e2e.yml b/.github/workflows/single-table-migration-e2e.yml index 3d7fbfa7e..5874f6890 100644 --- a/.github/workflows/single-table-migration-e2e.yml +++ b/.github/workflows/single-table-migration-e2e.yml @@ -2,6 +2,9 @@ name: Run full e2e test of the migration to single table persister (see https:// on: workflow_dispatch: + push: + branches: + - feat/persistence-migration-path jobs: test-migration: From 8d672ec53cfb8449ae916ce1a08fe08f47d74ad9 Mon Sep 17 00:00:00 2001 From: zepatrik Date: Tue, 14 Sep 2021 14:18:28 +0200 Subject: [PATCH 05/18] chore: add legacy migrations to the templates and drop not-needed table --- install.sh | 0 .../20201110175414000000_relationtuple.cockroach.down.sql | 0 .../sql/20201110175414000000_relationtuple.cockroach.up.sql | 5 +++++ .../sql/20201110175414000000_relationtuple.mysql.down.sql | 0 .../sql/20201110175414000000_relationtuple.mysql.up.sql | 5 +++++ .../20201110175414000000_relationtuple.postgres.down.sql | 0 .../sql/20201110175414000000_relationtuple.postgres.up.sql | 5 +++++ .../sql/20201110175414000000_relationtuple.sqlite3.down.sql | 0 .../sql/20201110175414000000_relationtuple.sqlite3.up.sql | 5 +++++ .../20201110175414000001_relationtuple.cockroach.down.sql | 1 + .../sql/20201110175414000001_relationtuple.cockroach.up.sql | 0 .../sql/20201110175414000001_relationtuple.mysql.down.sql | 1 + .../sql/20201110175414000001_relationtuple.mysql.up.sql | 0 .../20201110175414000001_relationtuple.postgres.down.sql | 1 + .../sql/20201110175414000001_relationtuple.postgres.up.sql | 0 .../sql/20201110175414000001_relationtuple.sqlite3.down.sql | 1 + .../sql/20201110175414000001_relationtuple.sqlite3.up.sql | 0 .../20210914134624000000_legacy-cleanup.cockroach.down.sql | 5 +++++ .../20210914134624000000_legacy-cleanup.cockroach.up.sql | 1 + .../sql/20210914134624000000_legacy-cleanup.mysql.down.sql | 5 +++++ .../sql/20210914134624000000_legacy-cleanup.mysql.up.sql | 1 + .../20210914134624000000_legacy-cleanup.postgres.down.sql | 5 +++++ .../sql/20210914134624000000_legacy-cleanup.postgres.up.sql | 1 + .../20210914134624000000_legacy-cleanup.sqlite3.down.sql | 4 ++++ .../sql/20210914134624000000_legacy-cleanup.sqlite3.up.sql | 1 + .../20201110175414_relationtuple.down.sql | 0 .../{sql => templates}/20201110175414_relationtuple.up.sql | 0 .../templates/20210914134624_legacy-cleanup.down.fizz | 6 ++++++ .../templates/20210914134624_legacy-cleanup.up.fizz | 1 + 29 files changed, 54 insertions(+) mode change 100644 => 100755 install.sh create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.cockroach.down.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.cockroach.up.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.mysql.down.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.mysql.up.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.postgres.down.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.postgres.up.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.sqlite3.down.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.sqlite3.up.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.cockroach.down.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.cockroach.up.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.mysql.down.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.mysql.up.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.postgres.down.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.postgres.up.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.sqlite3.down.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.sqlite3.up.sql create mode 100644 internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.cockroach.down.sql create mode 100644 internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.cockroach.up.sql create mode 100644 internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.mysql.down.sql create mode 100644 internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.mysql.up.sql create mode 100644 internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.postgres.down.sql create mode 100644 internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.postgres.up.sql create mode 100644 internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.sqlite3.down.sql create mode 100644 internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.sqlite3.up.sql rename internal/persistence/sql/migrations/{sql => templates}/20201110175414_relationtuple.down.sql (100%) rename internal/persistence/sql/migrations/{sql => templates}/20201110175414_relationtuple.up.sql (100%) create mode 100644 internal/persistence/sql/migrations/templates/20210914134624_legacy-cleanup.down.fizz create mode 100644 internal/persistence/sql/migrations/templates/20210914134624_legacy-cleanup.up.fizz diff --git a/install.sh b/install.sh old mode 100644 new mode 100755 diff --git a/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.cockroach.down.sql b/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.cockroach.down.sql new file mode 100644 index 000000000..e69de29bb diff --git a/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.cockroach.up.sql b/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.cockroach.up.sql new file mode 100644 index 000000000..e6c51f372 --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.cockroach.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE keto_namespace +( + id INTEGER PRIMARY KEY, + schema_version INTEGER NOT NULL +); \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.mysql.down.sql b/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.mysql.down.sql new file mode 100644 index 000000000..e69de29bb diff --git a/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.mysql.up.sql b/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.mysql.up.sql new file mode 100644 index 000000000..e6c51f372 --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.mysql.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE keto_namespace +( + id INTEGER PRIMARY KEY, + schema_version INTEGER NOT NULL +); \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.postgres.down.sql b/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.postgres.down.sql new file mode 100644 index 000000000..e69de29bb diff --git a/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.postgres.up.sql b/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.postgres.up.sql new file mode 100644 index 000000000..e6c51f372 --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.postgres.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE keto_namespace +( + id INTEGER PRIMARY KEY, + schema_version INTEGER NOT NULL +); \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.sqlite3.down.sql b/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.sqlite3.down.sql new file mode 100644 index 000000000..e69de29bb diff --git a/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.sqlite3.up.sql b/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.sqlite3.up.sql new file mode 100644 index 000000000..e6c51f372 --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.sqlite3.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE keto_namespace +( + id INTEGER PRIMARY KEY, + schema_version INTEGER NOT NULL +); \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.cockroach.down.sql b/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.cockroach.down.sql new file mode 100644 index 000000000..ab89a03dc --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.cockroach.down.sql @@ -0,0 +1 @@ +DROP TABLE keto_namespace; \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.cockroach.up.sql b/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.cockroach.up.sql new file mode 100644 index 000000000..e69de29bb diff --git a/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.mysql.down.sql b/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.mysql.down.sql new file mode 100644 index 000000000..ab89a03dc --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.mysql.down.sql @@ -0,0 +1 @@ +DROP TABLE keto_namespace; \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.mysql.up.sql b/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.mysql.up.sql new file mode 100644 index 000000000..e69de29bb diff --git a/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.postgres.down.sql b/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.postgres.down.sql new file mode 100644 index 000000000..ab89a03dc --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.postgres.down.sql @@ -0,0 +1 @@ +DROP TABLE keto_namespace; \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.postgres.up.sql b/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.postgres.up.sql new file mode 100644 index 000000000..e69de29bb diff --git a/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.sqlite3.down.sql b/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.sqlite3.down.sql new file mode 100644 index 000000000..ab89a03dc --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.sqlite3.down.sql @@ -0,0 +1 @@ +DROP TABLE keto_namespace; \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.sqlite3.up.sql b/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.sqlite3.up.sql new file mode 100644 index 000000000..e69de29bb diff --git a/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.cockroach.down.sql b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.cockroach.down.sql new file mode 100644 index 000000000..fde3c6a94 --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.cockroach.down.sql @@ -0,0 +1,5 @@ +CREATE TABLE "keto_namespace" ( +"id" SERIAL NOT NULL, +PRIMARY KEY("id"), +"schema_version" int NOT NULL +); \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.cockroach.up.sql b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.cockroach.up.sql new file mode 100644 index 000000000..aa1782069 --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.cockroach.up.sql @@ -0,0 +1 @@ +DROP TABLE "keto_namespace"; \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.mysql.down.sql b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.mysql.down.sql new file mode 100644 index 000000000..8aef7cf1b --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.mysql.down.sql @@ -0,0 +1,5 @@ +CREATE TABLE `keto_namespace` ( +`id` INTEGER NOT NULL AUTO_INCREMENT, +PRIMARY KEY(`id`), +`schema_version` INTEGER NOT NULL +) ENGINE=InnoDB; \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.mysql.up.sql b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.mysql.up.sql new file mode 100644 index 000000000..5b1def1b7 --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.mysql.up.sql @@ -0,0 +1 @@ +DROP TABLE `keto_namespace`; \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.postgres.down.sql b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.postgres.down.sql new file mode 100644 index 000000000..fde3c6a94 --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.postgres.down.sql @@ -0,0 +1,5 @@ +CREATE TABLE "keto_namespace" ( +"id" SERIAL NOT NULL, +PRIMARY KEY("id"), +"schema_version" int NOT NULL +); \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.postgres.up.sql b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.postgres.up.sql new file mode 100644 index 000000000..aa1782069 --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.postgres.up.sql @@ -0,0 +1 @@ +DROP TABLE "keto_namespace"; \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.sqlite3.down.sql b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.sqlite3.down.sql new file mode 100644 index 000000000..30f9414cc --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.sqlite3.down.sql @@ -0,0 +1,4 @@ +CREATE TABLE "keto_namespace" ( +"id" INTEGER PRIMARY KEY AUTOINCREMENT, +"schema_version" INTEGER NOT NULL +); \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.sqlite3.up.sql b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.sqlite3.up.sql new file mode 100644 index 000000000..aa1782069 --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.sqlite3.up.sql @@ -0,0 +1 @@ +DROP TABLE "keto_namespace"; \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20201110175414_relationtuple.down.sql b/internal/persistence/sql/migrations/templates/20201110175414_relationtuple.down.sql similarity index 100% rename from internal/persistence/sql/migrations/sql/20201110175414_relationtuple.down.sql rename to internal/persistence/sql/migrations/templates/20201110175414_relationtuple.down.sql diff --git a/internal/persistence/sql/migrations/sql/20201110175414_relationtuple.up.sql b/internal/persistence/sql/migrations/templates/20201110175414_relationtuple.up.sql similarity index 100% rename from internal/persistence/sql/migrations/sql/20201110175414_relationtuple.up.sql rename to internal/persistence/sql/migrations/templates/20201110175414_relationtuple.up.sql diff --git a/internal/persistence/sql/migrations/templates/20210914134624_legacy-cleanup.down.fizz b/internal/persistence/sql/migrations/templates/20210914134624_legacy-cleanup.down.fizz new file mode 100644 index 000000000..d2df585b1 --- /dev/null +++ b/internal/persistence/sql/migrations/templates/20210914134624_legacy-cleanup.down.fizz @@ -0,0 +1,6 @@ +create_table("keto_namespace") { + t.Column("id", "int", { primary: true }) + t.Column("schema_version", "int") + + t.DisableTimestamps() +} diff --git a/internal/persistence/sql/migrations/templates/20210914134624_legacy-cleanup.up.fizz b/internal/persistence/sql/migrations/templates/20210914134624_legacy-cleanup.up.fizz new file mode 100644 index 000000000..432652b55 --- /dev/null +++ b/internal/persistence/sql/migrations/templates/20210914134624_legacy-cleanup.up.fizz @@ -0,0 +1 @@ +drop_table("keto_namespace") From 5bae7fcb612ddd52f7024250059cc0a53e1b396d Mon Sep 17 00:00:00 2001 From: zepatrik Date: Tue, 14 Sep 2021 14:56:10 +0200 Subject: [PATCH 06/18] chore: remove --all-namespaces from migrate up command --- cmd/migrate/up.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cmd/migrate/up.go b/cmd/migrate/up.go index 029a8cfeb..4936faa01 100644 --- a/cmd/migrate/up.go +++ b/cmd/migrate/up.go @@ -16,13 +16,10 @@ import ( ) const ( - FlagYes = "yes" - FlagAllNamespace = "all-namespaces" + FlagYes = "yes" ) func newUpCmd() *cobra.Command { - var allNamespaces bool - cmd := &cobra.Command{ Use: "up", Short: "Migrate the database up", @@ -51,7 +48,6 @@ func newUpCmd() *cobra.Command { } RegisterYesFlag(cmd.Flags()) - cmd.Flags().BoolVar(&allNamespaces, FlagAllNamespace, false, "migrate all pending namespaces as well") cmdx.RegisterFormatFlags(cmd.Flags()) From 86bce8f9d89a482a814dc8c48a2de0f84521e576 Mon Sep 17 00:00:00 2001 From: aeneasr <3372410+aeneasr@users.noreply.github.com> Date: Tue, 14 Sep 2021 12:58:00 +0000 Subject: [PATCH 07/18] autogen(docs): generate cli docs --- docs/docs/cli/keto-migrate-up.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/docs/cli/keto-migrate-up.md b/docs/docs/cli/keto-migrate-up.md index 0e90f2c4c..e0239ecda 100644 --- a/docs/docs/cli/keto-migrate-up.md +++ b/docs/docs/cli/keto-migrate-up.md @@ -26,11 +26,10 @@ keto migrate up [flags] ### Options ``` - --all-namespaces migrate all pending namespaces as well - -f, --format string Set the output format. One of table, json, and json-pretty. (default "default") - -h, --help help for up - -q, --quiet Be quiet with output printing. - -y, --yes yes to all questions, no user input required + -f, --format string Set the output format. One of table, json, and json-pretty. (default "default") + -h, --help help for up + -q, --quiet Be quiet with output printing. + -y, --yes yes to all questions, no user input required ``` ### Options inherited from parent commands From 1b9abbd8a9979fb61c835efe86fa111a6d9730e8 Mon Sep 17 00:00:00 2001 From: zepatrik Date: Tue, 14 Sep 2021 15:04:20 +0200 Subject: [PATCH 08/18] chore: up migration command: simplify and improve docs --- cmd/migrate/up.go | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/cmd/migrate/up.go b/cmd/migrate/up.go index 4936faa01..0ee573c3f 100644 --- a/cmd/migrate/up.go +++ b/cmd/migrate/up.go @@ -23,8 +23,15 @@ func newUpCmd() *cobra.Command { cmd := &cobra.Command{ Use: "up", Short: "Migrate the database up", - Long: "Migrate the database up.\n" + - "This does not affect namespaces. Use `keto namespace migrate up` for migrating namespaces.", + Long: `Run this command on a fresh SQL installation and when you upgrade Ory Keto to a new minor version. + +It is recommended to run this command close to the SQL instance (e.g. same subnet) instead of over the public internet. +This decreases risk of failure and decreases time required. + +### WARNING ### + +Before running this command on an existing database, create a back up! +`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { ctx := cmd.Context() @@ -39,7 +46,7 @@ func newUpCmd() *cobra.Command { return err } - if err := BoxUp(cmd, mb, ""); err != nil { + if err := BoxUp(cmd, mb); err != nil { return err } @@ -58,38 +65,38 @@ func RegisterYesFlag(flags *pflag.FlagSet) { flags.BoolP(FlagYes, "y", false, "yes to all questions, no user input required") } -func BoxUp(cmd *cobra.Command, mb *popx.MigrationBox, msgPrefix string) error { - _, _ = fmt.Fprintln(cmd.OutOrStdout(), msgPrefix+"Current status:") +func BoxUp(cmd *cobra.Command, mb *popx.MigrationBox) error { + _, _ = fmt.Fprintln(cmd.OutOrStdout(), "Current status:") s, err := mb.Status(cmd.Context()) if err != nil { - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "%sCould not get migration status: %+v\n", msgPrefix, err) + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not get migration status: %+v\n", err) return cmdx.FailSilently(cmd) } cmdx.PrintTable(cmd, s) if !s.HasPending() { - _, _ = fmt.Fprintln(cmd.OutOrStdout(), msgPrefix+"All migrations are already applied, there is nothing to do.") + _, _ = fmt.Fprintln(cmd.OutOrStdout(), "All migrations are already applied, there is nothing to do.") return nil } - if !flagx.MustGetBool(cmd, FlagYes) && !cmdx.AskForConfirmation(msgPrefix+"Are you sure that you want to apply this migration? Make sure to check the CHANGELOG.md for breaking changes beforehand.", cmd.InOrStdin(), cmd.OutOrStdout()) { - _, _ = fmt.Fprintln(cmd.OutOrStdout(), msgPrefix+"Aborting") + if !flagx.MustGetBool(cmd, FlagYes) && !cmdx.AskForConfirmation("Are you sure that you want to apply this migration? Make sure to check the CHANGELOG.md for breaking changes beforehand.", cmd.InOrStdin(), cmd.OutOrStdout()) { + _, _ = fmt.Fprintln(cmd.OutOrStdout(), "Aborting") return nil } - _, _ = fmt.Fprintln(cmd.OutOrStdout(), msgPrefix+"Applying migrations...") + _, _ = fmt.Fprintln(cmd.OutOrStdout(), "Applying migrations...") if err := mb.Up(cmd.Context()); err != nil { - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "%sCould not apply migrations: %+v\n", msgPrefix, err) + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "%sCould not apply migrations: %+v\n", err) return cmdx.FailSilently(cmd) } - _, _ = fmt.Fprintln(cmd.OutOrStdout(), msgPrefix+"Successfully applied all migrations:") + _, _ = fmt.Fprintln(cmd.OutOrStdout(), "Successfully applied all migrations:") s, err = mb.Status(cmd.Context()) if err != nil { - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "%sCould not get migration status: %+v\n", msgPrefix, err) + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not get migration status: %+v\n", err) return cmdx.FailSilently(cmd) } From a90879f40e3b7b8ad5b56cad1e0f9e01c1f4bd65 Mon Sep 17 00:00:00 2001 From: zepatrik Date: Tue, 14 Sep 2021 15:08:26 +0200 Subject: [PATCH 09/18] chore: down migration command: simplify and improve docs --- cmd/migrate/down.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/cmd/migrate/down.go b/cmd/migrate/down.go index d2163a009..852bba449 100644 --- a/cmd/migrate/down.go +++ b/cmd/migrate/down.go @@ -19,8 +19,7 @@ func newDownCmd() *cobra.Command { Use: "down ", Short: "Migrate the database down", Long: "Migrate the database down a specific amount of steps.\n" + - "Pass 0 steps to fully migrate down.\n" + - "This does not affect namespaces. Use `keto namespace migrate down` for migrating namespaces.", + "Pass 0 steps to fully migrate down.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { steps, err := strconv.ParseInt(args[0], 0, 0) @@ -39,7 +38,7 @@ func newDownCmd() *cobra.Command { return err } - return BoxDown(cmd, mb, int(steps), "") + return BoxDown(cmd, mb, int(steps)) }, } @@ -49,27 +48,27 @@ func newDownCmd() *cobra.Command { return cmd } -func BoxDown(cmd *cobra.Command, mb *popx.MigrationBox, steps int, msgPrefix string) error { +func BoxDown(cmd *cobra.Command, mb *popx.MigrationBox, steps int) error { s, err := mb.Status(cmd.Context()) if err != nil { - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "%sCould not get migration status: %+v\n", msgPrefix, err) + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not get migration status: %+v\n", err) return cmdx.FailSilently(cmd) } cmdx.PrintTable(cmd, s) - if !flagx.MustGetBool(cmd, FlagYes) && !cmdx.AskForConfirmation(msgPrefix+"Do you really want to migrate down? This will delete data.", cmd.InOrStdin(), cmd.OutOrStdout()) { - _, _ = fmt.Fprintln(cmd.OutOrStdout(), msgPrefix+"Migration aborted.") + if !flagx.MustGetBool(cmd, FlagYes) && !cmdx.AskForConfirmation("Do you really want to migrate down? This will delete data.", cmd.InOrStdin(), cmd.OutOrStdout()) { + _, _ = fmt.Fprintln(cmd.OutOrStdout(), "Migration aborted.") return nil } if err := mb.Down(cmd.Context(), steps); err != nil { - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "%sCould apply down migrations: %+v\n", msgPrefix, err) + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could apply down migrations: %+v\n", err) return cmdx.FailSilently(cmd) } s, err = mb.Status(cmd.Context()) if err != nil { - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "%sCould not get migration status: %+v\n", msgPrefix, err) + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not get migration status: %+v\n", err) return cmdx.FailSilently(cmd) } cmdx.PrintTable(cmd, s) From 4b7f46504f8b4cb7c380c25db617828fcb6ccb74 Mon Sep 17 00:00:00 2001 From: aeneasr <3372410+aeneasr@users.noreply.github.com> Date: Tue, 14 Sep 2021 13:15:50 +0000 Subject: [PATCH 10/18] autogen(docs): generate cli docs --- docs/docs/cli/keto-migrate-down.md | 3 +-- docs/docs/cli/keto-migrate-up.md | 12 ++++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/docs/cli/keto-migrate-down.md b/docs/docs/cli/keto-migrate-down.md index 466ca27fd..5f107dca9 100644 --- a/docs/docs/cli/keto-migrate-down.md +++ b/docs/docs/cli/keto-migrate-down.md @@ -17,8 +17,7 @@ Migrate the database down ### Synopsis Migrate the database down a specific amount of steps. Pass 0 steps to fully -migrate down. This does not affect namespaces. Use `keto namespace migrate down` -for migrating namespaces. +migrate down. ``` keto migrate down <steps> [flags] diff --git a/docs/docs/cli/keto-migrate-up.md b/docs/docs/cli/keto-migrate-up.md index e0239ecda..298214e8a 100644 --- a/docs/docs/cli/keto-migrate-up.md +++ b/docs/docs/cli/keto-migrate-up.md @@ -16,8 +16,16 @@ Migrate the database up ### Synopsis -Migrate the database up. This does not affect namespaces. Use -`keto namespace migrate up` for migrating namespaces. +Run this command on a fresh SQL installation and when you upgrade Ory Keto to a +new minor version. + +It is recommended to run this command close to the SQL instance (e.g. same +subnet) instead of over the public internet. This decreases risk of failure and +decreases time required. + +### WARNING + +Before running this command on an existing database, create a back up! ``` keto migrate up [flags] From c385f4fbd3d6ac5f3e0acb0e3b479c72541d19b4 Mon Sep 17 00:00:00 2001 From: zepatrik Date: Tue, 14 Sep 2021 17:30:40 +0200 Subject: [PATCH 11/18] fix: format string error --- cmd/migrate/up.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/migrate/up.go b/cmd/migrate/up.go index 0ee573c3f..ecb0d6531 100644 --- a/cmd/migrate/up.go +++ b/cmd/migrate/up.go @@ -88,7 +88,7 @@ func BoxUp(cmd *cobra.Command, mb *popx.MigrationBox) error { _, _ = fmt.Fprintln(cmd.OutOrStdout(), "Applying migrations...") if err := mb.Up(cmd.Context()); err != nil { - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "%sCould not apply migrations: %+v\n", err) + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not apply migrations: %+v\n", err) return cmdx.FailSilently(cmd) } From 5c7025c9043b4f91dcd8f1e4cb4de2c74756b7aa Mon Sep 17 00:00:00 2001 From: Patrik Date: Mon, 20 Sep 2021 12:06:31 +0200 Subject: [PATCH 12/18] docs: improve command descriptions Co-authored-by: hackerman <3372410+aeneasr@users.noreply.github.com> --- cmd/migrate/up.go | 2 +- cmd/namespace/migrate_legacy.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/migrate/up.go b/cmd/migrate/up.go index ecb0d6531..efc88428a 100644 --- a/cmd/migrate/up.go +++ b/cmd/migrate/up.go @@ -23,7 +23,7 @@ func newUpCmd() *cobra.Command { cmd := &cobra.Command{ Use: "up", Short: "Migrate the database up", - Long: `Run this command on a fresh SQL installation and when you upgrade Ory Keto to a new minor version. + Long: `Run this command on a fresh SQL installation and when you upgrade Ory Keto from version v0.7.0 and later. It is recommended to run this command close to the SQL instance (e.g. same subnet) instead of over the public internet. This decreases risk of failure and decreases time required. diff --git a/cmd/namespace/migrate_legacy.go b/cmd/namespace/migrate_legacy.go index 24d6799bf..e6101ffc0 100644 --- a/cmd/namespace/migrate_legacy.go +++ b/cmd/namespace/migrate_legacy.go @@ -18,8 +18,8 @@ import ( func NewMigrateLegacyCmd() *cobra.Command { cmd := &cobra.Command{ Use: "legacy []", - Short: "Migrate a namespace from a legacy table.", - Long: "Migrate a legacy (v0.6.0) table to the v0.7.0 table.\n" + + Short: "Migrate a namespace from v0.6.x to v0.7.x and later.", + Long: "Migrate a legacy namespaces from v0.6.x to the v0.7.x and later.\n" + "This step only has to be executed once.\n" + "If no namespace is specified, all legacy namespaces will be migrated.\n" + "Please ensure that namespace IDs did not change in the config file and you have a backup in case something goes wrong!", From f350d158f86b533d9c2756624bc51ac0ce1f8853 Mon Sep 17 00:00:00 2001 From: aeneasr <3372410+aeneasr@users.noreply.github.com> Date: Mon, 20 Sep 2021 10:07:48 +0000 Subject: [PATCH 13/18] autogen(docs): generate cli docs --- docs/docs/cli/keto-migrate-up.md | 4 ++-- docs/docs/cli/keto-namespace-migrate-legacy.md | 13 +++++++------ docs/docs/cli/keto-namespace-migrate.md | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/docs/cli/keto-migrate-up.md b/docs/docs/cli/keto-migrate-up.md index 298214e8a..8706ec394 100644 --- a/docs/docs/cli/keto-migrate-up.md +++ b/docs/docs/cli/keto-migrate-up.md @@ -16,8 +16,8 @@ Migrate the database up ### Synopsis -Run this command on a fresh SQL installation and when you upgrade Ory Keto to a -new minor version. +Run this command on a fresh SQL installation and when you upgrade Ory Keto from +version v0.7.0 and later. It is recommended to run this command close to the SQL instance (e.g. same subnet) instead of over the public internet. This decreases risk of failure and diff --git a/docs/docs/cli/keto-namespace-migrate-legacy.md b/docs/docs/cli/keto-namespace-migrate-legacy.md index 7d097acfd..db88f3e98 100644 --- a/docs/docs/cli/keto-namespace-migrate-legacy.md +++ b/docs/docs/cli/keto-namespace-migrate-legacy.md @@ -2,7 +2,8 @@ id: keto-namespace-migrate-legacy title: keto namespace migrate legacy description: - keto namespace migrate legacy Migrate a namespace from a legacy table. + keto namespace migrate legacy Migrate a namespace from v0.6.x to v0.7.x and + later. ---