-
Notifications
You must be signed in to change notification settings - Fork 344
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: migration to single table SQL schema (#707)
This change adds a migration path from Keto version v0.6.x to the new persistence structure introduced by #638. Every namespace has to be migrated separately, or you can use the CLI to detect and migrate all namespaces at once. Have a look at `keto help namespace migrate legacy` for all details. **Please make sure that you backup the database before running the migration command**. Please note that this migration might be a bit slower than usual, as we have to pull the data from the database, transcode it in Keto, and then write it to the new table structure. Versions of Keto >v0.7 will not include this migration script, so you will first have to migrate to v0.7 and move on from there.
- Loading branch information
Showing
56 changed files
with
1,085 additions
and
123 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| name: Run full e2e test of the migration to single table persister (see https://github.com/ory/keto/issues/628) | ||
|
|
||
| on: | ||
| workflow_dispatch: | ||
| push: | ||
| branches: | ||
| - feat/persistence-migration-path | ||
|
|
||
| jobs: | ||
| test-migration: | ||
| runs-on: ubuntu-latest | ||
| name: Test Migration | ||
| steps: | ||
| - uses: actions/checkout@v2 | ||
| - uses: actions/setup-go@v2 | ||
| with: | ||
| go-version: '1.16' | ||
| - name: Run test script | ||
| run: ./scripts/single-table-migration-e2e.sh | ||
| - uses: actions/upload-artifact@v2 | ||
| if: failure() | ||
| with: | ||
| name: sqlite-db | ||
| path: migrate_e2e.sqlite |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,118 @@ | ||
| package namespace | ||
|
|
||
| import ( | ||
| "fmt" | ||
|
|
||
| "github.com/ory/x/cmdx" | ||
| "github.com/ory/x/flagx" | ||
| "github.com/pkg/errors" | ||
| "github.com/spf13/cobra" | ||
|
|
||
| "github.com/ory/keto/cmd/migrate" | ||
| "github.com/ory/keto/internal/driver" | ||
| "github.com/ory/keto/internal/namespace" | ||
| "github.com/ory/keto/internal/persistence" | ||
| "github.com/ory/keto/internal/persistence/sql/migrations" | ||
| ) | ||
|
|
||
| func NewMigrateLegacyCmd() *cobra.Command { | ||
| downOnly := false | ||
|
|
||
| cmd := &cobra.Command{ | ||
| Use: "legacy [<namespace-name>]", | ||
| Short: "Migrate a namespace from v0.6.x to v0.7.x and later.", | ||
| Long: "Migrate a legacy namespaces from v0.6.x to the v0.7.x and later.\n" + | ||
| "This step only has to be executed once.\n" + | ||
| "If no namespace is specified, all legacy namespaces will be migrated.\n" + | ||
| "Please ensure that namespace IDs did not change in the config file and you have a backup in case something goes wrong!", | ||
| Args: cobra.MaximumNArgs(1), | ||
| RunE: func(cmd *cobra.Command, args []string) error { | ||
| reg, err := driver.NewDefaultRegistry(cmd.Context(), cmd.Flags(), false) | ||
| if errors.Is(err, persistence.ErrNetworkMigrationsMissing) { | ||
| _, _ = fmt.Fprintln(cmd.ErrOrStderr(), "Migrations were not applied yet, please apply them first using `keto migrate up`.") | ||
| return cmdx.FailSilently(cmd) | ||
| } else if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| migrator := migrations.NewToSingleTableMigrator(reg) | ||
|
|
||
| var nn []*namespace.Namespace | ||
| if len(args) == 1 { | ||
| nm, err := reg.Config().NamespaceManager() | ||
| if err != nil { | ||
| _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "There seems to be a problem with the config: %s\n", err.Error()) | ||
| return cmdx.FailSilently(cmd) | ||
| } | ||
| n, err := nm.GetNamespaceByName(cmd.Context(), args[0]) | ||
| if err != nil { | ||
| _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "There seems to be a problem with the config: %s\n", err.Error()) | ||
| return cmdx.FailSilently(cmd) | ||
| } | ||
|
|
||
| nn = []*namespace.Namespace{n} | ||
|
|
||
| if !flagx.MustGetBool(cmd, migrate.FlagYes) && | ||
| !cmdx.AskForConfirmation( | ||
| fmt.Sprintf("Are you sure that you want to migrate the namespace '%s'?", args[0]), | ||
| cmd.InOrStdin(), cmd.OutOrStdout()) { | ||
| _, _ = fmt.Fprintln(cmd.OutOrStdout(), "OK, aborting.") | ||
| return nil | ||
| } | ||
| } else { | ||
| nn, err = migrator.LegacyNamespaces(cmd.Context()) | ||
| if err != nil { | ||
| _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not get legacy namespaces: %s\n", err.Error()) | ||
| return cmdx.FailSilently(cmd) | ||
| } | ||
|
|
||
| if len(nn) == 0 { | ||
| _, _ = fmt.Fprintln(cmd.OutOrStdout(), "Could not find legacy namespaces, there seems nothing to be done.") | ||
| return nil | ||
| } | ||
|
|
||
| var names string | ||
| for _, n := range nn { | ||
| names += " " + n.Name + "\n" | ||
| } | ||
| if !flagx.MustGetBool(cmd, migrate.FlagYes) && | ||
| !cmdx.AskForConfirmation( | ||
| fmt.Sprintf("I found the following legacy namespaces:\n%sDo you want to migrate all of them?", names), | ||
| cmd.InOrStdin(), cmd.OutOrStdout()) { | ||
| _, _ = fmt.Fprintln(cmd.OutOrStdout(), "OK, aborting.") | ||
| return nil | ||
| } | ||
| } | ||
|
|
||
| for _, n := range nn { | ||
| if !downOnly { | ||
| if err := migrator.MigrateNamespace(cmd.Context(), n); err != nil { | ||
| _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Encountered error while migrating: %s\nAborting.\n", err.Error()) | ||
| if errors.Is(err, migrations.ErrInvalidTuples(nil)) { | ||
| _, _ = fmt.Fprintln(cmd.ErrOrStderr(), "Please see https://github.com/ory/keto/issues/661 for why this happens and how to resolve this.") | ||
| } | ||
| return cmdx.FailSilently(cmd) | ||
| } | ||
| } | ||
| if flagx.MustGetBool(cmd, migrate.FlagYes) || | ||
| cmdx.AskForConfirmation( | ||
| fmt.Sprintf("Do you want to migrate namespace %s down? This will delete all data in the legacy table.", n.Name), | ||
| cmd.InOrStdin(), cmd.OutOrStdout()) { | ||
| if err := migrator.MigrateDown(cmd.Context(), n); err != nil { | ||
| _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not migrate down: %s\n", err.Error()) | ||
| return cmdx.FailSilently(cmd) | ||
| } | ||
| _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Successfully migrated down namespace %s.\n", n.Name) | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| }, | ||
| } | ||
|
|
||
| migrate.RegisterYesFlag(cmd.Flags()) | ||
| registerPackageFlags(cmd.Flags()) | ||
| cmd.Flags().BoolVar(&downOnly, "down-only", false, "Migrate legacy namespace(s) only down.") | ||
|
|
||
| return cmd | ||
| } |
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,90 @@ | ||
| package namespace | ||
|
|
||
| import ( | ||
| "context" | ||
| "io" | ||
| "os" | ||
| "path" | ||
| "testing" | ||
|
|
||
| "github.com/ory/keto/internal/relationtuple" | ||
|
|
||
| "github.com/ory/x/cmdx" | ||
| "github.com/stretchr/testify/assert" | ||
| "github.com/stretchr/testify/require" | ||
|
|
||
| "github.com/ory/keto/internal/driver" | ||
| "github.com/ory/keto/internal/driver/config" | ||
| "github.com/ory/keto/internal/namespace" | ||
| "github.com/ory/keto/internal/x/dbx" | ||
| ) | ||
|
|
||
| func TestMigrateLegacy(t *testing.T) { | ||
| setup := func(t *testing.T) (*cmdx.CommandExecuter, *driver.RegistryDefault) { | ||
| fp := path.Join(t.TempDir(), "db.sqlite") | ||
| dst, err := os.Create(fp) | ||
| require.NoError(t, err) | ||
| defer dst.Close() | ||
| src, err := os.Open("migrate_legacy_snapshot.sqlite") | ||
| require.NoError(t, err) | ||
| defer src.Close() | ||
| _, err = io.Copy(dst, src) | ||
| require.NoError(t, err) | ||
|
|
||
| reg := driver.NewTestRegistry(t, &dbx.DsnT{ | ||
| Name: "sqlite", | ||
| Conn: "sqlite://" + fp + "?_fk=true", | ||
| MigrateUp: true, | ||
| MigrateDown: false, | ||
| }) | ||
| nspaces := []*namespace.Namespace{ | ||
| { | ||
| ID: 0, | ||
| Name: "a", | ||
| }, | ||
| { | ||
| ID: 1, | ||
| Name: "b", | ||
| }, | ||
| } | ||
| require.NoError(t, reg.Config().Set(config.KeyNamespaces, nspaces)) | ||
|
|
||
| c := &cmdx.CommandExecuter{ | ||
| New: NewMigrateLegacyCmd, | ||
| Ctx: context.WithValue(context.Background(), driver.RegistryContextKey, reg), | ||
| } | ||
|
|
||
| return c, reg | ||
| } | ||
|
|
||
| t.Run("case=invalid subject", func(t *testing.T) { | ||
| c, reg := setup(t) | ||
|
|
||
| conn, err := reg.PopConnection() | ||
| require.NoError(t, err) | ||
| require.NoError(t, conn.RawQuery("insert into keto_0000000000_relation_tuples (shard_id, object, relation, subject, commit_time) values ('foo', 'obj', 'rel', 'invalid#subject', 'now')").Exec()) | ||
|
|
||
| stdErr := c.ExecExpectedErr(t, "-y", "a") | ||
| assert.Contains(t, stdErr, "found non-deserializable relationtuples") | ||
| assert.Contains(t, stdErr, "invalid#subject") | ||
|
|
||
| assert.Contains(t, c.ExecNoErr(t, "-y", "--down-only", "a"), "Successfully migrated down") | ||
| }) | ||
|
|
||
| t.Run("case=migrates down only", func(t *testing.T) { | ||
| c, reg := setup(t) | ||
|
|
||
| conn, err := reg.PopConnection() | ||
| require.NoError(t, err) | ||
| require.NoError(t, conn.RawQuery("insert into keto_0000000000_relation_tuples (shard_id, object, relation, subject, commit_time) values ('foo', 'obj', 'rel', 'sub', 'now')").Exec()) | ||
|
|
||
| c.ExecNoErr(t, "-y", "--down-only", "a") | ||
|
|
||
| rts, _, err := reg.RelationTupleManager().GetRelationTuples(context.Background(), &relationtuple.RelationQuery{ | ||
| Namespace: "a", | ||
| Object: "obj", | ||
| }) | ||
| require.NoError(t, err) | ||
| assert.Len(t, rts, 0) | ||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.