From 00713bc3d8c48dfb431c423b690b553c8fdd7c75 Mon Sep 17 00:00:00 2001 From: Patrik Date: Fri, 24 Sep 2021 12:05:45 +0200 Subject: [PATCH] feat: migration to single table SQL schema (#707) This change adds a migration path from Keto version v0.6.x to the new persistence structure introduced by https://github.com/ory/keto/pull/638. Every namespace has to be migrated separately, or you can use the CLI to detect and migrate all namespaces at once. Have a look at `keto help namespace migrate legacy` for all details. **Please make sure that you backup the database before running the migration command**. Please note that this migration might be a bit slower than usual, as we have to pull the data from the database, transcode it in Keto, and then write it to the new table structure. Versions of Keto >v0.7 will not include this migration script, so you will first have to migrate to v0.7 and move on from there. --- .../workflows/single-table-migration-e2e.yml | 24 ++ cmd/migrate/down.go | 17 +- cmd/migrate/up.go | 39 +-- cmd/namespace/migrate_legacy.go | 118 +++++++ cmd/namespace/migrate_legacy_snapshot.sqlite | Bin 0 -> 77824 bytes cmd/namespace/migrate_legacy_test.go | 90 ++++++ cmd/namespace/root.go | 2 +- docs/docs/cli/keto-migrate-down.md | 3 +- docs/docs/cli/keto-migrate-up.md | 21 +- .../docs/cli/keto-namespace-migrate-legacy.md | 50 +++ docs/docs/cli/keto-namespace-migrate.md | 29 +- docs/docs/cli/keto-namespace.md | 1 + docs/sidebar.json | 2 + go.mod | 2 +- go.sum | 8 +- install.sh | 0 internal/driver/pop_connection.go | 76 +++++ internal/driver/registry.go | 11 +- internal/driver/registry_default.go | 64 +--- internal/driver/registry_factory.go | 5 + .../20210128140414_namespace.down.sql | 0 .../20210128140414_namespace.mysql.down.sql | 0 .../20210128140414_namespace.up.sql | 0 .../sql/migrations/single_table.go | 296 ++++++++++++++++++ .../sql/migrations/single_table_test.go | 220 +++++++++++++ ...414000000_relationtuple.cockroach.down.sql | 0 ...75414000000_relationtuple.cockroach.up.sql | 5 + ...0175414000000_relationtuple.mysql.down.sql | 0 ...110175414000000_relationtuple.mysql.up.sql | 5 + ...5414000000_relationtuple.postgres.down.sql | 0 ...175414000000_relationtuple.postgres.up.sql | 5 + ...75414000000_relationtuple.sqlite3.down.sql | 0 ...0175414000000_relationtuple.sqlite3.up.sql | 5 + ...414000001_relationtuple.cockroach.down.sql | 1 + ...75414000001_relationtuple.cockroach.up.sql | 0 ...0175414000001_relationtuple.mysql.down.sql | 1 + ...110175414000001_relationtuple.mysql.up.sql | 0 ...5414000001_relationtuple.postgres.down.sql | 1 + ...175414000001_relationtuple.postgres.up.sql | 0 ...75414000001_relationtuple.sqlite3.down.sql | 1 + ...0175414000001_relationtuple.sqlite3.up.sql | 0 ...24000000_legacy-cleanup.cockroach.down.sql | 5 + ...4624000000_legacy-cleanup.cockroach.up.sql | 1 + ...134624000000_legacy-cleanup.mysql.down.sql | 5 + ...14134624000000_legacy-cleanup.mysql.up.sql | 1 + ...624000000_legacy-cleanup.postgres.down.sql | 5 + ...34624000000_legacy-cleanup.postgres.up.sql | 1 + ...4624000000_legacy-cleanup.sqlite3.down.sql | 4 + ...134624000000_legacy-cleanup.sqlite3.up.sql | 1 + .../20201110175414_relationtuple.down.sql | 0 .../20201110175414_relationtuple.up.sql | 0 .../20210914134624_legacy-cleanup.down.fizz | 6 + .../20210914134624_legacy-cleanup.up.fizz | 1 + internal/persistence/sql/persister.go | 9 +- internal/persistence/sql/relationtuples.go | 14 +- scripts/single-table-migration-e2e.sh | 53 ++++ 56 files changed, 1085 insertions(+), 123 deletions(-) create mode 100644 .github/workflows/single-table-migration-e2e.yml create mode 100644 cmd/namespace/migrate_legacy.go create mode 100644 cmd/namespace/migrate_legacy_snapshot.sqlite create mode 100644 cmd/namespace/migrate_legacy_test.go create mode 100644 docs/docs/cli/keto-namespace-migrate-legacy.md mode change 100644 => 100755 install.sh create mode 100644 internal/driver/pop_connection.go rename internal/persistence/sql/{ => migrations}/namespace_migrations/20210128140414_namespace.down.sql (100%) rename internal/persistence/sql/{ => migrations}/namespace_migrations/20210128140414_namespace.mysql.down.sql (100%) rename internal/persistence/sql/{ => migrations}/namespace_migrations/20210128140414_namespace.up.sql (100%) create mode 100644 internal/persistence/sql/migrations/single_table.go create mode 100644 internal/persistence/sql/migrations/single_table_test.go create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.cockroach.down.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.cockroach.up.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.mysql.down.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.mysql.up.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.postgres.down.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.postgres.up.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.sqlite3.down.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.sqlite3.up.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.cockroach.down.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.cockroach.up.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.mysql.down.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.mysql.up.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.postgres.down.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.postgres.up.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.sqlite3.down.sql create mode 100644 internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.sqlite3.up.sql create mode 100644 internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.cockroach.down.sql create mode 100644 internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.cockroach.up.sql create mode 100644 internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.mysql.down.sql create mode 100644 internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.mysql.up.sql create mode 100644 internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.postgres.down.sql create mode 100644 internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.postgres.up.sql create mode 100644 internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.sqlite3.down.sql create mode 100644 internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.sqlite3.up.sql rename internal/persistence/sql/migrations/{sql => templates}/20201110175414_relationtuple.down.sql (100%) rename internal/persistence/sql/migrations/{sql => templates}/20201110175414_relationtuple.up.sql (100%) create mode 100644 internal/persistence/sql/migrations/templates/20210914134624_legacy-cleanup.down.fizz create mode 100644 internal/persistence/sql/migrations/templates/20210914134624_legacy-cleanup.up.fizz create mode 100755 scripts/single-table-migration-e2e.sh diff --git a/.github/workflows/single-table-migration-e2e.yml b/.github/workflows/single-table-migration-e2e.yml new file mode 100644 index 000000000..5874f6890 --- /dev/null +++ b/.github/workflows/single-table-migration-e2e.yml @@ -0,0 +1,24 @@ +name: Run full e2e test of the migration to single table persister (see https://github.com/ory/keto/issues/628) + +on: + workflow_dispatch: + push: + branches: + - feat/persistence-migration-path + +jobs: + test-migration: + runs-on: ubuntu-latest + name: Test Migration + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-go@v2 + with: + go-version: '1.16' + - name: Run test script + run: ./scripts/single-table-migration-e2e.sh + - uses: actions/upload-artifact@v2 + if: failure() + with: + name: sqlite-db + path: migrate_e2e.sqlite diff --git a/cmd/migrate/down.go b/cmd/migrate/down.go index d2163a009..852bba449 100644 --- a/cmd/migrate/down.go +++ b/cmd/migrate/down.go @@ -19,8 +19,7 @@ func newDownCmd() *cobra.Command { Use: "down ", Short: "Migrate the database down", Long: "Migrate the database down a specific amount of steps.\n" + - "Pass 0 steps to fully migrate down.\n" + - "This does not affect namespaces. Use `keto namespace migrate down` for migrating namespaces.", + "Pass 0 steps to fully migrate down.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { steps, err := strconv.ParseInt(args[0], 0, 0) @@ -39,7 +38,7 @@ func newDownCmd() *cobra.Command { return err } - return BoxDown(cmd, mb, int(steps), "") + return BoxDown(cmd, mb, int(steps)) }, } @@ -49,27 +48,27 @@ func newDownCmd() *cobra.Command { return cmd } -func BoxDown(cmd *cobra.Command, mb *popx.MigrationBox, steps int, msgPrefix string) error { +func BoxDown(cmd *cobra.Command, mb *popx.MigrationBox, steps int) error { s, err := mb.Status(cmd.Context()) if err != nil { - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "%sCould not get migration status: %+v\n", msgPrefix, err) + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not get migration status: %+v\n", err) return cmdx.FailSilently(cmd) } cmdx.PrintTable(cmd, s) - if !flagx.MustGetBool(cmd, FlagYes) && !cmdx.AskForConfirmation(msgPrefix+"Do you really want to migrate down? This will delete data.", cmd.InOrStdin(), cmd.OutOrStdout()) { - _, _ = fmt.Fprintln(cmd.OutOrStdout(), msgPrefix+"Migration aborted.") + if !flagx.MustGetBool(cmd, FlagYes) && !cmdx.AskForConfirmation("Do you really want to migrate down? This will delete data.", cmd.InOrStdin(), cmd.OutOrStdout()) { + _, _ = fmt.Fprintln(cmd.OutOrStdout(), "Migration aborted.") return nil } if err := mb.Down(cmd.Context(), steps); err != nil { - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "%sCould apply down migrations: %+v\n", msgPrefix, err) + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could apply down migrations: %+v\n", err) return cmdx.FailSilently(cmd) } s, err = mb.Status(cmd.Context()) if err != nil { - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "%sCould not get migration status: %+v\n", msgPrefix, err) + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not get migration status: %+v\n", err) return cmdx.FailSilently(cmd) } cmdx.PrintTable(cmd, s) diff --git a/cmd/migrate/up.go b/cmd/migrate/up.go index 029a8cfeb..efc88428a 100644 --- a/cmd/migrate/up.go +++ b/cmd/migrate/up.go @@ -16,18 +16,22 @@ import ( ) const ( - FlagYes = "yes" - FlagAllNamespace = "all-namespaces" + FlagYes = "yes" ) func newUpCmd() *cobra.Command { - var allNamespaces bool - cmd := &cobra.Command{ Use: "up", Short: "Migrate the database up", - Long: "Migrate the database up.\n" + - "This does not affect namespaces. Use `keto namespace migrate up` for migrating namespaces.", + Long: `Run this command on a fresh SQL installation and when you upgrade Ory Keto from version v0.7.0 and later. + +It is recommended to run this command close to the SQL instance (e.g. same subnet) instead of over the public internet. +This decreases risk of failure and decreases time required. + +### WARNING ### + +Before running this command on an existing database, create a back up! +`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { ctx := cmd.Context() @@ -42,7 +46,7 @@ func newUpCmd() *cobra.Command { return err } - if err := BoxUp(cmd, mb, ""); err != nil { + if err := BoxUp(cmd, mb); err != nil { return err } @@ -51,7 +55,6 @@ func newUpCmd() *cobra.Command { } RegisterYesFlag(cmd.Flags()) - cmd.Flags().BoolVar(&allNamespaces, FlagAllNamespace, false, "migrate all pending namespaces as well") cmdx.RegisterFormatFlags(cmd.Flags()) @@ -62,38 +65,38 @@ func RegisterYesFlag(flags *pflag.FlagSet) { flags.BoolP(FlagYes, "y", false, "yes to all questions, no user input required") } -func BoxUp(cmd *cobra.Command, mb *popx.MigrationBox, msgPrefix string) error { - _, _ = fmt.Fprintln(cmd.OutOrStdout(), msgPrefix+"Current status:") +func BoxUp(cmd *cobra.Command, mb *popx.MigrationBox) error { + _, _ = fmt.Fprintln(cmd.OutOrStdout(), "Current status:") s, err := mb.Status(cmd.Context()) if err != nil { - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "%sCould not get migration status: %+v\n", msgPrefix, err) + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not get migration status: %+v\n", err) return cmdx.FailSilently(cmd) } cmdx.PrintTable(cmd, s) if !s.HasPending() { - _, _ = fmt.Fprintln(cmd.OutOrStdout(), msgPrefix+"All migrations are already applied, there is nothing to do.") + _, _ = fmt.Fprintln(cmd.OutOrStdout(), "All migrations are already applied, there is nothing to do.") return nil } - if !flagx.MustGetBool(cmd, FlagYes) && !cmdx.AskForConfirmation(msgPrefix+"Are you sure that you want to apply this migration? Make sure to check the CHANGELOG.md for breaking changes beforehand.", cmd.InOrStdin(), cmd.OutOrStdout()) { - _, _ = fmt.Fprintln(cmd.OutOrStdout(), msgPrefix+"Aborting") + if !flagx.MustGetBool(cmd, FlagYes) && !cmdx.AskForConfirmation("Are you sure that you want to apply this migration? Make sure to check the CHANGELOG.md for breaking changes beforehand.", cmd.InOrStdin(), cmd.OutOrStdout()) { + _, _ = fmt.Fprintln(cmd.OutOrStdout(), "Aborting") return nil } - _, _ = fmt.Fprintln(cmd.OutOrStdout(), msgPrefix+"Applying migrations...") + _, _ = fmt.Fprintln(cmd.OutOrStdout(), "Applying migrations...") if err := mb.Up(cmd.Context()); err != nil { - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "%sCould not apply migrations: %+v\n", msgPrefix, err) + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not apply migrations: %+v\n", err) return cmdx.FailSilently(cmd) } - _, _ = fmt.Fprintln(cmd.OutOrStdout(), msgPrefix+"Successfully applied all migrations:") + _, _ = fmt.Fprintln(cmd.OutOrStdout(), "Successfully applied all migrations:") s, err = mb.Status(cmd.Context()) if err != nil { - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "%sCould not get migration status: %+v\n", msgPrefix, err) + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not get migration status: %+v\n", err) return cmdx.FailSilently(cmd) } diff --git a/cmd/namespace/migrate_legacy.go b/cmd/namespace/migrate_legacy.go new file mode 100644 index 000000000..70f4192ab --- /dev/null +++ b/cmd/namespace/migrate_legacy.go @@ -0,0 +1,118 @@ +package namespace + +import ( + "fmt" + + "github.com/ory/x/cmdx" + "github.com/ory/x/flagx" + "github.com/pkg/errors" + "github.com/spf13/cobra" + + "github.com/ory/keto/cmd/migrate" + "github.com/ory/keto/internal/driver" + "github.com/ory/keto/internal/namespace" + "github.com/ory/keto/internal/persistence" + "github.com/ory/keto/internal/persistence/sql/migrations" +) + +func NewMigrateLegacyCmd() *cobra.Command { + downOnly := false + + cmd := &cobra.Command{ + Use: "legacy []", + Short: "Migrate a namespace from v0.6.x to v0.7.x and later.", + Long: "Migrate a legacy namespaces from v0.6.x to the v0.7.x and later.\n" + + "This step only has to be executed once.\n" + + "If no namespace is specified, all legacy namespaces will be migrated.\n" + + "Please ensure that namespace IDs did not change in the config file and you have a backup in case something goes wrong!", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + reg, err := driver.NewDefaultRegistry(cmd.Context(), cmd.Flags(), false) + if errors.Is(err, persistence.ErrNetworkMigrationsMissing) { + _, _ = fmt.Fprintln(cmd.ErrOrStderr(), "Migrations were not applied yet, please apply them first using `keto migrate up`.") + return cmdx.FailSilently(cmd) + } else if err != nil { + return err + } + + migrator := migrations.NewToSingleTableMigrator(reg) + + var nn []*namespace.Namespace + if len(args) == 1 { + nm, err := reg.Config().NamespaceManager() + if err != nil { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "There seems to be a problem with the config: %s\n", err.Error()) + return cmdx.FailSilently(cmd) + } + n, err := nm.GetNamespaceByName(cmd.Context(), args[0]) + if err != nil { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "There seems to be a problem with the config: %s\n", err.Error()) + return cmdx.FailSilently(cmd) + } + + nn = []*namespace.Namespace{n} + + if !flagx.MustGetBool(cmd, migrate.FlagYes) && + !cmdx.AskForConfirmation( + fmt.Sprintf("Are you sure that you want to migrate the namespace '%s'?", args[0]), + cmd.InOrStdin(), cmd.OutOrStdout()) { + _, _ = fmt.Fprintln(cmd.OutOrStdout(), "OK, aborting.") + return nil + } + } else { + nn, err = migrator.LegacyNamespaces(cmd.Context()) + if err != nil { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not get legacy namespaces: %s\n", err.Error()) + return cmdx.FailSilently(cmd) + } + + if len(nn) == 0 { + _, _ = fmt.Fprintln(cmd.OutOrStdout(), "Could not find legacy namespaces, there seems nothing to be done.") + return nil + } + + var names string + for _, n := range nn { + names += " " + n.Name + "\n" + } + if !flagx.MustGetBool(cmd, migrate.FlagYes) && + !cmdx.AskForConfirmation( + fmt.Sprintf("I found the following legacy namespaces:\n%sDo you want to migrate all of them?", names), + cmd.InOrStdin(), cmd.OutOrStdout()) { + _, _ = fmt.Fprintln(cmd.OutOrStdout(), "OK, aborting.") + return nil + } + } + + for _, n := range nn { + if !downOnly { + if err := migrator.MigrateNamespace(cmd.Context(), n); err != nil { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Encountered error while migrating: %s\nAborting.\n", err.Error()) + if errors.Is(err, migrations.ErrInvalidTuples(nil)) { + _, _ = fmt.Fprintln(cmd.ErrOrStderr(), "Please see https://github.com/ory/keto/issues/661 for why this happens and how to resolve this.") + } + return cmdx.FailSilently(cmd) + } + } + if flagx.MustGetBool(cmd, migrate.FlagYes) || + cmdx.AskForConfirmation( + fmt.Sprintf("Do you want to migrate namespace %s down? This will delete all data in the legacy table.", n.Name), + cmd.InOrStdin(), cmd.OutOrStdout()) { + if err := migrator.MigrateDown(cmd.Context(), n); err != nil { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not migrate down: %s\n", err.Error()) + return cmdx.FailSilently(cmd) + } + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Successfully migrated down namespace %s.\n", n.Name) + } + } + + return nil + }, + } + + migrate.RegisterYesFlag(cmd.Flags()) + registerPackageFlags(cmd.Flags()) + cmd.Flags().BoolVar(&downOnly, "down-only", false, "Migrate legacy namespace(s) only down.") + + return cmd +} diff --git a/cmd/namespace/migrate_legacy_snapshot.sqlite b/cmd/namespace/migrate_legacy_snapshot.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..db19ae2c166212dcadd09fbad68e13736458eb97 GIT binary patch literal 77824 zcmeI*O>^2t7{GDK*a+hn{E`{2JJXAmAvL&8nszd2NkUx@u3Zz;HiwP|W2K=n#z8o3 zPaRFCz4imO*Is(x$UHtiXmm{RwAF6b zI^qLmO;J_lQy~;ZNy_g{`Ry%h^2NOOM*gV=UXOd3RNnvO-<2g@*--ydG8e1=tyWh4 zSbnvj&)n~_5zF)f_TGk;HOs2~ym4`6{`lPNw2ZvB z6pXfcUh6bk7e?pm^4zqHE6Z#fmf0~HXHji8A681mlakm!d{p{IOtWEebQslMY_(3m zH|w38@J6WHAFXTInvCS}M3Hp+PY}n(RB`k-J4F=wFAEthTP~}=s-4>Dxfy{Kv3jjH z;FIEmgHjY^dMhpDVm+(1&m@NP`em(Mmv6W3?CLppWj7{g)rggz|Hzfrl{-rVJ$G*3 z(Q{(^rl{90#HlInUDTSUby=&MA9r{3p0~;SqSY4n+>~pa-4{2v-Oa`=YEWZU$K| z&SiZwYFC|>``(CfHwqCA)@@@|%l=SR?VtN8>PH_u?3#`5+U{{@886MYy*i{(42rHf4lAO&_6GQX=-a+6%6FxPM$7C@&Em{TStr zrVdZH%gb7}>O|H*L?d?Srb9%VmGHqD(Zd%PUll74KQC6q*6v+h93GvB!|K67P6Q`J zR$uwoy;b5->9b<>;6&tgdv9rc+0Rc=_Csx#!@1|3PV%9~1LqTePR6$%S@zT8%YJx= z{rWt3u!dSLO%+Ffvr|MN%YJHn*$<5e_ zS<1OGvkh&M@}D=}qWlLtrNXH@qNW6U_@&W1%lKIw(a|qP?)gcU|Kud)f8?&ODgVLt zS}(YX%73tC za(KNz9BHF!*;{fX$L^F3HynOHH!y9(YyHi>OA@WflBQ-}De{X20R#|0009ILKmY** z5I_I{1ZF{CT}|kzo%~L|P$=XJAKu^lFhsb*d)@{0ul1Q0*~0R#|0009ILKmY** zW

+p)ZcQ|9_)o-poXR1c3ko2q1s}0tg_000IagfB*tR0*ORw(fR-X@}Ykg1Q0*~ z0R#|0009ILKmY**5SU>Bp8ucW$t5%d5I_I{1Q0*~0R#|00D*W4c+daKhyQo(7IyPI z{~ymhAZi2jBi$9VDy9RUOoKmY**5I_I{1Q0+VRszoF|MC0(Vm0LiivR)$Abs!Ai~s@%Abjgli~s@%Ab + +## keto namespace migrate legacy + +Migrate a namespace from v0.6.x to v0.7.x and later. + +### Synopsis + +Migrate a legacy namespaces from v0.6.x to the v0.7.x and later. This step only +has to be executed once. If no namespace is specified, all legacy namespaces +will be migrated. Please ensure that namespace IDs did not change in the config +file and you have a backup in case something goes wrong! + +``` +keto namespace migrate legacy [<namespace-name>] [flags] +``` + +### Options + +``` + --down-only Migrate legacy namespace(s) only down. + -f, --format string Set the output format. One of table, json, and json-pretty. (default "default") + -h, --help help for legacy + -q, --quiet Be quiet with output printing. + --read-remote string Remote URL of the read API endpoint. (default "127.0.0.1:4466") + --write-remote string Remote URL of the write API endpoint. (default "127.0.0.1:4467") + -y, --yes yes to all questions, no user input required +``` + +### Options inherited from parent commands + +``` + -c, --config strings Config files to load, overwriting in the order specified. (default [/home/circleci/keto.yml]) +``` + +### SEE ALSO + +- [keto namespace migrate](keto-namespace-migrate) - Migrate a namespace diff --git a/docs/docs/cli/keto-namespace-migrate.md b/docs/docs/cli/keto-namespace-migrate.md index bcfebd470..e8dee9cdb 100644 --- a/docs/docs/cli/keto-namespace-migrate.md +++ b/docs/docs/cli/keto-namespace-migrate.md @@ -4,17 +4,30 @@ title: keto namespace migrate description: keto namespace migrate Migrate a namespace --- + + ## keto namespace migrate -Deprecated: this step is not required anymore since -https://github.com/ory/keto/pull/638 +Migrate a namespace + +### Options + +``` + -h, --help help for migrate +``` + +### Options inherited from parent commands + +``` + -c, --config strings Config files to load, overwriting in the order specified. (default [/home/circleci/keto.yml]) +``` ### SEE ALSO - [keto namespace](keto-namespace) - Read and manipulate namespaces -- [keto namespace migrate down](keto-namespace-migrate-down) - Migrate a - namespace down -- [keto namespace migrate status](keto-namespace-migrate-status) - Get the - current namespace migration status -- [keto namespace migrate up](keto-namespace-migrate-up) - Migrate a namespace - up +- [keto namespace migrate legacy](keto-namespace-migrate-legacy) - Migrate a + namespace from v0.6.x to v0.7.x and later. diff --git a/docs/docs/cli/keto-namespace.md b/docs/docs/cli/keto-namespace.md index 5f7c6b20b..a47d60528 100644 --- a/docs/docs/cli/keto-namespace.md +++ b/docs/docs/cli/keto-namespace.md @@ -29,5 +29,6 @@ Read and manipulate namespaces ### SEE ALSO - [keto](keto) - Global and consistent permission and authorization server +- [keto namespace migrate](keto-namespace-migrate) - Migrate a namespace - [keto namespace validate](keto-namespace-validate) - Validate namespace definitions diff --git a/docs/sidebar.json b/docs/sidebar.json index 4b64b3508..dbbafc7c5 100644 --- a/docs/sidebar.json +++ b/docs/sidebar.json @@ -50,6 +50,8 @@ "cli/keto-migrate-status", "cli/keto-migrate-up", "cli/keto-namespace", + "cli/keto-namespace-migrate", + "cli/keto-namespace-migrate-legacy", "cli/keto-namespace-validate", "cli/keto-relation-tuple", "cli/keto-relation-tuple-create", diff --git a/go.mod b/go.mod index 3a337823c..4351583e1 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/ory/herodot v0.9.6 github.com/ory/jsonschema/v3 v3.0.3 github.com/ory/keto/proto v0.0.0-00010101000000-000000000000 - github.com/ory/x v0.0.275 + github.com/ory/x v0.0.280 github.com/pelletier/go-toml v1.8.1 github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index b72c098e1..3a5faaff1 100644 --- a/go.sum +++ b/go.sum @@ -660,8 +660,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/knadh/koanf v0.14.1-0.20201201075439-e0853799f9ec/go.mod h1:H5mEFsTeWizwFXHKtsITL5ipsLTuAMQoGuQpp+1JL9U= -github.com/knadh/koanf v1.0.0 h1:tGQ1L53Tp4uPx6agVGBN1U7A4f83ZpH3SwZNG+BkfqI= -github.com/knadh/koanf v1.0.0/go.mod h1:vrMMuhIH0k7EoxiMbVfFlRvJYmxcT2Eha3DH8Tx5+X4= +github.com/knadh/koanf v1.2.2 h1:CydaDM/2mtza/ytVVnj4iQSVPjDq+pSV2vWMFDiQS08= +github.com/knadh/koanf v1.2.2/go.mod h1:xpPTwMhsA/aaQLAilyCCqfpEiY1gpa160AiCuWHJUjY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -865,8 +865,8 @@ github.com/ory/x v0.0.93/go.mod h1:lfcTaGXpTZs7IEQAW00r9EtTCOxD//SiP5uWtNiz31g= github.com/ory/x v0.0.110/go.mod h1:DJfkE3GdakhshNhw4zlKoRaL/ozg/lcTahA9OCih2BE= github.com/ory/x v0.0.127/go.mod h1:FwUujfFuCj5d+xgLn4fGMYPnzriR5bdAIulFXMtnK0M= github.com/ory/x v0.0.205/go.mod h1:A1s4iwmFIppRXZLF3J9GGWeY/HpREVm0Dk5z/787iek= -github.com/ory/x v0.0.275 h1:rFNmrmXUEGforCf+T1MUTk0VzJbMLsyZ31h/vV0II4w= -github.com/ory/x v0.0.275/go.mod h1:6bZ1d2RwvlC3iQOasMCQnliBC1DOLKLc/buxk4ync+Q= +github.com/ory/x v0.0.280 h1:yh/tzyFZtQlK2vI7G9StY25V36FGlRMPUXnwvrlKsCE= +github.com/ory/x v0.0.280/go.mod h1:APpShLyJcVzKw1kTgrHI+j/L9YM+8BRjHlcYObc7C1U= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/parnurzeal/gorequest v0.2.15/go.mod h1:3Kh2QUMJoqw3icWAecsyzkpY7UzRfDhbRdTjtNwNiUE= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= diff --git a/install.sh b/install.sh old mode 100644 new mode 100755 diff --git a/internal/driver/pop_connection.go b/internal/driver/pop_connection.go new file mode 100644 index 000000000..ef48ba532 --- /dev/null +++ b/internal/driver/pop_connection.go @@ -0,0 +1,76 @@ +package driver + +import ( + "time" + + "github.com/cenkalti/backoff/v3" + "github.com/gobuffalo/pop/v5" + "github.com/luna-duclos/instrumentedsql" + "github.com/luna-duclos/instrumentedsql/opentracing" + "github.com/ory/x/sqlcon" + "github.com/pkg/errors" +) + +func (r *RegistryDefault) PopConnectionWithOpts(popOpts ...func(*pop.ConnectionDetails)) (*pop.Connection, error) { + tracer := r.Tracer() + + var opts []instrumentedsql.Opt + if tracer.IsLoaded() { + opts = []instrumentedsql.Opt{ + instrumentedsql.WithTracer(opentracing.NewTracer(true)), + instrumentedsql.WithOmitArgs(), + } + } + pool, idlePool, connMaxLifetime, connMaxIdleTime, cleanedDSN := sqlcon.ParseConnectionOptions(r.Logger(), r.Config().DSN()) + connDetails := &pop.ConnectionDetails{ + URL: sqlcon.FinalizeDSN(r.Logger(), cleanedDSN), + IdlePool: idlePool, + ConnMaxLifetime: connMaxLifetime, + ConnMaxIdleTime: connMaxIdleTime, + Pool: pool, + UseInstrumentedDriver: tracer != nil && tracer.IsLoaded(), + InstrumentedDriverOptions: opts, + } + for _, o := range popOpts { + o(connDetails) + } + + bc := backoff.NewExponentialBackOff() + bc.MaxElapsedTime = time.Minute * 5 + bc.Reset() + + var conn *pop.Connection + if err := backoff.Retry(func() (err error) { + conn, err = pop.NewConnection(connDetails) + if err != nil { + r.Logger().WithError(err).Error("Unable to connect to database, retrying.") + return errors.WithStack(err) + } + + if err := conn.Open(); err != nil { + r.Logger().WithError(err).Error("Unable to open the database connection, retrying.") + return errors.WithStack(err) + } + + if err := conn.Store.(interface{ Ping() error }).Ping(); err != nil { + r.Logger().WithError(err).Error("Unable to ping the database connection, retrying.") + return errors.WithStack(err) + } + + return nil + }, bc); err != nil { + return nil, errors.WithStack(err) + } + + return conn, nil +} + +// PopConnection returns the standard connection that is kept for the whole time. +func (r *RegistryDefault) PopConnection() (*pop.Connection, error) { + if r.conn == nil { + var err error + r.conn, err = r.PopConnectionWithOpts() + return r.conn, err + } + return r.conn, nil +} diff --git a/internal/driver/registry.go b/internal/driver/registry.go index c3e35deea..09f83472e 100644 --- a/internal/driver/registry.go +++ b/internal/driver/registry.go @@ -4,6 +4,8 @@ import ( "context" "net/http" + "github.com/gobuffalo/pop/v5" + "github.com/ory/keto/internal/driver/config" "github.com/spf13/cobra" @@ -35,6 +37,10 @@ type ( expand.EngineProvider check.EngineProvider persistence.Migrator + persistence.Provider + + PopConnection() (*pop.Connection, error) + PopConnectionWithOpts(f ...func(*pop.ConnectionDetails)) (*pop.Connection, error) HealthHandler() *healthx.Handler Tracer() *tracing.Tracer @@ -54,4 +60,7 @@ type ( contextKeys string ) -const LogrusHookContextKey contextKeys = "logrus hook" +const ( + LogrusHookContextKey contextKeys = "logrus hook" + RegistryContextKey contextKeys = "registry" +) diff --git a/internal/driver/registry_default.go b/internal/driver/registry_default.go index 0f1a2093c..9cb7cbdd9 100644 --- a/internal/driver/registry_default.go +++ b/internal/driver/registry_default.go @@ -4,16 +4,11 @@ import ( "context" "net/http" "sync" - "time" "github.com/ory/x/networkx" - "github.com/cenkalti/backoff/v3" "github.com/gobuffalo/pop/v5" - "github.com/luna-duclos/instrumentedsql" - "github.com/luna-duclos/instrumentedsql/opentracing" "github.com/ory/x/popx" - "github.com/ory/x/sqlcon" "github.com/pkg/errors" "github.com/ory/x/dbal" @@ -153,6 +148,13 @@ func (r *RegistryDefault) RelationTupleManager() relationtuple.Manager { return r.p } +func (r *RegistryDefault) Persister() persistence.Persister { + if r.p == nil { + panic("no persister, but expected to have one") + } + return r.p +} + func (r *RegistryDefault) PermissionEngine() *check.Engine { if r.ce == nil { r.ce = check.NewEngine(r) @@ -201,58 +203,6 @@ func (r *RegistryDefault) MigrateDown(ctx context.Context) error { return mb.Up(ctx) } -func (r *RegistryDefault) PopConnection() (*pop.Connection, error) { - if r.conn == nil { - tracer := r.Tracer() - - var opts []instrumentedsql.Opt - if tracer.IsLoaded() { - opts = []instrumentedsql.Opt{ - instrumentedsql.WithTracer(opentracing.NewTracer(true)), - instrumentedsql.WithOmitArgs(), - } - } - pool, idlePool, connMaxLifetime, connMaxIdleTime, cleanedDSN := sqlcon.ParseConnectionOptions(r.Logger(), r.Config().DSN()) - connDetails := &pop.ConnectionDetails{ - URL: sqlcon.FinalizeDSN(r.Logger(), cleanedDSN), - IdlePool: idlePool, - ConnMaxLifetime: connMaxLifetime, - ConnMaxIdleTime: connMaxIdleTime, - Pool: pool, - UseInstrumentedDriver: tracer != nil && tracer.IsLoaded(), - InstrumentedDriverOptions: opts, - } - - bc := backoff.NewExponentialBackOff() - bc.MaxElapsedTime = time.Minute * 5 - bc.Reset() - - if err := backoff.Retry(func() (err error) { - conn, err := pop.NewConnection(connDetails) - if err != nil { - r.Logger().WithError(err).Error("Unable to connect to database, retrying.") - return errors.WithStack(err) - } - - if err := conn.Open(); err != nil { - r.Logger().WithError(err).Error("Unable to open the database connection, retrying.") - return errors.WithStack(err) - } - - if err := conn.Store.(interface{ Ping() error }).Ping(); err != nil { - r.Logger().WithError(err).Error("Unable to ping the database connection, retrying.") - return errors.WithStack(err) - } - - r.conn = conn - return nil - }, bc); err != nil { - return nil, errors.WithStack(err) - } - } - return r.conn, nil -} - func (r *RegistryDefault) determineNetwork(ctx context.Context) (*networkx.Network, error) { c, err := r.PopConnection() if err != nil { diff --git a/internal/driver/registry_factory.go b/internal/driver/registry_factory.go index 5fe3ad330..395c063ca 100644 --- a/internal/driver/registry_factory.go +++ b/internal/driver/registry_factory.go @@ -18,6 +18,11 @@ import ( ) func NewDefaultRegistry(ctx context.Context, flags *pflag.FlagSet, withoutNetwork bool) (Registry, error) { + reg, ok := ctx.Value(RegistryContextKey).(Registry) + if ok { + return reg, nil + } + hook, ok := ctx.Value(LogrusHookContextKey).(logrus.Hook) var opts []logrusx.Option diff --git a/internal/persistence/sql/namespace_migrations/20210128140414_namespace.down.sql b/internal/persistence/sql/migrations/namespace_migrations/20210128140414_namespace.down.sql similarity index 100% rename from internal/persistence/sql/namespace_migrations/20210128140414_namespace.down.sql rename to internal/persistence/sql/migrations/namespace_migrations/20210128140414_namespace.down.sql diff --git a/internal/persistence/sql/namespace_migrations/20210128140414_namespace.mysql.down.sql b/internal/persistence/sql/migrations/namespace_migrations/20210128140414_namespace.mysql.down.sql similarity index 100% rename from internal/persistence/sql/namespace_migrations/20210128140414_namespace.mysql.down.sql rename to internal/persistence/sql/migrations/namespace_migrations/20210128140414_namespace.mysql.down.sql diff --git a/internal/persistence/sql/namespace_migrations/20210128140414_namespace.up.sql b/internal/persistence/sql/migrations/namespace_migrations/20210128140414_namespace.up.sql similarity index 100% rename from internal/persistence/sql/namespace_migrations/20210128140414_namespace.up.sql rename to internal/persistence/sql/migrations/namespace_migrations/20210128140414_namespace.up.sql diff --git a/internal/persistence/sql/migrations/single_table.go b/internal/persistence/sql/migrations/single_table.go new file mode 100644 index 000000000..f879ca491 --- /dev/null +++ b/internal/persistence/sql/migrations/single_table.go @@ -0,0 +1,296 @@ +package migrations + +import ( + "context" + "embed" + "encoding/json" + "fmt" + "time" + + "github.com/gobuffalo/pop/v5" + "github.com/gofrs/uuid" + "github.com/ory/x/popx" + "github.com/ory/x/sqlcon" + "github.com/pkg/errors" + + "github.com/ory/keto/internal/driver/config" + "github.com/ory/keto/internal/namespace" + "github.com/ory/keto/internal/persistence" + "github.com/ory/keto/internal/persistence/sql" + "github.com/ory/keto/internal/relationtuple" + "github.com/ory/keto/internal/x" +) + +// Partially transferred from tree at https://github.com/ory/keto/tree/88cedc35b5bcb79ee54e361e00b9d7f60f27b431 +// Right before https://github.com/ory/keto/pull/638 + +type ( + dependencies interface { + persistence.Provider + x.LoggerProvider + config.Provider + + PopConnection() (*pop.Connection, error) + PopConnectionWithOpts(...func(*pop.ConnectionDetails)) (*pop.Connection, error) + } + toSingleTableMigrator struct { + d dependencies + perPage int + } + + relationTuple struct { + // An ID field is required to make pop happy. The actual ID is a composite primary key. + ID string `db:"shard_id" json:"-"` + Object string `db:"object" json:"object"` + Relation string `db:"relation" json:"relation"` + Subject string `db:"subject" json:"subject"` + CommitTime time.Time `db:"commit_time" json:"commit_time"` + Namespace *namespace.Namespace `db:"-" json:"-"` + } + relationTuples []*relationTuple + contextKey string + + ErrInvalidTuples []*relationTuple +) + +var ( + //go:embed namespace_migrations/*.sql + namespaceMigrations embed.FS +) + +const namespaceCtxKey contextKey = "namespace" + +func tableFromNamespace(n *namespace.Namespace) string { + return fmt.Sprintf("keto_%0.10d_relation_tuples", n.ID) +} + +func namespaceIDFromTable(t string) (nID int32, err error) { + _, err = fmt.Sscanf(t, "keto_%d_relation_tuples", &nID) + return nID, errors.WithStack(err) +} + +func migrationTableFromNamespace(n *namespace.Namespace) string { + return fmt.Sprintf("keto_namespace_%0.10d_migrations", n.ID) +} + +func namespaceTableFromContext(ctx context.Context) string { + n, ok := ctx.Value(namespaceCtxKey).(*namespace.Namespace) + if n == nil || !ok { + panic("namespace context key not set") + } + return tableFromNamespace(n) +} + +func (e ErrInvalidTuples) Error() string { + msg := "found non-deserializable relationtuples: " + raw, err := json.Marshal(e) + if err != nil { + msg += "internal error: " + err.Error() + } else { + msg += string(raw) + } + return msg +} + +func (e ErrInvalidTuples) Is(other error) bool { + _, ok := other.(ErrInvalidTuples) + return ok +} + +func (r *relationTuple) toInternal() (*relationtuple.InternalRelationTuple, error) { + if r == nil { + return nil, nil + } + + sub, err := relationtuple.SubjectFromString(r.Subject) + if err != nil { + return nil, err + } + + return &relationtuple.InternalRelationTuple{ + Relation: r.Relation, + Object: r.Object, + Namespace: r.Namespace.Name, + Subject: sub, + }, nil +} + +func (relationTuples) TableName(ctx context.Context) string { + return namespaceTableFromContext(ctx) +} + +func (relationTuple) TableName(ctx context.Context) string { + return namespaceTableFromContext(ctx) +} + +func NewToSingleTableMigrator(d dependencies) *toSingleTableMigrator { + return &toSingleTableMigrator{ + d: d, + perPage: 100, + } +} + +func (m *toSingleTableMigrator) namespaceMigrationBox(n *namespace.Namespace) (*popx.MigrationBox, error) { + c, err := m.d.PopConnectionWithOpts(func(d *pop.ConnectionDetails) { + d.Options = map[string]string{ + "migration_table_name": migrationTableFromNamespace(n), + } + }) + if err != nil { + return nil, err + } + + return popx.NewMigrationBox( + namespaceMigrations, + popx.NewMigrator(c, m.d.Logger(), nil, 0), + popx.WithTemplateValues(map[string]interface{}{ + "tableName": tableFromNamespace(n), + }), + ) +} + +func (m *toSingleTableMigrator) getOldRelationTuples(ctx context.Context, n *namespace.Namespace, page, perPage int) (relationTuples, bool, error) { + q := m.d.Persister().Connection(ctx). + WithContext(context.WithValue(ctx, namespaceCtxKey, n)). + Order("object, relation, subject, commit_time"). + Paginate(page, perPage) + + var res relationTuples + if err := q.All(&res); err != nil { + return nil, false, sqlcon.HandleError(err) + } + for _, r := range res { + r.Namespace = n + } + return res, q.Paginator.Page < q.Paginator.TotalPages, nil +} + +func (m *toSingleTableMigrator) insertOldRelationTuples(ctx context.Context, n *namespace.Namespace, rs ...*relationtuple.InternalRelationTuple) error { + for _, r := range rs { + if r.Subject == nil { + return errors.New("subject is not allowed to be nil") + } + + m.d.Logger().WithFields(r.ToLoggerFields()).Trace("creating in legacy database") + + if err := m.d.Persister().Connection(context.WithValue(ctx, namespaceCtxKey, n)).Create(&relationTuple{ + ID: "testing only", + Object: r.Object, + Relation: r.Relation, + Subject: r.Subject.String(), + CommitTime: time.Now(), + }); err != nil { + return err + } + } + return nil +} + +func (m *toSingleTableMigrator) MigrateNamespace(ctx context.Context, n *namespace.Namespace) error { + p, ok := m.d.Persister().(*sql.Persister) + if !ok { + panic("got unexpected persister") + } + + var irrecoverableRTs ErrInvalidTuples + + for done, page := false, 1; !done; { + if err := p.Transaction(ctx, func(ctx context.Context, _ *pop.Connection) error { + rs, hasNext, err := m.getOldRelationTuples(ctx, n, page, m.perPage) + if err != nil { + return err + } + + for _, r := range rs { + ri, err := r.toInternal() + if err != nil { + m.d.Logger().WithField("relation_tuple", r).WithField("hint", "").WithError(err).Warn("Skipping relation tuple, it seems to be in a broken state. Please recreate it manually.") + irrecoverableRTs = append(irrecoverableRTs, r) + continue + } + rt := &sql.RelationTuple{ + ID: uuid.Must(uuid.NewV4()), + CommitTime: r.CommitTime, + } + if err := rt.FromInternal(ctx, p, ri); err != nil { + return err + } + + m.d.Logger().WithFields(ri.ToLoggerFields()).Debug("creating in new table...") + if err := sqlcon.HandleError( + p.CreateWithNetwork(ctx, rt), + ); err != nil { + return err + } + } + if !hasNext { + done = true + return nil + } + + page++ + return nil + }); err != nil { + return err + } + } + + if len(irrecoverableRTs) != 0 { + return irrecoverableRTs + } + return nil +} + +func (m *toSingleTableMigrator) LegacyNamespaces(ctx context.Context) ([]*namespace.Namespace, error) { + c, err := m.d.PopConnection() + if err != nil { + return nil, err + } + + var query *pop.Query + switch d := c.Dialect.Name(); d { + case "sqlite3": + query = c.RawQuery("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'keto_%_relation_tuples'") + case "postgres": + query = c.RawQuery("SELECT tablename FROM pg_catalog.pg_tables WHERE tablename LIKE 'keto_%_relation_tuples'") + case "cockroach", "mysql": + query = c.RawQuery("SELECT table_name FROM information_schema.tables WHERE table_name LIKE 'keto_%_relation_tuples'") + default: + panic("got unknown database dialect " + d) + } + + var tableNames []string + if err := sqlcon.HandleError(query.All(&tableNames)); err != nil { + return nil, err + } + + nm, err := m.d.Config().NamespaceManager() + if err != nil { + return nil, err + } + + namespaces := make([]*namespace.Namespace, len(tableNames)) + for i := range tableNames { + cID, err := namespaceIDFromTable(tableNames[i]) + if err != nil { + return nil, err + } + namespaces[i], err = nm.GetNamespaceByConfigID(ctx, cID) + if err != nil { + return nil, err + } + } + + return namespaces, nil +} + +func (m *toSingleTableMigrator) MigrateDown(ctx context.Context, n *namespace.Namespace) error { + mb, err := m.namespaceMigrationBox(n) + if err != nil { + return err + } + if err := mb.Down(ctx, 0); err != nil { + return errors.WithStack(err) + } + return nil +} diff --git a/internal/persistence/sql/migrations/single_table_test.go b/internal/persistence/sql/migrations/single_table_test.go new file mode 100644 index 000000000..f9e0b5296 --- /dev/null +++ b/internal/persistence/sql/migrations/single_table_test.go @@ -0,0 +1,220 @@ +package migrations + +import ( + "context" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/ory/keto/internal/driver" + "github.com/ory/keto/internal/driver/config" + "github.com/ory/keto/internal/namespace" + "github.com/ory/keto/internal/relationtuple" + "github.com/ory/keto/internal/x" + "github.com/ory/keto/internal/x/dbx" +) + +func TestToSingleTableMigrator(t *testing.T) { + const debugOnDisk = false + + for _, dsn := range dbx.GetDSNs(t, debugOnDisk) { + t.Run("db="+dsn.Name, func(t *testing.T) { + r := driver.NewTestRegistry(t, dsn) + ctx := context.Background() + var nn []*namespace.Namespace + m := NewToSingleTableMigrator(r) + + setup := func(t *testing.T) *namespace.Namespace { + n := &namespace.Namespace{ + Name: t.Name(), + ID: int32(len(nn)), + } + + nn = append(nn, n) + + mb, err := m.namespaceMigrationBox(n) + require.NoError(t, err) + require.NoError(t, mb.Up(ctx)) + + t.Cleanup(func() { + if debugOnDisk { + return + } + require.NoError(t, mb.Down(ctx, -1)) + }) + + require.NoError(t, r.Config().Set(config.KeyNamespaces, nn)) + + return n + } + + t.Run("case=simple tuples", func(t *testing.T) { + n := setup(t) + + // insert tuples into the old table + sID := &relationtuple.InternalRelationTuple{ + Namespace: n.Name, + Object: "a", + Relation: "a", + Subject: &relationtuple.SubjectID{ID: "a"}, + } + sSet := &relationtuple.InternalRelationTuple{ + Namespace: n.Name, + Object: "b", + Relation: "b", + Subject: &relationtuple.SubjectSet{ + Namespace: n.Name, + Object: "b", + Relation: "b", + }, + } + require.NoError(t, m.insertOldRelationTuples(ctx, n, sID, sSet)) + + // get the tuple from the old table + oldRts, next, err := m.getOldRelationTuples(ctx, n, 0, 100) + require.NoError(t, err) + assert.False(t, next) + require.Len(t, oldRts, 2) + for i, r := range []*relationtuple.InternalRelationTuple{sID, sSet} { + assert.Equal(t, r.Namespace, oldRts[i].Namespace.Name) + assert.Equal(t, r.Object, oldRts[i].Object) + assert.Equal(t, r.Relation, oldRts[i].Relation) + assert.Equal(t, r.Subject.String(), oldRts[i].Subject) + } + + // migrate to new table + require.NoError(t, m.MigrateNamespace(ctx, n)) + + // get the tuple from the new table + rts, nextToken, err := r.RelationTupleManager().GetRelationTuples(ctx, &relationtuple.RelationQuery{Namespace: n.Name}) + require.NoError(t, err) + assert.Equal(t, "", nextToken) + require.Len(t, rts, 2) + assert.Equal(t, sID, rts[0]) + assert.Equal(t, sSet, rts[1]) + }) + + t.Run("case=paginates", func(t *testing.T) { + n := setup(t) + + defer func(old int) { + m.perPage = old + }(m.perPage) + m.perPage = 1 + + rts := make([]*relationtuple.InternalRelationTuple, 10) + for i := range rts { + rts[i] = &relationtuple.InternalRelationTuple{ + Namespace: n.Name, + Object: strconv.Itoa(i), + Relation: strconv.Itoa(i), + Subject: &relationtuple.SubjectID{ID: strconv.Itoa(i)}, + } + } + + require.NoError(t, m.insertOldRelationTuples(ctx, n, rts...)) + require.NoError(t, m.MigrateNamespace(ctx, n)) + + migrated, nextToken, err := r.RelationTupleManager().GetRelationTuples(ctx, &relationtuple.RelationQuery{Namespace: n.Name}, x.WithSize(len(rts))) + require.NoError(t, err) + assert.Equal(t, "", nextToken) + assert.Equal(t, rts, migrated) + }) + + t.Run("case=non-deserializable tuple", func(t *testing.T) { + n := setup(t) + + require.NoError(t, m.insertOldRelationTuples(ctx, n, &relationtuple.InternalRelationTuple{ + Namespace: n.Name, + Object: "o0", + Relation: "r", + Subject: &relationtuple.SubjectID{ID: "invalid#subject-id"}, + }, &relationtuple.InternalRelationTuple{ + Namespace: n.Name, + Object: "o1", + Relation: "r", + Subject: &relationtuple.SubjectID{ID: "invalid#subject-id"}, + })) + err := m.MigrateNamespace(ctx, n) + require.Error(t, err) + invalid, ok := err.(ErrInvalidTuples) + require.True(t, ok) + assert.Len(t, invalid, 2) + }) + }) + } +} + +func TestToSingleTableMigrator_HasLegacyTable(t *testing.T) { + const debugOnDisk = false + + for _, dsn := range dbx.GetDSNs(t, debugOnDisk) { + t.Run("db="+dsn.Name, func(t *testing.T) { + t.Run("case=simple detection", func(t *testing.T) { + ctx := context.Background() + reg := driver.NewTestRegistry(t, dsn) + m := NewToSingleTableMigrator(reg) + + nspaces := []*namespace.Namespace{{ + ID: 3, + Name: "foo", + }} + require.NoError(t, reg.Config().Set(config.KeyNamespaces, nspaces)) + + // expect to not report legacy table + legacyNamespaces, err := m.LegacyNamespaces(ctx) + require.NoError(t, err) + assert.Len(t, legacyNamespaces, 0) + + // migrate legacy table up + mb, err := m.namespaceMigrationBox(nspaces[0]) + require.NoError(t, err) + require.NoError(t, mb.Up(ctx)) + + // expect to report legacy table + legacyNamespaces, err = m.LegacyNamespaces(ctx) + require.NoError(t, err) + assert.Equal(t, nspaces, legacyNamespaces) + + // migrate legacy down + require.NoError(t, mb.Down(ctx, -1)) + + // expect to not report legacy + legacyNamespaces, err = m.LegacyNamespaces(ctx) + require.NoError(t, err) + assert.Len(t, legacyNamespaces, 0) + }) + + t.Run("case=multiple namespaces", func(t *testing.T) { + ctx := context.Background() + reg := driver.NewTestRegistry(t, dsn) + m := NewToSingleTableMigrator(reg) + + nspaces := []*namespace.Namespace{{ + ID: 0, + Name: "a", + }, { + ID: 1, + Name: "b", + }, { + ID: 2, + Name: "c", + }} + require.NoError(t, reg.Config().Set(config.KeyNamespaces, nspaces)) + + for _, n := range nspaces { + // migrate legacy table up + mb, err := m.namespaceMigrationBox(n) + require.NoError(t, err) + require.NoError(t, mb.Up(ctx)) + } + + ln, err := m.LegacyNamespaces(ctx) + require.NoError(t, err) + assert.Equal(t, nspaces, ln) + }) + }) + } +} diff --git a/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.cockroach.down.sql b/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.cockroach.down.sql new file mode 100644 index 000000000..e69de29bb diff --git a/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.cockroach.up.sql b/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.cockroach.up.sql new file mode 100644 index 000000000..e6c51f372 --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.cockroach.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE keto_namespace +( + id INTEGER PRIMARY KEY, + schema_version INTEGER NOT NULL +); \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.mysql.down.sql b/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.mysql.down.sql new file mode 100644 index 000000000..e69de29bb diff --git a/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.mysql.up.sql b/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.mysql.up.sql new file mode 100644 index 000000000..e6c51f372 --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.mysql.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE keto_namespace +( + id INTEGER PRIMARY KEY, + schema_version INTEGER NOT NULL +); \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.postgres.down.sql b/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.postgres.down.sql new file mode 100644 index 000000000..e69de29bb diff --git a/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.postgres.up.sql b/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.postgres.up.sql new file mode 100644 index 000000000..e6c51f372 --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.postgres.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE keto_namespace +( + id INTEGER PRIMARY KEY, + schema_version INTEGER NOT NULL +); \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.sqlite3.down.sql b/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.sqlite3.down.sql new file mode 100644 index 000000000..e69de29bb diff --git a/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.sqlite3.up.sql b/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.sqlite3.up.sql new file mode 100644 index 000000000..e6c51f372 --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20201110175414000000_relationtuple.sqlite3.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE keto_namespace +( + id INTEGER PRIMARY KEY, + schema_version INTEGER NOT NULL +); \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.cockroach.down.sql b/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.cockroach.down.sql new file mode 100644 index 000000000..ab89a03dc --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.cockroach.down.sql @@ -0,0 +1 @@ +DROP TABLE keto_namespace; \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.cockroach.up.sql b/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.cockroach.up.sql new file mode 100644 index 000000000..e69de29bb diff --git a/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.mysql.down.sql b/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.mysql.down.sql new file mode 100644 index 000000000..ab89a03dc --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.mysql.down.sql @@ -0,0 +1 @@ +DROP TABLE keto_namespace; \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.mysql.up.sql b/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.mysql.up.sql new file mode 100644 index 000000000..e69de29bb diff --git a/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.postgres.down.sql b/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.postgres.down.sql new file mode 100644 index 000000000..ab89a03dc --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.postgres.down.sql @@ -0,0 +1 @@ +DROP TABLE keto_namespace; \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.postgres.up.sql b/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.postgres.up.sql new file mode 100644 index 000000000..e69de29bb diff --git a/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.sqlite3.down.sql b/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.sqlite3.down.sql new file mode 100644 index 000000000..ab89a03dc --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.sqlite3.down.sql @@ -0,0 +1 @@ +DROP TABLE keto_namespace; \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.sqlite3.up.sql b/internal/persistence/sql/migrations/sql/20201110175414000001_relationtuple.sqlite3.up.sql new file mode 100644 index 000000000..e69de29bb diff --git a/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.cockroach.down.sql b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.cockroach.down.sql new file mode 100644 index 000000000..fde3c6a94 --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.cockroach.down.sql @@ -0,0 +1,5 @@ +CREATE TABLE "keto_namespace" ( +"id" SERIAL NOT NULL, +PRIMARY KEY("id"), +"schema_version" int NOT NULL +); \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.cockroach.up.sql b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.cockroach.up.sql new file mode 100644 index 000000000..aa1782069 --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.cockroach.up.sql @@ -0,0 +1 @@ +DROP TABLE "keto_namespace"; \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.mysql.down.sql b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.mysql.down.sql new file mode 100644 index 000000000..8aef7cf1b --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.mysql.down.sql @@ -0,0 +1,5 @@ +CREATE TABLE `keto_namespace` ( +`id` INTEGER NOT NULL AUTO_INCREMENT, +PRIMARY KEY(`id`), +`schema_version` INTEGER NOT NULL +) ENGINE=InnoDB; \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.mysql.up.sql b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.mysql.up.sql new file mode 100644 index 000000000..5b1def1b7 --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.mysql.up.sql @@ -0,0 +1 @@ +DROP TABLE `keto_namespace`; \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.postgres.down.sql b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.postgres.down.sql new file mode 100644 index 000000000..fde3c6a94 --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.postgres.down.sql @@ -0,0 +1,5 @@ +CREATE TABLE "keto_namespace" ( +"id" SERIAL NOT NULL, +PRIMARY KEY("id"), +"schema_version" int NOT NULL +); \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.postgres.up.sql b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.postgres.up.sql new file mode 100644 index 000000000..aa1782069 --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.postgres.up.sql @@ -0,0 +1 @@ +DROP TABLE "keto_namespace"; \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.sqlite3.down.sql b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.sqlite3.down.sql new file mode 100644 index 000000000..30f9414cc --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.sqlite3.down.sql @@ -0,0 +1,4 @@ +CREATE TABLE "keto_namespace" ( +"id" INTEGER PRIMARY KEY AUTOINCREMENT, +"schema_version" INTEGER NOT NULL +); \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.sqlite3.up.sql b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.sqlite3.up.sql new file mode 100644 index 000000000..aa1782069 --- /dev/null +++ b/internal/persistence/sql/migrations/sql/20210914134624000000_legacy-cleanup.sqlite3.up.sql @@ -0,0 +1 @@ +DROP TABLE "keto_namespace"; \ No newline at end of file diff --git a/internal/persistence/sql/migrations/sql/20201110175414_relationtuple.down.sql b/internal/persistence/sql/migrations/templates/20201110175414_relationtuple.down.sql similarity index 100% rename from internal/persistence/sql/migrations/sql/20201110175414_relationtuple.down.sql rename to internal/persistence/sql/migrations/templates/20201110175414_relationtuple.down.sql diff --git a/internal/persistence/sql/migrations/sql/20201110175414_relationtuple.up.sql b/internal/persistence/sql/migrations/templates/20201110175414_relationtuple.up.sql similarity index 100% rename from internal/persistence/sql/migrations/sql/20201110175414_relationtuple.up.sql rename to internal/persistence/sql/migrations/templates/20201110175414_relationtuple.up.sql diff --git a/internal/persistence/sql/migrations/templates/20210914134624_legacy-cleanup.down.fizz b/internal/persistence/sql/migrations/templates/20210914134624_legacy-cleanup.down.fizz new file mode 100644 index 000000000..d2df585b1 --- /dev/null +++ b/internal/persistence/sql/migrations/templates/20210914134624_legacy-cleanup.down.fizz @@ -0,0 +1,6 @@ +create_table("keto_namespace") { + t.Column("id", "int", { primary: true }) + t.Column("schema_version", "int") + + t.DisableTimestamps() +} diff --git a/internal/persistence/sql/migrations/templates/20210914134624_legacy-cleanup.up.fizz b/internal/persistence/sql/migrations/templates/20210914134624_legacy-cleanup.up.fizz new file mode 100644 index 000000000..432652b55 --- /dev/null +++ b/internal/persistence/sql/migrations/templates/20210914134624_legacy-cleanup.up.fizz @@ -0,0 +1 @@ +drop_table("keto_namespace") diff --git a/internal/persistence/sql/persister.go b/internal/persistence/sql/persister.go index b44aeeb9d..c80119a42 100644 --- a/internal/persistence/sql/persister.go +++ b/internal/persistence/sql/persister.go @@ -38,7 +38,6 @@ type ( config.Provider x.LoggerProvider - Tracer() *tracing.Tracer PopConnection() (*pop.Connection, error) } ) @@ -51,16 +50,10 @@ var ( //go:embed migrations/sql/*.sql migrations embed.FS - ////go:embed namespace_migrations/*.sql - //namespaceMigrations embed.FS - _ persistence.Persister = &Persister{} ) func NewPersister(reg dependencies, nid uuid.UUID) (*Persister, error) { - //pop.SetLogger(reg.Logger().PopLogger) - - pop.Debug = true conn, err := reg.PopConnection() if err != nil { return nil, err @@ -102,7 +95,7 @@ func (p *Persister) QueryWithNetwork(ctx context.Context) *pop.Query { return p.Connection(ctx).Where("nid = ?", p.NetworkID(ctx)) } -func (p *Persister) transaction(ctx context.Context, f func(ctx context.Context, c *pop.Connection) error) error { +func (p *Persister) Transaction(ctx context.Context, f func(ctx context.Context, c *pop.Connection) error) error { return popx.Transaction(ctx, p.conn.WithContext(ctx), f) } diff --git a/internal/persistence/sql/relationtuples.go b/internal/persistence/sql/relationtuples.go index 16044ed9c..608531671 100644 --- a/internal/persistence/sql/relationtuples.go +++ b/internal/persistence/sql/relationtuples.go @@ -112,7 +112,7 @@ func (r *RelationTuple) insertSubject(ctx context.Context, p *Persister, s relat return nil } -func (r *RelationTuple) fromInternal(ctx context.Context, p *Persister, rt *relationtuple.InternalRelationTuple) error { +func (r *RelationTuple) FromInternal(ctx context.Context, p *Persister, rt *relationtuple.InternalRelationTuple) error { n, err := p.GetNamespaceByName(ctx, rt.Namespace) if err != nil { return err @@ -125,7 +125,7 @@ func (r *RelationTuple) fromInternal(ctx context.Context, p *Persister, rt *rela return r.insertSubject(ctx, p, rt.Subject) } -func (p *Persister) insertRelationTuple(ctx context.Context, rel *relationtuple.InternalRelationTuple) error { +func (p *Persister) InsertRelationTuple(ctx context.Context, rel *relationtuple.InternalRelationTuple) error { if rel.Subject == nil { return errors.WithStack(relationtuple.ErrNilSubject) } @@ -136,7 +136,7 @@ func (p *Persister) insertRelationTuple(ctx context.Context, rel *relationtuple. ID: uuid.Must(uuid.NewV4()), CommitTime: time.Now(), } - if err := rt.fromInternal(ctx, p, rel); err != nil { + if err := rt.FromInternal(ctx, p, rel); err != nil { return err } @@ -176,7 +176,7 @@ func (p *Persister) whereSubject(ctx context.Context, q *pop.Query, sub relation } func (p *Persister) DeleteRelationTuples(ctx context.Context, rs ...*relationtuple.InternalRelationTuple) error { - return p.transaction(ctx, func(ctx context.Context, c *pop.Connection) error { + return p.Transaction(ctx, func(ctx context.Context, c *pop.Connection) error { for _, r := range rs { n, err := p.GetNamespaceByName(ctx, r.Namespace) if err != nil { @@ -258,9 +258,9 @@ func (p *Persister) GetRelationTuples(ctx context.Context, query *relationtuple. } func (p *Persister) WriteRelationTuples(ctx context.Context, rs ...*relationtuple.InternalRelationTuple) error { - return p.transaction(ctx, func(ctx context.Context, _ *pop.Connection) error { + return p.Transaction(ctx, func(ctx context.Context, _ *pop.Connection) error { for _, r := range rs { - if err := p.insertRelationTuple(ctx, r); err != nil { + if err := p.InsertRelationTuple(ctx, r); err != nil { return err } } @@ -269,7 +269,7 @@ func (p *Persister) WriteRelationTuples(ctx context.Context, rs ...*relationtupl } func (p *Persister) TransactRelationTuples(ctx context.Context, ins []*relationtuple.InternalRelationTuple, del []*relationtuple.InternalRelationTuple) error { - return p.transaction(ctx, func(ctx context.Context, _ *pop.Connection) error { + return p.Transaction(ctx, func(ctx context.Context, _ *pop.Connection) error { if err := p.WriteRelationTuples(ctx, ins...); err != nil { return err } diff --git a/scripts/single-table-migration-e2e.sh b/scripts/single-table-migration-e2e.sh new file mode 100755 index 000000000..5d2a8ffbd --- /dev/null +++ b/scripts/single-table-migration-e2e.sh @@ -0,0 +1,53 @@ +#!/bin/bash +set -euxo pipefail + +rm migrate_e2e.sqlite || true + +bash <(curl https://raw.githubusercontent.com/ory/keto/master/install.sh) -b . v0.6.0-alpha.3 + +export DSN="sqlite://./migrate_e2e.sqlite?_fk=true" +export KETO_READ_REMOTE="127.0.0.1:4466" +export KETO_WRITE_REMOTE="127.0.0.1:4467" + +config="$(mktemp --tmpdir keto.XXXXXX.yml)" +echo " +log: + level: debug + +namespaces: + - id: 0 + name: a + - id: 1 + name: b +" >> "$config" + +./keto migrate up -y -c "$config" --all-namespaces + +./keto serve all -c "$config" & +keto_server_pid=$! + +function teardown() { + kill $keto_server_pid || true +} +trap teardown EXIT + +jq '[range(300)] | map({namespace: (if . % 2 == 0 then "a" else "b" end), object: . | tostring, relation: "view", subject: "user"})' <(echo '{}') \ + | ./keto relation-tuple create -q - + +kill $keto_server_pid + +go build -tags sqlite -o keto_new . + +./keto_new migrate up -y -c "$config" +./keto_new namespace migrate legacy -y -c "$config" + +./keto_new serve all -c "$config" & +keto_server_pid=$! + +for i in {0..300..24} ; do + diff <(echo 'Allowed') <(./keto_new check user view a "$i") +done + +for i in {1..300..24} ; do + diff <(echo 'Allowed') <(./keto_new check user view b "$i") +done