refactor: persistence table structure (#638)

This big refactoring greatly reduces operation complexity and paves the way for upcoming performance improvements.
From now on the relation tuples from all namespaces are stored in the same table, instead of having tables per namespace. A migration path will be provided separately.

.bin/go.mod

@@ -4,8 +4,6 @@ go 1.16
 
 replace github.com/goreleaser/nfpm => github.com/goreleaser/nfpm v1.10.2
 
-replace github.com/ory/kratos-client-go => github.com/ory/kratos-client-go v0.5.4-alpha.1.0.20210210170256-960b093d8bf9
-
 replace github.com/ory/kratos/corp => github.com/ory/kratos/corp v0.0.0-20210118092700-c2358be1e867
 
 replace github.com/oleiade/reflections => github.com/oleiade/reflections v1.0.1
@@ -15,7 +13,6 @@ require (
 	github.com/go-swagger/go-swagger v0.26.1
 	github.com/goreleaser/godownloader v0.1.1-0.20200426152203-fd8ad8f7dd78
 	github.com/mattn/goveralls v0.0.7
-	github.com/ory/cli v0.0.49
 	github.com/ory/go-acc v0.2.6
 	github.com/pseudomuto/protoc-gen-doc v1.4.1
 	golang.org/x/tools v0.1.0

.bin/indirect_pins.go

@@ -7,7 +7,6 @@ import (
 	_ "github.com/mattn/goveralls"
 	_ "golang.org/x/tools/cmd/goimports"
 
-	_ "github.com/ory/cli"
 	_ "github.com/ory/go-acc"
 
 	// Protobuf and gRPC related tools

.gitignore

@@ -2,3 +2,4 @@ node_modules/
 .idea/
 coverage.txt
 dist/
+**/*.sqlite

Makefile

@@ -6,7 +6,6 @@ export PWD := $(shell pwd)
 GO_DEPENDENCIES = github.com/go-swagger/go-swagger/cmd/swagger \
 				  golang.org/x/tools/cmd/goimports \
 				  github.com/mattn/goveralls \
-				  github.com/ory/cli \
 				  github.com/ory/go-acc \
 				  github.com/bufbuild/buf/cmd/buf \
 				  google.golang.org/protobuf/cmd/protoc-gen-go \
@@ -16,12 +15,16 @@ GO_DEPENDENCIES = github.com/go-swagger/go-swagger/cmd/swagger \
 
 define make-go-dependency
   # go install is responsible for not re-building when the code hasn't changed
-  .bin/$(notdir $1): go.mod go.sum Makefile
+  .bin/$(notdir $1): .bin/go.mod .bin/go.sum Makefile
 		cd .bin; GOBIN=$(PWD)/.bin/ go install $1
 endef
 $(foreach dep, $(GO_DEPENDENCIES), $(eval $(call make-go-dependency, $(dep))))
 $(call make-lint-dependency)
 
+.bin/ory: Makefile
+		bash <(curl https://raw.githubusercontent.com/ory/cli/master/install.sh) -b .bin v0.0.57
+		touch -a -m .bin/ory
+
 node_modules: package.json package-lock.json Makefile
 		npm ci
 
@@ -95,3 +98,11 @@ test-docs-samples:
 		npm i \
 		&& \
 		npm test
+
+.PHONY: migrations-render
+migrations-render: .bin/ory
+		ory dev pop migration render internal/persistence/sql/migrations/templates internal/persistence/sql/migrations/sql
+
+.PHONY: migrations-render-replace
+migrations-render-replace: .bin/ory
+		ory dev pop migration render -r internal/persistence/sql/migrations/templates internal/persistence/sql/migrations/sql

cmd/client/test_helpers.go

@@ -4,6 +4,8 @@ import (
 	"net"
 	"testing"
 
+	"github.com/ory/keto/internal/driver/config"
+
 	"github.com/ory/x/cmdx"
 	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/require"
@@ -35,8 +37,9 @@ const (
 
 func NewTestServer(t *testing.T, rw ServerType, nspaces []*namespace.Namespace, newCmd func() *cobra.Command) *TestServer {
 	ts := &TestServer{
-		Reg: driver.NewMemoryTestRegistry(t, nspaces),
+		Reg: driver.NewSqliteTestRegistry(t, false),
 	}
+	require.NoError(t, ts.Reg.Config().Set(config.KeyNamespaces, nspaces))
 
 	switch rw {
 	case ReadServer:

cmd/migrate/down.go

@@ -29,12 +29,12 @@ func newDownCmd() *cobra.Command {
 				return fmt.Errorf("malformed argument %s for <steps>: %+v", args[0], err)
 			}
 
-			reg, err := driver.NewDefaultRegistry(cmd.Context(), cmd.Flags())
+			reg, err := driver.NewDefaultRegistry(cmd.Context(), cmd.Flags(), true)
 			if err != nil {
 				return err
 			}
 
-			mb, err := reg.Migrator().MigrationBox(cmd.Context())
+			mb, err := reg.MigrationBox()
 			if err != nil {
 				return err
 			}

cmd/migrate/migrate_test.go

@@ -3,10 +3,13 @@ package migrate
 import (
 	"bytes"
 	"context"
-	"regexp"
 	"strings"
 	"testing"
 
+	"github.com/ory/x/dbal"
+
+	"github.com/ory/keto/internal/x/dbx"
+
 	"github.com/ory/x/cmdx"
 	"github.com/ory/x/configx"
 	"github.com/sirupsen/logrus/hooks/test"
@@ -17,7 +20,6 @@ import (
 	"github.com/ory/keto/internal/driver"
 	"github.com/ory/keto/internal/driver/config"
 	"github.com/ory/keto/internal/namespace"
-	"github.com/ory/keto/internal/x"
 )
 
 func assertAllApplied(t *testing.T, status string) {
@@ -54,14 +56,14 @@ func TestMigrate(t *testing.T) {
 		}
 	}
 
-	for _, dsn := range x.GetDSNs(t) {
-		if dsn.Name == "memory" {
+	for _, dsn := range dbx.GetDSNs(t, false) {
+		if dbal.IsMemorySQLite(dsn.Conn) {
 			t.Run("dsn=memory", func(t *testing.T) {
 				t.Run("case=auto migrates", func(t *testing.T) {
 					hook := &test.Hook{}
 					ctx := context.WithValue(context.Background(), driver.LogrusHookContextKey, hook)
 
-					cf := x.ConfigFile(t, map[string]interface{}{
+					cf := dbx.ConfigFile(t, map[string]interface{}{
 						config.KeyDSN:        dsn.Conn,
 						config.KeyNamespaces: nspaces,
 						"log.level":          "debug",
@@ -78,7 +80,7 @@ func TestMigrate(t *testing.T) {
 				hook := &test.Hook{}
 				ctx := context.WithValue(context.Background(), driver.LogrusHookContextKey, hook)
 
-				cf := x.ConfigFile(t, map[string]interface{}{
+				cf := dbx.ConfigFile(t, map[string]interface{}{
 					config.KeyDSN:        dsn.Conn,
 					config.KeyNamespaces: nspaces,
 					"log.level":          "debug",
@@ -125,25 +127,6 @@ func TestMigrate(t *testing.T) {
 					assertNoneApplied(t, parts[0])
 					assertAllApplied(t, parts[1])
 				})
-
-				t.Run("case=applies namespace migrations on flag", func(t *testing.T) {
-					out := cmd.ExecNoErr(t, "up", "--"+FlagYes, "--"+FlagAllNamespace)
-
-					t.Cleanup(func() {
-						// migrate all down
-						t.Log(cmd.ExecNoErr(t, "down", "0", "--"+FlagYes))
-					})
-
-					parts := regexp.MustCompile("(?s)Current status:(.*)Successfully applied all migrations(.*)Going to migrate namespaces(.*)Successfully applied all migrations(.*)Current status(.*)Successfully applied all migrations(.*)").FindStringSubmatch(out)
-					require.Len(t, parts, 7)
-
-					assertNoneApplied(t, parts[1])
-					assertAllApplied(t, parts[2])
-					assertNoneApplied(t, parts[3])
-					assertAllApplied(t, parts[4])
-					assertNoneApplied(t, parts[5])
-					assertAllApplied(t, parts[6])
-				})
 			})
 		}
 	}

cmd/migrate/status.go

@@ -20,12 +20,12 @@ func newStatusCmd() *cobra.Command {
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			ctx := cmd.Context()
 
-			reg, err := driver.NewDefaultRegistry(ctx, cmd.Flags())
+			reg, err := driver.NewDefaultRegistry(ctx, cmd.Flags(), true)
 			if err != nil {
 				return err
 			}
 
-			mb, err := reg.Migrator().MigrationBox(ctx)
+			mb, err := reg.MigrationBox()
 			if err != nil {
 				return err
 			}

cmd/migrate/up.go

@@ -9,8 +9,6 @@ import (
 
 	"github.com/spf13/pflag"
 
-	"github.com/pkg/errors"
-
 	"github.com/ory/x/cmdx"
 	"github.com/spf13/cobra"
 
@@ -34,12 +32,12 @@ func newUpCmd() *cobra.Command {
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			ctx := cmd.Context()
 
-			reg, err := driver.NewDefaultRegistry(ctx, cmd.Flags())
+			reg, err := driver.NewDefaultRegistry(ctx, cmd.Flags(), true)
 			if err != nil {
 				return err
 			}
 
-			mb, err := reg.Migrator().MigrationBox(ctx)
+			mb, err := reg.MigrationBox()
 			if err != nil {
 				return err
 			}
@@ -48,35 +46,6 @@ func newUpCmd() *cobra.Command {
 				return err
 			}
 
-			if !allNamespaces {
-				// everything is done already
-				return nil
-			}
-
-			_, _ = fmt.Fprintln(cmd.OutOrStdout(), "\nGoing to migrate namespaces.")
-
-			nm, err := reg.Config().NamespaceManager()
-			if err != nil {
-				return errors.Wrap(err, "could not get the namespace manager")
-			}
-
-			nspaces, err := nm.Namespaces(cmd.Context())
-			if err != nil {
-				_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not get namespaces: %+v\n", err)
-				return cmdx.FailSilently(cmd)
-			}
-
-			for _, nspace := range nspaces {
-				mb, err := reg.NamespaceMigrator().NamespaceMigrationBox(ctx, nspace)
-				if err != nil {
-					return err
-				}
-
-				if err := BoxUp(cmd, mb, "[namespace="+nspace.Name+"] "); err != nil {
-					return err
-				}
-			}
-
 			return nil
 		},
 	}

cmd/namespace/migrate_down.go

@@ -1,56 +1,21 @@
 package namespace
 
 import (
-	"fmt"
-	"strconv"
-
 	"github.com/ory/keto/cmd/migrate"
 
-	"github.com/ory/x/cmdx"
 	"github.com/spf13/cobra"
-
-	"github.com/ory/keto/internal/driver"
 )
 
 func NewMigrateDownCmd() *cobra.Command {
 	cmd := &cobra.Command{
-		Use:   "down <namespace-name> <steps>",
-		Short: "Migrate a namespace down",
+		Deprecated: "This step is not necessary anymore, see https://github.com/ory/keto/pull/638",
+		Use:        "down <namespace-name> <steps>",
+		Short:      "Migrate a namespace down",
 		Long: "Migrate a namespace down.\n" +
 			"Pass 0 steps to fully migrate down.",
 		Args: cobra.ExactArgs(2),
-		RunE: func(cmd *cobra.Command, args []string) error {
-			ctx := cmd.Context()
-
-			steps, err := strconv.ParseInt(args[1], 0, 0)
-			if err != nil {
-				// return this error so it gets printed along the usage
-				return fmt.Errorf("malformed argument %s for <steps>: %+v", args[0], err)
-			}
-
-			reg, err := driver.NewDefaultRegistry(ctx, cmd.Flags())
-			if err != nil {
-				return err
-			}
-
-			nm, err := reg.Config().NamespaceManager()
-			if err != nil {
-				_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Could not initialize the namespace manager: %+v\n", err)
-				return cmdx.FailSilently(cmd)
-			}
-
-			n, err := nm.GetNamespace(ctx, args[0])
-			if err != nil {
-				_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Could not find the namespace with name \"%s\": %+v\n", args[0], err)
-				return cmdx.FailSilently(cmd)
-			}
-
-			mb, err := reg.NamespaceMigrator().NamespaceMigrationBox(ctx, n)
-			if err != nil {
-				return err
-			}
-
-			return migrate.BoxDown(cmd, mb, int(steps), "[namespace="+n.Name+"] ")
+		RunE: func(_ *cobra.Command, _ []string) error {
+			return nil
 		},
 	}
 

cmd/namespace/migrate_status.go

@@ -1,45 +1,19 @@
 package namespace
 
 import (
-	"fmt"
-
 	"github.com/ory/x/cmdx"
 	"github.com/spf13/cobra"
-
-	"github.com/ory/keto/cmd/migrate"
-	"github.com/ory/keto/internal/driver"
 )
 
 func NewMigrateStatusCmd() *cobra.Command {
 	cmd := &cobra.Command{
-		Use:   "status <namespace-name>",
-		Short: "Get the current namespace migration status",
-		Long:  "Get the current migration status of one specific namespace.\nDoes not apply any changes.",
-		Args:  cobra.ExactArgs(1),
-		RunE: func(cmd *cobra.Command, args []string) error {
-			reg, err := driver.NewDefaultRegistry(cmd.Context(), cmd.Flags())
-			if err != nil {
-				return err
-			}
-
-			nm, err := reg.Config().NamespaceManager()
-			if err != nil {
-				_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Could not initialize the namespace manager: %+v\n", err)
-				return cmdx.FailSilently(cmd)
-			}
-
-			n, err := nm.GetNamespace(cmd.Context(), args[0])
-			if err != nil {
-				_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Could not find the namespace with name \"%s\": %+v\n", args[0], err)
-				return cmdx.FailSilently(cmd)
-			}
-
-			mb, err := reg.NamespaceMigrator().NamespaceMigrationBox(cmd.Context(), n)
-			if err != nil {
-				return err
-			}
-
-			return migrate.BoxStatus(cmd, mb, "[namespace="+n.Name+"] ")
+		Deprecated: "This step is not necessary anymore, see https://github.com/ory/keto/pull/638",
+		Use:        "status <namespace-name>",
+		Short:      "Get the current namespace migration status",
+		Long:       "Get the current migration status of one specific namespace.\nDoes not apply any changes.",
+		Args:       cobra.ExactArgs(1),
+		RunE: func(_ *cobra.Command, _ []string) error {
+			return nil
 		},
 	}