feat: allow to apply namespace migrations together with regular migra…

…tions (#441)

cmd/migrate/down.go

@@ -27,11 +27,22 @@ func newDownCmd() *cobra.Command {
 			if err != nil {
 				return err
 			}
+
+			if err := reg.Migrator().MigrationStatus(ctx, cmd.OutOrStdout()); err != nil {
+				_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not get migration status: %+v\n", err)
+				return cmdx.FailSilently(cmd)
+			}
+
 			if err := reg.Migrator().MigrateDown(ctx, int(steps)); err != nil {
 				_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could apply down migrations: %+v\n", err)
 				return cmdx.FailSilently(cmd)
 			}
 
+			if err := reg.Migrator().MigrationStatus(ctx, cmd.OutOrStdout()); err != nil {
+				_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not get migration status: %+v\n", err)
+				return cmdx.FailSilently(cmd)
+			}
+
 			return nil
 		},
 	}

cmd/migrate/migrate_test.go

@@ -0,0 +1,150 @@
+package migrate
+
+import (
+	"bytes"
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/ory/x/cmdx"
+	"github.com/ory/x/configx"
+	"github.com/sirupsen/logrus/hooks/test"
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/ory/keto/internal/driver"
+	"github.com/ory/keto/internal/driver/config"
+	"github.com/ory/keto/internal/namespace"
+	"github.com/ory/keto/internal/x"
+)
+
+func assertAllApplied(t *testing.T, status string) {
+	assert.NotContains(t, status, "Pending")
+	assert.Contains(t, status, "Applied")
+}
+
+func assertNoneApplied(t *testing.T, status string) {
+	assert.Contains(t, status, "Pending")
+	assert.NotContains(t, status, "Applied")
+}
+
+func TestMigrate(t *testing.T) {
+	nspaces := []*namespace.Namespace{
+		{
+			Name: "default",
+			ID:   0,
+		},
+		{
+			Name: "other",
+			ID:   1,
+		},
+	}
+
+	newCmd := func(ctx context.Context, persistentArgs ...string) *cmdx.CommandExecuter {
+		return &cmdx.CommandExecuter{
+			New: func() *cobra.Command {
+				cmd := newMigrateCmd()
+				configx.RegisterFlags(cmd.PersistentFlags())
+				return cmd
+			},
+			Ctx:            ctx,
+			PersistentArgs: persistentArgs,
+		}
+	}
+
+	for _, dsn := range x.GetDSNs(t) {
+		if dsn.Name == "memory" {
+			t.Run("dsn=memory", func(t *testing.T) {
+				t.Run("case=auto migrates", func(t *testing.T) {
+					hook := &test.Hook{}
+					ctx := context.WithValue(context.Background(), driver.LogrusHookContextKey, hook)
+
+					cf := x.ConfigFile(t, map[string]interface{}{
+						config.KeyDSN:        dsn.Conn,
+						config.KeyNamespaces: nspaces,
+						"log.level":          "debug",
+					})
+
+					cmd := newCmd(ctx, "-c", cf)
+
+					out := cmd.ExecNoErr(t, "up", "--"+FlagYes)
+					assert.Contains(t, out, "All migrations are already applied, there is nothing to do.")
+				})
+			})
+		} else {
+			t.Run("dsn="+dsn.Name, func(t *testing.T) {
+				hook := &test.Hook{}
+				ctx := context.WithValue(context.Background(), driver.LogrusHookContextKey, hook)
+
+				cf := x.ConfigFile(t, map[string]interface{}{
+					config.KeyDSN:        dsn.Conn,
+					config.KeyNamespaces: nspaces,
+					"log.level":          "debug",
+				})
+
+				cmd := newCmd(ctx, "-c", cf)
+
+				t.Run("case=aborts on no", func(t *testing.T) {
+					stdOut, stdErr, err := cmd.Exec(bytes.NewBufferString("n\n"), "up")
+					require.NoError(t, err, "%s %s", stdOut, stdErr)
+
+					assert.Contains(t, stdOut, "Pending", "%s %s", stdOut, stdErr)
+					assert.NotContains(t, stdOut, "Applied", "%s %s", stdOut, stdErr)
+					assert.Contains(t, stdOut, "Aborting", "%s %s", stdOut, stdErr)
+				})
+
+				t.Run("case=applies on yes input", func(t *testing.T) {
+					stdOut, stdErr, err := cmd.Exec(bytes.NewBufferString("y\n"), "up")
+					require.NoError(t, err, "%s %s", stdOut, stdErr)
+
+					t.Cleanup(func() {
+						// migrate all down
+						cmd.ExecNoErr(t, "down", "0")
+					})
+
+					parts := strings.Split(stdOut, "Do you want to apply above planned migrations?")
+					require.Len(t, parts, 2)
+
+					assertNoneApplied(t, parts[0])
+					assertAllApplied(t, parts[1])
+				})
+
+				t.Run("case=applies on yes flag", func(t *testing.T) {
+					out := cmd.ExecNoErr(t, "up", "--"+FlagYes)
+
+					t.Cleanup(func() {
+						// migrate all down
+						cmd.ExecNoErr(t, "down", "0")
+					})
+
+					parts := strings.Split(out, "Applying migrations...")
+					require.Len(t, parts, 2)
+
+					assertNoneApplied(t, parts[0])
+					assertAllApplied(t, parts[1])
+				})
+
+				t.Run("case=applies namespace migrations on flag", func(t *testing.T) {
+					out := cmd.ExecNoErr(t, "up", "--"+FlagYes, "--"+FlagAllNamespace)
+
+					t.Cleanup(func() {
+						// migrate all down
+						cmd.ExecNoErr(t, "down", "0")
+					})
+
+					parts := strings.Split(out, "Applying migrations...")
+					require.Len(t, parts, 2)
+					innerParts := strings.Split(parts[1], "Going to migrate namespaces")
+					require.Len(t, innerParts, 2)
+
+					assertNoneApplied(t, parts[0])
+					assertAllApplied(t, innerParts[0])
+
+					assert.Contains(t, innerParts[1], "Successfully migrated namespace "+nspaces[0].Name)
+					assert.Contains(t, innerParts[1], "Successfully migrated namespace "+nspaces[1].Name)
+				})
+			})
+		}
+	}
+}

cmd/migrate/root.go

@@ -3,19 +3,17 @@ package migrate
 import "github.com/spf13/cobra"
 
 func newMigrateCmd() *cobra.Command {
-	return &cobra.Command{
+	cmd := &cobra.Command{
 		Use: "migrate",
 	}
-}
-
-func RegisterCommandsRecursive(parent *cobra.Command) {
-	migrateCmd := newMigrateCmd()
-
-	migrateCmd.AddCommand(
+	cmd.AddCommand(
 		newStatusCmd(),
 		newUpCmd(),
 		newDownCmd(),
 	)
+	return cmd
+}
 
-	parent.AddCommand(migrateCmd)
+func RegisterCommandsRecursive(parent *cobra.Command) {
+	parent.AddCommand(newMigrateCmd())
 }

cmd/migrate/up.go

@@ -1,18 +1,26 @@
 package migrate
 
 import (
+	"bytes"
 	"fmt"
+	"strings"
+
+	"github.com/pkg/errors"
 
 	"github.com/ory/x/cmdx"
-	"github.com/ory/x/flagx"
 	"github.com/spf13/cobra"
 
 	"github.com/ory/keto/internal/driver"
 )
 
-const FlagYes = "yes"
+const (
+	FlagYes          = "yes"
+	FlagAllNamespace = "all-namespaces"
+)
 
 func newUpCmd() *cobra.Command {
+	var yes, allNamespaces bool
+
 	cmd := &cobra.Command{
 		Use: "up",
 		RunE: func(cmd *cobra.Command, _ []string) error {
@@ -22,25 +30,90 @@ func newUpCmd() *cobra.Command {
 			if err != nil {
 				return err
 			}
-			if err := reg.Migrator().MigrationStatus(ctx, cmd.OutOrStdout()); err != nil {
-				_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Could not get migration status: %+v\n", err)
+
+			_, _ = fmt.Fprintln(cmd.OutOrStdout(), "Current status:")
+
+			status := &bytes.Buffer{}
+			if err := reg.Migrator().MigrationStatus(ctx, status); err != nil {
+				_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not get migration status: %+v\n", err)
 				return cmdx.FailSilently(cmd)
 			}
+			_, _ = cmd.OutOrStdout().Write(status.Bytes())
+
+			if !strings.Contains(status.String(), "Pending") {
+				_, _ = fmt.Fprintln(cmd.OutOrStdout(), "All migrations are already applied, there is nothing to do.")
+				return nil
+			}
 
-			if !flagx.MustGetBool(cmd, FlagYes) && !cmdx.AskForConfirmation("Do you want to apply above planned migrations?", cmd.InOrStdin(), cmd.OutOrStdout()) {
+			if !yes && !cmdx.AskForConfirmation("Do you want to apply above planned migrations?", cmd.InOrStdin(), cmd.OutOrStdout()) {
 				_, _ = fmt.Fprintln(cmd.OutOrStdout(), "Aborting")
 				return nil
 			}
 
+			_, _ = fmt.Fprintln(cmd.OutOrStdout(), "Applying migrations...")
+
 			if err := reg.Migrator().MigrateUp(ctx); err != nil {
-				_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Could not apply migrations: %+v\n", err)
+				_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not apply migrations: %+v\n", err)
+				return cmdx.FailSilently(cmd)
+			}
+
+			_, _ = fmt.Fprintln(cmd.OutOrStdout(), "Successfully applied all migrations:")
+
+			if err := reg.Migrator().MigrationStatus(ctx, cmd.OutOrStdout()); err != nil {
+				_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not get migration status: %+v\n", err)
 				return cmdx.FailSilently(cmd)
 			}
+
+			if !allNamespaces {
+				// everything is done already
+				return nil
+			}
+
+			_, _ = fmt.Fprintln(cmd.OutOrStdout(), "\nGoing to migrate namespaces.")
+
+			nm, err := reg.Config().NamespaceManager()
+			if err != nil {
+				return errors.Wrap(err, "could not get the namespace manager")
+			}
+
+			nspaces, err := nm.Namespaces(cmd.Context())
+			if err != nil {
+				_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not get namespaces: %+v\n", err)
+				return cmdx.FailSilently(cmd)
+			}
+
+			for _, nspace := range nspaces {
+				status := &bytes.Buffer{}
+				if err := reg.NamespaceMigrator().NamespaceStatus(cmd.Context(), status, nspace); err != nil {
+					_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not get migration status for namespace %s: %+v\n", nspace.Name, err)
+					return cmdx.FailSilently(cmd)
+				}
+				_, _ = cmd.OutOrStdout().Write(status.Bytes())
+
+				if !strings.Contains(status.String(), "Pending") {
+					_, _ = fmt.Fprintf(cmd.OutOrStdout(), "All migrations are already applied for namespace %s, there is nothing to do.\n", nspace.Name)
+					continue
+				}
+
+				if !yes && !cmdx.AskForConfirmation(fmt.Sprintf("Do you want to apply above planned migrations for namespace %s?", nspace.Name), cmd.InOrStdin(), cmd.OutOrStdout()) {
+					_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Skipping namespace %s\n", nspace.Name)
+					continue
+				}
+
+				if err := reg.NamespaceMigrator().MigrateNamespaceUp(cmd.Context(), nspace); err != nil {
+					_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not apply namespace migrations for namespace %s: %+v\n", nspace.Name, err)
+					return cmdx.FailSilently(cmd)
+				}
+
+				_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Successfully migrated namespace %s\n.", nspace.Name)
+			}
+
 			return nil
 		},
 	}
 
-	cmd.Flags().BoolP(FlagYes, "y", false, "yes to all questions, no user input required")
+	cmd.Flags().BoolVarP(&yes, FlagYes, "y", false, "yes to all questions, no user input required")
+	cmd.Flags().BoolVar(&allNamespaces, FlagAllNamespace, false, "migrate all pending namespaces as well")
 
 	return cmd
 }

internal/e2e/helpers.go

@@ -3,8 +3,6 @@ package e2e
 import (
 	"bytes"
 	"context"
-	"io/ioutil"
-	"path/filepath"
 	"strings"
 	"testing"
 
@@ -22,27 +20,10 @@ import (
 
 	"github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/require"
-	"github.com/tidwall/sjson"
 
 	"github.com/ory/keto/internal/driver"
 )
 
-func configFile(t testing.TB, values map[string]interface{}) string {
-	dir := t.TempDir()
-	fn := filepath.Join(dir, "keto.yml")
-
-	c := []byte("{}")
-	for key, val := range values {
-		var err error
-		c, err = sjson.SetBytes(c, key, val)
-		require.NoError(t, err)
-	}
-
-	require.NoError(t, ioutil.WriteFile(fn, c, 0600))
-
-	return fn
-}
-
 func setup(t testing.TB) (*test.Hook, context.Context) {
 	hook := &test.Hook{}
 	ctx, cancel := context.WithCancel(context.WithValue(context.Background(), driver.LogrusHookContextKey, hook))
@@ -65,7 +46,7 @@ func newInitializedReg(t testing.TB, dsn *x.DsnT, nspaces []*namespace.Namespace
 	configx.RegisterConfigFlag(flags, nil)
 
 	require.NoError(t, flags.Parse(
-		[]string{"--" + configx.FlagConfig, configFile(t, map[string]interface{}{
+		[]string{"--" + configx.FlagConfig, x.ConfigFile(t, map[string]interface{}{
 			config.KeyDSN:          dsn.Conn,
 			config.KeyNamespaces:   nspaces,
 			"log.level":            "debug",