feat: Add mapping table for encoding subject into UUIDs

cmd/migrate/migrate_uuid_mapping.go

@@ -0,0 +1,49 @@
+package migrate
+
+import (
+	"fmt"
+
+	"github.com/ory/x/cmdx"
+	"github.com/ory/x/flagx"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+
+	"github.com/ory/keto/internal/driver"
+	"github.com/ory/keto/internal/persistence"
+	"github.com/ory/keto/internal/persistence/sql/migrations"
+)
+
+func newMigrateUUIDMappingCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "uuid-mapping",
+		Short: "Migrate the non-UUID subject and object names to UUIDs.",
+		Long: `Migrate the non-UUID subject and object names to UUIDs.
+This step only has to be executed once.
+Please ensure that you have a backup in case something goes wrong!"`,
+		Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			reg, err := driver.NewDefaultRegistry(cmd.Context(), cmd.Flags(), false)
+			if errors.Is(err, persistence.ErrNetworkMigrationsMissing) {
+				_, _ = fmt.Fprintln(cmd.ErrOrStderr(),
+					"Migrations were not applied yet, please apply them first using `keto migrate up`.")
+				return cmdx.FailSilently(cmd)
+			} else if err != nil {
+				return err
+			}
+
+			if !flagx.MustGetBool(cmd, FlagYes) &&
+				!cmdx.AskForConfirmation(
+					"Are you sure you want to migrate the subject and object names to UUIDs?",
+					cmd.InOrStdin(), cmd.OutOrStdout()) {
+				_, _ = fmt.Fprintln(cmd.OutOrStdout(), "OK, aborting.")
+				return nil
+			}
+
+			migrator := migrations.NewToUUIDMappingMigrator(reg)
+			return migrator.MigrateUUIDMappings(cmd.Context())
+		},
+	}
+	RegisterYesFlag(cmd.Flags())
+
+	return cmd
+}

cmd/migrate/root.go

@@ -13,6 +13,7 @@ func newMigrateCmd() *cobra.Command {
 		newStatusCmd(),
 		newUpCmd(),
 		newDownCmd(),
+		newMigrateUUIDMappingCmd(),
 	)
 	return cmd
 }

internal/driver/registry.go

@@ -7,6 +7,7 @@ import (
 	"github.com/gobuffalo/pop/v6"
 
 	"github.com/ory/keto/internal/driver/config"
+	"github.com/ory/keto/internal/uuidmapping"
 
 	"github.com/spf13/cobra"
 
@@ -34,6 +35,7 @@ type (
 		x.WriterProvider
 
 		relationtuple.ManagerProvider
+		uuidmapping.ManagerProvider
 		expand.EngineProvider
 		check.EngineProvider
 		persistence.Migrator

internal/driver/registry_default.go

@@ -31,6 +31,7 @@ import (
 	grpcHealthV1 "google.golang.org/grpc/health/grpc_health_v1"
 
 	"github.com/ory/keto/internal/driver/config"
+	"github.com/ory/keto/internal/uuidmapping"
 
 	"github.com/ory/herodot"
 	"github.com/ory/x/healthx"
@@ -149,6 +150,13 @@ func (r *RegistryDefault) RelationTupleManager() relationtuple.Manager {
 	return r.p
 }
 
+func (r *RegistryDefault) UUIDMappingManager() uuidmapping.Manager {
+	if r.p == nil {
+		panic("no relation tuple manager, but expected to have one")
+	}
+	return r.p
+}
+
 func (r *RegistryDefault) Persister() persistence.Persister {
 	if r.p == nil {
 		panic("no persister, but expected to have one")

internal/persistence/definitions.go

@@ -9,11 +9,13 @@ import (
 	"github.com/gobuffalo/pop/v6"
 
 	"github.com/ory/keto/internal/relationtuple"
+	"github.com/ory/keto/internal/uuidmapping"
 )
 
 type (
 	Persister interface {
 		relationtuple.Manager
+		uuidmapping.Manager
 
 		Connection(ctx context.Context) *pop.Connection
 	}

internal/persistence/sql/full_test.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/ory/keto/internal/uuidmapping"
+
 	"github.com/gofrs/uuid"
 
 	"github.com/ory/keto/internal/x/dbx"
@@ -66,6 +68,11 @@ func TestPersister(t *testing.T) {
 				// same registry, but different persisters only differing in the network ID
 				relationtuple.IsolationTest(t, p0, p1, addNamespace(r, nspaces))
 			})
+
+			t.Run("uuidmapping.ManagerTest", func(t *testing.T) {
+				p, _, _ := setup(t, dsn)
+				uuidmapping.ManagerTest(t, p)
+			})
 		})
 	}
 }

internal/persistence/sql/migrations/single_table_test.go

@@ -226,7 +226,7 @@ func TestToSingleTableMigrator_HasLegacyTable(t *testing.T) {
 
 				ln, err := m.LegacyNamespaces(ctx)
 				require.NoError(t, err)
-				assert.Equal(t, nspaces, ln)
+				assert.ElementsMatch(t, nspaces, ln)
 			})
 		})
 	}

internal/persistence/sql/migrations/sql/20220110200400000000_create-uuid-mapping-table.cockroach.up.sql

@@ -0,0 +1,7 @@
+CREATE TABLE keto_uuid_mappings
+(
+    id                       UUID NOT NULL,
+    string_representation    TEXT NOT NULL CHECK (string_representation <> ''),
+
+    PRIMARY KEY (id)
+);

internal/persistence/sql/migrations/sql/20220110200400000000_create-uuid-mapping-table.mysql.up.sql

@@ -0,0 +1,7 @@
+CREATE TABLE keto_uuid_mappings
+(
+    id                       VARCHAR(64) NOT NULL,
+    string_representation    TEXT        NOT NULL CHECK (string_representation <> ''),
+
+    PRIMARY KEY (id)
+);

internal/persistence/sql/migrations/sql/20220110200400000000_create-uuid-mapping-table.postgres.up.sql

@@ -0,0 +1,7 @@
+CREATE TABLE keto_uuid_mappings
+(
+    id                       UUID NOT NULL,
+    string_representation    TEXT NOT NULL CHECK (string_representation <> ''),
+
+    PRIMARY KEY (id)
+);

internal/persistence/sql/migrations/sql/20220110200400000000_create-uuid-mapping-table.sqlite3.up.sql

@@ -0,0 +1,7 @@
+CREATE TABLE keto_uuid_mappings
+(
+    id                       UUID NOT NULL,
+    string_representation    TEXT NOT NULL CHECK (string_representation <> ''),
+
+    PRIMARY KEY (id)
+);

internal/persistence/sql/migrations/sql/20220110200400000001_create-uuid-mapping-table.cockroach.down.sql

@@ -0,0 +1 @@
+DROP TABLE keto_uuid_mappings;

internal/persistence/sql/migrations/sql/20220110200400000001_create-uuid-mapping-table.mysql.down.sql

@@ -0,0 +1 @@
+DROP TABLE keto_uuid_mappings;

internal/persistence/sql/migrations/sql/20220110200400000001_create-uuid-mapping-table.postgres.down.sql

@@ -0,0 +1 @@
+DROP TABLE keto_uuid_mappings;

internal/persistence/sql/migrations/sql/20220110200400000001_create-uuid-mapping-table.sqlite3.down.sql

@@ -0,0 +1 @@
+DROP TABLE keto_uuid_mappings;

internal/persistence/sql/migrations/templates/20220110200400_create-uuid-mapping-table.down.sql

@@ -0,0 +1 @@
+DROP TABLE keto_uuid_mappings;

internal/persistence/sql/migrations/templates/20220110200400_create-uuid-mapping-table.mysql.up.sql

@@ -0,0 +1,7 @@
+CREATE TABLE keto_uuid_mappings
+(
+    id                       VARCHAR(64) NOT NULL,
+    string_representation    TEXT        NOT NULL CHECK (string_representation <> ''),
+
+    PRIMARY KEY (id)
+);

internal/persistence/sql/migrations/templates/20220110200400_create-uuid-mapping-table.up.sql

@@ -0,0 +1,7 @@
+CREATE TABLE keto_uuid_mappings
+(
+    id                       UUID NOT NULL,
+    string_representation    TEXT NOT NULL CHECK (string_representation <> ''),
+
+    PRIMARY KEY (id)
+);

internal/persistence/sql/migrations/uuid_mapping.go

@@ -0,0 +1,111 @@
+package migrations
+
+import (
+	"context"
+
+	"github.com/gobuffalo/pop/v6"
+	"github.com/gofrs/uuid"
+	"github.com/ory/x/sqlcon"
+
+	"github.com/ory/keto/internal/persistence/sql"
+)
+
+type (
+	toUUIDMappingMigrator struct {
+		d       dependencies
+		perPage int
+	}
+)
+
+// NewToUUIDMappingMigrator creates a new UUID mapping migrator.
+func NewToUUIDMappingMigrator(d dependencies) *toUUIDMappingMigrator {
+	return &toUUIDMappingMigrator{d: d, perPage: 100}
+}
+
+// MigrateUUIDMappings migrates to UUID-mapped subject IDs for all relation
+// tuples in the database.
+func (m *toUUIDMappingMigrator) MigrateUUIDMappings(ctx context.Context) error {
+	p, ok := m.d.Persister().(*sql.Persister)
+	if !ok {
+		panic("got unexpected persister")
+	}
+
+	return p.Transaction(ctx, func(ctx context.Context, c *pop.Connection) error {
+		for page := 1; ; page++ {
+			relationTuples, hasNext, err := m.getRelationTuples(ctx, page)
+			if err != nil {
+				return err
+			}
+
+			if err := p.Connection(ctx).All(&relationTuples); err != nil {
+				return err
+			}
+
+			for _, rt := range relationTuples {
+				if err := m.migrateSubjectID(ctx, rt); err != nil {
+					return err
+				}
+				if err := m.migrateSubjectSetObject(ctx, rt); err != nil {
+					return err
+				}
+				if err := m.migrateObject(ctx, rt); err != nil {
+					return err
+				}
+				if err := c.Update(rt); err != nil {
+					return err
+				}
+			}
+
+			if !hasNext {
+				break
+			}
+		}
+		return nil
+	})
+}
+
+func (m *toUUIDMappingMigrator) migrateSubjectID(ctx context.Context, rt *sql.RelationTuple) error {
+	_, err := uuid.FromString(rt.SubjectID.String)
+	if err == nil || !rt.SubjectID.Valid || rt.SubjectID.String == "" {
+		return nil
+	}
+
+	rt.SubjectID.String, err = m.addUUIDMapping(ctx, rt.SubjectID.String)
+	return err
+}
+
+func (m *toUUIDMappingMigrator) migrateSubjectSetObject(ctx context.Context, rt *sql.RelationTuple) error {
+	_, err := uuid.FromString(rt.SubjectSetObject.String)
+	if err == nil || !rt.SubjectSetObject.Valid || rt.SubjectSetObject.String == "" {
+		return nil
+	}
+
+	rt.SubjectSetObject.String, err = m.addUUIDMapping(ctx, rt.SubjectSetObject.String)
+	return err
+}
+
+func (m *toUUIDMappingMigrator) migrateObject(ctx context.Context, rt *sql.RelationTuple) error {
+	_, err := uuid.FromString(rt.Object)
+	if err == nil || rt.Object == "" {
+		return nil
+	}
+
+	rt.Object, err = m.addUUIDMapping(ctx, rt.Object)
+	return err
+}
+
+func (m *toUUIDMappingMigrator) addUUIDMapping(ctx context.Context, value string) (id string, err error) {
+	uid, err := m.d.Persister().ToUUID(ctx, value)
+	return uid.String(), err
+}
+
+func (m *toUUIDMappingMigrator) getRelationTuples(ctx context.Context, page int) (res []*sql.RelationTuple, hasNext bool, err error) {
+	q := m.d.Persister().Connection(ctx).
+		Order("nid, shard_id").
+		Paginate(page, m.perPage)
+
+	if err := q.All(&res); err != nil {
+		return nil, false, sqlcon.HandleError(err)
+	}
+	return res, q.Paginator.Page < q.Paginator.TotalPages, nil
+}