feat: map subject/object strings to UUIDs (#792)

In relation tuples and related API objects such as queries, subjects
(including subject sets) and objects are now automatically mapped to and
from UUIDs.  The mapping is done in each handler and for each protocol
(HTTP, gRPC) separately.

Below the handler, all objects and subjects are now UUIDs, but still
handled as strings, for the following reasons:
 * No duplication of datastructures (one for type `string`, one for type
   `uuid.UUID`).
 * No unnecessary copying, the mapping is done in-place and batched
   across multiple objects.

Additionally, the migration (adding the string values to the mapping
table and replacing the values with UUIDs) is now performed
automatically as part of the migrations.  Tests are in `migratest`,
which now also handles `GoMigrations` properly.  This method replaces
the dedicated command in `cmd/migrate/...` for the UUID mappings.

cmd/expand/root_test.go

@@ -4,7 +4,6 @@ import (
 	"testing"
 
 	"github.com/ory/x/cmdx"
-
 	"github.com/stretchr/testify/assert"
 
 	"github.com/ory/keto/cmd/client"
@@ -18,12 +17,16 @@ func TestExpandCommand(t *testing.T) {
 
 	t.Run("case=unknown tuple", func(t *testing.T) {
 		t.Run("format=JSON", func(t *testing.T) {
-			stdOut := ts.Cmd.ExecNoErr(t, "access", nspace.Name, "object", "--"+cmdx.FlagFormat, string(cmdx.FormatJSON))
+			stdOut := ts.Cmd.ExecNoErr(t,
+				"access", nspace.Name,
+				"object", "--"+cmdx.FlagFormat, string(cmdx.FormatJSON))
 			assert.Equal(t, "null\n", stdOut)
 		})
 
 		t.Run("format=default", func(t *testing.T) {
-			stdOut := ts.Cmd.ExecNoErr(t, "access", nspace.Name, "object", "--"+cmdx.FlagFormat, string(cmdx.FormatDefault))
+			stdOut := ts.Cmd.ExecNoErr(t,
+				"access", nspace.Name,
+				"object", "--"+cmdx.FlagFormat, string(cmdx.FormatDefault))
 			assert.Contains(t, stdOut, "empty tree")
 		})
 	})

cmd/migrate/migrate_test.go

@@ -105,7 +105,7 @@ func TestMigrate(t *testing.T) {
 
 					t.Cleanup(func() {
 						// migrate all down
-						t.Log(cmd.ExecNoErr(t, "down", "0", "--"+FlagYes))
+						t.Logf("cleanup:\n%s\n", cmd.ExecNoErr(t, "down", "0", "--"+FlagYes))
 					})
 
 					parts := strings.Split(stdOut, "Are you sure that you want to apply this migration?")
@@ -120,7 +120,7 @@ func TestMigrate(t *testing.T) {
 
 					t.Cleanup(func() {
 						// migrate all down
-						t.Log(cmd.ExecNoErr(t, "down", "0", "--"+FlagYes))
+						t.Logf("cleanup:\n%s\n", cmd.ExecNoErr(t, "down", "0", "--"+FlagYes))
 					})
 
 					parts := strings.Split(out, "Applying migrations...")

cmd/migrate/root.go

@@ -17,7 +17,6 @@ func newMigrateCmd(opts []ketoctx.Option) *cobra.Command {
 		newStatusCmd(opts),
 		newUpCmd(opts),
 		newDownCmd(opts),
-		newMigrateUUIDMappingCmd(),
 	)
 	return cmd
 }

contrib/docs-code-samples/list-api-display-objects/99-cleanup/index.js

@@ -30,7 +30,6 @@ readClient.listRelationTuples(readRequest, (err, resp) => {
   writeClient.transactRelationTuples(writeRequest, (err) => {
     if (err) {
       console.log('Unexpected err', err)
-      return 1
     }
   })
 })

internal/check/handler.go

@@ -5,18 +5,14 @@ import (
 	"encoding/json"
 	"net/http"
 
+	"github.com/julienschmidt/httprouter"
 	"github.com/ory/herodot"
 	"github.com/pkg/errors"
-
-	rts "github.com/ory/keto/proto/ory/keto/relation_tuples/v1alpha2"
-
 	"google.golang.org/grpc"
 
 	"github.com/ory/keto/internal/relationtuple"
-
-	"github.com/julienschmidt/httprouter"
-
 	"github.com/ory/keto/internal/x"
+	rts "github.com/ory/keto/proto/ory/keto/relation_tuples/v1alpha2"
 )
 
 type (
@@ -30,7 +26,10 @@ type (
 	}
 )
 
-var _ rts.CheckServiceServer = (*Handler)(nil)
+var (
+	_ rts.CheckServiceServer = (*Handler)(nil)
+	_ *getCheckRequest       = nil
+)
 
 func NewHandler(d handlerDependencies) *Handler {
 	return &Handler{d: d}
@@ -64,7 +63,6 @@ type RESTResponse struct {
 }
 
 // swagger:parameters getCheck postCheck
-// nolint:deadcode,unused
 type getCheckRequest struct {
 	// in:query
 	MaxDepth int `json:"max-depth"`
@@ -105,6 +103,10 @@ func (h *Handler) getCheck(w http.ResponseWriter, r *http.Request, _ httprouter.
 		return
 	}
 
+	if err := h.d.PermissionEngine().d.UUIDMappingManager().MapFieldsToUUID(r.Context(), tuple); err != nil {
+		h.d.Writer().WriteError(w, r, err)
+		return
+	}
 	allowed, err := h.d.PermissionEngine().SubjectIsAllowed(r.Context(), tuple, maxDepth)
 	if err != nil {
 		h.d.Writer().WriteError(w, r, err)
@@ -151,6 +153,10 @@ func (h *Handler) postCheck(w http.ResponseWriter, r *http.Request, _ httprouter
 		return
 	}
 
+	if err := h.d.PermissionEngine().d.UUIDMappingManager().MapFieldsToUUID(r.Context(), &tuple); err != nil {
+		h.d.Writer().WriteError(w, r, err)
+		return
+	}
 	allowed, err := h.d.PermissionEngine().SubjectIsAllowed(r.Context(), &tuple, maxDepth)
 	if err != nil {
 		h.d.Writer().WriteError(w, r, err)
@@ -171,6 +177,9 @@ func (h *Handler) Check(ctx context.Context, req *rts.CheckRequest) (*rts.CheckR
 		return nil, err
 	}
 
+	if err := h.d.PermissionEngine().d.UUIDMappingManager().MapFieldsToUUID(ctx, tuple); err != nil {
+		return nil, err
+	}
 	allowed, err := h.d.PermissionEngine().SubjectIsAllowed(ctx, tuple, int(req.MaxDepth))
 	// TODO add content change handling
 	if err != nil {

internal/check/handler_test.go

@@ -23,6 +23,8 @@ import (
 )
 
 func assertAllowed(t *testing.T, resp *http.Response) {
+	t.Helper()
+
 	body, err := io.ReadAll(resp.Body)
 	require.NoError(t, err)
 
@@ -97,7 +99,7 @@ func TestRESTHandler(t *testing.T) {
 			Relation:  "r",
 			Subject:   &relationtuple.SubjectID{ID: "s"},
 		}
-		require.NoError(t, reg.RelationTupleManager().WriteRelationTuples(context.Background(), rt))
+		relationtuple.MapAndWriteTuples(t, reg, rt)
 
 		q, err := rt.ToURLQuery()
 		require.NoError(t, err)

internal/driver/config/namespace_memory.go

@@ -34,7 +34,7 @@ func (s *memoryNamespaceManager) GetNamespaceByName(_ context.Context, name stri
 		}
 	}
 
-	return nil, errors.WithStack(herodot.ErrNotFound.WithReasonf("Unknown namespace with name %s.", name))
+	return nil, errors.WithStack(herodot.ErrNotFound.WithReasonf("Unknown namespace with name %q.", name))
 }
 
 func (s *memoryNamespaceManager) GetNamespaceByConfigID(_ context.Context, id int32) (*namespace.Namespace, error) {

internal/driver/pop_connection.go

@@ -63,6 +63,12 @@ func (r *RegistryDefault) PopConnectionWithOpts(ctx context.Context, popOpts ...
 		return nil, errors.WithStack(err)
 	}
 
+	// Close this connection when the context is closed.
+	go func() {
+		<-ctx.Done()
+		conn.Close()
+	}()
+
 	return conn.WithContext(ctx), nil
 }
 

internal/driver/registry.go

@@ -4,21 +4,15 @@ import (
 	"context"
 	"net/http"
 
+	"github.com/gobuffalo/pop/v6"
+	"github.com/ory/x/healthx"
 	"github.com/ory/x/otelx"
 	prometheus "github.com/ory/x/prometheusx"
-
-	"github.com/gobuffalo/pop/v6"
-
-	"github.com/ory/keto/internal/driver/config"
-	"github.com/ory/keto/internal/uuidmapping"
-
 	"github.com/spf13/cobra"
-
 	"google.golang.org/grpc"
 
-	"github.com/ory/x/healthx"
-
 	"github.com/ory/keto/internal/check"
+	"github.com/ory/keto/internal/driver/config"
 	"github.com/ory/keto/internal/expand"
 	"github.com/ory/keto/internal/persistence"
 	"github.com/ory/keto/internal/relationtuple"
@@ -34,7 +28,6 @@ type (
 		x.WriterProvider
 
 		relationtuple.ManagerProvider
-		uuidmapping.ManagerProvider
 		expand.EngineProvider
 		check.EngineProvider
 		persistence.Migrator

internal/driver/registry_default.go

@@ -8,6 +8,7 @@ import (
 	"github.com/gobuffalo/pop/v6"
 	"github.com/ory/herodot"
 	"github.com/ory/x/dbal"
+	"github.com/ory/x/fsx"
 	"github.com/ory/x/healthx"
 	"github.com/ory/x/logrusx"
 	"github.com/ory/x/metricsx"
@@ -24,8 +25,9 @@ import (
 	"github.com/ory/keto/internal/expand"
 	"github.com/ory/keto/internal/persistence"
 	"github.com/ory/keto/internal/persistence/sql"
+	_ "github.com/ory/keto/internal/persistence/sql/migrations"
+	"github.com/ory/keto/internal/persistence/sql/migrations/uuidmapping"
 	"github.com/ory/keto/internal/relationtuple"
-	"github.com/ory/keto/internal/uuidmapping"
 	"github.com/ory/keto/internal/x"
 	"github.com/ory/keto/ketoctx"
 	rts "github.com/ory/keto/proto/ory/keto/relation_tuples/v1alpha2"
@@ -152,7 +154,7 @@ func (r *RegistryDefault) RelationTupleManager() relationtuple.Manager {
 	return r.p
 }
 
-func (r *RegistryDefault) UUIDMappingManager() uuidmapping.Manager {
+func (r *RegistryDefault) UUIDMappingManager() relationtuple.UUIDMappingManager {
 	if r.p == nil {
 		panic("no relation tuple manager, but expected to have one")
 	}
@@ -186,10 +188,16 @@ func (r *RegistryDefault) MigrationBox(ctx context.Context) (*popx.MigrationBox,
 		if err != nil {
 			return nil, err
 		}
-		mb, err := sql.NewMigrationBox(c, r.Logger(), r.Tracer(ctx))
+
+		mb, err := popx.NewMigrationBox(
+			fsx.Merge(sql.Migrations, networkx.Migrations),
+			popx.NewMigrator(c, r.Logger(), r.Tracer(ctx), 0),
+			popx.WithGoMigrations(uuidmapping.Migrations),
+		)
 		if err != nil {
 			return nil, err
 		}
+
 		r.mb = mb
 	}
 	return r.mb, nil

internal/e2e/cases_test.go

@@ -5,17 +5,14 @@ import (
 	"strconv"
 	"testing"
 
-	"github.com/stretchr/testify/require"
-
 	"github.com/ory/herodot"
-
-	"github.com/ory/keto/internal/x"
-
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/ory/keto/internal/expand"
 	"github.com/ory/keto/internal/namespace"
 	"github.com/ory/keto/internal/relationtuple"
+	"github.com/ory/keto/internal/x"
 )
 
 func runCases(c client, addNamespace func(*testing.T, ...*namespace.Namespace)) func(*testing.T) {
@@ -188,7 +185,7 @@ func runCases(c client, addNamespace func(*testing.T, ...*namespace.Namespace))
 				Relation:  "rel",
 			}
 			resp := c.queryTuple(t, q)
-			require.Equal(t, resp.RelationTuples, rts)
+			require.ElementsMatch(t, resp.RelationTuples, rts)
 
 			c.deleteAllTuples(t, q)
 			resp = c.queryTuple(t, q)