From 6f5fa0526b5134816ae9907b2a42d17a5acb088b Mon Sep 17 00:00:00 2001 From: zepatrik Date: Tue, 15 Dec 2020 10:48:39 +0100 Subject: [PATCH 1/5] fix: minor bugs and cleanup Closes #370 --- .schema/config.schema.json | 79 ++++++++++++++-------- cmd/migrate/status.go | 3 +- cmd/migrate/up.go | 19 +++++- cmd/namespace/migrate.go | 16 +++-- cmd/root.go | 83 +----------------------- cmd/serve.go | 65 ++++++++++--------- cmd/server/serve.go | 6 +- cmd/version.go | 8 ++- internal/driver/config/buildinfo.go | 7 ++ internal/driver/config/provider.go | 3 +- internal/driver/config/provider_koanf.go | 26 ++++++-- internal/driver/registry.go | 5 +- internal/driver/registry_default.go | 22 +++---- internal/driver/registry_factory.go | 11 ++-- keto-namespaces/media.yml | 3 - keto.yml | 9 +++ 16 files changed, 179 insertions(+), 186 deletions(-) create mode 100644 internal/driver/config/buildinfo.go delete mode 100644 keto-namespaces/media.yml create mode 100644 keto.yml diff --git a/.schema/config.schema.json b/.schema/config.schema.json index 1c206ce93..3e136a237 100644 --- a/.schema/config.schema.json +++ b/.schema/config.schema.json @@ -197,35 +197,62 @@ }, "serve": { "type": "object", - "title": "HTTP REST API", "additionalProperties": false, "properties": { - "port": { - "type": "integer", - "default": 4456, - "title": "Port", - "description": "The port to listen on.", - "minimum": 1, - "maximum": 65535, - "examples": [ - 4456 - ] - }, - "host": { - "type": "string", - "default": "", - "examples": [ - "localhost", - "127.0.0.1" - ], - "title": "Host", - "description": "The network interface to listen on." - }, - "cors": { - "$ref": "#/definitions/cors" + "rest": { + "type": "object", + "title": "HTTP REST API", + "additionalProperties": false, + "properties": { + "port": { + "type": "integer", + "default": 4466, + "title": "Port", + "description": "The port to listen on.", + "minimum": 1, + "maximum": 65535 + }, + "host": { + "type": "string", + "default": "", + "examples": [ + "localhost", + "127.0.0.1" + ], + "title": "Host", + "description": "The network interface to listen on." + }, + "cors": { + "$ref": "#/definitions/cors" + }, + "tls": { + "$ref": "#/definitions/tlsx" + } + } }, - "tls": { - "$ref": "#/definitions/tlsx" + "grpc": { + "type": "object", + "additionalProperties": false, + "properties": { + "port": { + "type": "integer", + "default": 4467, + "title": "Port", + "description": "The port to listen on.", + "minimum": 1, + "maximum": 65535 + }, + "host": { + "type": "string", + "default": "", + "examples": [ + "localhost", + "127.0.0.1" + ], + "title": "Host", + "description": "The network interface to listen on." + } + } } } }, diff --git a/cmd/migrate/status.go b/cmd/migrate/status.go index b7cb7479d..30ae78892 100644 --- a/cmd/migrate/status.go +++ b/cmd/migrate/status.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/ory/x/cmdx" - "github.com/ory/x/logrusx" "github.com/spf13/cobra" "github.com/ory/keto/internal/driver" @@ -18,7 +17,7 @@ func newStatusCmd() *cobra.Command { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - reg := driver.NewDefaultRegistry(ctx, logrusx.New("keto", "test"), cmd.Flags(), "test", "adf", "today") + reg := driver.NewDefaultRegistry(ctx, cmd.Flags()) if err := reg.Migrator().MigrationStatus(ctx, cmd.OutOrStdout()); err != nil { _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Could not get migration status: %+v\n", err) return cmdx.FailSilently(cmd) diff --git a/cmd/migrate/up.go b/cmd/migrate/up.go index d7069346d..390d0a6f9 100644 --- a/cmd/migrate/up.go +++ b/cmd/migrate/up.go @@ -5,12 +5,14 @@ import ( "fmt" "github.com/ory/x/cmdx" - "github.com/ory/x/logrusx" + "github.com/ory/x/flagx" "github.com/spf13/cobra" "github.com/ory/keto/internal/driver" ) +const FlagYes = "yes" + func newUpCmd() *cobra.Command { cmd := &cobra.Command{ Use: "up", @@ -18,7 +20,17 @@ func newUpCmd() *cobra.Command { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - reg := driver.NewDefaultRegistry(ctx, logrusx.New("keto", "test"), cmd.Flags(), "test", "adf", "today") + reg := driver.NewDefaultRegistry(ctx, cmd.Flags()) + if err := reg.Migrator().MigrationStatus(ctx, cmd.OutOrStdout()); err != nil { + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Could not get migration status: %+v\n", err) + return cmdx.FailSilently(cmd) + } + + if !flagx.MustGetBool(cmd, FlagYes) && !cmdx.AskForConfirmation("Do you want to apply above planned migrations?", cmd.InOrStdin(), cmd.OutOrStdout()) { + _, _ = fmt.Fprintln(cmd.OutOrStdout(), "Aborting") + return nil + } + if err := reg.Migrator().MigrateUp(ctx); err != nil { _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Could not apply migrations: %+v\n", err) return cmdx.FailSilently(cmd) @@ -26,5 +38,8 @@ func newUpCmd() *cobra.Command { return nil }, } + + cmd.Flags().BoolP(FlagYes, "y", false, "yes to all questions, no user input required") + return cmd } diff --git a/cmd/namespace/migrate.go b/cmd/namespace/migrate.go index 2baf27d77..9c7db116c 100644 --- a/cmd/namespace/migrate.go +++ b/cmd/namespace/migrate.go @@ -7,7 +7,6 @@ import ( "github.com/ory/x/cmdx" "github.com/ory/x/flagx" - "github.com/ory/x/logrusx" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -17,18 +16,25 @@ import ( func NewMigrateCmd() *cobra.Command { cmd := &cobra.Command{ - Use: "migrate ", + Use: "migrate ", Short: "Migrate a namespace up.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - reg := driver.NewDefaultRegistry(ctx, logrusx.New("keto", "master"), cmd.Flags(), "master", "local", "today") + reg := driver.NewDefaultRegistry(ctx, cmd.Flags()) - n, err := validateNamespaceFile(cmd, args[0]) + nm, err := reg.Config().NamespaceManager() if err != nil { - return err + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Could not initialize the namespace manager: %+v\n", err) + return cmdx.FailSilently(cmd) + } + + n, err := nm.GetNamespace(ctx, args[0]) + if err != nil { + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Could not find the namespace with name \"%s\": %+v\n", args[0], err) + return cmdx.FailSilently(cmd) } status, err := reg.NamespaceMigrator().NamespaceStatus(ctx, n.ID) diff --git a/cmd/root.go b/cmd/root.go index 186e6d828..624dfbd7e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -20,27 +20,15 @@ import ( "os" "path/filepath" "runtime" - "strings" - - "github.com/ory/keto/cmd/migrate" "github.com/ory/x/cmdx" + "github.com/ory/x/configx" + "github.com/ory/keto/cmd/migrate" "github.com/ory/keto/cmd/namespace" - "github.com/ory/keto/cmd/relationtuple" "github.com/spf13/cobra" - - "github.com/ory/viper" - - "github.com/ory/x/logrusx" -) - -var ( - Version = "master" - Date = "undefined" - Commit = "undefined" ) // RootCmd represents the base command when called without any subcommands @@ -48,10 +36,6 @@ var RootCmd = &cobra.Command{ Use: "keto", } -var cfgFile string - -var logger = new(logrusx.Logger) - // Execute adds all child commands to the root command sets flags appropriately. // This is called by main.main(). It only needs to happen once to the rootCmd. func Execute() { @@ -64,16 +48,7 @@ func Execute() { } func init() { - cobra.OnInitialize(initConfig) - - // Here you will define your flags and configuration settings. - // Cobra supports Persistent Flags, which, if defined here, - // will be global for your application. - RootCmd.PersistentFlags().StringSlice("config", []string{}, "Config file (default is $HOME/.keto.yaml)") - - // Cobra also supports local flags, which will only run - // when this action is called directly. - RootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") + configx.RegisterConfigFlag(RootCmd.PersistentFlags(), []string{filepath.Join(userHomeDir(), "keto.yml")}) relationtuple.RegisterCommandRecursive(RootCmd) @@ -82,58 +57,6 @@ func init() { migrate.RegisterCommandRecursive(RootCmd) } -// initConfig reads in config file and ENV variables if set. -func initConfig() { - if cfgFile != "" { - // enable ability to specify config file via flag - viper.SetConfigFile(cfgFile) - } else { - path := absPathify("$HOME") - if _, err := os.Stat(filepath.Join(path, ".keto.yml")); err != nil { - _, _ = os.Create(filepath.Join(path, ".keto.yml")) - } - - viper.SetConfigType("yaml") - viper.SetConfigName(".keto") // name of config file (without extension) - viper.AddConfigPath("$HOME") // adding home directory as first search path - } - - viper.SetDefault("serve.port", "4466") - viper.SetDefault("log.level", "info") - - viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) - viper.AutomaticEnv() // read in environment variables that match - - *logger = *logrusx.New("ORY Keto", Version) - - // If a config file is found, read it in. - if err := viper.ReadInConfig(); err != nil { - fmt.Printf(`Config file not found because "%s"`, err) - fmt.Println("") - } -} - -func absPathify(inPath string) string { - if strings.HasPrefix(inPath, "$HOME") { - inPath = userHomeDir() + inPath[5:] - } - - if strings.HasPrefix(inPath, "$") { - end := strings.Index(inPath, string(os.PathSeparator)) - inPath = os.Getenv(inPath[1:end]) + inPath[end:] - } - - if filepath.IsAbs(inPath) { - return filepath.Clean(inPath) - } - - p, err := filepath.Abs(inPath) - if err == nil { - return filepath.Clean(p) - } - return "" -} - func userHomeDir() string { if runtime.GOOS == "windows" { home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH") diff --git a/cmd/serve.go b/cmd/serve.go index 6ba610f5b..e6392f04a 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -20,24 +20,21 @@ import ( "net" "net/http" "os" + "strconv" "sync" acl "github.com/ory/keto/api/keto/acl/v1alpha1" - "github.com/ory/keto/internal/expand" - - "github.com/ory/keto/internal/check" - "github.com/julienschmidt/httprouter" + "github.com/ory/graceful" "github.com/spf13/cobra" "google.golang.org/grpc" - "github.com/ory/graceful" - + "github.com/ory/keto/internal/check" "github.com/ory/keto/internal/driver" + "github.com/ory/keto/internal/driver/config" + "github.com/ory/keto/internal/expand" "github.com/ory/keto/internal/relationtuple" - - "github.com/ory/x/logrusx" ) // serveCmd represents the serve command @@ -51,19 +48,12 @@ var serveCmd = &cobra.Command{ ORY Keto can be configured using environment variables as well as a configuration file. For more information on configuration options, open the configuration documentation: ->> https://github.com/ory/keto/blob/` + Version + `/docs/config.yaml <<`, +>> https://github.com/ory/keto/blob/` + config.Version + `/docs/config.yaml <<`, Run: func(cmd *cobra.Command, args []string) { - /* #nosec G102 - TODO this will be configurable */ - lis, err := net.Listen("tcp", ":4467") - if err != nil { - fmt.Fprintf(cmd.ErrOrStderr(), "%+v\n", err) - os.Exit(1) - } - ctx, cancel := context.WithCancel(context.Background()) defer cancel() - reg := driver.NewDefaultRegistry(ctx, logrusx.New("keto", "master"), cmd.Flags(), "master", "local", "today") + reg := driver.NewDefaultRegistry(ctx, cmd.Flags()) wg := &sync.WaitGroup{} wg.Add(2) @@ -71,12 +61,19 @@ on configuration options, open the configuration documentation: go func() { defer wg.Done() + lis, err := net.Listen("tcp", reg.Config().GRPCListenOn()) + if err != nil { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "%+v\n", err) + return + } + s := grpc.NewServer() relS := relationtuple.NewGRPCServer(reg) acl.RegisterReadServiceServer(s, relS) - fmt.Println("going to serve GRPC on", lis.Addr().String()) + + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Serving GRPC on %s\n", lis.Addr().String()) if err := s.Serve(lis); err != nil { - fmt.Fprintf(cmd.ErrOrStderr(), "%+v\n", err) + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "%+v\n", err) } }() @@ -84,21 +81,18 @@ on configuration options, open the configuration documentation: defer wg.Done() router := httprouter.New() - rh := relationtuple.NewHandler(reg) - rh.RegisterPublicRoutes(router) - ch := check.NewHandler(reg) - ch.RegisterPublicRoutes(router) - eh := expand.NewHandler(reg) - eh.RegisterPublicRoutes(router) + relationtuple.NewHandler(reg).RegisterPublicRoutes(router) + check.NewHandler(reg).RegisterPublicRoutes(router) + expand.NewHandler(reg).RegisterPublicRoutes(router) server := graceful.WithDefaults(&http.Server{ - Addr: ":4466", + Addr: reg.Config().RESTListenOn(), Handler: router, }) - fmt.Println("going to serve REST on", server.Addr) + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Serving REST on %s\n", server.Addr) if err := server.ListenAndServe(); err != nil { - fmt.Fprintf(cmd.ErrOrStderr(), "%+v\n", err) + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "%+v\n", err) } }() @@ -109,8 +103,15 @@ on configuration options, open the configuration documentation: func init() { RootCmd.AddCommand(serveCmd) - // TODO - //disableTelemetryEnv := viperx.GetBool(logrusx.New("ORY Keto", Version), "sqa.opt_out", false, "DISABLE_TELEMETRY") - serveCmd.PersistentFlags().Bool("disable-telemetry", true, "Disable anonymized telemetry reports - for more information please visit https://www.ory.sh/docs/ecosystem/sqa") - serveCmd.PersistentFlags().Bool("sqa-opt-out", true, "Disable anonymized telemetry reports - for more information please visit https://www.ory.sh/docs/ecosystem/sqa") + disableTelemetry, err := strconv.ParseBool(os.Getenv("DISABLE_TELEMETRY")) + if err != nil { + disableTelemetry = true + } + sqaOptOut, err := strconv.ParseBool(os.Getenv("SQA_OPT_OUT")) + if err != nil { + sqaOptOut = true + } + + serveCmd.PersistentFlags().Bool("disable-telemetry", disableTelemetry, "Disable anonymized telemetry reports - for more information please visit https://www.ory.sh/docs/ecosystem/sqa") + serveCmd.PersistentFlags().Bool("sqa-opt-out", sqaOptOut, "Disable anonymized telemetry reports - for more information please visit https://www.ory.sh/docs/ecosystem/sqa") } diff --git a/cmd/server/serve.go b/cmd/server/serve.go index b518d97af..1c739f554 100644 --- a/cmd/server/serve.go +++ b/cmd/server/serve.go @@ -70,7 +70,7 @@ package server // c := corsx.Initialize(n, logger, "serve") // // server := graceful.WithDefaults(&http.Server{ -// Addr: d.Configuration().ListenOn(), +// Addr: d.Configuration().RESTListenOn(), // Handler: c, // }) // @@ -97,10 +97,10 @@ package server // // if err := graceful.Graceful(func() error { // if cert != nil { -// logger.Printf("Listening on https://%s", d.Configuration().ListenOn()) +// logger.Printf("Listening on https://%s", d.Configuration().RESTListenOn()) // return server.ListenAndServeTLS("", "") // } -// logger.Printf("Listening on http://%s", d.Configuration().ListenOn()) +// logger.Printf("Listening on http://%s", d.Configuration().RESTListenOn()) // return server.ListenAndServe() // }, server.Shutdown); err != nil { // logger.Fatalf("Unable to gracefully shutdown HTTP(s) server because %v", err) diff --git a/cmd/version.go b/cmd/version.go index fe6218495..b89b6b4cf 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -21,8 +21,12 @@ package cmd -import "github.com/ory/x/cmdx" +import ( + "github.com/ory/x/cmdx" + + "github.com/ory/keto/internal/driver/config" +) func init() { - RootCmd.AddCommand(cmdx.Version(&Version, &Commit, &Date)) + RootCmd.AddCommand(cmdx.Version(&config.Version, &config.Commit, &config.Date)) } diff --git a/internal/driver/config/buildinfo.go b/internal/driver/config/buildinfo.go new file mode 100644 index 000000000..bd2633440 --- /dev/null +++ b/internal/driver/config/buildinfo.go @@ -0,0 +1,7 @@ +package config + +var ( + Version = "master" + Date = "undefined" + Commit = "undefined" +) diff --git a/internal/driver/config/provider.go b/internal/driver/config/provider.go index dc3756ec1..4241b05f4 100644 --- a/internal/driver/config/provider.go +++ b/internal/driver/config/provider.go @@ -12,7 +12,8 @@ type Provider interface { namespace.ManagerProvider CORS() (cors.Options, bool) - ListenOn() string + RESTListenOn() string + GRPCListenOn() string DSN() string TracingServiceName() string TracingProvider() string diff --git a/internal/driver/config/provider_koanf.go b/internal/driver/config/provider_koanf.go index 2b716a963..4e5e318e8 100644 --- a/internal/driver/config/provider_koanf.go +++ b/internal/driver/config/provider_koanf.go @@ -25,8 +25,11 @@ import ( const ( KeyDSN = "dsn" - KeyHost = "serve.host" - KeyPort = "serve.port" + KeyRESTHost = "serve.rest.host" + KeyRESTPort = "serve.rest.port" + + KeyGRPCHost = "serve.grpc.host" + KeyGRPCPort = "serve.grpc.port" KeyNamespaces = "namespaces" ) @@ -64,7 +67,7 @@ func New(ctx context.Context, flags *pflag.FlagSet, l *logrusx.Logger) (Provider configx.WithStderrValidationReporter(), configx.WithImmutables(KeyDSN, "serve"), configx.OmitKeysFromTracing(KeyDSN), - configx.WithLogrusWatcher(l), + configx.WithLogrusWatcher(kp.l), configx.WithContext(ctx), configx.AttachWatcher(func(watcherx.Event, error) { // TODO this can be optimized to run only on changes related to namespace config @@ -74,6 +77,7 @@ func New(ctx context.Context, flags *pflag.FlagSet, l *logrusx.Logger) (Provider if err != nil { return nil, err } + l.UseConfig(kp.p) return kp, nil } @@ -100,11 +104,19 @@ func (k *KoanfProvider) Set(key string, v interface{}) { } } -func (k *KoanfProvider) ListenOn() string { +func (k *KoanfProvider) RESTListenOn() string { + return fmt.Sprintf( + "%s:%d", + k.p.StringF(KeyRESTHost, ""), + k.p.IntF(KeyRESTPort, 4466), + ) +} + +func (k *KoanfProvider) GRPCListenOn() string { return fmt.Sprintf( "%s:%d", - k.p.StringF(KeyHost, ""), - k.p.IntF(KeyPort, 4466), + k.p.StringF(KeyGRPCHost, ""), + k.p.IntF(KeyGRPCPort, 4467), ) } @@ -118,7 +130,7 @@ func (k *KoanfProvider) CORS() (cors.Options, bool) { } func (k *KoanfProvider) DSN() string { - dsn := k.p.StringF(KeyDSN, DSNMemory) + dsn := k.p.String(KeyDSN) if dsn == "memory" { return DSNMemory } diff --git a/internal/driver/registry.go b/internal/driver/registry.go index c3ef7e7e6..982c83666 100644 --- a/internal/driver/registry.go +++ b/internal/driver/registry.go @@ -3,25 +3,24 @@ package driver import ( "context" - "github.com/ory/x/dbal" "github.com/ory/x/healthx" "github.com/ory/x/tracing" "github.com/ory/keto/internal/check" + "github.com/ory/keto/internal/driver/config" "github.com/ory/keto/internal/expand" "github.com/ory/keto/internal/namespace" "github.com/ory/keto/internal/persistence" "github.com/ory/keto/internal/relationtuple" - "github.com/ory/keto/internal/x" ) type Registry interface { - dbal.Driver Init(context.Context) error BuildVersion() string BuildDate() string BuildHash() string + Config() config.Provider x.LoggerProvider x.WriterProvider diff --git a/internal/driver/registry_default.go b/internal/driver/registry_default.go index 9ba3d703c..fe1f1107e 100644 --- a/internal/driver/registry_default.go +++ b/internal/driver/registry_default.go @@ -36,33 +36,27 @@ type RegistryDefault struct { conn *pop.Connection c config.Provider hh *healthx.Handler - - version, hash, date string -} - -func (r *RegistryDefault) CanHandle(dsn string) bool { - return true -} - -func (r *RegistryDefault) Ping() error { - return r.conn.Open() } func (r *RegistryDefault) BuildVersion() string { - return r.version + return config.Version } func (r *RegistryDefault) BuildDate() string { - return r.date + return config.Date } func (r *RegistryDefault) BuildHash() string { - return r.hash + return config.Commit +} + +func (r *RegistryDefault) Config() config.Provider { + return r.c } func (r *RegistryDefault) HealthHandler() *healthx.Handler { if r.hh == nil { - r.hh = healthx.NewHandler(r.Writer(), r.version, healthx.ReadyCheckers{}) + r.hh = healthx.NewHandler(r.Writer(), config.Version, healthx.ReadyCheckers{}) } return r.hh diff --git a/internal/driver/registry_factory.go b/internal/driver/registry_factory.go index 7cce12025..ccfccaf5c 100644 --- a/internal/driver/registry_factory.go +++ b/internal/driver/registry_factory.go @@ -14,18 +14,17 @@ import ( "github.com/ory/keto/internal/driver/config" ) -func NewDefaultRegistry(ctx context.Context, l *logrusx.Logger, flags *pflag.FlagSet, version, hash, date string) Registry { +func NewDefaultRegistry(ctx context.Context, flags *pflag.FlagSet) Registry { + l := logrusx.New("ORY Keto", config.Version) + c, err := config.New(ctx, flags, l) if err != nil { l.WithError(err).Fatal("Unable to initialize config provider.") } r := &RegistryDefault{ - c: c, - l: l, - version: version, - hash: hash, - date: date, + c: c, + l: l, } if err = r.Init(ctx); err != nil { diff --git a/keto-namespaces/media.yml b/keto-namespaces/media.yml deleted file mode 100644 index 62cfc34ac..000000000 --- a/keto-namespaces/media.yml +++ /dev/null @@ -1,3 +0,0 @@ -name: media -id: 0 - diff --git a/keto.yml b/keto.yml new file mode 100644 index 000000000..c7b19441c --- /dev/null +++ b/keto.yml @@ -0,0 +1,9 @@ + +dsn: postgres://keto:password@localhost:5432/keto?sslmode=disable + +namespaces: + - name: foo + id: 3 + +log: + level: debug From dea716d695cae38dfa5b9dfd28730e59df1d0ac5 Mon Sep 17 00:00:00 2001 From: zepatrik Date: Fri, 18 Dec 2020 17:17:24 +0100 Subject: [PATCH 2/5] test: add initial e2e test --- .circleci/config.yml | 10 +- Makefile | 8 + cmd/migrate/root.go | 10 +- cmd/migrate/status.go | 9 +- cmd/migrate/up.go | 9 +- cmd/namespace/migrate.go | 9 +- cmd/relationtuple/root.go | 10 +- cmd/root.go | 36 ++- cmd/serve.go | 117 -------- cmd/server/serve.go | 265 +++++++++++------- cmd/version.go | 32 --- go.mod | 6 +- go.sum | 2 + internal/driver/config/provider.go | 2 +- internal/driver/config/provider_koanf.go | 9 +- internal/driver/config/provider_koanf_test.go | 8 +- internal/driver/registry.go | 46 +-- internal/driver/registry_default.go | 62 ++-- internal/driver/registry_factory.go | 34 ++- internal/e2e/full_test.go | 172 ++++++++++++ internal/e2e/helpers.go | 41 +++ internal/persistence/sql/namespace.go | 6 +- internal/persistence/sql/persister.go | 2 + internal/relationtuple/definitions.go | 23 ++ internal/relationtuple/handler.go | 2 +- scripts/test-e2e.sh | 52 ---- scripts/test-resetdb.sh | 25 ++ 27 files changed, 595 insertions(+), 412 deletions(-) delete mode 100644 cmd/serve.go delete mode 100644 cmd/version.go create mode 100644 internal/e2e/full_test.go create mode 100644 internal/e2e/helpers.go delete mode 100755 scripts/test-e2e.sh create mode 100755 scripts/test-resetdb.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index a4ef44fa4..ce5bed99f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,9 +16,10 @@ jobs: image: circleci/golang:1.15 environment: TEST_DATABASE_POSTGRESQL: postgres://test:test@localhost:5432/keto?sslmode=disable - TEST_DATABASE_MYSQL: root:test@(localhost:3306)/mysql?parseTime=true + TEST_DATABASE_MYSQL: root:test@(localhost:3306)/mysql?parseTime=true&multiStatements=true + TEST_DATABASE_COCKROACH: cockroach://root@localhost:26257/defaultdb?sslmode=disable - - image: postgres:9.5 + image: postgres:11.8 environment: POSTGRES_USER: test POSTGRES_PASSWORD: test @@ -27,6 +28,9 @@ jobs: image: mysql:8.0 environment: MYSQL_ROOT_PASSWORD: test + - + image: cockroachdb/cockroach:v20.1.0 + command: start --insecure working_directory: /go/src/github.com/ory/keto steps: - checkout @@ -41,7 +45,7 @@ jobs: # Tests - - run: go test -tags sqlite -race -short -v $(go list ./... | grep -v cmd) + run: go test -tags sqlite -race -short -v ./... - run: go-acc -o coverage.txt ./... -- -v -tags sqlite diff --git a/Makefile b/Makefile index 423ef5dec..f248400a5 100644 --- a/Makefile +++ b/Makefile @@ -94,3 +94,11 @@ buf-lint: deps # .PHONY: buf buf: buf-lint buf-gen + +.PHONY: reset-testdb +reset-testdb: + source scripts/test-resetdb.sh + +.PHONY: e2e-test +e2e-test: + go test -tags sqlite -failfast -v ./internal/e2e diff --git a/cmd/migrate/root.go b/cmd/migrate/root.go index 0b8b9e448..b521de9c5 100644 --- a/cmd/migrate/root.go +++ b/cmd/migrate/root.go @@ -2,11 +2,15 @@ package migrate import "github.com/spf13/cobra" -var migrateCmd = &cobra.Command{ - Use: "migrate", +func newMigrateCmd() *cobra.Command { + return &cobra.Command{ + Use: "migrate", + } } -func RegisterCommandRecursive(parent *cobra.Command) { +func RegisterCommandsRecursive(parent *cobra.Command) { + migrateCmd := newMigrateCmd() + migrateCmd.AddCommand(newStatusCmd(), newUpCmd()) parent.AddCommand(migrateCmd) diff --git a/cmd/migrate/status.go b/cmd/migrate/status.go index 30ae78892..a8a682d92 100644 --- a/cmd/migrate/status.go +++ b/cmd/migrate/status.go @@ -1,7 +1,6 @@ package migrate import ( - "context" "fmt" "github.com/ory/x/cmdx" @@ -14,10 +13,12 @@ func newStatusCmd() *cobra.Command { cmd := &cobra.Command{ Use: "status", RunE: func(cmd *cobra.Command, _ []string) error { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := cmd.Context() - reg := driver.NewDefaultRegistry(ctx, cmd.Flags()) + reg, err := driver.NewDefaultRegistry(ctx, cmd.Flags()) + if err != nil { + return err + } if err := reg.Migrator().MigrationStatus(ctx, cmd.OutOrStdout()); err != nil { _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Could not get migration status: %+v\n", err) return cmdx.FailSilently(cmd) diff --git a/cmd/migrate/up.go b/cmd/migrate/up.go index 390d0a6f9..fd5b291e2 100644 --- a/cmd/migrate/up.go +++ b/cmd/migrate/up.go @@ -1,7 +1,6 @@ package migrate import ( - "context" "fmt" "github.com/ory/x/cmdx" @@ -17,10 +16,12 @@ func newUpCmd() *cobra.Command { cmd := &cobra.Command{ Use: "up", RunE: func(cmd *cobra.Command, _ []string) error { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := cmd.Context() - reg := driver.NewDefaultRegistry(ctx, cmd.Flags()) + reg, err := driver.NewDefaultRegistry(ctx, cmd.Flags()) + if err != nil { + return err + } if err := reg.Migrator().MigrationStatus(ctx, cmd.OutOrStdout()); err != nil { _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Could not get migration status: %+v\n", err) return cmdx.FailSilently(cmd) diff --git a/cmd/namespace/migrate.go b/cmd/namespace/migrate.go index 9c7db116c..bbc8f1f2f 100644 --- a/cmd/namespace/migrate.go +++ b/cmd/namespace/migrate.go @@ -1,7 +1,6 @@ package namespace import ( - "context" "errors" "fmt" @@ -20,10 +19,12 @@ func NewMigrateCmd() *cobra.Command { Short: "Migrate a namespace up.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := cmd.Context() - reg := driver.NewDefaultRegistry(ctx, cmd.Flags()) + reg, err := driver.NewDefaultRegistry(ctx, cmd.Flags()) + if err != nil { + return err + } nm, err := reg.Config().NamespaceManager() if err != nil { diff --git a/cmd/relationtuple/root.go b/cmd/relationtuple/root.go index 744c816d4..1dd30f911 100644 --- a/cmd/relationtuple/root.go +++ b/cmd/relationtuple/root.go @@ -9,13 +9,17 @@ import ( "github.com/ory/keto/cmd/client" ) -var relationCmd = &cobra.Command{ - Use: "relation-tuple", +func newRelationCmd() *cobra.Command { + return &cobra.Command{ + Use: "relation-tuple", + } } var packageFlags = pflag.NewFlagSet("relation package flags", pflag.ContinueOnError) -func RegisterCommandRecursive(parent *cobra.Command) { +func RegisterCommandsRecursive(parent *cobra.Command) { + relationCmd := newRelationCmd() + parent.AddCommand(relationCmd) relationCmd.AddCommand(newGetCmd()) diff --git a/cmd/root.go b/cmd/root.go index 624dfbd7e..178ffa05d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -15,12 +15,16 @@ package cmd import ( + "context" "errors" "fmt" "os" "path/filepath" "runtime" + "github.com/ory/keto/cmd/server" + "github.com/ory/keto/internal/driver/config" + "github.com/ory/x/cmdx" "github.com/ory/x/configx" @@ -32,14 +36,30 @@ import ( ) // RootCmd represents the base command when called without any subcommands -var RootCmd = &cobra.Command{ - Use: "keto", +func NewRootCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "keto", + } + + configx.RegisterConfigFlag(cmd.PersistentFlags(), []string{filepath.Join(userHomeDir(), "keto.yml")}) + + relationtuple.RegisterCommandsRecursive(cmd) + namespace.RegisterCommandsRecursive(cmd) + migrate.RegisterCommandsRecursive(cmd) + server.RegisterCommandsRecursive(cmd) + + cmd.AddCommand(cmdx.Version(&config.Version, &config.Commit, &config.Date)) + + return cmd } // Execute adds all child commands to the root command sets flags appropriately. // This is called by main.main(). It only needs to happen once to the rootCmd. func Execute() { - if err := RootCmd.Execute(); err != nil { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + if err := NewRootCmd().ExecuteContext(ctx); err != nil { if !errors.Is(err, cmdx.ErrNoPrintButFail) { fmt.Println(err) } @@ -47,16 +67,6 @@ func Execute() { } } -func init() { - configx.RegisterConfigFlag(RootCmd.PersistentFlags(), []string{filepath.Join(userHomeDir(), "keto.yml")}) - - relationtuple.RegisterCommandRecursive(RootCmd) - - namespace.RegisterCommandsRecursive(RootCmd) - - migrate.RegisterCommandRecursive(RootCmd) -} - func userHomeDir() string { if runtime.GOOS == "windows" { home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH") diff --git a/cmd/serve.go b/cmd/serve.go deleted file mode 100644 index e6392f04a..000000000 --- a/cmd/serve.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright © 2018 NAME HERE -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "context" - "fmt" - "net" - "net/http" - "os" - "strconv" - "sync" - - acl "github.com/ory/keto/api/keto/acl/v1alpha1" - - "github.com/julienschmidt/httprouter" - "github.com/ory/graceful" - "github.com/spf13/cobra" - "google.golang.org/grpc" - - "github.com/ory/keto/internal/check" - "github.com/ory/keto/internal/driver" - "github.com/ory/keto/internal/driver/config" - "github.com/ory/keto/internal/expand" - "github.com/ory/keto/internal/relationtuple" -) - -// serveCmd represents the serve command -var serveCmd = &cobra.Command{ - Use: "serve", - Short: "Starts the server and serves the HTTP REST API", - Long: `This command opens a network port and listens to HTTP/2 API requests. - -## Configuration - -ORY Keto can be configured using environment variables as well as a configuration file. For more information -on configuration options, open the configuration documentation: - ->> https://github.com/ory/keto/blob/` + config.Version + `/docs/config.yaml <<`, - Run: func(cmd *cobra.Command, args []string) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - reg := driver.NewDefaultRegistry(ctx, cmd.Flags()) - - wg := &sync.WaitGroup{} - wg.Add(2) - - go func() { - defer wg.Done() - - lis, err := net.Listen("tcp", reg.Config().GRPCListenOn()) - if err != nil { - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "%+v\n", err) - return - } - - s := grpc.NewServer() - relS := relationtuple.NewGRPCServer(reg) - acl.RegisterReadServiceServer(s, relS) - - _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Serving GRPC on %s\n", lis.Addr().String()) - if err := s.Serve(lis); err != nil { - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "%+v\n", err) - } - }() - - go func() { - defer wg.Done() - - router := httprouter.New() - relationtuple.NewHandler(reg).RegisterPublicRoutes(router) - check.NewHandler(reg).RegisterPublicRoutes(router) - expand.NewHandler(reg).RegisterPublicRoutes(router) - - server := graceful.WithDefaults(&http.Server{ - Addr: reg.Config().RESTListenOn(), - Handler: router, - }) - - _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Serving REST on %s\n", server.Addr) - if err := server.ListenAndServe(); err != nil { - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "%+v\n", err) - } - }() - - wg.Wait() - }, -} - -func init() { - RootCmd.AddCommand(serveCmd) - - disableTelemetry, err := strconv.ParseBool(os.Getenv("DISABLE_TELEMETRY")) - if err != nil { - disableTelemetry = true - } - sqaOptOut, err := strconv.ParseBool(os.Getenv("SQA_OPT_OUT")) - if err != nil { - sqaOptOut = true - } - - serveCmd.PersistentFlags().Bool("disable-telemetry", disableTelemetry, "Disable anonymized telemetry reports - for more information please visit https://www.ory.sh/docs/ecosystem/sqa") - serveCmd.PersistentFlags().Bool("sqa-opt-out", sqaOptOut, "Disable anonymized telemetry reports - for more information please visit https://www.ory.sh/docs/ecosystem/sqa") -} diff --git a/cmd/server/serve.go b/cmd/server/serve.go index 1c739f554..32d23820f 100644 --- a/cmd/server/serve.go +++ b/cmd/server/serve.go @@ -1,110 +1,161 @@ -/* - * Copyright © 2015-2018 Aeneas Rekkas - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @author Aeneas Rekkas - * @Copyright 2015-2018 Aeneas Rekkas - * @license Apache-2.0 - * - */ - -package server - -// -//// RunServe runs the Keto API HTTP server -//func RunServe( -// logger *logrusx.Logger, -// version, commit string, date string, -//) func(cmd *cobra.Command, args []string) { -// return func(cmd *cobra.Command, args []string) { -// d := driver.NewDefaultDriver( -// logger, -// version, commit, date, -// ) -// -// router := httprouter.New() -// d.Registry().HealthHandler().SetRoutes(router, true) -// -// n := negroni.New() -// n.Use(reqlog.NewMiddlewareFromLogger(logger, "keto").ExcludePaths(healthx.ReadyCheckPath, healthx.AliveCheckPath)) +// Copyright © 2018 NAME HERE // -// if tracer := d.Registry().Tracer(); tracer.IsLoaded() { -// n.Use(tracer) -// } +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // -// metrics := metricsx.New(cmd, logger, -// &metricsx.Options{ -// Service: "ory-keto", -// ClusterID: metricsx.Hash(viper.GetString("DATABASE_URL")), -// IsDevelopment: viper.GetString("DATABASE_URL") != "memory", -// WriteKey: "jk32cFATnj9GKbQdFL7fBB9qtKZdX9j7", -// WhitelistedPaths: stringslice.Merge( -// healthx.RoutesToObserve(), -// ), -// BuildVersion: version, -// BuildTime: date, -// BuildHash: commit, -// Config: &analytics.Config{ -// Endpoint: "https://sqa.ory.sh", -// GzipCompressionLevel: 6, -// BatchMaxSize: 500 * 1000, -// BatchSize: 250, -// Interval: time.Hour * 24, -// }, -// }, -// ) -// n.Use(metrics) +// http://www.apache.org/licenses/LICENSE-2.0 // -// n.UseHandler(router) -// c := corsx.Initialize(n, logger, "serve") -// -// server := graceful.WithDefaults(&http.Server{ -// Addr: d.Configuration().RESTListenOn(), -// Handler: c, -// }) -// -// cert, err := tlsx.Certificate( -// viper.GetString("serve.tls.cert.base64"), -// viper.GetString("serve.tls.key.base64"), -// viper.GetString("serve.tls.cert.path"), -// viper.GetString("serve.tls.key.path"), -// ) -// if errors.Cause(err) == tlsx.ErrNoCertificatesConfigured { -// // do nothing -// } else if err != nil { -// cmdx.Must(err, "Unable to load HTTP TLS certificate(s): %s", err) -// } else { -// server.TLSConfig = &tls.Config{ -// Certificates: cert, -// MinVersion: tls.VersionTLS13, -// } -// } -// -// if d.Registry().Tracer().IsLoaded() { -// server.RegisterOnShutdown(d.Registry().Tracer().Close) -// } -// -// if err := graceful.Graceful(func() error { -// if cert != nil { -// logger.Printf("Listening on https://%s", d.Configuration().RESTListenOn()) -// return server.ListenAndServeTLS("", "") -// } -// logger.Printf("Listening on http://%s", d.Configuration().RESTListenOn()) -// return server.ListenAndServe() -// }, server.Shutdown); err != nil { -// logger.Fatalf("Unable to gracefully shutdown HTTP(s) server because %v", err) -// return -// } -// } -//} +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "context" + "fmt" + "net" + "net/http" + "os" + "strconv" + "sync" + + "github.com/pkg/errors" + + acl "github.com/ory/keto/api/keto/acl/v1alpha1" + + "github.com/julienschmidt/httprouter" + "github.com/ory/graceful" + "github.com/spf13/cobra" + "google.golang.org/grpc" + + "github.com/ory/keto/internal/check" + "github.com/ory/keto/internal/driver" + "github.com/ory/keto/internal/driver/config" + "github.com/ory/keto/internal/expand" + "github.com/ory/keto/internal/relationtuple" +) + +// serveCmd represents the serve command +func newServe() *cobra.Command { + cmd := &cobra.Command{ + Use: "serve", + Short: "Starts the server and serves the HTTP REST API", + Long: `This command opens a network port and listens to HTTP/2 API requests. + +## Configuration + +ORY Keto can be configured using environment variables as well as a configuration file. For more information +on configuration options, open the configuration documentation: + +>> https://github.com/ory/keto/blob/` + config.Version + `/docs/config.yaml <<`, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + reg, err := driver.NewDefaultRegistry(ctx, cmd.Flags()) + if err != nil { + return err + } + + wg := &sync.WaitGroup{} + // the two servers + the ctx.Done listener go routine + wg.Add(3) + + var grpcErr, restErr error + var grpcServer *grpc.Server + go func() { + defer wg.Done() + + lis, err := net.Listen("tcp", reg.Config().GRPCListenOn()) + if err != nil { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "%+v\n", err) + grpcErr = err + return + } + + grpcServer = grpc.NewServer() + relS := relationtuple.NewGRPCServer(reg) + acl.RegisterReadServiceServer(grpcServer, relS) + + reg.Logger().WithField("addr", lis.Addr().String()).Info("serving GRPC") + if err := grpcServer.Serve(lis); err != nil { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "%+v\n", err) + grpcErr = err + } + }() + + var restServer *http.Server + go func() { + defer wg.Done() + + router := httprouter.New() + relationtuple.NewHandler(reg).RegisterPublicRoutes(router) + check.NewHandler(reg).RegisterPublicRoutes(router) + expand.NewHandler(reg).RegisterPublicRoutes(router) + reg.HealthHandler().SetRoutes(router, false) + + restServer = graceful.WithDefaults(&http.Server{ + Addr: reg.Config().RESTListenOn(), + Handler: router, + }) + + reg.Logger().WithField("addr", restServer.Addr).Info("serving REST") + if err := restServer.ListenAndServe(); err != nil { + if errors.Is(err, http.ErrServerClosed) { + // this means the server got closed and should not be reported + return + } + + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "%+v\n", err) + restErr = err + } + }() + + go func() { + defer wg.Done() + + <-ctx.Done() + // ctx is done already, so we need a new context + err := restServer.Shutdown(context.Background()) + + if restErr != nil { + if err != nil { + restErr = errors.Wrap(restErr, err.Error()) + } + } else { + restErr = err + } + + grpcServer.GracefulStop() + }() + + wg.Wait() + + if grpcErr == nil && restErr == nil { + return nil + } + + return fmt.Errorf("GRPC Error: %+v\nREST Error: %+v", grpcErr, restErr) + }, + } + disableTelemetry, err := strconv.ParseBool(os.Getenv("DISABLE_TELEMETRY")) + if err != nil { + disableTelemetry = true + } + sqaOptOut, err := strconv.ParseBool(os.Getenv("SQA_OPT_OUT")) + if err != nil { + sqaOptOut = true + } + + cmd.Flags().Bool("disable-telemetry", disableTelemetry, "Disable anonymized telemetry reports - for more information please visit https://www.ory.sh/docs/ecosystem/sqa") + cmd.Flags().Bool("sqa-opt-out", sqaOptOut, "Disable anonymized telemetry reports - for more information please visit https://www.ory.sh/docs/ecosystem/sqa") + + return cmd +} + +func RegisterCommandsRecursive(parent *cobra.Command) { + parent.AddCommand(newServe()) +} diff --git a/cmd/version.go b/cmd/version.go deleted file mode 100644 index b89b6b4cf..000000000 --- a/cmd/version.go +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright © 2015-2018 Aeneas Rekkas - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @author Aeneas Rekkas - * @Copyright 2015-2018 Aeneas Rekkas - * @license Apache-2.0 - * - */ - -package cmd - -import ( - "github.com/ory/x/cmdx" - - "github.com/ory/keto/internal/driver/config" -) - -func init() { - RootCmd.AddCommand(cmdx.Version(&config.Version, &config.Commit, &config.Date)) -} diff --git a/go.mod b/go.mod index 90021652b..740122d43 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/HdrHistogram/hdrhistogram-go v1.0.1 // indirect github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect github.com/bufbuild/buf v0.31.1 + github.com/cenkalti/backoff v2.2.1+incompatible github.com/containerd/continuity v0.0.0-20200228182428-0f16d7a0959c // indirect github.com/ghodss/yaml v1.0.0 github.com/go-openapi/errors v0.19.4 @@ -26,8 +27,7 @@ require ( github.com/ory/graceful v0.1.1 github.com/ory/herodot v0.9.1 github.com/ory/jsonschema/v3 v3.0.1 - github.com/ory/viper v1.7.5 - github.com/ory/x v0.0.169 + github.com/ory/x v0.0.171 github.com/pelletier/go-toml v1.8.0 github.com/pkg/errors v0.9.1 github.com/rs/cors v1.6.0 @@ -40,7 +40,7 @@ require ( github.com/sqs/goreturns v0.0.0-20181028201513-538ac6014518 github.com/stretchr/testify v1.6.1 github.com/tidwall/gjson v1.6.0 - github.com/tidwall/sjson v1.1.1 // indirect + github.com/tidwall/sjson v1.1.1 github.com/uber/jaeger-lib v2.4.0+incompatible // indirect go.mongodb.org/mongo-driver v1.3.4 // indirect golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect diff --git a/go.sum b/go.sum index f77d8f716..f108527f4 100644 --- a/go.sum +++ b/go.sum @@ -970,6 +970,8 @@ github.com/ory/x v0.0.127/go.mod h1:FwUujfFuCj5d+xgLn4fGMYPnzriR5bdAIulFXMtnK0M= github.com/ory/x v0.0.128/go.mod h1:ykx1XOsl9taQtoW2yNvuxl/feEfTfrZTcbY1U7841tI= github.com/ory/x v0.0.169 h1:pIo6Y0b+I8nB8PnBVK6h7ESUE0cIiys85wLBfsmPi4o= github.com/ory/x v0.0.169/go.mod h1:8d8Mlj2/ho+80yGjhFOVgbRWdm4loEwon622JtsdZyo= +github.com/ory/x v0.0.171 h1:NFi2TycOG6KlNpqIorulJbezwzKoeck05UJbGZjcvMY= +github.com/ory/x v0.0.171/go.mod h1:8d8Mlj2/ho+80yGjhFOVgbRWdm4loEwon622JtsdZyo= github.com/parnurzeal/gorequest v0.2.15/go.mod h1:3Kh2QUMJoqw3icWAecsyzkpY7UzRfDhbRdTjtNwNiUE= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= diff --git a/internal/driver/config/provider.go b/internal/driver/config/provider.go index 4241b05f4..aba3439d8 100644 --- a/internal/driver/config/provider.go +++ b/internal/driver/config/provider.go @@ -18,7 +18,7 @@ type Provider interface { TracingServiceName() string TracingProvider() string TracingConfig() *tracing.Config - Set(key string, v interface{}) + Set(key string, v interface{}) error } const DSNMemory = "sqlite://:memory:?_fk=true" diff --git a/internal/driver/config/provider_koanf.go b/internal/driver/config/provider_koanf.go index 4e5e318e8..4a3f63f3c 100644 --- a/internal/driver/config/provider_koanf.go +++ b/internal/driver/config/provider_koanf.go @@ -63,7 +63,7 @@ func New(ctx context.Context, flags *pflag.FlagSet, l *logrusx.Logger) (Provider kp.p, err = configx.New( schema, - flags, + configx.WithFlags(flags), configx.WithStderrValidationReporter(), configx.WithImmutables(KeyDSN, "serve"), configx.OmitKeysFromTracing(KeyDSN), @@ -96,12 +96,15 @@ func (k *KoanfProvider) resetNamespaceManager() { k.nm = nil } -func (k *KoanfProvider) Set(key string, v interface{}) { - k.p.Set(key, v) +func (k *KoanfProvider) Set(key string, v interface{}) error { + if err := k.p.Set(key, v); err != nil { + return err + } if key == KeyNamespaces { k.resetNamespaceManager() } + return nil } func (k *KoanfProvider) RESTListenOn() string { diff --git a/internal/driver/config/provider_koanf_test.go b/internal/driver/config/provider_koanf_test.go index 5c8ce2f92..6fe7d612c 100644 --- a/internal/driver/config/provider_koanf_test.go +++ b/internal/driver/config/provider_koanf_test.go @@ -46,7 +46,7 @@ func TestKoanfNamespaceManager(t *testing.T) { return func(t *testing.T) { _, p := setup(t) - p.Set(KeyNamespaces, value) + require.NoError(t, p.Set(KeyNamespaces, value)) assertNamespaces(t, p, namespaces...) @@ -100,17 +100,17 @@ func TestKoanfNamespaceManager(t *testing.T) { Name: "n1", } - p.Set(KeyNamespaces, []*namespace.Namespace{n0}) + require.NoError(t, p.Set(KeyNamespaces, []*namespace.Namespace{n0})) assertNamespaces(t, p, n0) - p.Set(KeyNamespaces, []*namespace.Namespace{n1}) + require.NoError(t, p.Set(KeyNamespaces, []*namespace.Namespace{n1})) assertNamespaces(t, p, n1) }) t.Run("case=creates watcher manager when namespaces is string URL", func(t *testing.T) { _, p := setup(t) - p.Set(KeyNamespaces, "file://"+t.TempDir()) + require.NoError(t, p.Set(KeyNamespaces, "file://"+t.TempDir())) nm, err := p.NamespaceManager() require.NoError(t, err) diff --git a/internal/driver/registry.go b/internal/driver/registry.go index 982c83666..8bfd819ae 100644 --- a/internal/driver/registry.go +++ b/internal/driver/registry.go @@ -15,23 +15,29 @@ import ( "github.com/ory/keto/internal/x" ) -type Registry interface { - Init(context.Context) error - BuildVersion() string - BuildDate() string - BuildHash() string - Config() config.Provider - - x.LoggerProvider - x.WriterProvider - - relationtuple.ManagerProvider - namespace.MigratorProvider - expand.EngineProvider - check.EngineProvider - persistence.MigratorProvider - persistence.Provider - - HealthHandler() *healthx.Handler - Tracer() *tracing.Tracer -} +type ( + Registry interface { + Init(context.Context) error + BuildVersion() string + BuildDate() string + BuildHash() string + Config() config.Provider + + x.LoggerProvider + x.WriterProvider + + relationtuple.ManagerProvider + namespace.MigratorProvider + expand.EngineProvider + check.EngineProvider + persistence.MigratorProvider + persistence.Provider + + HealthHandler() *healthx.Handler + Tracer() *tracing.Tracer + } + + contextKeys string +) + +const LogrusHookContextKey contextKeys = "logrus hook" diff --git a/internal/driver/registry_default.go b/internal/driver/registry_default.go index fe1f1107e..9e1956fcd 100644 --- a/internal/driver/registry_default.go +++ b/internal/driver/registry_default.go @@ -2,6 +2,10 @@ package driver import ( "context" + "time" + + "github.com/cenkalti/backoff" + "github.com/ory/x/sqlcon" "github.com/ory/keto/internal/driver/config" @@ -28,14 +32,14 @@ var _ x.LoggerProvider = &RegistryDefault{} var _ Registry = &RegistryDefault{} type RegistryDefault struct { - p persistence.Persister - l *logrusx.Logger - w herodot.Writer - ce *check.Engine - ee *expand.Engine - conn *pop.Connection - c config.Provider - hh *healthx.Handler + p persistence.Persister + l *logrusx.Logger + w herodot.Writer + ce *check.Engine + ee *expand.Engine + conn *pop.Connection + c config.Provider + health *healthx.Handler } func (r *RegistryDefault) BuildVersion() string { @@ -55,11 +59,11 @@ func (r *RegistryDefault) Config() config.Provider { } func (r *RegistryDefault) HealthHandler() *healthx.Handler { - if r.hh == nil { - r.hh = healthx.NewHandler(r.Writer(), config.Version, healthx.ReadyCheckers{}) + if r.health == nil { + r.health = healthx.NewHandler(r.Writer(), config.Version, healthx.ReadyCheckers{}) } - return r.hh + return r.health } func (r *RegistryDefault) Tracer() *tracing.Tracer { @@ -68,7 +72,7 @@ func (r *RegistryDefault) Tracer() *tracing.Tracer { func (r *RegistryDefault) Logger() *logrusx.Logger { if r.l == nil { - r.l = logrusx.New("keto", "dev") + r.l = logrusx.New("ORY Keto", config.Version) } return r.l } @@ -111,16 +115,32 @@ func (r *RegistryDefault) Migrator() persistence.Migrator { } func (r *RegistryDefault) Init(ctx context.Context) error { - c, err := pop.NewConnection(&pop.ConnectionDetails{ - URL: r.c.DSN(), - }) - if err != nil { - return errors.WithStack(err) - } + bc := backoff.NewExponentialBackOff() + bc.MaxElapsedTime = time.Minute * 5 + bc.Reset() + + if err := backoff.Retry(func() error { + pool, idlePool, connMaxLifetime, cleanedDSN := sqlcon.ParseConnectionOptions(r.l, r.c.DSN()) + c, err := pop.NewConnection(&pop.ConnectionDetails{ + URL: sqlcon.FinalizeDSN(r.l, cleanedDSN), + IdlePool: idlePool, + ConnMaxLifetime: connMaxLifetime, + Pool: pool, + }) + if err != nil { + r.Logger().WithError(err).Warnf("Unable to connect to database, retrying.") + return errors.WithStack(err) + } + + r.conn = c + if err := c.Open(); err != nil { + r.Logger().WithError(err).Warnf("Unable to open the database connection, retrying.") + return errors.WithStack(err) + } - r.conn = c - if err := c.Open(); err != nil { - return errors.WithStack(err) + return nil + }, bc); err != nil { + return err } nm, err := r.c.NamespaceManager() diff --git a/internal/driver/registry_factory.go b/internal/driver/registry_factory.go index ccfccaf5c..621a10af2 100644 --- a/internal/driver/registry_factory.go +++ b/internal/driver/registry_factory.go @@ -4,9 +4,12 @@ import ( "context" "testing" + "github.com/pkg/errors" + + "github.com/sirupsen/logrus/hooks/test" + "github.com/ory/keto/internal/namespace" - "github.com/ory/x/configx" "github.com/ory/x/logrusx" "github.com/spf13/pflag" "github.com/stretchr/testify/require" @@ -14,12 +17,19 @@ import ( "github.com/ory/keto/internal/driver/config" ) -func NewDefaultRegistry(ctx context.Context, flags *pflag.FlagSet) Registry { - l := logrusx.New("ORY Keto", config.Version) +func NewDefaultRegistry(ctx context.Context, flags *pflag.FlagSet) (Registry, error) { + hook, ok := ctx.Value(LogrusHookContextKey).(*test.Hook) + + var opts []logrusx.Option + if ok { + opts = append(opts, logrusx.WithHook(hook)) + } + + l := logrusx.New("ORY Keto", config.Version, opts...) c, err := config.New(ctx, flags, l) if err != nil { - l.WithError(err).Fatal("Unable to initialize config provider.") + return nil, errors.Wrap(err, "unable to initialize config provider") } r := &RegistryDefault{ @@ -28,25 +38,21 @@ func NewDefaultRegistry(ctx context.Context, flags *pflag.FlagSet) Registry { } if err = r.Init(ctx); err != nil { - l.WithError(err).Fatal("Unable to initialize service registry.") + return nil, errors.Wrap(err, "unable to initialize service registry") } - return r + return r, nil } func NewMemoryTestRegistry(t *testing.T, namespaces []*namespace.Namespace) Registry { - l := logrusx.New("keto", "test") + l := logrusx.New("ORY Keto", "testing") ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) - flags := pflag.NewFlagSet("test flags", pflag.ContinueOnError) - configx.RegisterFlags(flags) - require.NoError(t, flags.Set("config", "")) - - c, err := config.New(ctx, flags, l) + c, err := config.New(ctx, nil, l) require.NoError(t, err) - c.Set(config.KeyDSN, config.DSNMemory) - c.Set(config.KeyNamespaces, namespaces) + require.NoError(t, c.Set(config.KeyDSN, config.DSNMemory)) + require.NoError(t, c.Set(config.KeyNamespaces, namespaces)) r := &RegistryDefault{ c: c, diff --git a/internal/e2e/full_test.go b/internal/e2e/full_test.go new file mode 100644 index 000000000..b0b655a32 --- /dev/null +++ b/internal/e2e/full_test.go @@ -0,0 +1,172 @@ +package e2e + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strings" + "testing" + "time" + + "github.com/ory/keto/cmd/migrate" + + "github.com/ory/x/cmdx" + "github.com/ory/x/healthx" + "github.com/ory/x/sqlcon/dockertest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" + + "github.com/ory/keto/cmd" + "github.com/ory/keto/internal/driver/config" + "github.com/ory/keto/internal/namespace" + "github.com/ory/keto/internal/relationtuple" +) + +type dsnT struct { + name string + conn string + pre func(*testing.T, *cmdx.CommandExecuter, []*namespace.Namespace) +} + +func migrateEverythingUp(t *testing.T, c *cmdx.CommandExecuter, nn []*namespace.Namespace) { + out := cmdx.ExecNoErrCtx(c.Ctx, t, c.New(), c.PersistentArgs[0], c.PersistentArgs[1], "migrate", "status") + if strings.Contains(out, "Pending") { + c.ExecNoErr(t, "migrate", "up", "--"+migrate.FlagYes) + } + + for _, n := range nn { + c.ExecNoErr(t, "namespace", "migrate", n.Name) + } +} + +func Test(t *testing.T) { + // we use a slice of structs here to always have the same execution order + dsns := []*dsnT{{ + name: "memory", + conn: "memory", + pre: func(t *testing.T, c *cmdx.CommandExecuter, nn []*namespace.Namespace) { + // check if migrations are auto applied for dsn=memory + out := c.ExecNoErr(t, "migrate", "status") + assert.Contains(t, out, "Applied") + assert.NotContains(t, out, "Pending") + + for _, n := range nn { + out = c.ExecNoErr(t, "namespace", "migrate", n.Name) + assert.Contains(t, out, "already migrated") + } + }, + }} + if !testing.Short() { + dsns = append(dsns, + &dsnT{ + name: "mysql", + conn: dockertest.RunTestMySQL(t), + pre: migrateEverythingUp, + }, + &dsnT{ + name: "postgres", + conn: dockertest.RunTestPostgreSQL(t), + pre: migrateEverythingUp, + }, + &dsnT{ + name: "cockroach", + conn: dockertest.RunTestCockroachDB(t), + pre: migrateEverythingUp, + }, + ) + } + + for _, dsn := range dsns { + t.Run(fmt.Sprintf("dsn=%s", dsn.name), func(t *testing.T) { + _, ctx := setup(t) + + nspaces := []*namespace.Namespace{{ + Name: "dreams", + ID: 0, + }} + + c := &cmdx.CommandExecuter{ + New: cmd.NewRootCmd, + Ctx: ctx, + PersistentArgs: []string{"--config", configFile(t, map[string]interface{}{ + config.KeyDSN: dsn.conn, + config.KeyNamespaces: nspaces, + "log.level": "debug", + })}, + } + + dsn.pre(t, c, nspaces) + + // start the server + serverCtx, serverCancel := context.WithCancel(ctx) + serverDoneChannel := make(chan struct{}) + go func() { + cmdx.ExecNoErrCtx(serverCtx, t, cmd.NewRootCmd(), append(c.PersistentArgs, "serve")...) + close(serverDoneChannel) + }() + + // wait for /health/ready + for _, err := http.Get("http://127.0.0.1:4466" + healthx.ReadyCheckPath); err != nil; _, err = http.Get("http://127.0.0.1:4466" + healthx.ReadyCheckPath) { + time.Sleep(10 * time.Millisecond) + } + + lndTuple := &relationtuple.InternalRelationTuple{ + Namespace: nspaces[0].Name, + Object: "last nights dream", + Relation: "see", + Subject: &relationtuple.SubjectID{ID: "me"}, + } + lndTupleEnc, err := json.Marshal(lndTuple) + require.NoError(t, err) + + // create a relation tuple -- TODO use CLI commands instead + relationTuple := &bytes.Buffer{} + require.NoError(t, json.NewEncoder(relationTuple).Encode(lndTuple)) + + //stdout, stderr, err := c.Exec(t, relationTuple, "relation-tuple", "create", "-", "--"+client.FlagRemoteURL, "127.0.0.1:4467") + //require.NoError(t, err, "stdout: %s\nstderr: %s", stdout, stderr) + //assert.Len(t, stderr, 0, stdout) + + req, err := http.NewRequest(http.MethodPut, "http://127.0.0.1:4466/relationtuple", bytes.NewBuffer(lndTupleEnc)) + require.NoError(t, err) + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + assert.Equal(t, http.StatusCreated, resp.StatusCode) + + resp, err = http.Get(fmt.Sprintf("http://127.0.0.1:4466/relationtuple?namespace=%s", nspaces[0].Name)) + require.NoError(t, err) + d, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + assert.Equal(t, string(lndTupleEnc), gjson.GetBytes(d, "relations.0").String()) + + //relationTuple.Reset() + //require.NoError(t, json.NewEncoder(relationTuple).Encode(&relationtuple.InternalRelationTuple{ + // Namespace: namesp.Name, + // Object: "last nights dream", + // Relation: "see", + // Subject: &relationtuple.SubjectID{ID: "nightmare"}, + //})) + // + //stdout, stderr, err = c.Exec(t, relationTuple, "relationtuple", "create", "-") + //require.NoError(t, err) + //assert.Len(t, stderr, 0, stdout) + + // try the check API to see whether the tuple is interpreted correctly + resp, err = http.Get(fmt.Sprintf("http://127.0.0.1:4466/check?%s", lndTuple.ToURLQuery().Encode())) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + // stop the server + serverCancel() + // wait for it to stop + <-serverDoneChannel + }) + } +} diff --git a/internal/e2e/helpers.go b/internal/e2e/helpers.go new file mode 100644 index 000000000..e76665ccb --- /dev/null +++ b/internal/e2e/helpers.go @@ -0,0 +1,41 @@ +package e2e + +import ( + "context" + "io/ioutil" + "path/filepath" + "testing" + + "github.com/sirupsen/logrus/hooks/test" + "github.com/stretchr/testify/require" + "github.com/tidwall/sjson" + + "github.com/ory/keto/internal/driver" +) + +func configFile(t *testing.T, values map[string]interface{}) string { + dir := t.TempDir() + fn := filepath.Join(dir, "keto.yml") + + c := []byte("{}") + for key, val := range values { + var err error + c, err = sjson.SetBytes(c, key, val) + require.NoError(t, err) + } + + require.NoError(t, ioutil.WriteFile(fn, c, 0600)) + + return fn +} + +func setup(t *testing.T) (*test.Hook, context.Context) { + hook := &test.Hook{} + ctx, cancel := context.WithCancel(context.WithValue(context.Background(), driver.LogrusHookContextKey, hook)) + t.Cleanup(func() { + cancel() + + }) + + return hook, ctx +} diff --git a/internal/persistence/sql/namespace.go b/internal/persistence/sql/namespace.go index d2c9241f3..c8b83361a 100644 --- a/internal/persistence/sql/namespace.go +++ b/internal/persistence/sql/namespace.go @@ -57,12 +57,12 @@ func (p *Persister) MigrateNamespaceUp(ctx context.Context, n *namespace.Namespa Version: mostRecentSchemaVersion, } - if err := c.RawQuery(fmt.Sprintf("INSERT INTO %s (id, schema_version) VALUES (?, ?)", nr.TableName()), nr.ID, nr.Version).Exec(); err != nil { + // first create the table because of cockroach limitations, see https://github.com/cockroachdb/cockroach/issues/54477 + if err := c.RawQuery(createStmt(n)).Exec(); err != nil { return errors.WithStack(err) } - return errors.WithStack( - c.RawQuery(createStmt(n)).Exec()) + return errors.WithStack(c.RawQuery(fmt.Sprintf("INSERT INTO %s (id, schema_version) VALUES (?, ?)", nr.TableName()), nr.ID, nr.Version).Exec()) }) } diff --git a/internal/persistence/sql/persister.go b/internal/persistence/sql/persister.go index 60976dfe9..4541f4a66 100644 --- a/internal/persistence/sql/persister.go +++ b/internal/persistence/sql/persister.go @@ -45,6 +45,8 @@ var ( ) func NewPersister(c *pop.Connection, l *logrusx.Logger, namespaces namespace.Manager) (*Persister, error) { + pop.SetLogger(l.PopLogger) + mb, err := pkgerx.NewMigrationBox(migrations, c, l) if err != nil { return nil, err diff --git a/internal/relationtuple/definitions.go b/internal/relationtuple/definitions.go index 5a9abfb1e..d5e30f28f 100644 --- a/internal/relationtuple/definitions.go +++ b/internal/relationtuple/definitions.go @@ -2,10 +2,13 @@ package relationtuple import ( "context" + "encoding/json" "fmt" "net/url" "strings" + "github.com/tidwall/sjson" + "github.com/pkg/errors" acl "github.com/ory/keto/api/keto/acl/v1alpha1" @@ -194,6 +197,17 @@ func (r *InternalRelationTuple) UnmarshalJSON(raw []byte) error { return nil } +func (r *InternalRelationTuple) MarshalJSON() ([]byte, error) { + type t InternalRelationTuple + + enc, err := json.Marshal((*t)(r)) + if err != nil { + return nil, errors.WithStack(err) + } + + return sjson.SetBytes(enc, "subject", r.Subject.String()) +} + func (r *InternalRelationTuple) FromGRPC(gr *acl.RelationTuple) *InternalRelationTuple { r.Subject = SubjectFromGRPC(gr.Subject) r.Object = gr.Object @@ -228,6 +242,15 @@ func (r *InternalRelationTuple) FromURLQuery(query url.Values) (*InternalRelatio return r, nil } +func (r *InternalRelationTuple) ToURLQuery() url.Values { + return url.Values{ + "namespace": []string{r.Namespace}, + "object": []string{r.Object}, + "relation": []string{r.Relation}, + "subject": []string{r.Subject.String()}, + } +} + func (q *RelationQuery) FromGRPC(query *acl.ListRelationTuplesRequest_Query) *RelationQuery { return &RelationQuery{ Namespace: query.Namespace, diff --git a/internal/relationtuple/handler.go b/internal/relationtuple/handler.go index 6d723dd81..f8e1411c2 100644 --- a/internal/relationtuple/handler.go +++ b/internal/relationtuple/handler.go @@ -27,7 +27,7 @@ type ( ) const ( - routeBase = "/relations" + routeBase = "/relationtuple" ) func NewHandler(d handlerDeps) *handler { diff --git a/scripts/test-e2e.sh b/scripts/test-e2e.sh deleted file mode 100755 index 80496843a..000000000 --- a/scripts/test-e2e.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -cd "$( dirname "${BASH_SOURCE[0]}" )/.." - -killall keto || true - -DATABASE_URL=memory keto serve --disable-telemetry & -while ! echo exit | nc 127.0.0.1 4466; do sleep 1; done - -# Explicitly run without endpoint to see if that's working properly. -export KETO_URL=http://127.0.0.1:4466/ -keto engines acp ory policies import regex ./tests/stubs/policies.json - -# And check if it's working without trailing slash -export KETO_URL=http://127.0.0.1:4466 -keto engines acp ory policies import exact ./tests/stubs/policies.json - -# One more for the glob endpoint -keto engines acp ory policies import glob ./tests/stubs/policies.json - -# Now explicitly check if that works with the --endpoint flag -keto engines --endpoint http://localhost:4466 acp ory roles import regex ./tests/stubs/roles.json -# And with slash -keto engines --endpoint http://localhost:4466/ acp ory roles import exact ./tests/stubs/roles.json -# And with globs -keto engines --endpoint http://localhost:4466/ acp ory roles import glob ./tests/stubs/roles.json - -# Importing data is done, let's perform some checks - -exit $(keto engines --endpoint http://localhost:4466 acp ory allowed regex peter-1 resources-11 actions-11 | grep -c '"allowed": false') -exit $(keto engines --endpoint http://localhost:4466 acp ory allowed regex maria-1 resources-11 actions-11 | grep -c '"allowed": false') -exit $(keto engines --endpoint http://localhost:4466 acp ory allowed regex group-1 resources-11 actions-11 | grep -c '"allowed": false') - -exit $(keto engines --endpoint http://localhost:4466 acp ory allowed regex not-exist resources-11 actions-11 | grep -c '"allowed": true') - -exit $(keto engines --endpoint http://localhost:4466 acp ory allowed exact peter-1 resources-11 actions-11 | grep -c '"allowed": false') -exit $(keto engines --endpoint http://localhost:4466 acp ory allowed exact maria-1 resources-11 actions-11 | grep -c '"allowed": false') -exit $(keto engines --endpoint http://localhost:4466 acp ory allowed exact group-1 resources-11 actions-11 | grep -c '"allowed": false') - -exit $(keto engines --endpoint http://localhost:4466 acp ory allowed exact not-exist resources-11 actions-11 | grep -c '"allowed": true') - -exit $(keto engines --endpoint http://localhost:4466 acp ory allowed glob peter-1 resources-11 actions-11 | grep -c '"allowed": false') -exit $(keto engines --endpoint http://localhost:4466 acp ory allowed glob maria-1 resources-11 actions-11 | grep -c '"allowed": false') -exit $(keto engines --endpoint http://localhost:4466 acp ory allowed glob group-1 resources-11 actions-11 | grep -c '"allowed": false') - -exit $(keto engines --endpoint http://localhost:4466 acp ory allowed glob not-exist resources-11 actions-11 | grep -c '"allowed": true') - - -kill %1 -exit 0 diff --git a/scripts/test-resetdb.sh b/scripts/test-resetdb.sh new file mode 100755 index 000000000..ff6b8974b --- /dev/null +++ b/scripts/test-resetdb.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +docker rm -f test_keto_postgres || true +docker rm -f test_keto_mysql || true +docker rm -f test_keto_cockroach || true + +postgres_port="$(docker port "$(docker run --name test_keto_postgres -e "POSTGRES_PASSWORD=secret" -e "POSTGRES_DB=postgres" -p 0:5432 -d postgres:11.8)" 5432 | sed 's/.*:\([0-9]*\)/\1/')" +mysql_port="$(docker port "$(docker run --name test_keto_mysql -e "MYSQL_ROOT_PASSWORD=secret" -p 0:3306 -d mysql:8.0)" 3306 | sed 's/.*:\([0-9]*\)/\1/')" +cockroach_port="$(docker port "$(docker run --name test_keto_cockroach -p 0:26257 -d cockroachdb/cockroach:v20.1.0 start --insecure)" 26257 | sed 's/.*:\([0-9]*\)/\1/')" + +TEST_DATABASE_POSTGRESQL=$(printf "postgres://postgres:secret@localhost:%s/postgres?sslmode=disable" "$postgres_port") +TEST_DATABASE_MYSQL=$(printf "mysql://root:secret@(localhost:%s)/mysql?parseTime=true&multiStatements=true" "$mysql_port") +TEST_DATABASE_COCKROACHDB=$(printf "cockroach://root@localhost:%s/defaultdb?sslmode=disable" "$cockroach_port") + +export TEST_DATABASE_POSTGRESQL +export TEST_DATABASE_MYSQL +export TEST_DATABASE_COCKROACHDB + +# undo set from above +set +e +set +u +set +x +set +o pipefail From 7814cfb6227dcfbc56e8301c8ede02e7ac29ac79 Mon Sep 17 00:00:00 2001 From: zepatrik Date: Fri, 18 Dec 2020 17:21:29 +0100 Subject: [PATCH 3/5] ci: fix mysql connection url --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ce5bed99f..c5495d0bd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,7 +16,7 @@ jobs: image: circleci/golang:1.15 environment: TEST_DATABASE_POSTGRESQL: postgres://test:test@localhost:5432/keto?sslmode=disable - TEST_DATABASE_MYSQL: root:test@(localhost:3306)/mysql?parseTime=true&multiStatements=true + TEST_DATABASE_MYSQL: mysql://root:test@(localhost:3306)/mysql?parseTime=true&multiStatements=true TEST_DATABASE_COCKROACH: cockroach://root@localhost:26257/defaultdb?sslmode=disable - image: postgres:11.8 From b1279d6ddadfdc41ec6a434cc8070680e42a4dea Mon Sep 17 00:00:00 2001 From: zepatrik Date: Fri, 18 Dec 2020 17:30:49 +0100 Subject: [PATCH 4/5] ci: fix cockroach env var name --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c5495d0bd..1cdeeb8d3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,7 +17,7 @@ jobs: environment: TEST_DATABASE_POSTGRESQL: postgres://test:test@localhost:5432/keto?sslmode=disable TEST_DATABASE_MYSQL: mysql://root:test@(localhost:3306)/mysql?parseTime=true&multiStatements=true - TEST_DATABASE_COCKROACH: cockroach://root@localhost:26257/defaultdb?sslmode=disable + TEST_DATABASE_COCKROACHDB: cockroach://root@localhost:26257/defaultdb?sslmode=disable - image: postgres:11.8 environment: From 9a0f013108208fc293ba4d9ad0f596c27d8b3a9c Mon Sep 17 00:00:00 2001 From: zepatrik Date: Mon, 21 Dec 2020 16:04:24 +0100 Subject: [PATCH 5/5] feat: separate GRPC and REST client in e2e test --- Makefile | 4 +- cmd/migrate/down.go | 40 +++++ cmd/migrate/root.go | 6 +- cmd/namespace/migrate_down.go | 50 ++++++ cmd/namespace/{migrate.go => migrate_up.go} | 4 +- cmd/namespace/root.go | 11 +- internal/check/handler.go | 4 +- internal/e2e/cases_test.go | 73 +++++++++ internal/e2e/full_suit_test.go | 137 ++++++++++++++++ internal/e2e/full_test.go | 172 -------------------- internal/e2e/grpc_client_test.go | 37 +++++ internal/e2e/helpers.go | 26 +++ internal/e2e/rest_client_test.go | 87 ++++++++++ internal/expand/engine.go | 5 +- internal/expand/handler.go | 4 +- internal/expand/tree.go | 38 ++++- internal/namespace/definitons.go | 1 + internal/persistence/definitions.go | 1 + internal/persistence/sql/namespace.go | 21 +++ internal/persistence/sql/persister.go | 6 +- internal/relationtuple/definitions.go | 39 ++++- internal/relationtuple/handler.go | 6 +- 22 files changed, 577 insertions(+), 195 deletions(-) create mode 100644 cmd/migrate/down.go create mode 100644 cmd/namespace/migrate_down.go rename cmd/namespace/{migrate.go => migrate_up.go} (97%) create mode 100644 internal/e2e/cases_test.go create mode 100644 internal/e2e/full_suit_test.go delete mode 100644 internal/e2e/full_test.go create mode 100644 internal/e2e/grpc_client_test.go create mode 100644 internal/e2e/rest_client_test.go diff --git a/Makefile b/Makefile index f248400a5..a04d2a0d7 100644 --- a/Makefile +++ b/Makefile @@ -99,6 +99,6 @@ buf: buf-lint buf-gen reset-testdb: source scripts/test-resetdb.sh -.PHONY: e2e-test -e2e-test: +.PHONY: test-e2e +test-e2e: go test -tags sqlite -failfast -v ./internal/e2e diff --git a/cmd/migrate/down.go b/cmd/migrate/down.go new file mode 100644 index 000000000..4e25bd268 --- /dev/null +++ b/cmd/migrate/down.go @@ -0,0 +1,40 @@ +package migrate + +import ( + "fmt" + "strconv" + + "github.com/ory/x/cmdx" + "github.com/spf13/cobra" + + "github.com/ory/keto/internal/driver" +) + +func newDownCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "down ", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + steps, err := strconv.ParseInt(args[0], 0, 0) + if err != nil { + // return this error so it gets printed along the usage + return fmt.Errorf("malformed argument %s for : %+v", args[0], err) + } + + reg, err := driver.NewDefaultRegistry(ctx, cmd.Flags()) + if err != nil { + return err + } + if err := reg.Migrator().MigrateDown(ctx, int(steps)); err != nil { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could apply down migrations: %+v\n", err) + return cmdx.FailSilently(cmd) + } + + return nil + }, + } + + return cmd +} diff --git a/cmd/migrate/root.go b/cmd/migrate/root.go index b521de9c5..c52eff6f4 100644 --- a/cmd/migrate/root.go +++ b/cmd/migrate/root.go @@ -11,7 +11,11 @@ func newMigrateCmd() *cobra.Command { func RegisterCommandsRecursive(parent *cobra.Command) { migrateCmd := newMigrateCmd() - migrateCmd.AddCommand(newStatusCmd(), newUpCmd()) + migrateCmd.AddCommand( + newStatusCmd(), + newUpCmd(), + newDownCmd(), + ) parent.AddCommand(migrateCmd) } diff --git a/cmd/namespace/migrate_down.go b/cmd/namespace/migrate_down.go new file mode 100644 index 000000000..4db138b58 --- /dev/null +++ b/cmd/namespace/migrate_down.go @@ -0,0 +1,50 @@ +package namespace + +import ( + "fmt" + + "github.com/ory/x/cmdx" + "github.com/spf13/cobra" + + "github.com/ory/keto/internal/driver" +) + +func NewMigrateDownCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "down ", + Short: "Migrate a namespace down.", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + reg, err := driver.NewDefaultRegistry(ctx, cmd.Flags()) + if err != nil { + return err + } + + nm, err := reg.Config().NamespaceManager() + if err != nil { + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Could not initialize the namespace manager: %+v\n", err) + return cmdx.FailSilently(cmd) + } + + n, err := nm.GetNamespace(ctx, args[0]) + if err != nil { + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Could not find the namespace with name \"%s\": %+v\n", args[0], err) + return cmdx.FailSilently(cmd) + } + + if err := reg.NamespaceMigrator().MigrateNamespaceDown(ctx, n, 0); err != nil { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not apply namespace migration: %+v\n", err) + return cmdx.FailSilently(cmd) + } + + return nil + }, + } + + registerYesFlag(cmd.Flags()) + registerPackageFlags(cmd.Flags()) + + return cmd +} diff --git a/cmd/namespace/migrate.go b/cmd/namespace/migrate_up.go similarity index 97% rename from cmd/namespace/migrate.go rename to cmd/namespace/migrate_up.go index bbc8f1f2f..4527563f5 100644 --- a/cmd/namespace/migrate.go +++ b/cmd/namespace/migrate_up.go @@ -13,9 +13,9 @@ import ( "github.com/ory/keto/internal/persistence" ) -func NewMigrateCmd() *cobra.Command { +func NewMigrateUpCmd() *cobra.Command { cmd := &cobra.Command{ - Use: "migrate ", + Use: "up ", Short: "Migrate a namespace up.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/namespace/root.go b/cmd/namespace/root.go index 31c13a7a9..c7ca1d076 100644 --- a/cmd/namespace/root.go +++ b/cmd/namespace/root.go @@ -14,9 +14,18 @@ func NewNamespaceCmd() *cobra.Command { } } +func NewMigrateCmd() *cobra.Command { + return &cobra.Command{ + Use: "migrate", + } +} + func RegisterCommandsRecursive(parent *cobra.Command) { rootCmd := NewNamespaceCmd() - rootCmd.AddCommand(NewMigrateCmd(), NewValidateCmd()) + migrateCmd := NewMigrateCmd() + migrateCmd.AddCommand(NewMigrateUpCmd(), NewMigrateDownCmd()) + + rootCmd.AddCommand(migrateCmd, NewValidateCmd()) parent.AddCommand(rootCmd) } diff --git a/internal/check/handler.go b/internal/check/handler.go index f1c3d9dfd..83c4d8176 100644 --- a/internal/check/handler.go +++ b/internal/check/handler.go @@ -33,10 +33,10 @@ func NewHandler(d handlerDependencies) *restHandler { return &restHandler{d: d} } -const routeBase = "/check" +const RouteBase = "/check" func (h *restHandler) RegisterPublicRoutes(router *httprouter.Router) { - router.GET(routeBase, h.getCheck) + router.GET(RouteBase, h.getCheck) } func (h *restHandler) getCheck(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { diff --git a/internal/e2e/cases_test.go b/internal/e2e/cases_test.go new file mode 100644 index 000000000..19fe1c133 --- /dev/null +++ b/internal/e2e/cases_test.go @@ -0,0 +1,73 @@ +package e2e + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/ory/keto/internal/expand" + "github.com/ory/keto/internal/namespace" + "github.com/ory/keto/internal/relationtuple" +) + +func runCases(c client, nspaces []*namespace.Namespace) func(*testing.T) { + return func(t *testing.T) { + t.Run("case=creates tuple and uses it then", func(t *testing.T) { + tuple := &relationtuple.InternalRelationTuple{ + Namespace: nspaces[0].Name, + Object: fmt.Sprintf("object for client %T", c), + Relation: "access", + Subject: &relationtuple.SubjectID{ID: "client"}, + } + + c.createTuple(t, tuple) + + allTuple := c.queryTuple(t, &relationtuple.RelationQuery{Namespace: tuple.Namespace}) + + assert.Contains(t, allTuple, tuple) + + // try the check API to see whether the tuple is interpreted correctly + assert.True(t, c.check(t, tuple)) + }) + + t.Run("case=expand API", func(t *testing.T) { + obj := fmt.Sprintf("tree for client %T", c) + rel := "expand" + + subjects := []string{"s1", "s2"} + expectedTree := &expand.Tree{ + Type: expand.Union, + Subject: &relationtuple.SubjectSet{ + Namespace: nspaces[0].Name, + Object: obj, + Relation: rel, + }, + Children: make([]*expand.Tree, len(subjects)), + } + + for i, subjectID := range subjects { + c.createTuple(t, &relationtuple.InternalRelationTuple{ + Namespace: nspaces[0].Name, + Object: obj, + Relation: rel, + Subject: &relationtuple.SubjectID{ID: subjectID}, + }) + expectedTree.Children[i] = &expand.Tree{ + Type: expand.Leaf, + Subject: &relationtuple.SubjectID{ID: subjectID}, + } + } + + actualTree := c.expand(t, expectedTree.Subject.(*relationtuple.SubjectSet), 100) + + assert.Equal(t, expectedTree.Type, actualTree.Type) + assert.Equal(t, expectedTree.Subject, actualTree.Subject) + assert.Equal(t, len(expectedTree.Children), len(actualTree.Children), "expected: %+v; actual: %+v", expectedTree.Children, actualTree.Children) + + for _, child := range expectedTree.Children { + assert.Contains(t, actualTree.Children, child) + } + }) + } +} diff --git a/internal/e2e/full_suit_test.go b/internal/e2e/full_suit_test.go new file mode 100644 index 000000000..47b3cb9bb --- /dev/null +++ b/internal/e2e/full_suit_test.go @@ -0,0 +1,137 @@ +package e2e + +import ( + "context" + "fmt" + "net/http" + "testing" + "time" + + "github.com/ory/keto/internal/expand" + + "github.com/ory/x/cmdx" + "github.com/ory/x/healthx" + "github.com/ory/x/sqlcon/dockertest" + "github.com/stretchr/testify/assert" + + "github.com/ory/keto/cmd" + cliclient "github.com/ory/keto/cmd/client" + "github.com/ory/keto/internal/driver/config" + "github.com/ory/keto/internal/namespace" + "github.com/ory/keto/internal/relationtuple" +) + +type ( + dsnT struct { + name string + conn string + pre func(*testing.T, *cmdx.CommandExecuter, []*namespace.Namespace) + } + client interface { + createTuple(t *testing.T, r *relationtuple.InternalRelationTuple) + queryTuple(t *testing.T, q *relationtuple.RelationQuery) []*relationtuple.InternalRelationTuple + check(t *testing.T, r *relationtuple.InternalRelationTuple) bool + expand(t *testing.T, r *relationtuple.SubjectSet, depth int) *expand.Tree + } +) + +func Test(t *testing.T) { + // we use a slice of structs here to always have the same execution order + dsns := []*dsnT{{ + name: "memory", + conn: "memory", + pre: func(t *testing.T, c *cmdx.CommandExecuter, nn []*namespace.Namespace) { + // check if migrations are auto applied for dsn=memory + out := c.ExecNoErr(t, "migrate", "status") + assert.Contains(t, out, "Applied") + assert.NotContains(t, out, "Pending") + + for _, n := range nn { + out = c.ExecNoErr(t, "namespace", "migrate", "up", n.Name) + assert.Contains(t, out, "already migrated") + } + }, + }} + if !testing.Short() { + dsns = append(dsns, + &dsnT{ + name: "mysql", + conn: dockertest.RunTestMySQL(t), + pre: migrateEverythingUp, + }, + &dsnT{ + name: "postgres", + conn: dockertest.RunTestPostgreSQL(t), + pre: migrateEverythingUp, + }, + &dsnT{ + name: "cockroach", + conn: dockertest.RunTestCockroachDB(t), + pre: migrateEverythingUp, + }, + ) + } + + for _, dsn := range dsns { + t.Run(fmt.Sprintf("dsn=%s", dsn.name), func(t *testing.T) { + // We initialize and migrate everything for each DSN once + _, ctx := setup(t) + + nspaces := []*namespace.Namespace{{ + Name: "dreams", + ID: 0, + }} + + c := &cmdx.CommandExecuter{ + New: cmd.NewRootCmd, + Ctx: ctx, + PersistentArgs: []string{"--config", configFile(t, map[string]interface{}{ + config.KeyDSN: dsn.conn, + config.KeyNamespaces: nspaces, + "log.level": "debug", + })}, + } + + dsn.pre(t, c, nspaces) + // Initialization done + + // Start the server + serverCtx, serverCancel := context.WithCancel(ctx) + serverDoneChannel := make(chan struct{}) + go func() { + cmdx.ExecNoErrCtx(serverCtx, t, cmd.NewRootCmd(), append(c.PersistentArgs, "serve")...) + close(serverDoneChannel) + }() + + // defer this to make sure it is shutdown on test failure as well + defer func() { + // stop the server + serverCancel() + // wait for it to stop + <-serverDoneChannel + }() + + // wait for /health/ready + for _, err := http.Get("http://127.0.0.1:4466" + healthx.ReadyCheckPath); err != nil; _, err = http.Get("http://127.0.0.1:4466" + healthx.ReadyCheckPath) { + time.Sleep(10 * time.Millisecond) + } + + // The test cases start here + // We execute every test with the GRPC client (using the client commands) and REST client + for i, cl := range []client{ + &grpcClient{c: &cmdx.CommandExecuter{ + New: cmd.NewRootCmd, + Ctx: ctx, + PersistentArgs: []string{"--" + cliclient.FlagRemoteURL, "127.0.0.1:4467"}, + }}, + &restClient{baseURL: "http://127.0.0.1:4466"}, + } { + // TODO remove once GRPC client and handler are implemented + if i == 0 { + continue + } + t.Run(fmt.Sprintf("client=%T", cl), runCases(cl, nspaces)) + } + }) + } +} diff --git a/internal/e2e/full_test.go b/internal/e2e/full_test.go deleted file mode 100644 index b0b655a32..000000000 --- a/internal/e2e/full_test.go +++ /dev/null @@ -1,172 +0,0 @@ -package e2e - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "strings" - "testing" - "time" - - "github.com/ory/keto/cmd/migrate" - - "github.com/ory/x/cmdx" - "github.com/ory/x/healthx" - "github.com/ory/x/sqlcon/dockertest" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/tidwall/gjson" - - "github.com/ory/keto/cmd" - "github.com/ory/keto/internal/driver/config" - "github.com/ory/keto/internal/namespace" - "github.com/ory/keto/internal/relationtuple" -) - -type dsnT struct { - name string - conn string - pre func(*testing.T, *cmdx.CommandExecuter, []*namespace.Namespace) -} - -func migrateEverythingUp(t *testing.T, c *cmdx.CommandExecuter, nn []*namespace.Namespace) { - out := cmdx.ExecNoErrCtx(c.Ctx, t, c.New(), c.PersistentArgs[0], c.PersistentArgs[1], "migrate", "status") - if strings.Contains(out, "Pending") { - c.ExecNoErr(t, "migrate", "up", "--"+migrate.FlagYes) - } - - for _, n := range nn { - c.ExecNoErr(t, "namespace", "migrate", n.Name) - } -} - -func Test(t *testing.T) { - // we use a slice of structs here to always have the same execution order - dsns := []*dsnT{{ - name: "memory", - conn: "memory", - pre: func(t *testing.T, c *cmdx.CommandExecuter, nn []*namespace.Namespace) { - // check if migrations are auto applied for dsn=memory - out := c.ExecNoErr(t, "migrate", "status") - assert.Contains(t, out, "Applied") - assert.NotContains(t, out, "Pending") - - for _, n := range nn { - out = c.ExecNoErr(t, "namespace", "migrate", n.Name) - assert.Contains(t, out, "already migrated") - } - }, - }} - if !testing.Short() { - dsns = append(dsns, - &dsnT{ - name: "mysql", - conn: dockertest.RunTestMySQL(t), - pre: migrateEverythingUp, - }, - &dsnT{ - name: "postgres", - conn: dockertest.RunTestPostgreSQL(t), - pre: migrateEverythingUp, - }, - &dsnT{ - name: "cockroach", - conn: dockertest.RunTestCockroachDB(t), - pre: migrateEverythingUp, - }, - ) - } - - for _, dsn := range dsns { - t.Run(fmt.Sprintf("dsn=%s", dsn.name), func(t *testing.T) { - _, ctx := setup(t) - - nspaces := []*namespace.Namespace{{ - Name: "dreams", - ID: 0, - }} - - c := &cmdx.CommandExecuter{ - New: cmd.NewRootCmd, - Ctx: ctx, - PersistentArgs: []string{"--config", configFile(t, map[string]interface{}{ - config.KeyDSN: dsn.conn, - config.KeyNamespaces: nspaces, - "log.level": "debug", - })}, - } - - dsn.pre(t, c, nspaces) - - // start the server - serverCtx, serverCancel := context.WithCancel(ctx) - serverDoneChannel := make(chan struct{}) - go func() { - cmdx.ExecNoErrCtx(serverCtx, t, cmd.NewRootCmd(), append(c.PersistentArgs, "serve")...) - close(serverDoneChannel) - }() - - // wait for /health/ready - for _, err := http.Get("http://127.0.0.1:4466" + healthx.ReadyCheckPath); err != nil; _, err = http.Get("http://127.0.0.1:4466" + healthx.ReadyCheckPath) { - time.Sleep(10 * time.Millisecond) - } - - lndTuple := &relationtuple.InternalRelationTuple{ - Namespace: nspaces[0].Name, - Object: "last nights dream", - Relation: "see", - Subject: &relationtuple.SubjectID{ID: "me"}, - } - lndTupleEnc, err := json.Marshal(lndTuple) - require.NoError(t, err) - - // create a relation tuple -- TODO use CLI commands instead - relationTuple := &bytes.Buffer{} - require.NoError(t, json.NewEncoder(relationTuple).Encode(lndTuple)) - - //stdout, stderr, err := c.Exec(t, relationTuple, "relation-tuple", "create", "-", "--"+client.FlagRemoteURL, "127.0.0.1:4467") - //require.NoError(t, err, "stdout: %s\nstderr: %s", stdout, stderr) - //assert.Len(t, stderr, 0, stdout) - - req, err := http.NewRequest(http.MethodPut, "http://127.0.0.1:4466/relationtuple", bytes.NewBuffer(lndTupleEnc)) - require.NoError(t, err) - resp, err := http.DefaultClient.Do(req) - require.NoError(t, err) - require.NoError(t, resp.Body.Close()) - assert.Equal(t, http.StatusCreated, resp.StatusCode) - - resp, err = http.Get(fmt.Sprintf("http://127.0.0.1:4466/relationtuple?namespace=%s", nspaces[0].Name)) - require.NoError(t, err) - d, err := ioutil.ReadAll(resp.Body) - require.NoError(t, err) - require.NoError(t, resp.Body.Close()) - assert.Equal(t, string(lndTupleEnc), gjson.GetBytes(d, "relations.0").String()) - - //relationTuple.Reset() - //require.NoError(t, json.NewEncoder(relationTuple).Encode(&relationtuple.InternalRelationTuple{ - // Namespace: namesp.Name, - // Object: "last nights dream", - // Relation: "see", - // Subject: &relationtuple.SubjectID{ID: "nightmare"}, - //})) - // - //stdout, stderr, err = c.Exec(t, relationTuple, "relationtuple", "create", "-") - //require.NoError(t, err) - //assert.Len(t, stderr, 0, stdout) - - // try the check API to see whether the tuple is interpreted correctly - resp, err = http.Get(fmt.Sprintf("http://127.0.0.1:4466/check?%s", lndTuple.ToURLQuery().Encode())) - require.NoError(t, err) - require.NoError(t, resp.Body.Close()) - assert.Equal(t, http.StatusOK, resp.StatusCode) - - // stop the server - serverCancel() - // wait for it to stop - <-serverDoneChannel - }) - } -} diff --git a/internal/e2e/grpc_client_test.go b/internal/e2e/grpc_client_test.go new file mode 100644 index 000000000..066d12ca2 --- /dev/null +++ b/internal/e2e/grpc_client_test.go @@ -0,0 +1,37 @@ +package e2e + +import ( + "testing" + + "github.com/ory/x/cmdx" + + "github.com/ory/keto/internal/expand" + "github.com/ory/keto/internal/relationtuple" +) + +type grpcClient struct { + c *cmdx.CommandExecuter +} + +var _ client = &grpcClient{} + +func (g *grpcClient) createTuple(t *testing.T, r *relationtuple.InternalRelationTuple) { + + //stdout, stderr, err := c.Exec(t, relationTuple, "relation-tuple", "create", "-", "--"+client.FlagRemoteURL, "127.0.0.1:4467") + //require.NoError(t, err, "stdout: %s\nstderr: %s", stdout, stderr) + //assert.Len(t, stderr, 0, stdout) + + panic("implement me") +} + +func (g *grpcClient) queryTuple(t *testing.T, q *relationtuple.RelationQuery) []*relationtuple.InternalRelationTuple { + panic("implement me") +} + +func (g *grpcClient) check(t *testing.T, r *relationtuple.InternalRelationTuple) bool { + panic("implement me") +} + +func (g *grpcClient) expand(t *testing.T, r *relationtuple.SubjectSet, depth int) *expand.Tree { + panic("implement me") +} diff --git a/internal/e2e/helpers.go b/internal/e2e/helpers.go index e76665ccb..24fa272a3 100644 --- a/internal/e2e/helpers.go +++ b/internal/e2e/helpers.go @@ -4,8 +4,14 @@ import ( "context" "io/ioutil" "path/filepath" + "strings" "testing" + "github.com/ory/x/cmdx" + + "github.com/ory/keto/cmd/migrate" + "github.com/ory/keto/internal/namespace" + "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/require" "github.com/tidwall/sjson" @@ -39,3 +45,23 @@ func setup(t *testing.T) (*test.Hook, context.Context) { return hook, ctx } + +func migrateEverythingUp(t *testing.T, c *cmdx.CommandExecuter, nn []*namespace.Namespace) { + out := c.ExecNoErr(t, "migrate", "status") + if strings.Contains(out, "Pending") { + c.ExecNoErr(t, "migrate", "up", "--"+migrate.FlagYes) + } + + for _, n := range nn { + c.ExecNoErr(t, "namespace", "migrate", "up", n.Name) + } + + // TODO enable when namespace migrations are done properly with driver specific statements + //t.Cleanup(func() { + // for _, n := range nn { + // c.ExecNoErr(t, "namespace", "migrate", "down", n.Name, "1") + // } + // + // c.ExecNoErr(t, "migrate", "down", "1") + //}) +} diff --git a/internal/e2e/rest_client_test.go b/internal/e2e/rest_client_test.go new file mode 100644 index 000000000..c3d782f39 --- /dev/null +++ b/internal/e2e/rest_client_test.go @@ -0,0 +1,87 @@ +package e2e + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" + + "github.com/ory/keto/internal/check" + "github.com/ory/keto/internal/expand" + "github.com/ory/keto/internal/relationtuple" +) + +var _ client = &restClient{} + +type restClient struct { + baseURL string +} + +func (rc *restClient) makeRequest(t *testing.T, method, path, body string) (string, int) { + var b io.Reader + if body != "" { + b = bytes.NewBufferString(body) + } + + // t.Logf("Requesting %s %s%s with body %#v", method, rc.baseURL, path, body) + req, err := http.NewRequest(method, rc.baseURL+path, b) + require.NoError(t, err) + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + respBody, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + + return string(respBody), resp.StatusCode +} + +func (rc *restClient) createTuple(t *testing.T, r *relationtuple.InternalRelationTuple) { + tEnc, err := json.Marshal(r) + require.NoError(t, err) + + body, code := rc.makeRequest(t, http.MethodPut, relationtuple.RouteBase, string(tEnc)) + assert.Equal(t, http.StatusCreated, code, body) +} + +func (rc *restClient) queryTuple(t *testing.T, q *relationtuple.RelationQuery) []*relationtuple.InternalRelationTuple { + body, code := rc.makeRequest(t, http.MethodGet, fmt.Sprintf("%s?%s", relationtuple.RouteBase, q.ToURLQuery().Encode()), "") + require.Equal(t, http.StatusOK, code, body) + + tuple := make([]*relationtuple.InternalRelationTuple, 0, gjson.Get(body, "relations.#").Int()) + require.NoError(t, json.Unmarshal([]byte(gjson.Get(body, "relations").Raw), &tuple)) + + return tuple +} + +func (rc *restClient) check(t *testing.T, r *relationtuple.InternalRelationTuple) bool { + body, code := rc.makeRequest(t, http.MethodGet, fmt.Sprintf("%s?%s", check.RouteBase, r.ToURLQuery().Encode()), "") + + if code == http.StatusOK { + assert.Equal(t, `"allowed"`, body) // JSON string, therefore quoted + return true + } + + assert.Equal(t, http.StatusForbidden, code) + assert.Equal(t, `"rejected"`, body) // JSON string, therefore quoted + return false +} + +func (rc *restClient) expand(t *testing.T, r *relationtuple.SubjectSet, depth int) *expand.Tree { + query := r.ToURLQuery() + query.Set("depth", fmt.Sprintf("%d", depth)) + + body, code := rc.makeRequest(t, http.MethodGet, fmt.Sprintf("%s?%s", expand.RouteBase, query.Encode()), "") + require.Equal(t, http.StatusOK, code, body) + + tree := &expand.Tree{} + require.NoError(t, json.Unmarshal([]byte(body), tree)) + + return tree +} diff --git a/internal/expand/engine.go b/internal/expand/engine.go index cbf178e1e..626255efb 100644 --- a/internal/expand/engine.go +++ b/internal/expand/engine.go @@ -35,8 +35,9 @@ func (e *Engine) BuildTree(ctx context.Context, subject relationtuple.Subject, r // TODO handle pagination rels, _, err := e.d.RelationTupleManager().GetRelationTuples(ctx, &relationtuple.RelationQuery{ - Relation: us.Relation, - Object: us.Object, + Relation: us.Relation, + Object: us.Object, + Namespace: us.Namespace, }) if err != nil { // TODO error handling diff --git a/internal/expand/handler.go b/internal/expand/handler.go index 85caa151e..fe3952b74 100644 --- a/internal/expand/handler.go +++ b/internal/expand/handler.go @@ -22,14 +22,14 @@ type ( } ) -const routeBase = "/expand" +const RouteBase = "/expand" func NewHandler(d handlerDependencies) *handler { return &handler{d: d} } func (h *handler) RegisterPublicRoutes(router *httprouter.Router) { - router.GET(routeBase, h.getCheck) + router.GET(RouteBase, h.getCheck) } func (h *handler) getCheck(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { diff --git a/internal/expand/tree.go b/internal/expand/tree.go index 17ae3c4f2..1a4c8aaed 100644 --- a/internal/expand/tree.go +++ b/internal/expand/tree.go @@ -1,7 +1,9 @@ package expand import ( - "errors" + "encoding/json" + + "github.com/pkg/errors" "github.com/ory/keto/internal/relationtuple" ) @@ -11,7 +13,7 @@ type ( Tree struct { Type NodeType `json:"type"` Subject relationtuple.Subject `json:"subject"` - Children []*Tree `json:"children"` + Children []*Tree `json:"children,omitempty"` } ) @@ -46,16 +48,40 @@ func (t NodeType) MarshalJSON() ([]byte, error) { func (t *NodeType) UnmarshalJSON(v []byte) error { switch string(v) { - case "union": + case `"union"`: *t = Union - case "exclusion": + case `"exclusion"`: *t = Exclusion - case "intersection": + case `"intersection"`: *t = Intersection - case "leaf": + case `"leaf"`: *t = Leaf default: return ErrUnknownNodeType } return nil } + +func (t *Tree) UnmarshalJSON(v []byte) error { + type node struct { + Type NodeType `json:"type"` + Children []*Tree `json:"children,omitempty"` + Subject string `json:"subject"` + } + + n := &node{} + if err := json.Unmarshal(v, n); err != nil { + return errors.WithStack(err) + } + + var err error + t.Subject, err = relationtuple.SubjectFromString(n.Subject) + if err != nil { + return err + } + + t.Type = n.Type + t.Children = n.Children + + return nil +} diff --git a/internal/namespace/definitons.go b/internal/namespace/definitons.go index 0d1eb596c..dd9ee42ca 100644 --- a/internal/namespace/definitons.go +++ b/internal/namespace/definitons.go @@ -20,6 +20,7 @@ type ( } Migrator interface { MigrateNamespaceUp(ctx context.Context, n *Namespace) error + MigrateNamespaceDown(ctx context.Context, n *Namespace, steps int) error NamespaceStatus(ctx context.Context, id int) (*Status, error) } Manager interface { diff --git a/internal/persistence/definitions.go b/internal/persistence/definitions.go index 2a4480c16..5313944af 100644 --- a/internal/persistence/definitions.go +++ b/internal/persistence/definitions.go @@ -17,6 +17,7 @@ type ( } Migrator interface { MigrateUp(context.Context) error + MigrateDown(context.Context, int) error MigrationStatus(context.Context, io.Writer) error } MigratorProvider interface { diff --git a/internal/persistence/sql/namespace.go b/internal/persistence/sql/namespace.go index c8b83361a..7c2ce836b 100644 --- a/internal/persistence/sql/namespace.go +++ b/internal/persistence/sql/namespace.go @@ -36,6 +36,13 @@ CREATE TABLE %[1]s CREATE INDEX %[1]s_object_idx ON %[1]s (object); CREATE INDEX %[1]s_user_set_idx ON %[1]s (object, relation); +` + namespaceDropStatement = ` +DROP INDEX %[1]s_user_set_idx; + +DROP INDEX %[1]s_object_idx; + +DROP TABLE %[1]s; ` mostRecentSchemaVersion = 1 @@ -49,6 +56,10 @@ func createStmt(n *namespace.Namespace) string { return fmt.Sprintf(namespaceCreateStatement, tableFromNamespace(n)) } +func dropStmt(n *namespace.Namespace) string { + return fmt.Sprintf(namespaceDropStatement, tableFromNamespace(n)) +} + func (p *Persister) MigrateNamespaceUp(ctx context.Context, n *namespace.Namespace) error { return p.transaction(ctx, func(ctx context.Context, c *pop.Connection) error { // TODO this is only creating new namespaces and not applying migrations @@ -66,6 +77,16 @@ func (p *Persister) MigrateNamespaceUp(ctx context.Context, n *namespace.Namespa }) } +func (p *Persister) MigrateNamespaceDown(ctx context.Context, n *namespace.Namespace, _ int) error { + return p.transaction(ctx, func(ctx context.Context, c *pop.Connection) error { + if err := c.RawQuery(dropStmt(n)).Exec(); err != nil { + return errors.WithStack(err) + } + + return errors.WithStack(c.RawQuery(fmt.Sprintf("DELETE FROM %s WHERE id = ?", (&namespaceRow{}).TableName()), n.ID).Exec()) + }) +} + func (p *Persister) NamespaceFromName(ctx context.Context, name string) (*namespace.Namespace, error) { return p.namespaces.GetNamespace(ctx, name) } diff --git a/internal/persistence/sql/persister.go b/internal/persistence/sql/persister.go index 4541f4a66..e6fbdc273 100644 --- a/internal/persistence/sql/persister.go +++ b/internal/persistence/sql/persister.go @@ -62,6 +62,10 @@ func (p *Persister) MigrateUp(_ context.Context) error { return p.mb.Up() } +func (p *Persister) MigrateDown(_ context.Context, steps int) error { + return p.mb.Down(steps) +} + func (p *Persister) MigrationStatus(_ context.Context, w io.Writer) error { return p.mb.Status(w) } @@ -74,7 +78,7 @@ func (p *Persister) connection(ctx context.Context) *pop.Connection { return tx.(*pop.Connection) } -func (p *Persister) transaction(ctx context.Context, f func(context.Context, *pop.Connection) error) error { +func (p *Persister) transaction(ctx context.Context, f func(ctx context.Context, c *pop.Connection) error) error { tx := ctx.Value(transactionContextKey) if tx != nil { return f(ctx, tx.(*pop.Connection)) diff --git a/internal/relationtuple/definitions.go b/internal/relationtuple/definitions.go index d5e30f28f..89b588e61 100644 --- a/internal/relationtuple/definitions.go +++ b/internal/relationtuple/definitions.go @@ -34,6 +34,8 @@ type ( internalRelations []*InternalRelationTuple } Subject interface { + json.Marshaler + String() string FromString(string) (Subject, error) Equals(interface{}) bool @@ -133,6 +135,14 @@ func (s *SubjectSet) FromURLQuery(values url.Values) *SubjectSet { return s } +func (s *SubjectSet) ToURLQuery() url.Values { + return url.Values{ + "namespace": []string{s.Namespace}, + "object": []string{s.Object}, + "relation": []string{s.Relation}, + } +} + func (s *SubjectID) ToGRPC() *acl.Subject { return &acl.Subject{ Ref: &acl.Subject_Id{ @@ -169,11 +179,19 @@ func (s *SubjectSet) Equals(v interface{}) bool { return uv.Relation == s.Relation && uv.Object == s.Object && uv.Namespace == s.Namespace } +func (s SubjectID) MarshalJSON() ([]byte, error) { + return []byte(`"` + s.String() + `"`), nil +} + +func (s SubjectSet) MarshalJSON() ([]byte, error) { + return []byte(`"` + s.String() + `"`), nil +} + func (r *InternalRelationTuple) String() string { return fmt.Sprintf("%s:%s#%s@%s", r.Namespace, r.Object, r.Relation, r.Subject) } -func (r *InternalRelationTuple) DeriveSubject() Subject { +func (r *InternalRelationTuple) DeriveSubject() *SubjectSet { return &SubjectSet{ Namespace: r.Namespace, Object: r.Object, @@ -280,6 +298,25 @@ func (q *RelationQuery) FromURLQuery(query url.Values) (*RelationQuery, error) { return q, nil } +func (q *RelationQuery) ToURLQuery() url.Values { + v := make(url.Values, 4) + + if q.Namespace != "" { + v.Add("namespace", q.Namespace) + } + if q.Relation != "" { + v.Add("relation", q.Relation) + } + if q.Object != "" { + v.Add("object", q.Object) + } + if q.Subject != nil { + v.Add("subject", q.Subject.String()) + } + + return v +} + func (q *RelationQuery) String() string { return fmt.Sprintf("namespace: %s; object: %s; relation: %s; subject: %s", q.Namespace, q.Object, q.Relation, q.Subject) } diff --git a/internal/relationtuple/handler.go b/internal/relationtuple/handler.go index f8e1411c2..1085b5179 100644 --- a/internal/relationtuple/handler.go +++ b/internal/relationtuple/handler.go @@ -27,7 +27,7 @@ type ( ) const ( - routeBase = "/relationtuple" + RouteBase = "/relationtuple" ) func NewHandler(d handlerDeps) *handler { @@ -43,8 +43,8 @@ func NewGRPCServer(d handlerDeps) *GRPCServer { } func (h *handler) RegisterPublicRoutes(router *httprouter.Router) { - router.GET(routeBase, h.getRelations) - router.PUT(routeBase, h.createRelation) + router.GET(RouteBase, h.getRelations) + router.PUT(RouteBase, h.createRelation) } func (h *handler) getRelations(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {