From 8844b0bf57434a2764dc64528dce9e6f9d1972a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Pablo=20Villaf=C3=A1=C3=B1ez?= Date: Wed, 15 Sep 2021 16:33:34 +0200 Subject: [PATCH 1/4] Expose configuration for the reva archiver --- storage/pkg/command/frontend.go | 7 +++++++ storage/pkg/config/config.go | 8 ++++++++ storage/pkg/flagset/frontend.go | 24 ++++++++++++++++++++++++ 3 files changed, 39 insertions(+) diff --git a/storage/pkg/command/frontend.go b/storage/pkg/command/frontend.go index ef686698b8f..1ad2ef05e56 100644 --- a/storage/pkg/command/frontend.go +++ b/storage/pkg/command/frontend.go @@ -147,6 +147,13 @@ func frontendConfigFromStruct(c *cli.Context, cfg *config.Config, filesCfg map[s "timeout": 86400, "insecure": true, }, + "archiver": map[string]interface{}{ + "prefix": cfg.Reva.Frontend.ArchiverPrefix, + "timeout": 86400, + "insecure": true, + "max_num_files": cfg.Reva.Archiver.MaxNumFiles, + "max_size": cfg.Reva.Archiver.MaxSize, + }, "datagateway": map[string]interface{}{ "prefix": cfg.Reva.Frontend.DatagatewayPrefix, "transfer_shared_secret": cfg.Reva.TransferSecret, diff --git a/storage/pkg/config/config.go b/storage/pkg/config/config.go index 4d73c02c762..7539eea6cb5 100644 --- a/storage/pkg/config/config.go +++ b/storage/pkg/config/config.go @@ -132,6 +132,7 @@ type FrontendPort struct { Port AppProviderPrefix string + ArchiverPrefix string DatagatewayPrefix string OCDavPrefix string OCSPrefix string @@ -401,6 +402,12 @@ type OCDav struct { DavFilesNamespace string } +// Archiver defines the available archiver configuration. +type Archiver struct { + MaxNumFiles int64 + MaxSize int64 +} + // Reva defines the available reva configuration. type Reva struct { // JWTSecret used to sign jwt tokens between services @@ -412,6 +419,7 @@ type Reva struct { UserGroupRest UserGroupRest UserOwnCloudSQL UserOwnCloudSQL OCDav OCDav + Archiver Archiver Storages StorageConfig // Ports are used to configure which services to start on which port Frontend FrontendPort diff --git a/storage/pkg/flagset/frontend.go b/storage/pkg/flagset/frontend.go index 2f7ec6fb151..e2ccc3eea8b 100644 --- a/storage/pkg/flagset/frontend.go +++ b/storage/pkg/flagset/frontend.go @@ -57,6 +57,23 @@ func FrontendWithConfig(cfg *config.Config) []cli.Flag { Destination: &cfg.Reva.OCDav.DavFilesNamespace, }, + // Archiver + + &cli.Int64Flag{ + Name: "archiver-max-num-files", + Value: flags.OverrideDefaultInt64(cfg.Reva.Archiver.MaxNumFiles, 10000), + Usage: "Maximum number of files to be included in the archiver", + EnvVars: []string{"STORAGE_ARCHIVER_MAX_NUM_FILES"}, + Destination: &cfg.Reva.Archiver.MaxNumFiles, + }, + &cli.Int64Flag{ + Name: "archiver-max-size", + Value: flags.OverrideDefaultInt64(cfg.Reva.Archiver.MaxSize, 1073741824), // 1GB + Usage: "Maximum size for the sum of the sizes of all the files included in the archive", + EnvVars: []string{"STORAGE_ARCHIVER_MAX_SIZE"}, + Destination: &cfg.Reva.Archiver.MaxSize, + }, + // Services // Frontend @@ -98,6 +115,13 @@ func FrontendWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_FRONTEND_APPPROVIDER_PREFIX"}, Destination: &cfg.Reva.Frontend.AppProviderPrefix, }, + &cli.StringFlag{ + Name: "archiver-prefix", + Value: flags.OverrideDefaultString(cfg.Reva.Frontend.ArchiverPrefix, "archiver"), + Usage: "archiver prefix", + EnvVars: []string{"STORAGE_FRONTEND_ARCHIVER_PREFIX"}, + Destination: &cfg.Reva.Frontend.ArchiverPrefix, + }, &cli.StringFlag{ Name: "datagateway-prefix", Value: flags.OverrideDefaultString(cfg.Reva.Frontend.DatagatewayPrefix, "data"), From 075bd7500c539e8f80de2a2e0a7986d0e3f20347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Pablo=20Villaf=C3=A1=C3=B1ez?= Date: Wed, 15 Sep 2021 17:33:16 +0200 Subject: [PATCH 2/4] Add changelog entry --- changelog/unreleased/expose-reva-archiver.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog/unreleased/expose-reva-archiver.md diff --git a/changelog/unreleased/expose-reva-archiver.md b/changelog/unreleased/expose-reva-archiver.md new file mode 100644 index 00000000000..0beb0bdf2a9 --- /dev/null +++ b/changelog/unreleased/expose-reva-archiver.md @@ -0,0 +1,5 @@ +Enhancement: Expose the reva archiver in OCIS + +The reva archiver can now be accessed through the storage frontend service + +https://github.com/owncloud/ocis/pull/2509 From c1dd0ea66e30eb9d81508f75c58a9cc5c194846e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Pablo=20Villaf=C3=A1=C3=B1ez?= Date: Thu, 16 Sep 2021 12:48:07 +0200 Subject: [PATCH 3/4] Add default route to the archiver --- proxy/pkg/proxy/proxy.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/proxy/pkg/proxy/proxy.go b/proxy/pkg/proxy/proxy.go index 0d8aea603c0..960c187c531 100644 --- a/proxy/pkg/proxy/proxy.go +++ b/proxy/pkg/proxy/proxy.go @@ -290,6 +290,10 @@ func defaultPolicies() []config.Policy { Endpoint: "/signin/", Backend: "http://localhost:9130", }, + { + Endpoint: "/archiver/", + Backend: "http://localhost:9140", + }, { Type: config.RegexRoute, Endpoint: "/ocs/v[12].php/cloud/(users?|groups)", // we have `user`, `users` and `groups` in ocis-ocs @@ -379,6 +383,10 @@ func defaultPolicies() []config.Policy { Endpoint: "/signin/", Backend: "http://localhost:9130", }, + { + Endpoint: "/archiver/", + Backend: "http://localhost:9140", + }, { Endpoint: "/ocs/", Backend: "https://demo.owncloud.com", From 889b30cd18deb45613a8cddf7d4587fd8509b20c Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Wed, 22 Sep 2021 13:51:58 +0200 Subject: [PATCH 4/4] make archiver routes match also "/archiver" instead of "/archiver/" only --- proxy/pkg/proxy/proxy.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proxy/pkg/proxy/proxy.go b/proxy/pkg/proxy/proxy.go index 960c187c531..e1b729df8d0 100644 --- a/proxy/pkg/proxy/proxy.go +++ b/proxy/pkg/proxy/proxy.go @@ -291,7 +291,7 @@ func defaultPolicies() []config.Policy { Backend: "http://localhost:9130", }, { - Endpoint: "/archiver/", + Endpoint: "/archiver", Backend: "http://localhost:9140", }, { @@ -384,7 +384,7 @@ func defaultPolicies() []config.Policy { Backend: "http://localhost:9130", }, { - Endpoint: "/archiver/", + Endpoint: "/archiver", Backend: "http://localhost:9140", }, {