Separate blobs from metadata in the ocis driver (#1452)

README.md

@@ -82,7 +82,7 @@ You can also read the [build from sources guide](https://reva.link/docs/getting-
     ../../../cmd/revad/revad -c gateway.toml &
     ../../../cmd/revad/revad -c shares.toml &
     ../../../cmd/revad/revad -c storage-home.toml &
-    ../../../cmd/revad/revad -c storage-oc.toml &
+    ../../../cmd/revad/revad -c storage-users.toml &
     ../../../cmd/revad/revad -c storage-publiclink.toml &
     ../../../cmd/revad/revad -c ldap-users.toml
     ```

changelog/unreleased/separate-blobs-from-metadata-in-ocis.md

@@ -0,0 +1,12 @@
+Change: Separate blobs from metadata in the ocis storage driver
+
+We changed the ocis storage driver to keep the file content separate from the
+metadata by storing the blobs in a separate directory. This allows for using
+a different (potentially faster) storage for the metadata.
+
+**Note** This change makes existing ocis storages incompatible with the new code.
+
+We also streamlined the ocis and the s3ng drivers so that most of the code is
+shared between them.
+
+https://github.com/cs3org/reva/pull/1452

pkg/storage/fs/ocis/blobstore/blobstore.go

@@ -0,0 +1,83 @@
+// Copyright 2018-2021 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package blobstore
+
+import (
+	"bufio"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+)
+
+// Blobstore provides an interface to an filesystem based blobstore
+type Blobstore struct {
+	root string
+}
+
+// New returns a new Blobstore
+func New(root string) (*Blobstore, error) {
+	err := os.MkdirAll(root, 0700)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Blobstore{
+		root: root,
+	}, nil
+}
+
+// Upload stores some data in the blobstore under the given key
+func (bs *Blobstore) Upload(key string, data io.Reader) error {
+	f, err := os.OpenFile(bs.path(key), os.O_CREATE|os.O_WRONLY, 0700)
+	if err != nil {
+		return errors.Wrapf(err, "could not open blob '%s' for writing", key)
+	}
+
+	w := bufio.NewWriter(f)
+	_, err = w.ReadFrom(data)
+	if err != nil {
+		return errors.Wrapf(err, "could not write blob '%s'", key)
+	}
+
+	return w.Flush()
+}
+
+// Download retrieves a blob from the blobstore for reading
+func (bs *Blobstore) Download(key string) (io.ReadCloser, error) {
+	file, err := os.Open(bs.path(key))
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not read blob '%s'", key)
+	}
+	return file, nil
+}
+
+// Delete deletes a blob from the blobstore
+func (bs *Blobstore) Delete(key string) error {
+	err := os.Remove(bs.path(key))
+	if err != nil {
+		return errors.Wrapf(err, "could not delete blob '%s'", key)
+	}
+	return nil
+}
+
+func (bs *Blobstore) path(key string) string {
+	return filepath.Join(bs.root, filepath.Clean(filepath.Join("/", key)))
+}

pkg/storage/fs/ocis/blobstore/blobstore_suite_test.go

@@ -0,0 +1,31 @@
+// Copyright 2018-2021 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package blobstore_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestBlobstore(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Blobstore Suite")
+}

pkg/storage/fs/ocis/blobstore/blobstore_test.go

@@ -0,0 +1,118 @@
+// Copyright 2018-2021 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package blobstore_test
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/cs3org/reva/pkg/storage/fs/ocis/blobstore"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Blobstore", func() {
+	var (
+		tmpRoot  string
+		key      string
+		blobPath string
+		data     []byte
+
+		bs *blobstore.Blobstore
+	)
+
+	BeforeEach(func() {
+		var err error
+		tmpRoot, err = ioutil.TempDir("", "reva-unit-tests-*-root")
+		Expect(err).ToNot(HaveOccurred())
+
+		data = []byte("1234567890")
+		key = "foo"
+		blobPath = path.Join(tmpRoot, "blobs", key)
+
+		bs, err = blobstore.New(path.Join(tmpRoot, "blobs"))
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		if strings.HasPrefix(tmpRoot, os.TempDir()) {
+			os.RemoveAll(tmpRoot)
+		}
+	})
+
+	It("creates the root directory if it doesn't exist", func() {
+		_, err := os.Stat(path.Join(tmpRoot, "blobs"))
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	Describe("Upload", func() {
+		It("writes the blob", func() {
+			err := bs.Upload(key, bytes.NewReader(data))
+			Expect(err).ToNot(HaveOccurred())
+
+			writtenBytes, err := ioutil.ReadFile(blobPath)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(writtenBytes).To(Equal(data))
+		})
+	})
+
+	Context("with an existing blob", func() {
+		BeforeEach(func() {
+			Expect(ioutil.WriteFile(blobPath, data, 0700)).To(Succeed())
+		})
+
+		Describe("Download", func() {
+			It("cleans the key", func() {
+				reader, err := bs.Download("../" + key)
+				Expect(err).ToNot(HaveOccurred())
+
+				readData, err := ioutil.ReadAll(reader)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(readData).To(Equal(data))
+			})
+
+			It("returns a reader to the blob", func() {
+				reader, err := bs.Download(key)
+				Expect(err).ToNot(HaveOccurred())
+
+				readData, err := ioutil.ReadAll(reader)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(readData).To(Equal(data))
+			})
+		})
+
+		Describe("Delete", func() {
+			It("deletes the blob", func() {
+				_, err := os.Stat(blobPath)
+				Expect(err).ToNot(HaveOccurred())
+
+				err = bs.Delete(key)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = os.Stat(blobPath)
+				Expect(err).To(HaveOccurred())
+			})
+		})
+	})
+
+})