feat(GODT-2385): Fallback for store read operations

Allow the user to specify a fallback reader when reading store files
from disk. This is a stop-gap feature until we have a proper migration
feature in Gluon to allow users of the library to read old cache files
from before the changes made in
90330fd.

store/disk.go

@@ -19,22 +19,32 @@ import (
 )
 
 type onDiskStore struct {
-	path string
-	gcm  cipher.AEAD
-	sem  *Semaphore
+	path     string
+	gcm      cipher.AEAD
+	sem      *Semaphore
+	fallback Fallback
 }
 
-func NewOnDiskStore(path string, pass []byte, opt ...Option) (Store, error) {
-	if err := os.MkdirAll(path, 0o700); err != nil {
+func NewCipher(pass []byte) (cipher.AEAD, error) {
+	aes, err := aes.NewCipher(hash(pass))
+	if err != nil {
 		return nil, err
 	}
 
-	aes, err := aes.NewCipher(hash(pass))
+	gcm, err := cipher.NewGCM(aes)
 	if err != nil {
 		return nil, err
 	}
 
-	gcm, err := cipher.NewGCM(aes)
+	return gcm, nil
+}
+
+func NewOnDiskStore(path string, pass []byte, opt ...Option) (Store, error) {
+	if err := os.MkdirAll(path, 0o700); err != nil {
+		return nil, err
+	}
+
+	gcm, err := NewCipher(pass)
 	if err != nil {
 		return nil, err
 	}
@@ -154,10 +164,28 @@ func (c *onDiskStore) Get(messageID imap.InternalMessageID) ([]byte, error) {
 
 	header := make([]byte, len(storeHeaderBytes))
 	if _, err := io.ReadFull(file, header); err != nil {
+		if errors.Is(err, io.ErrUnexpectedEOF) && c.fallback != nil {
+			result, err := c.readFromFallback(file)
+			if err != nil {
+				return nil, fmt.Errorf("failed to read from fallback: %w", err)
+			}
+
+			return result, nil
+		}
+
 		return nil, err
 	}
 
 	if !bytes.Equal(header, storeHeaderBytes) {
+		if c.fallback != nil {
+			result, err := c.readFromFallback(file)
+			if err != nil {
+				return nil, fmt.Errorf("failed to read from fallback: %w", err)
+			}
+
+			return result, nil
+		}
+
 		return nil, fmt.Errorf("file is not a valid store file")
 	}
 
@@ -296,6 +324,16 @@ func (*OnDiskStoreBuilder) Delete(path, userID string) error {
 	return os.RemoveAll(storePath)
 }
 
+func (c *onDiskStore) readFromFallback(file *os.File) ([]byte, error) {
+	if pos, err := file.Seek(0, 0); err != nil {
+		return nil, err
+	} else if pos != 0 {
+		return nil, fmt.Errorf("failed to rewind file to start")
+	}
+
+	return c.fallback.Read(c.gcm, file)
+}
+
 func getEncryptedBlockSize(aead cipher.AEAD, blockSize int) int {
 	return blockSize + aead.Overhead()
 }

store/fallback.go

@@ -0,0 +1,15 @@
+package store
+
+import (
+	"crypto/cipher"
+	"io"
+)
+
+// Fallback provides an interface to supply an alternative way to read a store file should the main route fail.
+// This is mainly intended to allow users of the library to read old store formats they may have kept on disk.
+// This is a stop-gap until a complete data migration cycle can be implemented in gluon.
+type Fallback interface {
+	Read(gcm cipher.AEAD, reader io.Reader) ([]byte, error)
+
+	Write(gcm cipher.AEAD, filepath string, data []byte) error
+}

store/fallback_v0/compressor.go

@@ -0,0 +1,6 @@
+package fallback_v0
+
+type Compressor interface {
+	Compress([]byte) ([]byte, error)
+	Decompress([]byte) ([]byte, error)
+}

store/fallback_v0/compressor_gzip.go

@@ -0,0 +1,43 @@
+package fallback_v0
+
+import (
+	"bytes"
+	"compress/gzip"
+)
+
+type GZipCompressor struct{}
+
+func (GZipCompressor) Compress(dec []byte) ([]byte, error) {
+	buf := new(bytes.Buffer)
+
+	zw := gzip.NewWriter(buf)
+
+	if _, err := zw.Write(dec); err != nil {
+		return nil, err
+	}
+
+	if err := zw.Close(); err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
+func (GZipCompressor) Decompress(cmp []byte) ([]byte, error) {
+	zr, err := gzip.NewReader(bytes.NewReader(cmp))
+	if err != nil {
+		return nil, err
+	}
+
+	buf := new(bytes.Buffer)
+
+	if _, err := buf.ReadFrom(zr); err != nil {
+		return nil, err
+	}
+
+	if err := zr.Close(); err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}

store/fallback_v0/compressor_zlib.go

@@ -0,0 +1,43 @@
+package fallback_v0
+
+import (
+	"bytes"
+	"compress/zlib"
+)
+
+type ZLibCompressor struct{}
+
+func (ZLibCompressor) Compress(dec []byte) ([]byte, error) {
+	buf := new(bytes.Buffer)
+
+	zw := zlib.NewWriter(buf)
+
+	if _, err := zw.Write(dec); err != nil {
+		return nil, err
+	}
+
+	if err := zw.Close(); err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
+func (ZLibCompressor) Decompress(cmp []byte) ([]byte, error) {
+	zr, err := zlib.NewReader(bytes.NewReader(cmp))
+	if err != nil {
+		return nil, err
+	}
+
+	buf := new(bytes.Buffer)
+
+	if _, err := buf.ReadFrom(zr); err != nil {
+		return nil, err
+	}
+
+	if err := zr.Close(); err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}

store/fallback_v0/disk.go

@@ -0,0 +1,68 @@
+package fallback_v0
+
+import (
+	"crypto/cipher"
+	"crypto/rand"
+	"io"
+	"os"
+
+	"github.com/ProtonMail/gluon/store"
+)
+
+type onDiskStoreV0 struct {
+	compressor Compressor
+}
+
+func NewOnDiskStoreV0() store.Fallback {
+	return &onDiskStoreV0{compressor: nil}
+}
+
+func NewOnDiskStoreV0WithCompressor(c Compressor) store.Fallback {
+	return &onDiskStoreV0{compressor: c}
+}
+
+func (d *onDiskStoreV0) Read(gcm cipher.AEAD, reader io.Reader) ([]byte, error) {
+	enc, err := io.ReadAll(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	b, err := gcm.Open(nil, enc[:gcm.NonceSize()], enc[gcm.NonceSize():], nil)
+	if err != nil {
+		return nil, err
+	}
+
+	if d.compressor != nil {
+		dec, err := d.compressor.Decompress(b)
+		if err != nil {
+			return nil, err
+		}
+
+		b = dec
+	}
+
+	return b, nil
+}
+
+func (d *onDiskStoreV0) Write(gcm cipher.AEAD, filepath string, data []byte) error {
+	nonce := make([]byte, gcm.NonceSize())
+
+	if _, err := rand.Read(nonce); err != nil {
+		return err
+	}
+
+	if d.compressor != nil {
+		enc, err := d.compressor.Compress(data)
+		if err != nil {
+			return err
+		}
+
+		data = enc
+	}
+
+	return os.WriteFile(
+		filepath,
+		gcm.Seal(nonce, nonce, data, nil),
+		0o600,
+	)
+}

store/option.go

@@ -17,3 +17,15 @@ type withSem struct {
 func (opt withSem) config(store *onDiskStore) {
 	store.sem = opt.sem
 }
+
+type withFallback struct {
+	f Fallback
+}
+
+func WithFallback(f Fallback) Option {
+	return &withFallback{f: f}
+}
+
+func (opt withFallback) config(store *onDiskStore) {
+	store.fallback = opt.f
+}

store/store_test.go

@@ -2,6 +2,7 @@ package store_test
 
 import (
 	"bytes"
+	"github.com/ProtonMail/gluon/store/fallback_v0"
 	"math/rand"
 	"os"
 	"path/filepath"
@@ -82,6 +83,62 @@ func TestStoreReadFailsIfHeaderDoesNotMatch(t *testing.T) {
 	require.Error(t, err)
 }
 
+func TestStoreFallbackRead(t *testing.T) {
+	fallbackStore := fallback_v0.NewOnDiskStoreV0WithCompressor(&fallback_v0.GZipCompressor{})
+
+	storeDir := t.TempDir()
+
+	password := []byte("pass")
+
+	fileContents := []byte("Hello world from gluon store")
+
+	id := imap.NewInternalMessageID()
+
+	{
+		// create old store file on disk
+		gcm, err := store.NewCipher(password)
+		require.NoError(t, err)
+
+		filepath := filepath.Join(storeDir, id.String())
+
+		require.NoError(t, fallbackStore.Write(gcm, filepath, fileContents))
+	}
+
+	// Reading file without fallback should fail.
+	{
+		store, err := store.NewOnDiskStore(
+			storeDir,
+			[]byte("pass"),
+			store.WithSemaphore(store.NewSemaphore(runtime.NumCPU())),
+		)
+		require.NoError(t, err)
+		defer func() {
+			require.NoError(t, store.Close())
+		}()
+
+		_, err = store.Get(id)
+		require.Error(t, err)
+	}
+
+	//
+	{
+		store, err := store.NewOnDiskStore(
+			storeDir,
+			[]byte("pass"),
+			store.WithSemaphore(store.NewSemaphore(runtime.NumCPU())),
+			store.WithFallback(fallbackStore),
+		)
+		require.NoError(t, err)
+		defer func() {
+			require.NoError(t, store.Close())
+		}()
+
+		b, err := store.Get(id)
+		require.NoError(t, err)
+		require.Equal(t, fileContents, b)
+	}
+}
+
 func TestOnDiskStore(t *testing.T) {
 	store, err := store.NewOnDiskStore(
 		t.TempDir(),