From 46dd797392278bd20d0c53c0833f099d4d78adab Mon Sep 17 00:00:00 2001 From: Yuvraj Date: Sat, 17 Jul 2021 03:53:22 +0530 Subject: [PATCH 1/3] Added default no limit on storage Signed-off-by: Yuvraj --- storage/config.go | 2 +- storage/stow_store.go | 2 +- storage/stow_store_test.go | 31 +++++++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/storage/config.go b/storage/config.go index a44c150a..518202e0 100644 --- a/storage/config.go +++ b/storage/config.go @@ -33,7 +33,7 @@ var ( defaultConfig = &Config{ Type: TypeS3, Limits: LimitsConfig{ - GetLimitMegabytes: 2, + GetLimitMegabytes: 0, }, Connection: ConnectionConfig{ Region: "us-east-1", diff --git a/storage/stow_store.go b/storage/stow_store.go index 0799d0eb..6a63ad74 100644 --- a/storage/stow_store.go +++ b/storage/stow_store.go @@ -230,7 +230,7 @@ func (s *StowStore) ReadRaw(ctx context.Context, reference DataReference) (io.Re return nil, err } - if sizeMbs := sizeBytes / MiB; sizeMbs > GetConfig().Limits.GetLimitMegabytes { + if sizeMbs := sizeBytes / MiB; sizeMbs > GetConfig().Limits.GetLimitMegabytes && GetConfig().Limits.GetLimitMegabytes != 0 { return nil, errors.Errorf(ErrExceedsLimit, "limit exceeded. %vmb > %vmb.", sizeMbs, GetConfig().Limits.GetLimitMegabytes) } diff --git a/storage/stow_store_test.go b/storage/stow_store_test.go index d291b5cb..bec30aa9 100644 --- a/storage/stow_store_test.go +++ b/storage/stow_store_test.go @@ -181,6 +181,8 @@ func TestStowStore_ReadRaw(t *testing.T) { t.Run("Exceeds limit", func(t *testing.T) { testScope := promutils.NewTestScope() fn := fQNFn["s3"] + GetConfig().Limits.GetLimitMegabytes = 2 + s, err := NewStowRawStore(fn(container), &mockStowLoc{ ContainerCb: func(id string) (stow.Container, error) { if id == container { @@ -207,6 +209,35 @@ func TestStowStore_ReadRaw(t *testing.T) { assert.NotNil(t, errors.Cause(err)) }) + t.Run("No Limit", func(t *testing.T) { + testScope := promutils.NewTestScope() + fn := fQNFn["s3"] + GetConfig().Limits.GetLimitMegabytes = 0 + + s, err := NewStowRawStore(fn(container), &mockStowLoc{ + ContainerCb: func(id string) (stow.Container, error) { + if id == container { + return newMockStowContainer(container), nil + } + return nil, fmt.Errorf("container is not supported") + }, + CreateContainerCb: func(name string) (stow.Container, error) { + if name == container { + return newMockStowContainer(container), nil + } + return nil, fmt.Errorf("container is not supported") + }, + }, false, testScope) + assert.NoError(t, err) + err = s.WriteRaw(context.TODO(), DataReference("s3://container/path"), 3*MiB, Options{}, bytes.NewReader([]byte{})) + assert.NoError(t, err) + metadata, err := s.Head(context.TODO(), DataReference("s3://container/path")) + assert.NoError(t, err) + assert.True(t, metadata.Exists()) + _, err = s.ReadRaw(context.TODO(), DataReference("s3://container/path")) + assert.Nil(t, err) + }) + t.Run("Happy Path multi-container enabled", func(t *testing.T) { testScope := promutils.NewTestScope() fn := fQNFn["s3"] From 609484419c8ac3b1e09ee71c7e97f205fbc3028c Mon Sep 17 00:00:00 2001 From: Yuvraj Date: Sat, 17 Jul 2021 23:24:14 +0530 Subject: [PATCH 2/3] More change Signed-off-by: Yuvraj --- storage/stow_store.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/storage/stow_store.go b/storage/stow_store.go index 6a63ad74..18f7fb7c 100644 --- a/storage/stow_store.go +++ b/storage/stow_store.go @@ -7,6 +7,8 @@ import ( "sync" "time" + "github.com/flyteorg/flytestdlib/errors" + "github.com/aws/aws-sdk-go/aws/awserr" s32 "github.com/aws/aws-sdk-go/service/s3" "github.com/graymeta/stow/azure" @@ -20,8 +22,6 @@ import ( "github.com/flyteorg/flytestdlib/logger" "github.com/flyteorg/flytestdlib/promutils/labeled" - "github.com/flyteorg/flytestdlib/errors" - "github.com/flyteorg/flytestdlib/promutils" "github.com/graymeta/stow" @@ -230,8 +230,10 @@ func (s *StowStore) ReadRaw(ctx context.Context, reference DataReference) (io.Re return nil, err } - if sizeMbs := sizeBytes / MiB; sizeMbs > GetConfig().Limits.GetLimitMegabytes && GetConfig().Limits.GetLimitMegabytes != 0 { - return nil, errors.Errorf(ErrExceedsLimit, "limit exceeded. %vmb > %vmb.", sizeMbs, GetConfig().Limits.GetLimitMegabytes) + if GetConfig().Limits.GetLimitMegabytes != 0 { + if sizeMbs := sizeBytes / MiB; sizeMbs > GetConfig().Limits.GetLimitMegabytes { + return nil, errors.Errorf(ErrExceedsLimit, "limit exceeded. %vmb > %vmb.", sizeMbs, GetConfig().Limits.GetLimitMegabytes) + } } return item.Open() From ef929ab04618ae3f453173f230fe28e9ab13309d Mon Sep 17 00:00:00 2001 From: Yuvraj Date: Mon, 26 Jul 2021 11:07:38 +0530 Subject: [PATCH 3/3] more changese Signed-off-by: Yuvraj --- storage/config.go | 2 +- storage/stow_store_test.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/storage/config.go b/storage/config.go index 518202e0..a44c150a 100644 --- a/storage/config.go +++ b/storage/config.go @@ -33,7 +33,7 @@ var ( defaultConfig = &Config{ Type: TypeS3, Limits: LimitsConfig{ - GetLimitMegabytes: 0, + GetLimitMegabytes: 2, }, Connection: ConnectionConfig{ Region: "us-east-1", diff --git a/storage/stow_store_test.go b/storage/stow_store_test.go index bec30aa9..87025ed7 100644 --- a/storage/stow_store_test.go +++ b/storage/stow_store_test.go @@ -181,7 +181,6 @@ func TestStowStore_ReadRaw(t *testing.T) { t.Run("Exceeds limit", func(t *testing.T) { testScope := promutils.NewTestScope() fn := fQNFn["s3"] - GetConfig().Limits.GetLimitMegabytes = 2 s, err := NewStowRawStore(fn(container), &mockStowLoc{ ContainerCb: func(id string) (stow.Container, error) {