diff --git a/.github/workflows/intergration-test.yml b/.github/workflows/intergration-test.yml new file mode 100644 index 0000000..542c7bd --- /dev/null +++ b/.github/workflows/intergration-test.yml @@ -0,0 +1,49 @@ +name: "Integration Test" + +on: [push, pull_request] + +jobs: + integration_test: + name: "Integration Test" + runs-on: ${{ matrix.os }} + + strategy: + matrix: + go: [ "1.15", "1.16" ] + hdfs-version: + - 3.2.2 + - 3.3.0 + - 3.3.1 + os: [ubuntu-latest] + + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go }} + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Checkout python env + uses: actions/setup-python@v2 + with: + python-version: '3.8' + + - name: Checkout java env + uses: actions/setup-java@v1 + with: + java-version: '11' + + - name: Setup-hdfs env + uses: beyondstorage/setup-hdfs@master + with: + hdfs-version: ${{ matrix.hdfs-version }} + + - run: curl ${{ env.HDFS_NAMENODE_ADDR }} + + - name: Test + env: + STORAGE_HDFS_INTEGRATION_TEST: "on" + STORAGE_HDFS_ENDPOINT: "tcp:127.0.0.1:9000" + run: make integration_test \ No newline at end of file diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index eeb0bc8..de4ecd4 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: go: [ "1.15", "1.16" ] - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest] steps: - name: Set up Go 1.x @@ -25,4 +25,4 @@ jobs: run: make build - name: Test - run: make test + run: make test \ No newline at end of file diff --git a/go.mod b/go.mod index 850ef60..c1fac14 100644 --- a/go.mod +++ b/go.mod @@ -6,12 +6,8 @@ require ( github.com/beyondstorage/go-endpoint v1.1.0 github.com/beyondstorage/go-integration-test/v4 v4.2.0 github.com/beyondstorage/go-storage/v4 v4.4.0 - github.com/colinmarc/hdfs/v2 v2.2.0 // indirect + github.com/colinmarc/hdfs/v2 v2.2.0 github.com/golang/protobuf v1.5.2 // indirect github.com/google/uuid v1.3.0 - github.com/jcmturner/aescts v2.0.0+incompatible // indirect - github.com/jcmturner/dnsutils v2.0.0+incompatible // indirect - github.com/jcmturner/goidentity v6.0.1+incompatible // indirect - github.com/jcmturner/gokrb5 v8.4.2+incompatible // indirect - github.com/jcmturner/rpc v1.1.1 // indirect + github.com/stretchr/testify v1.7.0 ) diff --git a/go.sum b/go.sum index 27bf842..4caa0b8 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,6 @@ github.com/beyondstorage/go-integration-test/v4 v4.2.0/go.mod h1:jLyYWSGUjQRH7U1 github.com/beyondstorage/go-storage/v4 v4.3.0/go.mod h1:0fdcRCzLKMQe7Ve4zPlyTGgoPYwuINiV79Gx9tCt9tQ= github.com/beyondstorage/go-storage/v4 v4.4.0 h1:sWURraKFjNR4qpwthr45cAGOIx6EOLrrJcz6su4Je30= github.com/beyondstorage/go-storage/v4 v4.4.0/go.mod h1:mc9VzBImjXDg1/1sLfta2MJH79elfM6m47ZZvZ+q/Uw= -github.com/colinmarc/hdfs v1.1.3 h1:662salalXLFmp+ctD+x0aG+xOg62lnVnOJHksXYpFBw= -github.com/colinmarc/hdfs v1.1.3/go.mod h1:0DumPviB681UcSuJErAbDIOx6SIaJWj463TymfZG02I= github.com/colinmarc/hdfs/v2 v2.2.0 h1:4AaIlTq+/sWmeqYhI0dX8bD4YrMQM990tRjm636FkGM= github.com/colinmarc/hdfs/v2 v2.2.0/go.mod h1:Wss6n3mtaZyRwWaqtSH+6ge01qT0rw9dJJmvoUnIQ/E= github.com/dave/dst v0.26.2 h1:lnxLAKI3tx7MgLNVDirFCsDTlTG9nKTk7GcptKcWSwY= @@ -35,31 +33,23 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v1.2.0 h1:S7P+1Hm5V/AT9cjEcUD5uDaQSX0OE577aCXgoaKpYbQ= github.com/gorilla/sessions v1.2.0/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/jcmturner/aescts v2.0.0+incompatible h1:NADOdvG7ba6APDycsRnElB5PGKPo1RYS0R9SPuH8uDI= -github.com/jcmturner/aescts v2.0.0+incompatible/go.mod h1:k9gJoDUf1GH5r2IBtBjwjDCoLELYxOcEhitdP8RL7qQ= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= -github.com/jcmturner/dnsutils v2.0.0+incompatible h1:OncvQxiEJbkPrG9UL/25GmHtvs6w4IvcpYT0+FBpP6M= -github.com/jcmturner/dnsutils v2.0.0+incompatible/go.mod h1:tqMo38L01jO8AKxT0S9OQVlGZu3dkEt+z5CA+LOhwB0= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= -github.com/jcmturner/goidentity v6.0.1+incompatible h1:I+jJ9JbbrqUAiMB8sNLXlginFJ2lPrxst/N3kb67Dko= -github.com/jcmturner/goidentity v6.0.1+incompatible/go.mod h1:1L/Wdvsh7UG1LqY2/yh6gzL7v/+3y7lXPHOAIlTaYio= github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= -github.com/jcmturner/gokrb5 v8.4.2+incompatible h1:MQW70Fbazv31g6URAXCjO2bGenIL0wVt3wqcpc0EjHI= -github.com/jcmturner/gokrb5 v8.4.2+incompatible/go.mod h1:0Q5eFyVvYsEsZ8xl1A/jUqhXvxUp/X9ELrJm+zieq5E= github.com/jcmturner/gokrb5/v8 v8.4.1 h1:IGSJfqBzMS6TA0oJ7DxXdyzPK563QHa8T2IqER2ggyQ= github.com/jcmturner/gokrb5/v8 v8.4.1/go.mod h1:T1hnNppQsBtxW0tCHMHTkAt8n/sABdzZgZdoFrZaZNM= -github.com/jcmturner/rpc v1.1.1 h1:QStZbZ/rHemt0CZuZCBCFHA2Ez60qVIXBPpa6dzLKvc= -github.com/jcmturner/rpc v1.1.1/go.mod h1:6LzLUsxSQz8DjmFJ/6oxXY2LFB5F1yi0Sp01Ano39sQ= github.com/jcmturner/rpc/v2 v2.0.2 h1:gMB4IwRXYsWw4Bc6o/az2HJgFUA1ffSh90i26ZJ6Xl0= github.com/jcmturner/rpc/v2 v2.0.2/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -86,7 +76,6 @@ github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIK github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -104,7 +93,6 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= @@ -125,7 +113,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190726230722-1bd56024c620/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs= @@ -141,11 +128,6 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= -gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= -gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= -gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= -gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= gopkg.in/src-d/go-billy.v4 v4.3.0 h1:KtlZ4c1OWbIs4jCv5ZXrTqG8EQocr0g/d4DjNg70aek= gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/storage.go b/storage.go index 0b6e441..ceea3bb 100644 --- a/storage.go +++ b/storage.go @@ -138,7 +138,7 @@ func (s *Storage) read(ctx context.Context, path string, w io.Writer, opt pairSt return 0, err } if opt.HasOffset { - _, err = f.Seek(opt.Offset, 0) + _, err := f.Seek(opt.Offset, 0) if err != nil { return 0, err } @@ -150,6 +150,9 @@ func (s *Storage) read(ctx context.Context, path string, w io.Writer, opt pairSt if opt.HasIoCallback { rc = iowrap.CallbackReader(rc, opt.IoCallback) } + if opt.HasSize { + return io.CopyN(w, f, opt.Size) + } return io.Copy(w, f) } @@ -225,4 +228,4 @@ func (s *Storage) writeAppend(ctx context.Context, o *Object, r io.Reader, size }() return io.CopyN(f, r, size) -} +} \ No newline at end of file diff --git a/tests/storage_test.go b/tests/storage_test.go index 8784bbb..3d2cf0c 100644 --- a/tests/storage_test.go +++ b/tests/storage_test.go @@ -33,4 +33,4 @@ func TestAppender(t *testing.T) { t.Skipf("STORAGE_HDFS_INTEGRATION_TEST is not 'on', skipped") } tests.TestAppender(t, setupTest(t)) -} +} \ No newline at end of file diff --git a/tests/utils_test.go b/tests/utils_test.go index 058cfab..8ac5a60 100644 --- a/tests/utils_test.go +++ b/tests/utils_test.go @@ -23,4 +23,4 @@ func setupTest(t *testing.T) types.Storager { } return store -} +} \ No newline at end of file diff --git a/utils.go b/utils.go index c1f8860..ae23e1b 100644 --- a/utils.go +++ b/utils.go @@ -98,7 +98,7 @@ func (s *Storage) getAbsPath(path string) string { if filepath.IsAbs(path) { return path } - return s.workDir + path + return filepath.Join(s.workDir, path) } func (s *Storage) formatError(op string, err error, path ...string) error { @@ -115,4 +115,4 @@ func (s *Storage) formatError(op string, err error, path ...string) error { func (s *Storage) newObject(done bool) *types.Object { return types.NewObject(s, done) -} +} \ No newline at end of file diff --git a/utils_test.go b/utils_test.go new file mode 100644 index 0000000..cdbecde --- /dev/null +++ b/utils_test.go @@ -0,0 +1,62 @@ +package hdfs + +import ( + "errors" + "os" + "testing" + + "github.com/beyondstorage/go-storage/v4/services" + "github.com/stretchr/testify/assert" +) + +func TestFormatOsError(t *testing.T) { + testErr := errors.New("test error") + tests := []struct { + name string + input error + expected error + }{ + { + "not found", + os.ErrNotExist, + services.ErrObjectNotExist, + }, + { + "not supported error", + testErr, + services.ErrUnexpected, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := formatError(tt.input) + assert.True(t, errors.Is(err, tt.expected)) + }) + } +} + +func TestGetAbsPath(t *testing.T) { + cases := []struct { + name string + base string + path string + expectedPath string + }{ + {"direct path", "", "abc", "abc"}, + {"under direct path", "", "root/abc", "root/abc"}, + {"under direct path", "", "root/abc/", "root/abc"}, + {"under root", "/", "abc", "/abc"}, + {"under exist dir", "/root", "abc", "/root/abc"}, + {"under new dir", "/root", "abc/", "/root/abc"}, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + client := Storage{workDir: tt.base} + + getPath := client.getAbsPath(tt.path) + assert.Equal(t, tt.expectedPath, getPath) + }) + } +} \ No newline at end of file