feat: Set up integration tests

.github/workflows/intergration-test.yml

@@ -0,0 +1,49 @@
+name: "Integration Test"
+
+on: [push, pull_request]
+
+jobs:
+  integration_test:
+    name: "Integration Test"
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        go: [ "1.15", "1.16" ]
+        hdfs-version:
+          - 3.2.2
+          - 3.3.0
+          - 3.3.1
+        os: [ubuntu-latest]
+
+    steps:
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go }}
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Checkout python env
+        uses: actions/setup-python@v2
+        with:
+            python-version: '3.8'
+
+      - name: Checkout java env
+        uses: actions/setup-java@v1
+        with:
+            java-version: '11'
+
+      - name: Setup-hdfs env
+        uses: beyondstorage/setup-hdfs@master
+        with:
+            hdfs-version: ${{ matrix.hdfs-version }}
+
+      - run: curl ${{ env.HDFS_NAMENODE_ADDR }}
+
+      - name: Test
+        env:
+          STORAGE_HDFS_INTEGRATION_TEST: "on"
+          STORAGE_HDFS_ENDPOINT: "tcp:127.0.0.1:9000"
+        run: make integration_test

.github/workflows/unit-test.yml

@@ -10,7 +10,7 @@ jobs:
     strategy:
       matrix:
         go: [ "1.15", "1.16" ]
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest]
 
     steps:
       - name: Set up Go 1.x
@@ -25,4 +25,4 @@ jobs:
         run: make build
 
       - name: Test
-        run: make test
+        run: make test

go.mod

@@ -6,12 +6,8 @@ require (
 	github.com/beyondstorage/go-endpoint v1.1.0
 	github.com/beyondstorage/go-integration-test/v4 v4.2.0
 	github.com/beyondstorage/go-storage/v4 v4.4.0
-	github.com/colinmarc/hdfs/v2 v2.2.0 // indirect
+	github.com/colinmarc/hdfs/v2 v2.2.0
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/uuid v1.3.0
-	github.com/jcmturner/aescts v2.0.0+incompatible // indirect
-	github.com/jcmturner/dnsutils v2.0.0+incompatible // indirect
-	github.com/jcmturner/goidentity v6.0.1+incompatible // indirect
-	github.com/jcmturner/gokrb5 v8.4.2+incompatible // indirect
-	github.com/jcmturner/rpc v1.1.1 // indirect
+	github.com/stretchr/testify v1.7.0
 )

go.sum

@@ -7,8 +7,6 @@ github.com/beyondstorage/go-integration-test/v4 v4.2.0/go.mod h1:jLyYWSGUjQRH7U1
 github.com/beyondstorage/go-storage/v4 v4.3.0/go.mod h1:0fdcRCzLKMQe7Ve4zPlyTGgoPYwuINiV79Gx9tCt9tQ=
 github.com/beyondstorage/go-storage/v4 v4.4.0 h1:sWURraKFjNR4qpwthr45cAGOIx6EOLrrJcz6su4Je30=
 github.com/beyondstorage/go-storage/v4 v4.4.0/go.mod h1:mc9VzBImjXDg1/1sLfta2MJH79elfM6m47ZZvZ+q/Uw=
-github.com/colinmarc/hdfs v1.1.3 h1:662salalXLFmp+ctD+x0aG+xOg62lnVnOJHksXYpFBw=
-github.com/colinmarc/hdfs v1.1.3/go.mod h1:0DumPviB681UcSuJErAbDIOx6SIaJWj463TymfZG02I=
 github.com/colinmarc/hdfs/v2 v2.2.0 h1:4AaIlTq+/sWmeqYhI0dX8bD4YrMQM990tRjm636FkGM=
 github.com/colinmarc/hdfs/v2 v2.2.0/go.mod h1:Wss6n3mtaZyRwWaqtSH+6ge01qT0rw9dJJmvoUnIQ/E=
 github.com/dave/dst v0.26.2 h1:lnxLAKI3tx7MgLNVDirFCsDTlTG9nKTk7GcptKcWSwY=
@@ -35,31 +33,23 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ=
 github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
+github.com/gorilla/sessions v1.2.0 h1:S7P+1Hm5V/AT9cjEcUD5uDaQSX0OE577aCXgoaKpYbQ=
 github.com/gorilla/sessions v1.2.0/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
 github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE=
 github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/jcmturner/aescts v2.0.0+incompatible h1:NADOdvG7ba6APDycsRnElB5PGKPo1RYS0R9SPuH8uDI=
-github.com/jcmturner/aescts v2.0.0+incompatible/go.mod h1:k9gJoDUf1GH5r2IBtBjwjDCoLELYxOcEhitdP8RL7qQ=
 github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
 github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
-github.com/jcmturner/dnsutils v2.0.0+incompatible h1:OncvQxiEJbkPrG9UL/25GmHtvs6w4IvcpYT0+FBpP6M=
-github.com/jcmturner/dnsutils v2.0.0+incompatible/go.mod h1:tqMo38L01jO8AKxT0S9OQVlGZu3dkEt+z5CA+LOhwB0=
 github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
 github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
 github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
 github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
-github.com/jcmturner/goidentity v6.0.1+incompatible h1:I+jJ9JbbrqUAiMB8sNLXlginFJ2lPrxst/N3kb67Dko=
-github.com/jcmturner/goidentity v6.0.1+incompatible/go.mod h1:1L/Wdvsh7UG1LqY2/yh6gzL7v/+3y7lXPHOAIlTaYio=
 github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o=
 github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=
-github.com/jcmturner/gokrb5 v8.4.2+incompatible h1:MQW70Fbazv31g6URAXCjO2bGenIL0wVt3wqcpc0EjHI=
-github.com/jcmturner/gokrb5 v8.4.2+incompatible/go.mod h1:0Q5eFyVvYsEsZ8xl1A/jUqhXvxUp/X9ELrJm+zieq5E=
 github.com/jcmturner/gokrb5/v8 v8.4.1 h1:IGSJfqBzMS6TA0oJ7DxXdyzPK563QHa8T2IqER2ggyQ=
 github.com/jcmturner/gokrb5/v8 v8.4.1/go.mod h1:T1hnNppQsBtxW0tCHMHTkAt8n/sABdzZgZdoFrZaZNM=
-github.com/jcmturner/rpc v1.1.1 h1:QStZbZ/rHemt0CZuZCBCFHA2Ez60qVIXBPpa6dzLKvc=
-github.com/jcmturner/rpc v1.1.1/go.mod h1:6LzLUsxSQz8DjmFJ/6oxXY2LFB5F1yi0Sp01Ano39sQ=
 github.com/jcmturner/rpc/v2 v2.0.2 h1:gMB4IwRXYsWw4Bc6o/az2HJgFUA1ffSh90i26ZJ6Xl0=
 github.com/jcmturner/rpc/v2 v2.0.2/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
@@ -86,7 +76,6 @@ github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIK
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -104,7 +93,6 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
@@ -125,7 +113,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190726230722-1bd56024c620/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs=
@@ -141,11 +128,6 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo=
-gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q=
-gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4=
-gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM=
-gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8=
 gopkg.in/src-d/go-billy.v4 v4.3.0 h1:KtlZ4c1OWbIs4jCv5ZXrTqG8EQocr0g/d4DjNg70aek=
 gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

storage.go

@@ -138,7 +138,7 @@ func (s *Storage) read(ctx context.Context, path string, w io.Writer, opt pairSt
 		return 0, err
 	}
 	if opt.HasOffset {
-		_, err = f.Seek(opt.Offset, 0)
+		_, err := f.Seek(opt.Offset, 0)
 		if err != nil {
 			return 0, err
 		}
@@ -150,6 +150,9 @@ func (s *Storage) read(ctx context.Context, path string, w io.Writer, opt pairSt
 	if opt.HasIoCallback {
 		rc = iowrap.CallbackReader(rc, opt.IoCallback)
 	}
+	if opt.HasSize {
+		return io.CopyN(w, f, opt.Size)
+	}
 
 	return io.Copy(w, f)
 }
@@ -225,4 +228,4 @@ func (s *Storage) writeAppend(ctx context.Context, o *Object, r io.Reader, size
 	}()
 
 	return io.CopyN(f, r, size)
-}
+}

tests/storage_test.go

@@ -33,4 +33,4 @@ func TestAppender(t *testing.T) {
 		t.Skipf("STORAGE_HDFS_INTEGRATION_TEST is not 'on', skipped")
 	}
 	tests.TestAppender(t, setupTest(t))
-}
+}

tests/utils_test.go

@@ -23,4 +23,4 @@ func setupTest(t *testing.T) types.Storager {
 	}
 
 	return store
-}
+}

utils.go

@@ -98,7 +98,7 @@ func (s *Storage) getAbsPath(path string) string {
 	if filepath.IsAbs(path) {
 		return path
 	}
-	return s.workDir + path
+	return filepath.Join(s.workDir, path)
 }
 
 func (s *Storage) formatError(op string, err error, path ...string) error {
@@ -115,4 +115,4 @@ func (s *Storage) formatError(op string, err error, path ...string) error {
 
 func (s *Storage) newObject(done bool) *types.Object {
 	return types.NewObject(s, done)
-}
+}

utils_test.go

@@ -0,0 +1,62 @@
+package hdfs
+
+import (
+	"errors"
+	"os"
+	"testing"
+
+	"github.com/beyondstorage/go-storage/v4/services"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFormatOsError(t *testing.T) {
+	testErr := errors.New("test error")
+	tests := []struct {
+		name     string
+		input    error
+		expected error
+	}{
+		{
+			"not found",
+			os.ErrNotExist,
+			services.ErrObjectNotExist,
+		},
+		{
+			"not supported error",
+			testErr,
+			services.ErrUnexpected,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := formatError(tt.input)
+			assert.True(t, errors.Is(err, tt.expected))
+		})
+	}
+}
+
+func TestGetAbsPath(t *testing.T) {
+	cases := []struct {
+		name         string
+		base         string
+		path         string
+		expectedPath string
+	}{
+		{"direct path", "", "abc", "abc"},
+		{"under direct path", "", "root/abc", "root/abc"},
+		{"under direct path", "", "root/abc/", "root/abc"},
+		{"under root", "/", "abc", "/abc"},
+		{"under exist dir", "/root", "abc", "/root/abc"},
+		{"under new dir", "/root", "abc/", "/root/abc"},
+	}
+
+	for _, tt := range cases {
+		t.Run(tt.name, func(t *testing.T) {
+			client := Storage{workDir: tt.base}
+
+			getPath := client.getAbsPath(tt.path)
+			assert.Equal(t, tt.expectedPath, getPath)
+		})
+	}
+}