enha: integration testing

vesoft-inc · May 22, 2023 · e31f3c1 · e31f3c1
1 parent 9f55c86
commit e31f3c1
Show file tree

Hide file tree

Showing 44 changed files with 1,301 additions and 152 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -8,7 +8,7 @@ on:
     branches:
       - 'master'
     tags:
-      - 'v*'
+      - 'v[0-9]+.[0-9]+.[0-9]+*'
 
 permissions:
   contents: write
@@ -27,27 +27,31 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v4
         with:
           go-version: 1.19
-          cache: true
       - run: |
             go env -w GOFLAGS=-buildvcs=false
             git config --global --add safe.directory $(pwd)
+      - run: |
+            curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+            chmod +x /usr/local/bin/docker-compose
       - run: make lint
       - run: make test
+      - run: make test-it # integration-testing
       - uses: codecov/codecov-action@v2
       - uses: docker/login-action@v1
         if: success() && startsWith(github.ref, 'refs/tags/')
         with:
-          registry: ${{ secrets.HARBOR_REGISTRY }}
-          username: ${{ secrets.HARBOR_USERNAME }}
-          password: ${{ secrets.HARBOR_PASSWORD }}
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
       - uses: goreleaser/goreleaser-action@v2
         if: success() && startsWith(github.ref, 'refs/tags/')
         with:
           version: latest
           distribution: goreleaser
           args: release --rm-dist
         env:
+          GORELEASER_CURRENT_TAG: ${{ github.ref_name }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          DOCKER_IMAGE_REPO: vesoft/
diff --git a/.goreleaser.yaml b/.goreleaser.yaml
@@ -1,7 +1,7 @@
 project_name: nebula-importer
 
 release:
-  name_template: "Nebula Importer {{.Version}}"
+  name_template: "NebulaGraph Importer {{.Version}}"
 
 before:
   hooks:
@@ -33,25 +33,38 @@ builds:
       - "7"
 
 archives:
-  - name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
+  - id: archives
+    name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
+    wrap_in_directory: true
     replacements:
       darwin: Darwin
       linux: Linux
       windows: Windows
       386: i386
       amd64: x86_64
     files:
-      - README.md
+      - none* # https://goreleaser.com/customization/archive/?h=archives#packaging-only-the-binaries
     format_overrides:
       - goos: windows
         format: zip
+  - id: binary
+    name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
+    format: binary
+    replacements:
+      darwin: Darwin
+      linux: Linux
+      windows: Windows
+      386: i386
+      amd64: x86_64
+    files:
+      - none* # https://goreleaser.com/customization/archive/?h=archives#packaging-only-the-binaries
 
 dockers:
   - &dockers
     image_templates:
-      - "reg.vesoft-inc.com/vesoft-ent/{{ .ProjectName }}:{{ .Version }}-amd64"
+      - "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-amd64"
     use: buildx
-    dockerfile: Dockerfile.goreleaser
+    dockerfile: Dockerfile
     build_flag_templates:
       - "--platform=linux/amd64"
       - "--pull"
@@ -64,7 +77,7 @@ dockers:
       - "--label=org.opencontainers.image.revision={{ .FullCommit }}"
   - <<: *dockers
     image_templates:
-      - "reg.vesoft-inc.com/vesoft-ent/{{ .ProjectName }}:{{ .Version }}-armv7"
+      - "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-armv7"
     goarch: arm
     goarm: 7
     build_flag_templates:
@@ -79,7 +92,7 @@ dockers:
       - "--label=org.opencontainers.image.revision={{ .FullCommit }}"
   - <<: *dockers
     image_templates:
-      - "reg.vesoft-inc.com/vesoft-ent/{{ .ProjectName }}:{{ .Version }}-arm64v8"
+      - "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-arm64v8"
     goarch: arm64
     build_flag_templates:
       - "--platform=linux/arm64/v8"
@@ -93,19 +106,37 @@ dockers:
       - "--label=org.opencontainers.image.revision={{ .FullCommit }}"
 docker_manifests:
   - &docker_manifests
-    name_template: "reg.vesoft-inc.com/vesoft-ent/{{ .ProjectName }}:latest"
+    name_template: "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}"
+    image_templates:
+      - "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-amd64"
+      - "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-armv7"
+      - "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-arm64v8"
+  - <<: *docker_manifests
+    name_template: "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:v{{ .Major }}"
+    skip_push: auto
     image_templates:
-      - "reg.vesoft-inc.com/vesoft-ent/{{ .ProjectName }}:{{ .Version }}-amd64"
-      - "reg.vesoft-inc.com/vesoft-ent/{{ .ProjectName }}:{{ .Version }}-armv7"
-      - "reg.vesoft-inc.com/vesoft-ent/{{ .ProjectName }}:{{ .Version }}-arm64v8"
+      - "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-amd64"
+      - "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-armv7"
+      - "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-arm64v8"
   - <<: *docker_manifests
-    name_template: "reg.vesoft-inc.com/vesoft-ent/{{ .ProjectName }}:{{ .Version }}"
+    name_template: "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:v{{ .Major }}.{{ .Minor }}"
+    skip_push: auto
+    image_templates:
+      - "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-amd64"
+      - "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-armv7"
+      - "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-arm64v8"
+  - <<: *docker_manifests
+    name_template: "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:latest"
+    image_templates:
+      - "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-amd64"
+      - "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-armv7"
+      - "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-arm64v8"
 
 nfpms:
 - file_name_template: '{{ .ConventionalFileName }}'
   id: packages
   homepage: https://github.com/vesoft-inc/nebula-importer
-  description: Nebula Importer.
+  description: NebulaGraph Importer.
   maintainer: Vee Zhang <[email protected]>
   vendor: Vesoft Inc.
   bindir: /usr/bin

diff --git a/Dockerfile b/Dockerfile
@@ -1,24 +1,7 @@
-FROM reg.vesoft-inc.com/ci/golang:1.18-alpine AS builder
-
-LABEL stage=gobuilder
-
-ENV CGO_ENABLED 0
-ENV GOOS linux
-WORKDIR /build/zero
-
-ADD go.mod .
-ADD go.sum .
-COPY pkg pkg
-COPY cmd cmd
-RUN go mod download
-
-RUN go build -ldflags="-s -w" -o /usr/bin/nebula-importer ./cmd/nebula-importer
-
-FROM reg.vesoft-inc.com/ci/alpine
+FROM reg.vesoft-inc.com/proxy/library/alpine
 
 RUN apk update --no-cache && apk add --no-cache ca-certificates tzdata
-ENV TZ Asia/Shanghai
 
-COPY --from=builder /usr/bin/nebula-importer /usr/bin/nebula-importer
+ADD nebula-importer /usr/local/bin/nebula-importer
 
-ENTRYPOINT ["/usr/bin/nebula-importer"]
+ENTRYPOINT ["/usr/local/bin/nebula-importer"]
diff --git a/Makefile b/Makefile
@@ -44,6 +44,9 @@ build:
 test:
 	go test -gcflags=all="-l" -race -coverprofile=coverage.txt -covermode=atomic ./pkg/...
 
+test-it: # integration-testing
+	docker-compose -f integration-testing/docker-compose.yaml up --build --exit-code-from importer
+
 docker-build:
 	docker build -t "${DOCKER_REPO}/nebula-importer:${IMAGE_TAG}" -f Dockerfile .
 

diff --git a/README.md b/README.md
@@ -3,9 +3,9 @@
 [![GolangCI](https://golangci.com/badges/github.com/vesoft-inc/nebula-importer.svg)](https://golangci.com/r/github.com/vesoft-inc/nebula-importer)
 [![GoDoc](https://godoc.org/github.com/vesoft-inc/nebula-importer?status.svg)](https://godoc.org/github.com/vesoft-inc/nebula-importer)
 
-# What is Nebula Importer?
+# What is NebulaGraph Importer?
 
-**Nebula Importer** is a tool to import data into [NebulaGraph](https://github.com/vesoft-inc/nebula).
+**NebulaGraph Importer** is a tool to import data into [NebulaGraph](https://github.com/vesoft-inc/nebula).
 
 ## Features
 
@@ -28,6 +28,7 @@ Download the packages on the [Releases page](https://github.com/vesoft-inc/nebul
 You can choose according to your needs, the following installation packages are supported:
 
 * binary
+* archives
 * apk
 * deb
 * rpm
@@ -41,8 +42,17 @@ $ go install github.com/vesoft-inc/nebula-importer/cmd/nebula-importer@latest
 ### From docker
 
 ```shell
-$ docker pull vesoft/nebula-importer
-$ docker run -it -v <path-to-config>:/config.yaml --rm vesoft/nebula-importer -c /config.yaml
+$ docker pull vesoft/nebula-importer:<version>
+$ docker run --rm -ti \
+      --network=host \
+      -v <config_file>:<config_file> \
+      -v <data_dir>:<data_dir> \
+      vesoft/nebula-importer:<version>
+      --config <config_file>
+
+# config_file: the absolute path to the configuration file.
+# data_dir: the absolute path to the data directory, ignore if not a local file.
+# version: the version of NebulaGraph Importer.
 ```
 
 ### From Source Code
@@ -57,12 +67,12 @@ You can find a binary named `nebula-importer` in `bin` directory.
 
 ## Configuration Instructions
 
-`Nebula Importer`'s configuration file is in YAML format. You can find some examples in [examples](examples/).
+`NebulaGraph Importer`'s configuration file is in YAML format. You can find some examples in [examples](examples/).
 
 Configuration options are divided into four groups:
 
 * `client` is configuration options related to the NebulaGraph connection client.
-* `manager` is global control configuration options related to Nebula Importer.
+* `manager` is global control configuration options related to NebulaGraph Importer.
 * `log` is configuration options related to printing logs.
 * `sources` is the data source configuration items.
 
@@ -100,14 +110,19 @@ client:
   hooks:
     before:
       - statements:
-          - statements1
-          - statements2
-        wait: 10s
+          - UPDATE CONFIGS storage:wal_ttl=3600;
+          - UPDATE CONFIGS storage:rocksdb_column_family_options = { disable_auto_compactions = true };
       - statements:
-          - statements3
+          - |
+            DROP SPACE IF EXISTS basic_int_examples;
+            CREATE SPACE IF NOT EXISTS basic_int_examples(partition_num=5, replica_factor=1, vid_type=int);
+            USE basic_int_examples;
+        wait: 10s
     after:
       - statements:
-          - statements4
+          - |
+            UPDATE CONFIGS storage:wal_ttl=86400;
+            UPDATE CONFIGS storage:rocksdb_column_family_options = { disable_auto_compactions = false };
 ```
 
 * `manager.spaceName`: **Required**. Specifies which space the data is imported into.
@@ -132,7 +147,7 @@ log:
    - logs/nebula-importer.log
 ```
 
-* `log.level`: **Optional**. Specifies the log level. The default value is `INFO`.
+* `log.level`: **Optional**. Specifies the log level, optional values is `DEBUG`, `INFO`, `WARN`, `ERROR`, `PANIC` or `FATAL`. The default value is `INFO`.
 * `log.console`: **Optional**. Specifies whether to print logs to the console. The default value is `true`.
 * `log.files`: **Optional**. Specifies which files to print logs to.
 
@@ -168,16 +183,16 @@ s3:
   region: <region>
   bucket: <bucket>
   key: <key>
-  accessKey: <accessKey>
-  secretKey: <secretKey>
+  accessKeyID: <Access Key ID>
+  accessKeySecret: <Access Key Secret>
 ```
 
 * `endpoint`: **Optional**. The endpoint of s3 service, can be omitted if using aws s3.
 * `region`: **Required**. The region of s3 service.
 * `bucket`: **Required**. The bucket of file in s3 service.
 * `key`: **Required**. The object key of file in s3 service.
-* `accessKey`: **Optional**. The access key of s3 service. If it is public data, no need to configure.
-* `secretKey`: **Optional**. The secret key of s3 service. If it is public data, no need to configure.
+* `accessKeyID`: **Optional**. The `Access Key ID` of s3 service. If it is public data, no need to configure.
+* `accessKeySecret`: **Optional**. The `Access Key Secret` of s3 service. If it is public data, no need to configure.
 
 #### oss
 
@@ -188,15 +203,15 @@ oss:
   endpoint: <endpoint>
   bucket: <bucket>
   key: <key>
-  accessKey: <accessKey>
-  secretKey: <secretKey>
+  accessKeyID: <Access Key ID>
+  accessKeySecret: <Access Key Secret>
 ```
 
 * `endpoint`: **Required**. The endpoint of oss service.
 * `bucket`: **Required**. The bucket of file in oss service.
 * `key`: **Required**. The object key of file in oss service.
-* `accessKey`: **Required**. The access key of oss service.
-* `secretKey`: **Required**. The secret key of oss service.
+* `accessKeyID`: **Required**. The `Access Key ID` of oss service.
+* `accessKeySecret`: **Required**. The `Access Key Secret` of oss service.
 
 #### ftp
 

diff --git a/docs/configuration-reference.md b/docs/configuration-reference.md
@@ -1,4 +1,4 @@
-# Nebula Importer Configuration Description
+# NebulaGraph Importer Configuration Description
 
 | options                                      | description                                                               | default    |
 | :--                                          | :--                                                                       | :--        |
@@ -12,7 +12,7 @@
 | client.retry                                 | The failed retrying times to execute nGQL queries in NebulaGraph client.  | 3          |
 | client.retryInitialInterval                  | The initialization interval retrying.                                     | 1s         |
 |                                              |                                                                           |            |
-| manager                                      | The global control configuration options related to Nebula Importer.      | -          |
+| manager                                      | The global control configuration options related to NebulaGraph Importer. | -          |
 | manager.spaceName                            | Specifies which space the data is imported into.                          | -          |
 | manager.batch                                | Specifies the batch size for all sources of the inserted data.            | 128        |
 | manager.readerConcurrency                    | Specifies the concurrency of reader to read from sources.                 | 50         |
@@ -36,13 +36,13 @@
 | sources[].s3.region                          | The region of s3 service.                                                 | -          |
 | sources[].s3.bucket                          | The bucket of file in s3 service.                                         | -          |
 | sources[].s3.key                             | The object key of file in s3 service.                                     | -          |
-| sources[].s3.accessKey                       | The access key of s3 service.                                             | -          |
-| sources[].s3.secretKey                       | The secret key of s3 service.                                             | -          |
+| sources[].s3.accessKeyID                     | The `Access Key ID` of s3 service.                                        | -          |
+| sources[].s3.accessKeySecret                 | The `Access Key Secret` of s3 service.                                    | -          |
 | sources[].oss.endpoint                       | The endpoint of oss service.                                              | -          |
 | sources[].oss.bucket                         | The bucket of file in oss service.                                        | -          |
 | sources[].oss.key                            | The object key of file in oss service.                                    | -          |
-| sources[].oss.accessKey                      | The access key of oss service.                                            | -          |
-| sources[].oss.secretKey                      | The secret key of oss service.                                            | -          |
+| sources[].oss.accessKeyID                    | The `Access Key ID` of oss service.                                       | -          |
+| sources[].oss.accessKeySecret                | The `Access Key Secret` of oss service.                                   | -          |
 | sources[].ftp.host                           | The host of ftp service.                                                  | -          |
 | sources[].ftp.host                           | The port of ftp service.                                                  | -          |
 | sources[].ftp.user                           | The user of ftp service.                                                  | -          |

diff --git a/examples/oss/oss.v3.yaml b/examples/oss/oss.v3.yaml
@@ -32,10 +32,10 @@ log:
 sources:
   - oss:
       endpoint: https://oss-cn-hangzhou.aliyuncs.com
-      bucket: bucketName
-      key: objectKey
-      accessKey: accessKey
-      secretKey: secretKey
+      bucket: bucket-name
+      key: object-key
+      accessKeyID: "Access Key ID"
+      accessKeySecret: "Access Key Secret"
     csv:
       delimiter: "\t"
     tags:

diff --git a/examples/s3/s3.v3.yaml b/examples/s3/s3.v3.yaml
@@ -34,8 +34,8 @@ sources:
       region: us-east-1
       bucket: gdelt-open-data
       key: events/20190918.export.csv
-      # accessKey: ""
-      # secretKey: ""
+      # accessKeyID: ""
+      # accessKeySecret: ""
     csv:
       delimiter: "\t"
     tags:
@@ -52,8 +52,8 @@ sources:
       region: us-east-1
       bucket: gdelt-open-data
       key: events/20190918.export.csv
-      accessKey: "accessKey"
-      secretKey: "secretKey"
+      accessKeyID: "Access Key ID"
+      accessKeySecret: "Access Key Secret"
     csv:
       delimiter: "\t"
     tags: