Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

enha: integration testing #267

Merged
merged 1 commit into from
May 22, 2023
Merged

enha: integration testing #267

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 10 additions & 6 deletions .github/workflows/build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ on:
branches:
- 'master'
tags:
- 'v*'
- 'v[0-9]+.[0-9]+.[0-9]+*'

permissions:
contents: write
Expand All @@ -27,27 +27,31 @@ jobs:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: actions/setup-go@v3
- uses: actions/setup-go@v4
with:
go-version: 1.19
cache: true
- run: |
go env -w GOFLAGS=-buildvcs=false
git config --global --add safe.directory $(pwd)
- run: |
curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
chmod +x /usr/local/bin/docker-compose
- run: make lint
- run: make test
- run: make test-it # integration-testing
- uses: codecov/codecov-action@v2
- uses: docker/login-action@v1
if: success() && startsWith(github.ref, 'refs/tags/')
with:
registry: ${{ secrets.HARBOR_REGISTRY }}
username: ${{ secrets.HARBOR_USERNAME }}
password: ${{ secrets.HARBOR_PASSWORD }}
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- uses: goreleaser/goreleaser-action@v2
if: success() && startsWith(github.ref, 'refs/tags/')
with:
version: latest
distribution: goreleaser
args: release --rm-dist
env:
GORELEASER_CURRENT_TAG: ${{ github.ref_name }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
DOCKER_IMAGE_REPO: vesoft/
57 changes: 44 additions & 13 deletions .goreleaser.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
project_name: nebula-importer

release:
name_template: "Nebula Importer {{.Version}}"
name_template: "NebulaGraph Importer {{.Version}}"

before:
hooks:
Expand Down Expand Up @@ -33,25 +33,38 @@ builds:
- "7"

archives:
- name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
- id: archives
name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
wrap_in_directory: true
replacements:
darwin: Darwin
linux: Linux
windows: Windows
386: i386
amd64: x86_64
files:
- README.md
- none* # https://goreleaser.com/customization/archive/?h=archives#packaging-only-the-binaries
format_overrides:
- goos: windows
format: zip
- id: binary
name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
format: binary
replacements:
darwin: Darwin
linux: Linux
windows: Windows
386: i386
amd64: x86_64
files:
- none* # https://goreleaser.com/customization/archive/?h=archives#packaging-only-the-binaries

dockers:
- &dockers
image_templates:
- "reg.vesoft-inc.com/vesoft-ent/{{ .ProjectName }}:{{ .Version }}-amd64"
- "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-amd64"
use: buildx
dockerfile: Dockerfile.goreleaser
dockerfile: Dockerfile
build_flag_templates:
- "--platform=linux/amd64"
- "--pull"
Expand All @@ -64,7 +77,7 @@ dockers:
- "--label=org.opencontainers.image.revision={{ .FullCommit }}"
- <<: *dockers
image_templates:
- "reg.vesoft-inc.com/vesoft-ent/{{ .ProjectName }}:{{ .Version }}-armv7"
- "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-armv7"
goarch: arm
goarm: 7
build_flag_templates:
Expand All @@ -79,7 +92,7 @@ dockers:
- "--label=org.opencontainers.image.revision={{ .FullCommit }}"
- <<: *dockers
image_templates:
- "reg.vesoft-inc.com/vesoft-ent/{{ .ProjectName }}:{{ .Version }}-arm64v8"
- "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-arm64v8"
goarch: arm64
build_flag_templates:
- "--platform=linux/arm64/v8"
Expand All @@ -93,19 +106,37 @@ dockers:
- "--label=org.opencontainers.image.revision={{ .FullCommit }}"
docker_manifests:
- &docker_manifests
name_template: "reg.vesoft-inc.com/vesoft-ent/{{ .ProjectName }}:latest"
name_template: "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}"
image_templates:
- "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-amd64"
- "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-armv7"
- "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-arm64v8"
- <<: *docker_manifests
name_template: "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:v{{ .Major }}"
skip_push: auto
image_templates:
- "reg.vesoft-inc.com/vesoft-ent/{{ .ProjectName }}:{{ .Version }}-amd64"
- "reg.vesoft-inc.com/vesoft-ent/{{ .ProjectName }}:{{ .Version }}-armv7"
- "reg.vesoft-inc.com/vesoft-ent/{{ .ProjectName }}:{{ .Version }}-arm64v8"
- "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-amd64"
- "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-armv7"
- "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-arm64v8"
- <<: *docker_manifests
name_template: "reg.vesoft-inc.com/vesoft-ent/{{ .ProjectName }}:{{ .Version }}"
name_template: "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:v{{ .Major }}.{{ .Minor }}"
skip_push: auto
image_templates:
- "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-amd64"
- "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-armv7"
- "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-arm64v8"
- <<: *docker_manifests
name_template: "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:latest"
image_templates:
- "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-amd64"
- "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-armv7"
- "{{ .Env.DOCKER_IMAGE_REPO }}{{ .ProjectName }}:{{ .Tag }}-arm64v8"

nfpms:
- file_name_template: '{{ .ConventionalFileName }}'
id: packages
homepage: https://github.com/vesoft-inc/nebula-importer
description: Nebula Importer.
description: NebulaGraph Importer.
maintainer: Vee Zhang <[email protected]>
vendor: Vesoft Inc.
bindir: /usr/bin
Expand Down
23 changes: 3 additions & 20 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,24 +1,7 @@
FROM reg.vesoft-inc.com/ci/golang:1.18-alpine AS builder

LABEL stage=gobuilder

ENV CGO_ENABLED 0
ENV GOOS linux
WORKDIR /build/zero

ADD go.mod .
ADD go.sum .
COPY pkg pkg
COPY cmd cmd
RUN go mod download

RUN go build -ldflags="-s -w" -o /usr/bin/nebula-importer ./cmd/nebula-importer

FROM reg.vesoft-inc.com/ci/alpine
FROM reg.vesoft-inc.com/proxy/library/alpine

RUN apk update --no-cache && apk add --no-cache ca-certificates tzdata
ENV TZ Asia/Shanghai

COPY --from=builder /usr/bin/nebula-importer /usr/bin/nebula-importer
ADD nebula-importer /usr/local/bin/nebula-importer

ENTRYPOINT ["/usr/bin/nebula-importer"]
ENTRYPOINT ["/usr/local/bin/nebula-importer"]
3 changes: 3 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ build:
test:
go test -gcflags=all="-l" -race -coverprofile=coverage.txt -covermode=atomic ./pkg/...

test-it: # integration-testing
docker-compose -f integration-testing/docker-compose.yaml up --build --exit-code-from importer

docker-build:
docker build -t "${DOCKER_REPO}/nebula-importer:${IMAGE_TAG}" -f Dockerfile .

Expand Down
55 changes: 35 additions & 20 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@
[![GolangCI](https://golangci.com/badges/github.com/vesoft-inc/nebula-importer.svg)](https://golangci.com/r/github.com/vesoft-inc/nebula-importer)
[![GoDoc](https://godoc.org/github.com/vesoft-inc/nebula-importer?status.svg)](https://godoc.org/github.com/vesoft-inc/nebula-importer)

# What is Nebula Importer?
# What is NebulaGraph Importer?

**Nebula Importer** is a tool to import data into [NebulaGraph](https://github.com/vesoft-inc/nebula).
**NebulaGraph Importer** is a tool to import data into [NebulaGraph](https://github.com/vesoft-inc/nebula).

## Features

Expand All @@ -28,6 +28,7 @@ Download the packages on the [Releases page](https://github.com/vesoft-inc/nebul
You can choose according to your needs, the following installation packages are supported:

* binary
* archives
* apk
* deb
* rpm
Expand All @@ -41,8 +42,17 @@ $ go install github.com/vesoft-inc/nebula-importer/cmd/nebula-importer@latest
### From docker

```shell
$ docker pull vesoft/nebula-importer
$ docker run -it -v <path-to-config>:/config.yaml --rm vesoft/nebula-importer -c /config.yaml
$ docker pull vesoft/nebula-importer:<version>
$ docker run --rm -ti \
--network=host \
-v <config_file>:<config_file> \
-v <data_dir>:<data_dir> \
vesoft/nebula-importer:<version>
--config <config_file>

# config_file: the absolute path to the configuration file.
# data_dir: the absolute path to the data directory, ignore if not a local file.
# version: the version of NebulaGraph Importer.
```

### From Source Code
Expand All @@ -57,12 +67,12 @@ You can find a binary named `nebula-importer` in `bin` directory.

## Configuration Instructions

`Nebula Importer`'s configuration file is in YAML format. You can find some examples in [examples](examples/).
`NebulaGraph Importer`'s configuration file is in YAML format. You can find some examples in [examples](examples/).

Configuration options are divided into four groups:

* `client` is configuration options related to the NebulaGraph connection client.
* `manager` is global control configuration options related to Nebula Importer.
* `manager` is global control configuration options related to NebulaGraph Importer.
* `log` is configuration options related to printing logs.
* `sources` is the data source configuration items.

Expand Down Expand Up @@ -100,14 +110,19 @@ client:
hooks:
before:
- statements:
- statements1
- statements2
wait: 10s
- UPDATE CONFIGS storage:wal_ttl=3600;
- UPDATE CONFIGS storage:rocksdb_column_family_options = { disable_auto_compactions = true };
- statements:
- statements3
- |
DROP SPACE IF EXISTS basic_int_examples;
CREATE SPACE IF NOT EXISTS basic_int_examples(partition_num=5, replica_factor=1, vid_type=int);
USE basic_int_examples;
wait: 10s
after:
- statements:
- statements4
- |
UPDATE CONFIGS storage:wal_ttl=86400;
UPDATE CONFIGS storage:rocksdb_column_family_options = { disable_auto_compactions = false };
```

* `manager.spaceName`: **Required**. Specifies which space the data is imported into.
Expand All @@ -132,7 +147,7 @@ log:
- logs/nebula-importer.log
```

* `log.level`: **Optional**. Specifies the log level. The default value is `INFO`.
* `log.level`: **Optional**. Specifies the log level, optional values is `DEBUG`, `INFO`, `WARN`, `ERROR`, `PANIC` or `FATAL`. The default value is `INFO`.
* `log.console`: **Optional**. Specifies whether to print logs to the console. The default value is `true`.
* `log.files`: **Optional**. Specifies which files to print logs to.

Expand Down Expand Up @@ -168,16 +183,16 @@ s3:
region: <region>
bucket: <bucket>
key: <key>
accessKey: <accessKey>
secretKey: <secretKey>
accessKeyID: <Access Key ID>
accessKeySecret: <Access Key Secret>
```

* `endpoint`: **Optional**. The endpoint of s3 service, can be omitted if using aws s3.
* `region`: **Required**. The region of s3 service.
* `bucket`: **Required**. The bucket of file in s3 service.
* `key`: **Required**. The object key of file in s3 service.
* `accessKey`: **Optional**. The access key of s3 service. If it is public data, no need to configure.
* `secretKey`: **Optional**. The secret key of s3 service. If it is public data, no need to configure.
* `accessKeyID`: **Optional**. The `Access Key ID` of s3 service. If it is public data, no need to configure.
* `accessKeySecret`: **Optional**. The `Access Key Secret` of s3 service. If it is public data, no need to configure.

#### oss

Expand All @@ -188,15 +203,15 @@ oss:
endpoint: <endpoint>
bucket: <bucket>
key: <key>
accessKey: <accessKey>
secretKey: <secretKey>
accessKeyID: <Access Key ID>
accessKeySecret: <Access Key Secret>
```

* `endpoint`: **Required**. The endpoint of oss service.
* `bucket`: **Required**. The bucket of file in oss service.
* `key`: **Required**. The object key of file in oss service.
* `accessKey`: **Required**. The access key of oss service.
* `secretKey`: **Required**. The secret key of oss service.
* `accessKeyID`: **Required**. The `Access Key ID` of oss service.
* `accessKeySecret`: **Required**. The `Access Key Secret` of oss service.

#### ftp

Expand Down
12 changes: 6 additions & 6 deletions docs/configuration-reference.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Nebula Importer Configuration Description
# NebulaGraph Importer Configuration Description

| options | description | default |
| :-- | :-- | :-- |
Expand All @@ -12,7 +12,7 @@
| client.retry | The failed retrying times to execute nGQL queries in NebulaGraph client. | 3 |
| client.retryInitialInterval | The initialization interval retrying. | 1s |
| | | |
| manager | The global control configuration options related to Nebula Importer. | - |
| manager | The global control configuration options related to NebulaGraph Importer. | - |
| manager.spaceName | Specifies which space the data is imported into. | - |
| manager.batch | Specifies the batch size for all sources of the inserted data. | 128 |
| manager.readerConcurrency | Specifies the concurrency of reader to read from sources. | 50 |
Expand All @@ -36,13 +36,13 @@
| sources[].s3.region | The region of s3 service. | - |
| sources[].s3.bucket | The bucket of file in s3 service. | - |
| sources[].s3.key | The object key of file in s3 service. | - |
| sources[].s3.accessKey | The access key of s3 service. | - |
| sources[].s3.secretKey | The secret key of s3 service. | - |
| sources[].s3.accessKeyID | The `Access Key ID` of s3 service. | - |
| sources[].s3.accessKeySecret | The `Access Key Secret` of s3 service. | - |
| sources[].oss.endpoint | The endpoint of oss service. | - |
| sources[].oss.bucket | The bucket of file in oss service. | - |
| sources[].oss.key | The object key of file in oss service. | - |
| sources[].oss.accessKey | The access key of oss service. | - |
| sources[].oss.secretKey | The secret key of oss service. | - |
| sources[].oss.accessKeyID | The `Access Key ID` of oss service. | - |
| sources[].oss.accessKeySecret | The `Access Key Secret` of oss service. | - |
| sources[].ftp.host | The host of ftp service. | - |
| sources[].ftp.host | The port of ftp service. | - |
| sources[].ftp.user | The user of ftp service. | - |
Expand Down
8 changes: 4 additions & 4 deletions examples/oss/oss.v3.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,10 @@ log:
sources:
- oss:
endpoint: https://oss-cn-hangzhou.aliyuncs.com
bucket: bucketName
key: objectKey
accessKey: accessKey
secretKey: secretKey
bucket: bucket-name
key: object-key
accessKeyID: "Access Key ID"
accessKeySecret: "Access Key Secret"
csv:
delimiter: "\t"
tags:
Expand Down
8 changes: 4 additions & 4 deletions examples/s3/s3.v3.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ sources:
region: us-east-1
bucket: gdelt-open-data
key: events/20190918.export.csv
# accessKey: ""
# secretKey: ""
# accessKeyID: ""
# accessKeySecret: ""
csv:
delimiter: "\t"
tags:
Expand All @@ -52,8 +52,8 @@ sources:
region: us-east-1
bucket: gdelt-open-data
key: events/20190918.export.csv
accessKey: "accessKey"
secretKey: "secretKey"
accessKeyID: "Access Key ID"
accessKeySecret: "Access Key Secret"
csv:
delimiter: "\t"
tags:
Expand Down
Loading