Merge 2.0 branch to master (#102)

* nebula-importer supports nebula-graph v2-alpha-preview (#98) * feat: use nebula-go/v2 as nebula-graph client mod. * feat: wrap the VID value in double quotes when in the INSERT VERTEX statement. * feat: update metad, storaged, graphd to v2 version in docker-compose.yaml. * feat: quotes vid value. * feat: use vesoft/nebula-console:v2-preview-nightly as importer docker image. * feat: update ci/bootstrap.sh because nebula-console:v2-preview-nightly is based on alpine. * feat: remove uuid function, and use "(string)hash" instead of hash as vid value. * feat: use ubuntu-18.04 as github ci server. * feat: increase afterPeriod wait seconds in example.yaml. * feat: logger the importer init proceduces. * feat: add beforePeriod to wait postStart commands competition. * feat: use ubuntu-latest as github ci server. * feat: revert to previous afterPeriod. * feat: remove afterPeriod logger info. * style: just for re-trigger upstream PR CI. * Remove vendors (#101) * Remove vendor * Add vendorbuild target * Ignore vendor * Continue cleanup * Support nebula 2.0 data import (#99) * Use nebula go client 2.0 * Try to fix importer tests * Fix vid type error * Add more tests * Rename runner file * Fix crash * Fix Ci * Cleanup * Cleanup vendors * Restore 1.0 test csv data * Fix logger Co-authored-by: laishzh <[email protected]>
vesoft-inc · Dec 10, 2020 · 30cb255 · 30cb255
1 parent 5d1476d
commit 30cb255
Show file tree

Hide file tree

Showing 114 changed files with 307 additions and 25,438 deletions.
diff --git a/.gitignore b/.gitignore
@@ -4,7 +4,6 @@
 *.dll
 *.so
 *.dylib
-nebula-importer
 
 # Test binary, build with `go test -c`
 *.test
@@ -15,4 +14,6 @@ nebula-importer
 # macOS
 .DS_Store
 
-examples/err/
+err/
+vendor/
+nebula-importer
diff --git a/Makefile b/Makefile
@@ -4,6 +4,15 @@
 default: build
 
 build: clean fmt
+	@cd cmd; \
+	go build -o nebula-importer; \
+	mv nebula-importer ..;
+	@echo "nebula-importer has been outputed to $$(pwd)/nebula-importer";
+
+vendor: clean fmt
+	@cd cmd; go mod vendor
+
+vendorbuild: vendor
 	@cd cmd; \
 	go build -mod vendor -o nebula-importer; \
 	mv nebula-importer ..;

diff --git a/ci/bootstrap.sh b/ci/bootstrap.sh
@@ -5,19 +5,21 @@ set -e
 addr=$1
 port=$2
 
-curl -fsSL https://studygolang.com/dl/golang/go1.13.4.linux-amd64.tar.gz -o go1.13.4.linux-amd64.tar.gz
-tar zxf go1.13.4.linux-amd64.tar.gz -C /usr/local/
-
-export GOROOT=/usr/local/go
 export GOPATH=/usr/local/nebula/
-export PATH=$PATH:$GOROOT/bin:$GOPATH/bin
 export GO111MODULE=on
 
-pushd ./importer/cmd
-go build -mod vendor -o ../../nebula-importer
-popd
+# build nebula-console
+wget "https://github.com/vesoft-inc/nebula-console/archive/master.zip" -O nebula-console.zip
+unzip nebula-console.zip -d .
+mv nebula-console-* nebula-console
+cd nebula-console
+go build -o nebula-console
+
+cd /usr/local/nebula/importer/cmd
+go build -o ../../nebula-importer
+cd /usr/local/nebula
 
-until echo "quit" | ./bin/nebula -u user -p password --addr=$addr --port=$port &> /dev/null; do
+until echo "quit" | /usr/local/nebula/nebula-console/nebula-console -u user -p password --addr=$addr --port=$port &> /dev/null; do
   echo "nebula graph is unavailable - sleeping"
   sleep 2
 done

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -1,19 +1,19 @@
 version: '3.4'
 services:
   metad:
-    image: vesoft/nebula-metad:nightly
+    image: vesoft/nebula-metad:v2-preview-nightly
     environment:
       USER: root
     command:
-      - --meta_server_addrs=172.29.1.1:45500
-      - --local_ip=172.29.1.1
-      - --ws_ip=172.29.1.1
+      - --meta_server_addrs=metad:45500
+      - --local_ip=metad
+      - --ws_ip=metad
       - --port=45500
     volumes:
       - metadata:/usr/local/nebula/data/meta
       - logsdata:/usr/local/nebula/logs
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://172.29.1.1:11000/status"]
+      test: ["CMD", "curl", "-f", "http://metad:11000/status"]
       interval: 15s
       timeout: 10s
       retries: 3
@@ -23,25 +23,24 @@ services:
       - 11002
     networks:
       nebula-net:
-        ipv4_address: 172.29.1.1
     restart: on-failure
 
   storaged:
-    image: vesoft/nebula-storaged:nightly
+    image: vesoft/nebula-storaged:v2-preview-nightly
     environment:
       USER: root
     command:
-      - --meta_server_addrs=172.29.1.1:45500
-      - --local_ip=172.29.2.1
-      - --ws_ip=172.29.2.1
+      - --meta_server_addrs=metad:45500
+      - --local_ip=storaged
+      - --ws_ip=storaged
       - --port=44500
     volumes:
       - storagedata:/usr/local/nebula/data/storage
       - logsdata:/usr/local/nebula/logs
     depends_on:
       - metad
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://172.29.2.1:12000/status"]
+      test: ["CMD", "curl", "-f", "http://storaged:12000/status"]
       interval: 15s
       timeout: 10s
       retries: 3
@@ -51,24 +50,23 @@ services:
       - 12002
     networks:
       nebula-net:
-        ipv4_address: 172.29.2.1
     restart: on-failure
 
   graphd1:
-    image: vesoft/nebula-graphd:nightly
+    image: vesoft/nebula-graphd:v2-preview-nightly
     environment:
       USER: root
     command:
-      - --meta_server_addrs=172.29.1.1:45500
+      - --meta_server_addrs=metad:45500
       - --port=3699
-      - --ws_ip=172.29.3.1
+      - --ws_ip=graphd1
       - --log_dir=/usr/local/nebula/logs1
     volumes:
       - logsdata:/usr/local/nebula/logs1
     depends_on:
       - metad
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://172.29.3.1:13000/status"]
+      test: ["CMD", "curl", "-f", "http://graphd1:13000/status"]
       interval: 30s
       timeout: 10s
       retries: 3
@@ -79,25 +77,24 @@ services:
       - 13002
     networks:
       nebula-net:
-        ipv4_address: 172.29.3.1
     restart: on-failure
 
   graphd2:
-    image: vesoft/nebula-graphd:nightly
+    image: vesoft/nebula-graphd:v2-preview-nightly
     environment:
       USER: root
     command:
-      - --meta_server_addrs=172.29.1.1:45500
+      - --meta_server_addrs=metad:45500
       - --port=3699
-      - --ws_ip=172.29.3.2
+      - --ws_ip=graphd2
       - --log_dir=/usr/local/nebula/logs2
       - --enable_authorize=true
     volumes:
       - logsdata:/usr/local/nebula/logs2
     depends_on:
       - metad
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://172.29.3.2:13000/status"]
+      test: ["CMD", "curl", "-f", "http://graphd2:13000/status"]
       interval: 30s
       timeout: 10s
       retries: 3
@@ -108,30 +105,25 @@ services:
       - 13002
     networks:
       nebula-net:
-        ipv4_address: 172.29.3.2
     restart: on-failure
 
   importer:
-    image: vesoft/nebula-console:nightly
+    image: golang:alpine
     volumes:
       - .:/usr/local/nebula/importer:rw
+    working_dir: /usr/local/nebula
     entrypoint:
       - ./importer/ci/bootstrap.sh
-      - "172.29.3.1"
+      - "graphd1"
       - "3699"
     depends_on:
       - graphd1
       - graphd2
     networks:
       nebula-net:
-        ipv4_address: 172.29.4.1
 
 networks:
   nebula-net:
-    ipam:
-      driver: default
-      config:
-        - subnet: 172.29.0.0/16
 
 volumes:
   storagedata: