diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 974509d8d35..517f78b1a6b 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -199,3 +199,92 @@ jobs: with: name: ${{ matrix.os }}-${{ matrix.compiler }}-nebula-test-logs path: ./build/server_*/logs/ + + standalone: + name: standalone-build + needs: lint + runs-on: [self-hosted, nebula] + strategy: + fail-fast: false + matrix: + os: + - centos7 + compiler: + - gcc-9.3 + env: + CCACHE_DIR: /tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }} + CCACHE_MAXSIZE: 8G + container: + image: vesoft/nebula-dev:${{ matrix.os }} + volumes: + - /tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}:/tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }} + options: --cap-add=SYS_PTRACE + steps: + - uses: webiny/action-post-run@2.0.1 + with: + run: sh -c "find . -mindepth 1 -delete" + - uses: actions/checkout@v2 + - name: Prepare environment + id: prepare + run: | + [ -d build/ ] && rm -rf build/* || mkdir -p build + make init -C tests + - name: CMake + id: cmake + run: | + case ${{ matrix.compiler }} in + gcc-*) + case ${{ matrix.os }} in + centos7) + # build with Release type + cmake \ + -DCMAKE_CXX_COMPILER=$TOOLSET_GCC_DIR/bin/g++ \ + -DCMAKE_C_COMPILER=$TOOLSET_GCC_DIR/bin/gcc \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_TESTING=on \ + -DENABLE_STANDALONE_VERSION=on \ + -GNinja \ + -B build + echo "::set-output name=j::10" + ;; + esac + ;; + esac + - name: Make + run: | + ccache -z + ninja -j $(nproc) + ccache -s + working-directory: build/ + - name: CTest + env: + ASAN_OPTIONS: fast_unwind_on_malloc=1 + run: ctest -j $(($(nproc)/2+1)) --timeout 400 --output-on-failure + working-directory: build/ + timeout-minutes: 20 + - name: Setup Cluster + run: | + make standalone-up + working-directory: tests/ + timeout-minutes: 60 + - name: TCK + run: | + make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} standalone-tck + working-directory: tests/ + timeout-minutes: 60 + - name: LDBC + run: | + make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} ldbc + working-directory: tests/ + timeout-minutes: 60 + - name: Down cluster + run: | + make RM_DIR=false down + working-directory: tests/ + timeout-minutes: 2 + - name: Upload logs + uses: actions/upload-artifact@v2 + if: ${{ failure() }} + with: + name: ${{ matrix.os }}-${{ matrix.compiler }}-nebula-test-logs + path: ./build/server_*/logs/ diff --git a/src/clients/storage/StorageClientBase-inl.h b/src/clients/storage/StorageClientBase-inl.h index 6341e6dd74e..eb0c43472e0 100644 --- a/src/clients/storage/StorageClientBase-inl.h +++ b/src/clients/storage/StorageClientBase-inl.h @@ -240,16 +240,18 @@ void StorageClientBase::getResponseImpl( DCHECK(!!ioThreadPool_); evb = ioThreadPool_->getEventBase(); } + auto reqPtr = std::make_shared>(std::move(request.first), + std::move(request.second)); folly::via( evb, - [evb, request = std::move(request), remoteFunc = std::move(remoteFunc), pro, this]() mutable { - auto host = request.first; + [evb, request = std::move(reqPtr), remoteFunc = std::move(remoteFunc), pro, this]() mutable { + auto host = request->first; auto client = clientsMan_->client(host, evb, false, FLAGS_storage_client_timeout_ms); - auto spaceId = request.second.get_space_id(); - auto partsId = getReqPartsId(request.second); - remoteFunc(client.get(), request.second) + auto spaceId = request->second.get_space_id(); + auto partsId = getReqPartsId(request->second); + remoteFunc(client.get(), request->second) .via(evb) - .thenValue([spaceId, pro, this](Response&& resp) mutable { + .thenValue([spaceId, pro, request, this](Response&& resp) mutable { auto& result = resp.get_result(); for (auto& code : result.get_failed_parts()) { VLOG(3) << "Failure! Failed part " << code.get_part_id() << ", failed code " diff --git a/src/daemons/CMakeLists.txt b/src/daemons/CMakeLists.txt index 2bc9e8fe659..fbe4d7808a1 100644 --- a/src/daemons/CMakeLists.txt +++ b/src/daemons/CMakeLists.txt @@ -242,6 +242,7 @@ nebula_add_executable( $ $ $ + $ ${storage_meta_deps} ${common_deps} LIBRARIES diff --git a/src/daemons/StandAloneDaemon.cpp b/src/daemons/StandAloneDaemon.cpp index c2d1b211a74..c1929f1af6c 100644 --- a/src/daemons/StandAloneDaemon.cpp +++ b/src/daemons/StandAloneDaemon.cpp @@ -163,6 +163,13 @@ int main(int argc, char *argv[]) { return EXIT_FAILURE; } + // load the time zone data + status = nebula::time::Timezone::init(); + if (!status.ok()) { + LOG(ERROR) << status; + return EXIT_FAILURE; + } + // Initialize the global timezone, it's only used for datetime type compute // won't affect the process timezone. status = nebula::time::Timezone::initializeGlobalTimezone(); diff --git a/src/graph/executor/test/StorageServerStub.cpp b/src/graph/executor/test/StorageServerStub.cpp index 87b2a22528d..9ace772300d 100644 --- a/src/graph/executor/test/StorageServerStub.cpp +++ b/src/graph/executor/test/StorageServerStub.cpp @@ -23,7 +23,6 @@ std::shared_ptr instance_ = nullptr; void GraphStorageLocalServer::setThreadManager( std::shared_ptr threadManager) { - // lock? threadManager_ = threadManager; } diff --git a/src/storage/GraphStorageLocalServer.cpp b/src/storage/GraphStorageLocalServer.cpp index 9feb7bfcf55..34046ce69ca 100644 --- a/src/storage/GraphStorageLocalServer.cpp +++ b/src/storage/GraphStorageLocalServer.cpp @@ -26,7 +26,6 @@ std::shared_ptr instance_ = nullptr; void GraphStorageLocalServer::setThreadManager( std::shared_ptr threadManager) { - // lock? threadManager_ = threadManager; } diff --git a/src/storage/stats/StorageStats.cpp b/src/storage/stats/StorageStats.cpp index 40f5181c195..6e75ab10825 100644 --- a/src/storage/stats/StorageStats.cpp +++ b/src/storage/stats/StorageStats.cpp @@ -21,8 +21,10 @@ void initStorageStats() { kNumTagsDeleted = stats::StatsManager::registerStats("num_tags_deleted", "rate, sum"); kNumVerticesDeleted = stats::StatsManager::registerStats("num_vertices_deleted", "rate, sum"); +#ifndef BUILD_STANDALONE initMetaClientStats(); initKVStats(); +#endif } } // namespace nebula diff --git a/tests/Makefile b/tests/Makefile index 0275c0f51f0..6f1e9c78711 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -26,7 +26,10 @@ PASSWORD_LOCK_TIME_IN_SECS ?= 0 gherkin_fmt = ~/.local/bin/reformat-gherkin run_test = PYTHONPATH=$$PYTHONPATH:$(CURR_DIR)/.. $(CURR_DIR)/nebula-test-run.py test_without_skip = python3 -m pytest -m "not skip" +test_without_skip_sa = python3 -m pytest -m "not skip and not distonly" test_j = $(test_without_skip) -n$(J) +test_j_sa = $(test_without_skip_sa) -n$(J) + install-deps: pip3 install --user -U setuptools wheel -i $(PYPI_MIRROR) @@ -73,6 +76,18 @@ up: clean --ca_signed=$(CA_SIGNED) \ --containerized=$(CONTAINERIZED) +standalone-up: clean + @mkdir -p $(CURR_DIR)/.pytest + $(run_test) --cmd=start_standalone \ + --build_dir=$(BUILD_DIR) \ + --debug=$(DEBUG) \ + --multi_graphd=false \ + --enable_ssl=$(ENABLE_SSL) \ + --enable_graph_ssl=$(ENABLE_GRAPH_SSL) \ + --enable_meta_ssl=$(ENABLE_META_SSL) \ + --ca_signed=$(CA_SIGNED) \ + --containerized=$(CONTAINERIZED) + down: $(run_test) --cmd=stop --rm_dir=$(RM_DIR) @@ -92,6 +107,9 @@ slow-query: currdir $(test_j) tck/steps/test_kill_slow_query_via_same_service.py && \ $(test_j) tck/steps/test_kill_slow_query_via_different_service.py +standalone-tck: jobs + $(test_j_sa) tck/steps/test_tck.py + tck: jobs slow-query $(test_j) tck/steps/test_tck.py @@ -99,6 +117,7 @@ ldbc: currdir $(test_j) tck/steps/test_ldbc.py test-all: test tck ldbc +test-standalone-all: standalone-tck ldbc fail: currdir python3 -m pytest \ diff --git a/tests/common/nebula_service.py b/tests/common/nebula_service.py index 679f516ea33..352448322ba 100644 --- a/tests/common/nebula_service.py +++ b/tests/common/nebula_service.py @@ -27,12 +27,20 @@ class NebulaProcess(object): - def __init__(self, name, ports, suffix_index=0, params=None): + def __init__(self, name, ports, suffix_index=0, params=None, is_standalone=False): + self.is_sa = is_standalone if params is None: params = {} - assert len(ports) == 4, 'should have 4 ports but have {}'.format(len(ports)) - self.name = name - self.tcp_port, self.tcp_internal_port, self.http_port, self.https_port = ports + if is_standalone == False: + assert len(ports) == 4, 'should have 4 ports but have {}'.format(len(ports)) + self.name = name + self.tcp_port, self.tcp_internal_port, self.http_port, self.https_port = ports + else: + assert len(ports) == 12, 'should have 12 ports but have {}'.format(len(ports)) + self.name = name + self.tcp_port, self.tcp_internal_port, self.http_port, self.https_port = ports[0:4] + self.meta_port, self.meta_tcp_internal_port, self.meta_http_port, self.meta_https_port = ports[4:8] + self.storage_port, self.storage_tcp_internal_port, self.storage_http_port, self.storage_https_port = ports[8:12] self.suffix_index = suffix_index self.params = params self.host = '127.0.0.1' @@ -45,13 +53,28 @@ def update_meta_server_addrs(self, address): self.update_param({'meta_server_addrs': address}) def _format_nebula_command(self): - process_params = { - 'log_dir': 'logs{}'.format(self.suffix_index), - 'pid_file': 'pids{}/nebula-{}.pid'.format(self.suffix_index, self.name), - 'port': self.tcp_port, - 'ws_http_port': self.http_port, - 'ws_h2_port': self.https_port, - } + if self.is_sa == False: + process_params = { + 'log_dir': 'logs{}'.format(self.suffix_index), + 'pid_file': 'pids{}/nebula-{}.pid'.format(self.suffix_index, self.name), + 'port': self.tcp_port, + 'ws_http_port': self.http_port, + 'ws_h2_port': self.https_port, + } + else: + process_params = { + 'log_dir': 'logs{}'.format(self.suffix_index), + 'pid_file': 'pids{}/nebula-{}.pid'.format(self.suffix_index, self.name), + 'port': self.tcp_port, + 'ws_http_port': self.http_port, + 'ws_h2_port': self.https_port, + 'meta_port': self.meta_port, + 'ws_meta_http_port': self.meta_http_port, + 'ws_meta_h2_port': self.meta_https_port, + 'storage_port': self.storage_port, + 'ws_storage_http_port': self.storage_http_port, + 'ws_storage_h2_port': self.storage_https_port, + } # data path if self.name.upper() != 'GRAPHD': process_params['data_path'] = 'data{}/{}'.format( @@ -109,6 +132,7 @@ def __init__( graphd_num=1, ca_signed=False, debug_log=True, + use_standalone=False, **kwargs, ): assert graphd_num > 0 and metad_num > 0 and storaged_num > 0 @@ -144,8 +168,41 @@ def __init__( self.ports_per_process = 4 self.lock_file = os.path.join(TMP_DIR, "cluster_port.lock") self.delimiter = "\n" - self._make_params(**kwargs) - self.init_process() + + if use_standalone == False: + self._make_params(**kwargs) + self.init_process() + else: + self._make_sa_params(**kwargs) + self.init_standalone() + + def init_standalone(self): + process_count = self.metad_num + self.storaged_num + self.graphd_num + ports_count = process_count * self.ports_per_process + self.all_ports = self._find_free_port(ports_count) + print(self.all_ports) + index = 0 + standalone = NebulaProcess( + "standalone", + self.all_ports[index : index + ports_count ], + index, + self.graphd_param, + is_standalone=True + ) + self.graphd_processes.append(standalone) + self.all_processes = ( + self.graphd_processes + ) + # update meta address + meta_server_addrs = ','.join( + [ + '{}:{}'.format(process.host, process.meta_port) + for process in self.graphd_processes + ] + ) + + for p in self.all_processes: + p.update_meta_server_addrs(meta_server_addrs) def init_process(self): process_count = self.metad_num + self.storaged_num + self.graphd_num @@ -239,6 +296,38 @@ def _make_params(self, **kwargs): for p in [self.metad_param, self.storaged_param, self.graphd_param]: p.update(kwargs) + def _make_sa_params(self, **kwargs): + _params = { + 'heartbeat_interval_secs': 1, + 'expired_time_factor': 60, + } + if self.ca_signed: + _params['cert_path'] = 'share/resources/test.derive.crt' + _params['key_path'] = 'share/resources/test.derive.key' + _params['ca_path'] = 'share/resources/test.ca.pem' + + else: + _params['cert_path'] = 'share/resources/test.ca.pem' + _params['key_path'] = 'share/resources/test.ca.key' + _params['password_path'] = 'share/resources/test.ca.password' + + if self.debug_log: + _params['v'] = '4' + + self.graphd_param = copy.copy(_params) + self.graphd_param['local_config'] = 'false' + self.graphd_param['enable_authorize'] = 'true' + self.graphd_param['system_memory_high_watermark_ratio'] = '0.95' + self.graphd_param['num_rows_to_check_memory'] = '4' + self.graphd_param['session_reclaim_interval_secs'] = '2' + # Login retry + self.graphd_param['failed_login_attempts'] = '5' + self.graphd_param['password_lock_time_in_secs'] = '10' + self.graphd_param['raft_heartbeat_interval_secs'] = '30' + self.graphd_param['skip_wait_in_rate_limiter'] = 'true' + for p in [self.metad_param, self.storaged_param, self.graphd_param]: + p.update(kwargs) + def set_work_dir(self, work_dir): self.work_dir = work_dir @@ -268,6 +357,32 @@ def _copy_nebula_conf(self): shutil.copy(self.src_dir + '/tests/cert/test.derive.key', resources_dir) shutil.copy(self.src_dir + '/tests/cert/test.derive.crt', resources_dir) + def _copy_standalone_conf(self): + bin_path = self.build_dir + '/bin/' + conf_path = self.src_dir + '/conf/' + + for item in ['nebula-standalone']: + shutil.copy(bin_path + item, self.work_dir + '/bin/') + shutil.copy( + conf_path + '{}.conf.default'.format(item), + self.work_dir + '/conf/{}.conf'.format(item), + ) + + resources_dir = self.work_dir + '/share/resources/' + os.makedirs(resources_dir) + + # timezone file + shutil.copy( + self.build_dir + '/../resources/date_time_zonespec.csv', resources_dir + ) + shutil.copy(self.build_dir + '/../resources/gflags.json', resources_dir) + # cert files + shutil.copy(self.src_dir + '/tests/cert/test.ca.key', resources_dir) + shutil.copy(self.src_dir + '/tests/cert/test.ca.pem', resources_dir) + shutil.copy(self.src_dir + '/tests/cert/test.ca.password', resources_dir) + shutil.copy(self.src_dir + '/tests/cert/test.derive.key', resources_dir) + shutil.copy(self.src_dir + '/tests/cert/test.derive.crt', resources_dir) + @staticmethod def is_port_in_use(port): with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: @@ -330,6 +445,24 @@ def _telnet_port(self, port): result = sk.connect_ex(('127.0.0.1', port)) return result == 0 + def install_standalone(self, work_dir=None): + if work_dir is not None: + self.work_dir = work_dir + print("workdir not exist") + if os.path.exists(self.work_dir): + shutil.rmtree(self.work_dir) + os.mkdir(self.work_dir) + print("work directory: " + self.work_dir) + os.chdir(self.work_dir) + installed_files = ['bin', 'conf', 'scripts'] + for f in installed_files: + os.mkdir(self.work_dir + '/' + f) + self._copy_standalone_conf() + max_suffix = max([self.graphd_num, self.storaged_num, self.metad_num]) + for i in range(max_suffix): + os.mkdir(self.work_dir + '/logs{}'.format(i)) + os.mkdir(self.work_dir + '/pids{}'.format(i)) + def install(self, work_dir=None): if work_dir is not None: self.work_dir = work_dir @@ -424,6 +557,64 @@ def start(self): return [p.tcp_port for p in self.graphd_processes] + def start_standalone(self): + os.chdir(self.work_dir) + start_time = time.time() + for p in self.all_processes: + print('start stand alone process') + p.start() + + config = Config() + config.max_connection_pool_size = 20 + config.timeout = 60000 + # init connection pool + client_pool = ConnectionPool() + # assert client_pool.init([("127.0.0.1", int(self.graphd_port))], config) + ssl_config = get_ssl_config(self.is_graph_ssl, self.ca_signed) + print("begin to add hosts") + ok = False + # wait graph is ready, and then add hosts + for _ in range(20): + try: + ok = client_pool.init( + [("127.0.0.1", self.graphd_processes[0].tcp_port)], + config, + ssl_config, + ) + if ok: + break + except: + pass + time.sleep(1) + + assert ok, "graph is not ready" + # get session from the pool + client = client_pool.get_session('root', 'nebula') + + hosts = ",".join( + [ + "127.0.0.1:{}".format(str(storaged.storage_port)) + for storaged in self.graphd_processes + ] + ) + cmd = "ADD HOSTS {}".format(hosts) + print("add hosts cmd is {}".format(cmd)) + resp = client.execute(cmd) + assert resp.is_succeeded(), resp.error_msg() + client.release() + + # wait nebula start + server_ports = [p.tcp_port for p in self.all_processes] + if not self._check_servers_status(server_ports): + self._collect_pids() + self.kill_all(signal.SIGKILL) + elapse = time.time() - start_time + raise Exception(f'nebula servers not ready in {elapse}s') + + self._collect_pids() + + return [p.tcp_port for p in self.graphd_processes] + def _collect_pids(self): for pf in glob.glob(self.work_dir + '/pid*/*.pid'): with open(pf) as f: diff --git a/tests/nebula-test-run.py b/tests/nebula-test-run.py index 3f7ee40e77f..b131f84acbf 100755 --- a/tests/nebula-test-run.py +++ b/tests/nebula-test-run.py @@ -152,6 +152,52 @@ def start_nebula(nb, configs): f.write(json.dumps(data)) print('Start nebula successfully') +def start_standalone(nb, configs): + if configs.address is not None and configs.address != "": + print('test remote nebula graph, address is {}'.format(configs.address)) + if len(configs.address.split(':')) != 2: + raise Exception('Invalid address, address is {}'.format(configs.address)) + address, port = configs.address.split(':') + ports = [int(port)] + else: + print('Start standalone version') + nb.install_standalone() + address = "localhost" + ports = nb.start_standalone() + + is_graph_ssl = opt_is(configs.enable_ssl, "true") or opt_is( + configs.enable_graph_ssl, "true" + ) + ca_signed = opt_is(configs.enable_ssl, "true") + # Load csv data + pool = get_conn_pool(address, ports[0], get_ssl_config(is_graph_ssl, ca_signed)) + sess = pool.get_session(configs.user, configs.password) + + if not os.path.exists(TMP_DIR): + os.mkdir(TMP_DIR) + + with open(SPACE_TMP_PATH, "w") as f: + spaces = [] + folder = os.path.join(CURR_PATH, "data") + for space in os.listdir(folder): + if not os.path.exists(os.path.join(folder, space, "config.yaml")): + continue + data_dir = os.path.join(folder, space) + space_desc = load_csv_data(sess, data_dir, space) + spaces.append(space_desc.__dict__) + f.write(json.dumps(spaces)) + + with open(NB_TMP_PATH, "w") as f: + data = { + "ip": "localhost", + "port": ports, + "work_dir": nb.work_dir, + "enable_ssl": configs.enable_ssl, + "enable_graph_ssl": configs.enable_graph_ssl, + "ca_signed": configs.ca_signed, + } + f.write(json.dumps(data)) + print('Start standalone successfully') def stop_nebula(nb, configs=None): if configs.address is not None and configs.address != "": @@ -174,11 +220,18 @@ def stop_nebula(nb, configs=None): parser = init_parser() (configs, opts) = parser.parse_args() + if opt_is(configs.cmd, "start_standalone"): + graphd_inst = 1 + is_standalone = True + else: + graphd_inst = 2 + is_standalone = False + # Setup nebula graph service nebula_svc = NebulaService( configs.build_dir, NEBULA_HOME, - graphd_num=2, + graphd_num=graphd_inst, storaged_num=1, debug_log=opt_is(configs.debug, "true"), ca_signed=opt_is(configs.ca_signed, "true"), @@ -186,12 +239,15 @@ def stop_nebula(nb, configs=None): enable_graph_ssl=configs.enable_graph_ssl, enable_meta_ssl=configs.enable_meta_ssl, containerized=configs.containerized, + use_standalone=is_standalone ) if opt_is(configs.cmd, "start"): start_nebula(nebula_svc, configs) elif opt_is(configs.cmd, "stop"): stop_nebula(nebula_svc, configs) + elif opt_is(configs.cmd, "start_standalone"): + start_standalone(nebula_svc, configs) else: raise ValueError(f"Invalid parser args: {configs.cmd}") except Exception as x: diff --git a/tests/pytest.ini b/tests/pytest.ini index 78ccb975c58..6b1f1433936 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -14,6 +14,7 @@ python_files = query/stateless/test_keyword.py query/stateless/test_lookup.py markers = + "distonly: mark tests can only work on distribution version." PytestUnknownMarkWarning uncompatible diff --git a/tests/tck/features/admin/Authentication.feature b/tests/tck/features/admin/Authentication.feature index 5dccbe3e19f..246cdfff724 100644 --- a/tests/tck/features/admin/Authentication.feature +++ b/tests/tck/features/admin/Authentication.feature @@ -10,6 +10,7 @@ Feature: Test Authentication graphd:password_lock_time_in_secs=5 """ + @distonly Scenario: Test login with invalid password When executing query: """ @@ -62,6 +63,7 @@ Feature: Test Authentication Invalid password, remaining attempts: 4 """ + @distonly Scenario: Test login with invalid password multi users When executing query: """ diff --git a/tests/tck/features/admin/Sessions.feature b/tests/tck/features/admin/Sessions.feature index da1181fd5b7..f3cbae5e922 100644 --- a/tests/tck/features/admin/Sessions.feature +++ b/tests/tck/features/admin/Sessions.feature @@ -6,6 +6,7 @@ Feature: Test sessions Background: Given a nebulacluster with 3 graphd and 1 metad and 1 storaged + @distonly Scenario: Show sessions When executing query: """ @@ -32,6 +33,7 @@ Feature: Test sessions | /\d+/ | "root" | "s1" | /.*/ | /.*/ | "127.0.0.1:${cluster.graphd_processes[0].tcp_port}" | 0 | /.*(127\.0\.0\.1)$/ | | /\d+/ | "user1" | "" | /.*/ | /.*/ | "127.0.0.1:${cluster.graphd_processes[1].tcp_port}" | 0 | /.*(127\.0\.0\.1)$/ | + @distonly Scenario: Show local sessions When executing query: """