From e642c056e74ed3cb6c727d80dd642f1a983f8732 Mon Sep 17 00:00:00 2001 From: Yee <2520865+yixinglu@users.noreply.github.com> Date: Fri, 8 Oct 2021 09:54:59 +0800 Subject: [PATCH] Improve memory watermark detection (#2885) * Add MemoryUtils to compute available memory * Improve file reader * Benchmark fstream and popen * Replace Memory with MemoryUtils * Add sysinfo bench * Need not to close fstream * Cleanup staled codes * Use bg thread to check memory watermark in query engine * Replace stringPrintf with sformat * Refactor storage detail * Reuse env of job * Fix insert test * Refactor bg thread for updating memory watermark * check memory watermark in executor start * format Makefile * Use literal string * Update code coverage badge * Cleanup Makefile * Check memory in iterator and executor * Fix compile error * Cleanup Makefile * Fix test and todo * Fix test * Fix tck tests * Fix containerized test * Refactor nebula test script * Optimize check * Try to fix workflow * Restore volumes of container * Fix compile error * Fix compile * Address comments * Fix error * Fix ci --- .github/workflows/nightly.yml | 2 +- .github/workflows/pull_request.yml | 12 +-- README-CN.md | 20 +++-- README.md | 7 +- docker/Dockerfile | 6 +- docker/Dockerfile.graphd | 2 +- docker/Dockerfile.metad | 2 +- docker/Dockerfile.storaged | 2 +- resources/gflags.json | 1 + src/clients/storage/StorageClientBase-inl.h | 2 +- src/common/CMakeLists.txt | 1 + src/common/base/CMakeLists.txt | 1 - src/common/base/Memory.cpp | 31 ------- src/common/base/Memory.h | 44 ---------- src/common/base/test/CMakeLists.txt | 7 -- src/common/base/test/MemoryTest.cpp | 29 ------- src/common/geo/io/wkt/test/CMakeLists.txt | 1 + src/common/memory/CMakeLists.txt | 12 +++ src/common/memory/MemoryUtils.cpp | 83 +++++++++++++++++++ src/common/memory/MemoryUtils.h | 36 ++++++++ src/common/memory/test/CMakeLists.txt | 21 +++++ src/common/memory/test/MemoryUtilsTest.cpp | 49 +++++++++++ .../memory/test/SysInfoReadBenchmark.cpp | 61 ++++++++++++++ src/common/thrift/ThriftClientManager.h | 6 +- src/daemons/CMakeLists.txt | 1 + src/daemons/GraphDaemon.cpp | 1 + src/graph/context/Iterator.cpp | 44 ++++++++-- src/graph/context/Iterator.h | 35 +++++--- src/graph/context/Result.cpp | 8 +- src/graph/context/Result.h | 13 +++ src/graph/context/test/CMakeLists.txt | 2 +- src/graph/executor/Executor.cpp | 26 +++--- src/graph/executor/Executor.h | 2 + src/graph/executor/StorageAccessExecutor.cpp | 16 ++++ src/graph/executor/StorageAccessExecutor.h | 18 ++-- src/graph/executor/query/FilterExecutor.cpp | 9 +- src/graph/executor/query/GetEdgesExecutor.cpp | 4 +- .../executor/query/GetNeighborsExecutor.cpp | 24 ++---- src/graph/executor/query/GetPropExecutor.h | 1 - .../executor/query/GetVerticesExecutor.cpp | 4 +- .../executor/query/IndexScanExecutor.cpp | 3 +- src/graph/executor/test/CMakeLists.txt | 1 + src/graph/optimizer/test/CMakeLists.txt | 2 +- src/graph/planner/test/CMakeLists.txt | 2 +- src/graph/service/GraphFlags.cpp | 4 +- src/graph/service/GraphFlags.h | 4 +- src/graph/service/GraphService.cpp | 8 +- src/graph/service/GraphService.h | 1 - src/graph/service/QueryEngine.cpp | 29 ++++++- src/graph/service/QueryEngine.h | 12 +-- src/graph/util/Utils.h | 6 +- src/graph/util/test/CMakeLists.txt | 1 + src/graph/validator/test/CMakeLists.txt | 1 + src/graph/visitor/test/CMakeLists.txt | 1 + src/parser/test/CMakeLists.txt | 2 +- src/version/Version.cpp.in | 6 +- tests/Makefile | 50 +++++++---- tests/admin/test_configs.py | 29 ++++--- tests/common/nebula_service.py | 55 ++++++------ tests/nebula-test-run.py | 27 ++++-- .../tck/features/insert/Insert.IntVid.feature | 2 +- tests/tck/features/match/Base.feature | 2 +- tests/tck/features/schema/Schema.feature | 2 +- 63 files changed, 584 insertions(+), 312 deletions(-) delete mode 100644 src/common/base/Memory.cpp delete mode 100644 src/common/base/Memory.h delete mode 100644 src/common/base/test/MemoryTest.cpp create mode 100644 src/common/memory/CMakeLists.txt create mode 100644 src/common/memory/MemoryUtils.cpp create mode 100644 src/common/memory/MemoryUtils.h create mode 100644 src/common/memory/test/CMakeLists.txt create mode 100644 src/common/memory/test/MemoryUtilsTest.cpp create mode 100644 src/common/memory/test/SysInfoReadBenchmark.cpp diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5c2955d0a62..f477bb6d05b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -128,7 +128,7 @@ jobs: timeout-minutes: 20 - name: Setup cluster run: | - make ENABLE_SSL=true CA_SIGNED=true up + make CONTAINERIZED=true ENABLE_SSL=true CA_SIGNED=true up working-directory: tests/ timeout-minutes: 2 - name: Pytest diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 9f9f2d4c40b..93edc0147b0 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -52,11 +52,11 @@ jobs: exclude: - os: centos7 compiler: clang-10 + env: + CCACHE_DIR: /tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }} + CCACHE_MAXSIZE: 8G container: image: vesoft/nebula-dev:${{ matrix.os }} - env: - CCACHE_DIR: /tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }} - CCACHE_MAXSIZE: 8G volumes: - /tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}:/tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }} options: --cap-add=SYS_PTRACE @@ -132,17 +132,17 @@ jobs: case ${{ matrix.os }} in centos7) # normal cluster - make up + make CONTAINERIZED=true up ;; ubuntu2004) # ssl cluster - make ENABLE_SSL=true CA_SIGNED=true up + make CONTAINERIZED=true ENABLE_SSL=true CA_SIGNED=true up ;; esac ;; clang-*) # graph ssl only cluster - make ENABLE_SSL=false ENABLE_GRAPH_SSL=true up + make CONTAINERIZED=true ENABLE_SSL=false ENABLE_GRAPH_SSL=true up ;; esac working-directory: tests/ diff --git a/README-CN.md b/README-CN.md index 5ebd7c510a6..8e01ef6dba8 100644 --- a/README-CN.md +++ b/README-CN.md @@ -5,14 +5,22 @@

- WeiXin - Zhihu - SegmentFault - Sina Weibo - + + WeiXin + + + Zhihu + + + SegmentFault + + + Sina Weibo + + nebula star - + nebula fork

diff --git a/README.md b/README.md index a4caadb058c..5cd64044bab 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,16 @@
A distributed, scalable, lightning-fast graph database

+ + code coverage + nightly build - + nebula star - + nebula fork
diff --git a/docker/Dockerfile b/docker/Dockerfile index 3b29632ee17..ba9588ab4ee 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,7 +20,7 @@ RUN rpm -ivh *.rpm \ EXPOSE 9669 19669 19670 -ENTRYPOINT ["/usr/local/nebula/bin/nebula-graphd", "--flagfile=/usr/local/nebula/etc/nebula-graphd.conf", "--daemonize=false"] +ENTRYPOINT ["/usr/local/nebula/bin/nebula-graphd", "--flagfile=/usr/local/nebula/etc/nebula-graphd.conf", "--daemonize=false", "--containerized=true"] FROM centos:7 as metad @@ -35,7 +35,7 @@ RUN rpm -ivh *.rpm \ EXPOSE 9559 9560 19559 19560 -ENTRYPOINT ["/usr/local/nebula/bin/nebula-metad", "--flagfile=/usr/local/nebula/etc/nebula-metad.conf", "--daemonize=false"] +ENTRYPOINT ["/usr/local/nebula/bin/nebula-metad", "--flagfile=/usr/local/nebula/etc/nebula-metad.conf", "--daemonize=false", "--containerized=true"] FROM centos:7 as storaged @@ -50,7 +50,7 @@ RUN rpm -ivh *.rpm \ EXPOSE 9777 9778 9779 9780 19779 19780 -ENTRYPOINT ["/usr/local/nebula/bin/nebula-storaged", "--flagfile=/usr/local/nebula/etc/nebula-storaged.conf", "--daemonize=false"] +ENTRYPOINT ["/usr/local/nebula/bin/nebula-storaged", "--flagfile=/usr/local/nebula/etc/nebula-storaged.conf", "--daemonize=false", "--containerized=true"] FROM centos:7 as tools diff --git a/docker/Dockerfile.graphd b/docker/Dockerfile.graphd index d9c83ee40f3..a896791c7ba 100644 --- a/docker/Dockerfile.graphd +++ b/docker/Dockerfile.graphd @@ -20,4 +20,4 @@ RUN rpm -ivh *.rpm \ EXPOSE 9669 19669 19670 -ENTRYPOINT ["/usr/local/nebula/bin/nebula-graphd", "--flagfile=/usr/local/nebula/etc/nebula-graphd.conf", "--daemonize=false"] +ENTRYPOINT ["/usr/local/nebula/bin/nebula-graphd", "--flagfile=/usr/local/nebula/etc/nebula-graphd.conf", "--daemonize=false", "--containerized=true"] diff --git a/docker/Dockerfile.metad b/docker/Dockerfile.metad index d762ca81e95..6b0a246007c 100644 --- a/docker/Dockerfile.metad +++ b/docker/Dockerfile.metad @@ -20,4 +20,4 @@ RUN rpm -ivh *.rpm \ EXPOSE 9559 9560 19559 19560 -ENTRYPOINT ["/usr/local/nebula/bin/nebula-metad", "--flagfile=/usr/local/nebula/etc/nebula-metad.conf", "--daemonize=false"] +ENTRYPOINT ["/usr/local/nebula/bin/nebula-metad", "--flagfile=/usr/local/nebula/etc/nebula-metad.conf", "--daemonize=false", "--containerized=true"] diff --git a/docker/Dockerfile.storaged b/docker/Dockerfile.storaged index 38418fc92ef..4d22252dbd2 100644 --- a/docker/Dockerfile.storaged +++ b/docker/Dockerfile.storaged @@ -20,4 +20,4 @@ RUN rpm -ivh *.rpm \ EXPOSE 9777 9778 9779 9780 19779 19780 -ENTRYPOINT ["/usr/local/nebula/bin/nebula-storaged", "--flagfile=/usr/local/nebula/etc/nebula-storaged.conf", "--daemonize=false"] +ENTRYPOINT ["/usr/local/nebula/bin/nebula-storaged", "--flagfile=/usr/local/nebula/etc/nebula-storaged.conf", "--daemonize=false", "--containerized=true"] diff --git a/resources/gflags.json b/resources/gflags.json index d033e847501..48683f75048 100644 --- a/resources/gflags.json +++ b/resources/gflags.json @@ -11,6 +11,7 @@ "custom_filter_interval_secs", "accept_partial_success", "system_memory_high_watermark_ratio", + "num_rows_to_check_memory", "session_idle_timeout_secs", "session_reclaim_interval_secs", "max_allowed_connections", diff --git a/src/clients/storage/StorageClientBase-inl.h b/src/clients/storage/StorageClientBase-inl.h index c8adf7028aa..65eb546b381 100644 --- a/src/clients/storage/StorageClientBase-inl.h +++ b/src/clients/storage/StorageClientBase-inl.h @@ -139,8 +139,8 @@ folly::SemiFuture> StorageClientBase::c // then-callback will be executed on the same IO thread .via(evb) .then([this, context, host, spaceId, start](folly::Try&& val) { - auto& r = context->findRequest(host); if (val.hasException()) { + auto& r = context->findRequest(host); LOG(ERROR) << "Request to " << host << " failed: " << val.exception().what(); auto parts = getReqPartsId(r); context->resp.appendFailedParts(parts, nebula::cpp2::ErrorCode::E_RPC_FAILURE); diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 469771ab431..dc6948c8198 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -27,3 +27,4 @@ nebula_add_subdirectory(plugin) nebula_add_subdirectory(utils) nebula_add_subdirectory(ssl) nebula_add_subdirectory(geo) +nebula_add_subdirectory(memory) diff --git a/src/common/base/CMakeLists.txt b/src/common/base/CMakeLists.txt index e49d9964319..2909758b165 100644 --- a/src/common/base/CMakeLists.txt +++ b/src/common/base/CMakeLists.txt @@ -16,7 +16,6 @@ nebula_add_library( SignalHandler.cpp SlowOpTracker.cpp StringValue.cpp - Memory.cpp ${gdb_debug_script} ) diff --git a/src/common/base/Memory.cpp b/src/common/base/Memory.cpp deleted file mode 100644 index 1d148bae472..00000000000 --- a/src/common/base/Memory.cpp +++ /dev/null @@ -1,31 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ - -#include "common/base/Memory.h" - -#include - -#include "common/base/Logging.h" - -namespace nebula { - -StatusOr> MemInfo::make() { - std::unique_ptr mem(new MemInfo); - NG_RETURN_IF_ERROR(mem->init()); - return mem; -} - -MemInfo::MemInfo() noexcept { info_ = std::make_unique(); } - -Status MemInfo::init() { - if (sysinfo(info_.get()) == -1) { - auto err = errno; - return Status::Error("Fail to call sysinfo to get memory info, errno: %d", err); - } - return Status::OK(); -} - -} // namespace nebula diff --git a/src/common/base/Memory.h b/src/common/base/Memory.h deleted file mode 100644 index 134d56c640e..00000000000 --- a/src/common/base/Memory.h +++ /dev/null @@ -1,44 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ - -#ifndef COMMON_BASE_MEMORY_H_ -#define COMMON_BASE_MEMORY_H_ - -#include - -#include -#include - -#include "common/base/StatusOr.h" -#include "common/cpp/helpers.h" - -namespace nebula { - -class MemInfo final : protected cpp::NonCopyable, protected cpp::NonMovable { - public: - static StatusOr> make(); - - uint64_t totalInKB() const { return (info_->totalram * info_->mem_unit) >> 10; } - - uint64_t freeInKB() const { return (info_->freeram * info_->mem_unit) >> 10; } - - uint64_t bufferInKB() const { return (info_->bufferram * info_->mem_unit) >> 10; } - - uint64_t usedInKB() const { return totalInKB() - freeInKB() - bufferInKB(); } - - bool hitsHighWatermark(float ratio = 0.8f) const { return usedInKB() > totalInKB() * ratio; } - - private: - MemInfo() noexcept; - - Status init(); - - std::unique_ptr info_; -}; - -} // namespace nebula - -#endif // COMMON_BASE_MEMORY_H_ diff --git a/src/common/base/test/CMakeLists.txt b/src/common/base/test/CMakeLists.txt index dbd03c5de3b..1e621ae6137 100644 --- a/src/common/base/test/CMakeLists.txt +++ b/src/common/base/test/CMakeLists.txt @@ -94,13 +94,6 @@ nebula_add_test( LIBRARIES gtest gtest_main ) -nebula_add_test( - NAME memory_test - SOURCES MemoryTest.cpp - OBJECTS $ - LIBRARIES gtest gtest_main -) - nebula_add_executable( NAME range_vs_transform_bm SOURCES RangeVsTransformBenchmark.cpp diff --git a/src/common/base/test/MemoryTest.cpp b/src/common/base/test/MemoryTest.cpp deleted file mode 100644 index 9c2a6a65095..00000000000 --- a/src/common/base/test/MemoryTest.cpp +++ /dev/null @@ -1,29 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ - -#include - -#include "common/base/Memory.h" - -namespace nebula { - -TEST(MemInfoTest, TestMemInfo) { - auto status = MemInfo::make(); - ASSERT(status.ok()); - auto mem = std::move(status).value(); - ASSERT_GE(mem->totalInKB(), mem->usedInKB()); - ASSERT_GE(mem->totalInKB(), mem->freeInKB()); - ASSERT_GE(mem->totalInKB(), mem->bufferInKB()); - ASSERT_EQ(mem->totalInKB(), mem->usedInKB() + mem->freeInKB() + mem->bufferInKB()); - - float ratio = static_cast(mem->usedInKB() - 100) / mem->totalInKB(); - ASSERT(mem->hitsHighWatermark(ratio)); - - ratio = static_cast(mem->usedInKB() + 100) / mem->totalInKB(); - ASSERT_FALSE(mem->hitsHighWatermark(ratio)); -} - -} // namespace nebula diff --git a/src/common/geo/io/wkt/test/CMakeLists.txt b/src/common/geo/io/wkt/test/CMakeLists.txt index 3223d53f98a..808082b9d60 100644 --- a/src/common/geo/io/wkt/test/CMakeLists.txt +++ b/src/common/geo/io/wkt/test/CMakeLists.txt @@ -44,6 +44,7 @@ set(WKT_PARSER_TEST_LIBS $ $ $ + $ ) nebula_add_test( diff --git a/src/common/memory/CMakeLists.txt b/src/common/memory/CMakeLists.txt new file mode 100644 index 00000000000..ebff9abc064 --- /dev/null +++ b/src/common/memory/CMakeLists.txt @@ -0,0 +1,12 @@ +# Copyright (c) 2021 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + + +nebula_add_library( + memory_obj OBJECT + MemoryUtils.cpp +) + +nebula_add_subdirectory(test) diff --git a/src/common/memory/MemoryUtils.cpp b/src/common/memory/MemoryUtils.cpp new file mode 100644 index 00000000000..1e238d0efe2 --- /dev/null +++ b/src/common/memory/MemoryUtils.cpp @@ -0,0 +1,83 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#include "common/memory/MemoryUtils.h" + +#include +#include + +#include +#include +#include + +#include "common/fs/FileUtils.h" + +DEFINE_bool(containerized, false, "Whether run this process inside the docker container"); +DEFINE_double(system_memory_high_watermark_ratio, 0.8, "high watermark ratio of system memory"); + +using nebula::fs::FileUtils; + +namespace nebula { + +static const std::regex reMemAvailable(R"(^Mem(Available|Total):\s+(\d+)\skB$)"); +static const std::regex reTotalCache(R"(^total_(cache|inactive_file)\s+(\d+)$)"); + +std::atomic_bool MemoryUtils::kHitMemoryHighWatermark{false}; + +StatusOr MemoryUtils::hitsHighWatermark() { + double available = 0.0, total = 0.0; + if (FLAGS_containerized) { + FileUtils::FileLineIterator iter("/sys/fs/cgroup/memory/memory.stat", &reTotalCache); + uint64_t cacheSize = 0; + for (; iter.valid(); ++iter) { + auto& sm = iter.matched(); + cacheSize += std::stoul(sm[2].str(), NULL); + } + + auto limitStatus = MemoryUtils::readSysContents("/sys/fs/cgroup/memory/memory.limit_in_bytes"); + NG_RETURN_IF_ERROR(limitStatus); + uint64_t limitInBytes = std::move(limitStatus).value(); + + auto usageStatus = MemoryUtils::readSysContents("/sys/fs/cgroup/memory/memory.usage_in_bytes"); + NG_RETURN_IF_ERROR(usageStatus); + uint64_t usageInBytes = std::move(usageStatus).value(); + + total = static_cast(limitInBytes); + available = static_cast(limitInBytes - usageInBytes + cacheSize); + } else { + FileUtils::FileLineIterator iter("/proc/meminfo", &reMemAvailable); + std::vector memorySize; + for (; iter.valid(); ++iter) { + auto& sm = iter.matched(); + memorySize.emplace_back(std::stoul(sm[2].str(), NULL) << 10); + } + CHECK_EQ(memorySize.size(), 2U); + size_t i = 0, j = 1; + if (memorySize[0] < memorySize[1]) { + std::swap(i, j); + } + total = memorySize[i]; + available = memorySize[j]; + } + + auto hits = (1 - available / total) > FLAGS_system_memory_high_watermark_ratio; + LOG_IF_EVERY_N(WARNING, hits, 100) + << "Memory usage has hit the high watermark of system, available: " << available + << " vs. total: " << total << " in bytes."; + return hits; +} + +StatusOr MemoryUtils::readSysContents(const std::string& path) { + std::ifstream ifs(path); + if (!ifs) { + return Status::Error("Could not open the file: %s", path.c_str()); + } + uint64_t value = 0; + ifs >> value; + return value; +} + +} // namespace nebula diff --git a/src/common/memory/MemoryUtils.h b/src/common/memory/MemoryUtils.h new file mode 100644 index 00000000000..dc3f92d920b --- /dev/null +++ b/src/common/memory/MemoryUtils.h @@ -0,0 +1,36 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#pragma once + +#include +#include +#include + +#include "common/base/StatusOr.h" + +namespace nebula { + +/** + * MemoryUtils will compute the memory consumption of containerization and physical machine + * deployment: + * - physical machine: read the `/proc/meminfo' + * - container: read the `/sys/fs/memory/memory.{stat,usage_in_bytes,limit_in_bytes}' + */ +class MemoryUtils final { + public: + static StatusOr hitsHighWatermark(); + + static std::atomic_bool kHitMemoryHighWatermark; + + private: + MemoryUtils(const MemoryUtils &) = delete; + void operator=(const MemoryUtils &) = delete; + + static StatusOr readSysContents(const std::string &path); +}; + +} // namespace nebula diff --git a/src/common/memory/test/CMakeLists.txt b/src/common/memory/test/CMakeLists.txt new file mode 100644 index 00000000000..bb181fdda44 --- /dev/null +++ b/src/common/memory/test/CMakeLists.txt @@ -0,0 +1,21 @@ +# Copyright (c) 2021 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +nebula_add_executable( + NAME sys_info_read_bm + SOURCES SysInfoReadBenchmark.cpp + OBJECTS + LIBRARIES follybenchmark boost_regex ${THRIFT_LIBRARIES} ${PROXYGEN_LIBRARIES} +) + +nebula_add_test( + NAME memory_utils_test + SOURCES MemoryUtilsTest.cpp + OBJECTS + $ + $ + $ + LIBRARIES gtest gtest_main +) diff --git a/src/common/memory/test/MemoryUtilsTest.cpp b/src/common/memory/test/MemoryUtilsTest.cpp new file mode 100644 index 00000000000..318a64d534c --- /dev/null +++ b/src/common/memory/test/MemoryUtilsTest.cpp @@ -0,0 +1,49 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#include +#include + +#include "common/memory/MemoryUtils.h" + +DECLARE_bool(containerized); +DECLARE_double(system_memory_high_watermark_ratio); + +namespace nebula { + +TEST(MemoryHighWatermarkTest, TestHitsHighWatermarkInHost) { + FLAGS_containerized = false; + FLAGS_system_memory_high_watermark_ratio = 0.01; + auto status = MemoryUtils::hitsHighWatermark(); + ASSERT_TRUE(status.ok()); + ASSERT_TRUE(std::move(status).value()); +} + +TEST(MemoryHighWatermarkTest, TestNotHitsHighWatermarkInHost) { + FLAGS_containerized = false; + FLAGS_system_memory_high_watermark_ratio = 0.99; + auto status = MemoryUtils::hitsHighWatermark(); + ASSERT_TRUE(status.ok()); + ASSERT_FALSE(std::move(status).value()); +} + +TEST(MemoryHighWatermarkTest, DISABLED_TestHitsHighWatermarkInContainer) { + FLAGS_containerized = true; + FLAGS_system_memory_high_watermark_ratio = 0.01; + auto status = MemoryUtils::hitsHighWatermark(); + ASSERT_TRUE(status.ok()); + ASSERT_TRUE(std::move(status).value()); +} + +TEST(MemoryHighWatermarkTest, TestNotHitsHighWatermarkInContainer) { + FLAGS_containerized = true; + FLAGS_system_memory_high_watermark_ratio = 0.99; + auto status = MemoryUtils::hitsHighWatermark(); + ASSERT_TRUE(status.ok()); + ASSERT_FALSE(std::move(status).value()); +} + +} // namespace nebula diff --git a/src/common/memory/test/SysInfoReadBenchmark.cpp b/src/common/memory/test/SysInfoReadBenchmark.cpp new file mode 100644 index 00000000000..a8e892cdc3e --- /dev/null +++ b/src/common/memory/test/SysInfoReadBenchmark.cpp @@ -0,0 +1,61 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#include +#include + +#include +#include +#include + +BENCHMARK(Popen) { + auto pipe = popen("cat /sys/fs/cgroup/memory/memory.limit_in_bytes", "r"); + if (!pipe) { + std::cerr << ::strerror(errno); + } + uint64_t value = 0; + if (fscanf(pipe, "%lu", &value) != 2) { + std::cerr << ::strerror(errno); + } + if (pclose(pipe) < 0) { + std::cerr << ::strerror(errno); + } + folly::doNotOptimizeAway(value); +} + +BENCHMARK(Fstream) { + std::ifstream ifs("/sys/fs/cgroup/memory/memory.limit_in_bytes"); + if (!ifs.is_open()) { + std::cerr << "fail to open"; + } + uint64_t value; + ifs >> value; + ifs.close(); + folly::doNotOptimizeAway(value); +} + +BENCHMARK(SysInfo) { + struct sysinfo info; + if (sysinfo(&info) == -1) { + std::cerr << "fail to sysinfo: " << strerror(errno); + } + uint64_t total = info.totalram * info.mem_unit; + folly::doNotOptimizeAway(total); +} + +int main(int argc, char **argv) { + gflags::ParseCommandLineFlags(&argc, &argv, true); + folly::runBenchmarks(); + return 0; +} + +// ============================================================================ +// src/common/memory/test/SysInfoReadBenchmark.cpp relative time/iter iters/s +// ============================================================================ +// Popen 1.34ms 744.82 +// Fstream 4.44us 225.29K +// SysInfo 524.54ns 1.91M +// ============================================================================ diff --git a/src/common/thrift/ThriftClientManager.h b/src/common/thrift/ThriftClientManager.h index fa23b3678f3..8c70b0352ce 100644 --- a/src/common/thrift/ThriftClientManager.h +++ b/src/common/thrift/ThriftClientManager.h @@ -31,10 +31,8 @@ class ThriftClientManager final { } private: - using ClientMap = std::unordered_map, // - // pair - std::shared_ptr // Async thrift client - >; + using ClientMap = + std::unordered_map, std::shared_ptr >; folly::ThreadLocal clientMap_; // whether enable ssl diff --git a/src/daemons/CMakeLists.txt b/src/daemons/CMakeLists.txt index b9c9a9e1d82..13eaa0e673e 100644 --- a/src/daemons/CMakeLists.txt +++ b/src/daemons/CMakeLists.txt @@ -25,6 +25,7 @@ set(common_deps $ $ $ + $ $ $ $ diff --git a/src/daemons/GraphDaemon.cpp b/src/daemons/GraphDaemon.cpp index 43430942f2b..34f5850292a 100644 --- a/src/daemons/GraphDaemon.cpp +++ b/src/daemons/GraphDaemon.cpp @@ -42,6 +42,7 @@ extern Status setupBreakpad(); #endif DECLARE_string(flagfile); +DECLARE_bool(containerized); int main(int argc, char *argv[]) { google::SetVersionString(nebula::versionString()); diff --git a/src/graph/context/Iterator.cpp b/src/graph/context/Iterator.cpp index b995b2d8669..b3af28f5d86 100644 --- a/src/graph/context/Iterator.cpp +++ b/src/graph/context/Iterator.cpp @@ -8,12 +8,34 @@ #include "common/datatypes/Edge.h" #include "common/datatypes/Vertex.h" +#include "common/memory/MemoryUtils.h" #include "graph/util/SchemaUtil.h" #include "interface/gen-cpp2/common_types.h" + +DECLARE_int32(num_rows_to_check_memory); +DECLARE_double(system_memory_high_watermark_ratio); + namespace nebula { namespace graph { -GetNeighborsIter::GetNeighborsIter(std::shared_ptr value) - : Iterator(value, Kind::kGetNeighbors) { + +bool Iterator::hitsSysMemoryHighWatermark() const { + if (checkMemory_) { + if (numRowsModN_ >= FLAGS_num_rows_to_check_memory) { + numRowsModN_ -= FLAGS_num_rows_to_check_memory; + } + if (UNLIKELY(numRowsModN_ == 0)) { + if (MemoryUtils::kHitMemoryHighWatermark.load()) { + throw std::runtime_error( + folly::sformat("Used memory hits the high watermark({}) of total system memory.", + FLAGS_system_memory_high_watermark_ratio)); + } + } + } + return false; +} + +GetNeighborsIter::GetNeighborsIter(std::shared_ptr value, bool checkMemory) + : Iterator(value, Kind::kGetNeighbors, checkMemory) { if (value == nullptr) { return; } @@ -177,8 +199,8 @@ Status GetNeighborsIter::buildPropIndex(const std::string& props, } bool GetNeighborsIter::valid() const { - return valid_ && currentDs_ < dsIndices_.end() && currentRow_ < rowsUpperBound_ && - colIdx_ < currentDs_->colUpperBound; + return Iterator::valid() && valid_ && currentDs_ < dsIndices_.end() && + currentRow_ < rowsUpperBound_ && colIdx_ < currentDs_->colUpperBound; } void GetNeighborsIter::next() { @@ -186,6 +208,8 @@ void GetNeighborsIter::next() { return; } + numRowsModN_++; + if (noEdge_) { if (++currentRow_ < rowsUpperBound_) { return; @@ -519,7 +543,8 @@ void GetNeighborsIter::clearEdges() { } } -SequentialIter::SequentialIter(std::shared_ptr value) : Iterator(value, Kind::kSequential) { +SequentialIter::SequentialIter(std::shared_ptr value, bool checkMemory) + : Iterator(value, Kind::kSequential, checkMemory) { DCHECK(value->isDataSet()); auto& ds = value->mutableDataSet(); iter_ = ds.rows.begin(); @@ -538,7 +563,7 @@ SequentialIter::SequentialIter(std::unique_ptr left, std::unique_ptr> inputList) - : Iterator(inputList.front()->valuePtr(), Kind::kSequential) { + : Iterator(inputList.front()->valuePtr(), Kind::kSequential, inputList.front()->checkMemory()) { init(std::move(inputList)); } @@ -560,10 +585,11 @@ void SequentialIter::init(std::vector>&& iterators) { iter_ = rows_->begin(); } -bool SequentialIter::valid() const { return iter_ < rows_->end(); } +bool SequentialIter::valid() const { return Iterator::valid() && iter_ < rows_->end(); } void SequentialIter::next() { if (valid()) { + ++numRowsModN_; ++iter_; } } @@ -600,7 +626,8 @@ Value SequentialIter::getVertex(const std::string& name) const { return getColum Value SequentialIter::getEdge() const { return getColumn("EDGE"); } -PropIter::PropIter(std::shared_ptr value) : SequentialIter(value) { +PropIter::PropIter(std::shared_ptr value, bool checkMemory) + : SequentialIter(value, checkMemory) { DCHECK(value->isDataSet()); auto& ds = value->getDataSet(); auto status = makeDataSetIndex(ds); @@ -826,5 +853,6 @@ std::ostream& operator<<(std::ostream& os, Iterator::Kind kind) { os << " iterator"; return os; } + } // namespace graph } // namespace nebula diff --git a/src/graph/context/Iterator.h b/src/graph/context/Iterator.h index f3667d63487..86b3be66834 100644 --- a/src/graph/context/Iterator.h +++ b/src/graph/context/Iterator.h @@ -43,7 +43,8 @@ class Iterator { kProp, }; - explicit Iterator(std::shared_ptr value, Kind kind) : value_(value), kind_(kind) {} + explicit Iterator(std::shared_ptr value, Kind kind, bool checkMemory = false) + : checkMemory_(checkMemory), kind_(kind), numRowsModN_(0), value_(value) {} virtual ~Iterator() = default; @@ -51,7 +52,7 @@ class Iterator { virtual std::unique_ptr copy() const = 0; - virtual bool valid() const = 0; + virtual bool valid() const { return !hitsSysMemoryHighWatermark(); } virtual void next() = 0; @@ -75,7 +76,10 @@ class Iterator { // Reset iterator position to `pos' from begin. Must be sure that the `pos' // position is lower than `size()' before resetting - void reset(size_t pos = 0) { doReset(pos); } + void reset(size_t pos = 0) { + numRowsModN_ = 0; + doReset(pos); + } virtual void clear() = 0; @@ -126,22 +130,32 @@ class Iterator { virtual Value getEdge() const { return Value(); } + bool checkMemory() const { return checkMemory_; } + void setCheckMemory(bool checkMemory) { checkMemory_ = checkMemory; } + protected: virtual void doReset(size_t pos) = 0; + bool hitsSysMemoryHighWatermark() const; - std::shared_ptr value_; + bool checkMemory_{false}; Kind kind_; + mutable int64_t numRowsModN_{0}; + std::shared_ptr value_; }; class DefaultIter final : public Iterator { public: - explicit DefaultIter(std::shared_ptr value) : Iterator(value, Kind::kDefault) {} + explicit DefaultIter(std::shared_ptr value, bool checkMemory = false) + : Iterator(value, Kind::kDefault, checkMemory) {} std::unique_ptr copy() const override { return std::make_unique(*this); } - bool valid() const override { return !(counter_ > 0); } + bool valid() const override { return Iterator::valid() && !(counter_ > 0); } - void next() override { counter_++; } + void next() override { + numRowsModN_++; + counter_++; + } void erase() override { counter_--; } @@ -188,7 +202,7 @@ class DefaultIter final : public Iterator { class GetNeighborsIter final : public Iterator { public: - explicit GetNeighborsIter(std::shared_ptr value); + explicit GetNeighborsIter(std::shared_ptr value, bool checkMemory = false); std::unique_ptr copy() const override { auto copy = std::make_unique(*this); @@ -203,6 +217,7 @@ class GetNeighborsIter final : public Iterator { void clear() override { valid_ = false; dsIndices_.clear(); + reset(); } void erase() override; @@ -338,7 +353,7 @@ class GetNeighborsIter final : public Iterator { class SequentialIter : public Iterator { public: - explicit SequentialIter(std::shared_ptr value); + explicit SequentialIter(std::shared_ptr value, bool checkMemory = false); // Union multiple sequential iterators explicit SequentialIter(std::vector> inputList); @@ -437,7 +452,7 @@ class SequentialIter : public Iterator { class PropIter final : public SequentialIter { public: - explicit PropIter(std::shared_ptr value); + explicit PropIter(std::shared_ptr value, bool checkMemory = false); std::unique_ptr copy() const override { auto copy = std::make_unique(*this); diff --git a/src/graph/context/Result.cpp b/src/graph/context/Result.cpp index f90df637057..ea27d165920 100644 --- a/src/graph/context/Result.cpp +++ b/src/graph/context/Result.cpp @@ -25,13 +25,13 @@ ResultBuilder& ResultBuilder::iter(Iterator::Kind kind) { << "Must set value when creating non-default iterator"; switch (kind) { case Iterator::Kind::kDefault: - return iter(std::make_unique(core_.value)); + return iter(std::make_unique(core_.value, core_.checkMemory)); case Iterator::Kind::kSequential: - return iter(std::make_unique(core_.value)); + return iter(std::make_unique(core_.value, core_.checkMemory)); case Iterator::Kind::kGetNeighbors: - return iter(std::make_unique(core_.value)); + return iter(std::make_unique(core_.value, core_.checkMemory)); case Iterator::Kind::kProp: - return iter(std::make_unique(core_.value)); + return iter(std::make_unique(core_.value, core_.checkMemory)); default: LOG(FATAL) << "Invalid Iterator kind" << static_cast(kind); } diff --git a/src/graph/context/Result.h b/src/graph/context/Result.h index 47bab014286..40d779930ea 100644 --- a/src/graph/context/Result.h +++ b/src/graph/context/Result.h @@ -43,6 +43,13 @@ class Result final { Iterator* iterRef() { return core_.iter.get(); } + void checkMemory(bool checkMemory) { + core_.checkMemory = checkMemory; + if (core_.iter) { + core_.iter->setCheckMemory(checkMemory); + } + } + private: friend class ResultBuilder; friend class ExecutionContext; @@ -64,6 +71,7 @@ class Result final { return *this; } + bool checkMemory{false}; State state; std::string msg; std::shared_ptr value; @@ -86,6 +94,11 @@ class ResultBuilder final { return Result(std::move(core_)); } + ResultBuilder& checkMemory(bool checkMemory = false) { + core_.checkMemory = checkMemory; + return *this; + } + ResultBuilder& value(Value&& value) { core_.value = std::make_shared(std::move(value)); return *this; diff --git a/src/graph/context/test/CMakeLists.txt b/src/graph/context/test/CMakeLists.txt index fcf2cb5a08d..54f353abbca 100644 --- a/src/graph/context/test/CMakeLists.txt +++ b/src/graph/context/test/CMakeLists.txt @@ -42,7 +42,7 @@ SET(CONTEXT_TEST_LIBS $ $ $ - $ + $ ) nebula_add_test( diff --git a/src/graph/executor/Executor.cpp b/src/graph/executor/Executor.cpp index 81156c107a7..3ed9d88017a 100644 --- a/src/graph/executor/Executor.cpp +++ b/src/graph/executor/Executor.cpp @@ -11,8 +11,8 @@ #include -#include "common/base/Memory.h" #include "common/base/ObjectPool.h" +#include "common/memory/MemoryUtils.h" #include "graph/context/ExecutionContext.h" #include "graph/context/QueryContext.h" #include "graph/executor/ExecutionError.h" @@ -103,6 +103,7 @@ using folly::stringPrintf; DEFINE_bool(enable_lifetime_optimize, true, "Does enable the lifetime optimize."); +DECLARE_double(system_memory_high_watermark_ratio); namespace nebula { namespace graph { @@ -557,17 +558,9 @@ Status Executor::open() { << "ep: " << qctx()->plan()->id() << "query: " << qctx()->rctx()->query(); return Status::Error("Execution had been killed"); } - auto status = MemInfo::make(); - NG_RETURN_IF_ERROR(status); - auto mem = std::move(status).value(); - if (node_->isQueryNode() && mem->hitsHighWatermark(FLAGS_system_memory_high_watermark_ratio)) { - return Status::Error( - "Used memory(%ldKB) hits the high watermark(%lf) of total system " - "memory(%ldKB).", - mem->usedInKB(), - FLAGS_system_memory_high_watermark_ratio, - mem->totalInKB()); - } + + NG_RETURN_IF_ERROR(checkMemoryWatermark()); + numRows_ = 0; execTime_ = 0; totalDuration_.reset(); @@ -587,6 +580,14 @@ Status Executor::close() { return Status::OK(); } +Status Executor::checkMemoryWatermark() { + if (node_->isQueryNode() && MemoryUtils::kHitMemoryHighWatermark.load()) { + return Status::Error("Used memory hits the high watermark(%lf) of total system memory.", + FLAGS_system_memory_high_watermark_ratio); + } + return Status::OK(); +} + folly::Future Executor::start(Status status) const { return folly::makeFuture(std::move(status)).via(runner()); } @@ -613,6 +614,7 @@ Status Executor::finish(Result &&result) { if (!FLAGS_enable_lifetime_optimize || node()->outputVarPtr()->userCount.load(std::memory_order_relaxed) != 0) { numRows_ = result.size(); + result.checkMemory(node()->isQueryNode()); ectx_->setResult(node()->outputVar(), std::move(result)); } else { VLOG(1) << "Drop variable " << node()->outputVar(); diff --git a/src/graph/executor/Executor.h b/src/graph/executor/Executor.h index f2e8c23409e..7c253d67605 100644 --- a/src/graph/executor/Executor.h +++ b/src/graph/executor/Executor.h @@ -44,6 +44,8 @@ class Executor : private cpp::NonCopyable, private cpp::NonMovable { // Cleanup or reset executor some states after each execution virtual Status close(); + Status checkMemoryWatermark(); + QueryContext *qctx() const { return qctx_; } int64_t id() const { return id_; } diff --git a/src/graph/executor/StorageAccessExecutor.cpp b/src/graph/executor/StorageAccessExecutor.cpp index 703ab069872..b010376c9e5 100644 --- a/src/graph/executor/StorageAccessExecutor.cpp +++ b/src/graph/executor/StorageAccessExecutor.cpp @@ -6,11 +6,16 @@ #include "graph/executor/StorageAccessExecutor.h" +#include + #include "graph/context/Iterator.h" #include "graph/context/QueryExpressionContext.h" #include "graph/util/SchemaUtil.h" +#include "graph/util/Utils.h" #include "interface/gen-cpp2/meta_types.h" +using apache::thrift::optional_field_ref; + namespace nebula { namespace graph { @@ -77,5 +82,16 @@ DataSet StorageAccessExecutor::buildRequestDataSetByVidType(Iterator *iter, return internal::buildRequestDataSet(space, exprCtx, iter, expr, dedup); } +std::string StorageAccessExecutor::getStorageDetail( + optional_field_ref &> ref) const { + if (ref.has_value()) { + auto content = util::join(*ref, [](auto &iter) -> std::string { + return folly::sformat("{}:{}(us)", iter.first, iter.second); + }); + return "{" + content + "}"; + } + return ""; +} + } // namespace graph } // namespace nebula diff --git a/src/graph/executor/StorageAccessExecutor.h b/src/graph/executor/StorageAccessExecutor.h index 7d089592e06..5113dac3dfa 100644 --- a/src/graph/executor/StorageAccessExecutor.h +++ b/src/graph/executor/StorageAccessExecutor.h @@ -133,20 +133,18 @@ class StorageAccessExecutor : public Executor { auto &hostLatency = resp.hostLatency(); for (size_t i = 0; i < hostLatency.size(); ++i) { auto &info = hostLatency[i]; - auto &response = resp.responses()[i]; - stats.emplace(folly::stringPrintf("%s exec/total", std::get<0>(info).toString().c_str()), - folly::stringPrintf("%d(us)/%d(us)", std::get<1>(info), std::get<2>(info))); - if (response.result_ref()->latency_detail_us_ref().has_value()) { - std::string storageDetail = "{"; - for (auto iter : (*response.result_ref()->latency_detail_us_ref())) { - storageDetail += folly::stringPrintf("%s:%d(us),", iter.first.data(), iter.second); - } - storageDetail += "}"; - stats.emplace("storage_detail", storageDetail); + stats.emplace(folly::sformat("{} exec/total", std::get<0>(info).toString()), + folly::sformat("{}(us)/{}(us)", std::get<1>(info), std::get<2>(info))); + auto detail = getStorageDetail(resp.responses()[i].result_ref()->latency_detail_us_ref()); + if (!detail.empty()) { + stats.emplace("storage_detail", detail); } } } + std::string getStorageDetail( + apache::thrift::optional_field_ref &> ref) const; + bool isIntVidType(const SpaceInfo &space) const; DataSet buildRequestDataSetByVidType(Iterator *iter, Expression *expr, bool dedup); diff --git a/src/graph/executor/query/FilterExecutor.cpp b/src/graph/executor/query/FilterExecutor.cpp index cbd1a58879a..79e296aeb02 100644 --- a/src/graph/executor/query/FilterExecutor.cpp +++ b/src/graph/executor/query/FilterExecutor.cpp @@ -19,8 +19,9 @@ folly::Future FilterExecutor::execute() { Result result = ectx_->getResult(filter->inputVar()); auto* iter = result.iterRef(); if (iter == nullptr || iter->isDefaultIter()) { - LOG(ERROR) << "Internal Error: iterator is nullptr or DefaultIter"; - return Status::Error("Internal Error: iterator is nullptr or DefaultIter"); + auto status = Status::Error("iterator is nullptr or DefaultIter"); + LOG(ERROR) << status; + return status; } VLOG(2) << "Get input var: " << filter->inputVar() @@ -34,9 +35,7 @@ folly::Future FilterExecutor::execute() { while (iter->valid()) { auto val = condition->eval(ctx(iter)); if (val.isBadNull() || (!val.empty() && !val.isBool() && !val.isNull())) { - return Status::Error( - "Internal Error: Wrong type result, " - "the type should be NULL,EMPTY or BOOL"); + return Status::Error("Wrong type result, the type should be NULL, EMPTY or BOOL"); } if (val.empty() || val.isNull() || !val.getBool()) { if (UNLIKELY(filter->needStableFilter())) { diff --git a/src/graph/executor/query/GetEdgesExecutor.cpp b/src/graph/executor/query/GetEdgesExecutor.cpp index bd6432edea9..3bc5ee73c22 100644 --- a/src/graph/executor/query/GetEdgesExecutor.cpp +++ b/src/graph/executor/query/GetEdgesExecutor.cpp @@ -58,7 +58,6 @@ folly::Future GetEdgesExecutor::getEdges() { } auto edges = buildRequestDataSet(ge); - VLOG(1) << "Edges: " << edges; if (edges.rows.empty()) { // TODO: add test for empty input. @@ -82,8 +81,7 @@ folly::Future GetEdgesExecutor::getEdges() { .via(runner()) .ensure([this, getPropsTime]() { SCOPED_TIMER(&execTime_); - otherStats_.emplace("total_rpc", - folly::stringPrintf("%lu(us)", getPropsTime.elapsedInUSec())); + otherStats_.emplace("total_rpc", folly::sformat("{}(us)", getPropsTime.elapsedInUSec())); }) .thenValue([this, ge](StorageRpcResponse &&rpcResp) { SCOPED_TIMER(&execTime_); diff --git a/src/graph/executor/query/GetNeighborsExecutor.cpp b/src/graph/executor/query/GetNeighborsExecutor.cpp index 73cc0c5bf65..3cabd7101b9 100644 --- a/src/graph/executor/query/GetNeighborsExecutor.cpp +++ b/src/graph/executor/query/GetNeighborsExecutor.cpp @@ -64,8 +64,7 @@ folly::Future GetNeighborsExecutor::execute() { .via(runner()) .ensure([this, getNbrTime]() { SCOPED_TIMER(&execTime_); - otherStats_.emplace("total_rpc_time", - folly::stringPrintf("%lu(us)", getNbrTime.elapsedInUSec())); + otherStats_.emplace("total_rpc_time", folly::sformat("{}(us)", getNbrTime.elapsedInUSec())); }) .thenValue([this](StorageRpcResponse&& resp) { SCOPED_TIMER(&execTime_); @@ -78,16 +77,11 @@ folly::Future GetNeighborsExecutor::execute() { } auto& info = hostLatency[i]; otherStats_.emplace( - folly::stringPrintf("%s exec/total/vertices", std::get<0>(info).toString().c_str()), - folly::stringPrintf( - "%d(us)/%d(us)/%lu,", std::get<1>(info), std::get<2>(info), size)); - if (result.result.latency_detail_us_ref().has_value()) { - std::string storageDetail = "{"; - for (auto iter : (*result.result.latency_detail_us_ref())) { - storageDetail += folly::stringPrintf("%s:%d(us),", iter.first.data(), iter.second); - } - storageDetail += "}"; - otherStats_.emplace("storage_detail", storageDetail); + folly::sformat("{} exec/total/vertices", std::get<0>(info).toString()), + folly::sformat("{}(us)/{}(us)/{},", std::get<1>(info), std::get<2>(info), size)); + auto detail = getStorageDetail(result.result.latency_detail_us_ref()); + if (!detail.empty()) { + otherStats_.emplace("storage_detail", detail); } } return handleResponse(resp); @@ -101,7 +95,6 @@ Status GetNeighborsExecutor::handleResponse(RpcResponse& resps) { builder.state(result.value()); auto& responses = resps.responses(); - VLOG(2) << node_->toString() << ", Resp size: " << responses.size(); List list; for (auto& resp : responses) { auto dataset = resp.get_vertices(); @@ -110,11 +103,10 @@ Status GetNeighborsExecutor::handleResponse(RpcResponse& resps) { continue; } - VLOG(2) << "Resp row size: " << dataset->rows.size() << ", Resp: " << *dataset; list.values.emplace_back(std::move(*dataset)); } - builder.value(Value(std::move(list))); - return finish(builder.iter(Iterator::Kind::kGetNeighbors).build()); + builder.value(Value(std::move(list))).iter(Iterator::Kind::kGetNeighbors); + return finish(builder.build()); } } // namespace graph diff --git a/src/graph/executor/query/GetPropExecutor.h b/src/graph/executor/query/GetPropExecutor.h index 5e4e8959542..8d7dfaccbf5 100644 --- a/src/graph/executor/query/GetPropExecutor.h +++ b/src/graph/executor/query/GetPropExecutor.h @@ -41,7 +41,6 @@ class GetPropExecutor : public StorageAccessExecutor { DCHECK_EQ(colNames.size(), v.colSize()); v.colNames = colNames; } - VLOG(2) << "Dataset in get props: \n" << v << "\n"; return finish( ResultBuilder().value(std::move(v)).iter(Iterator::Kind::kProp).state(state).build()); } diff --git a/src/graph/executor/query/GetVerticesExecutor.cpp b/src/graph/executor/query/GetVerticesExecutor.cpp index ef3e4486d7f..0f7e5e8c7c9 100644 --- a/src/graph/executor/query/GetVerticesExecutor.cpp +++ b/src/graph/executor/query/GetVerticesExecutor.cpp @@ -26,7 +26,6 @@ folly::Future GetVerticesExecutor::getVertices() { GraphStorageClient *storageClient = qctx()->getStorageClient(); DataSet vertices = buildRequestDataSet(gv); - VLOG(1) << "vertices: " << vertices; if (vertices.rows.empty()) { // TODO: add test for empty input. return finish( @@ -49,8 +48,7 @@ folly::Future GetVerticesExecutor::getVertices() { .via(runner()) .ensure([this, getPropsTime]() { SCOPED_TIMER(&execTime_); - otherStats_.emplace("total_rpc", - folly::stringPrintf("%lu(us)", getPropsTime.elapsedInUSec())); + otherStats_.emplace("total_rpc", folly::sformat("{}(us)", getPropsTime.elapsedInUSec())); }) .thenValue([this, gv](StorageRpcResponse &&rpcResp) { SCOPED_TIMER(&execTime_); diff --git a/src/graph/executor/query/IndexScanExecutor.cpp b/src/graph/executor/query/IndexScanExecutor.cpp index d2cc3ec3a4c..c8791af6102 100644 --- a/src/graph/executor/query/IndexScanExecutor.cpp +++ b/src/graph/executor/query/IndexScanExecutor.cpp @@ -65,7 +65,7 @@ Status IndexScanExecutor::handleResp(storage::StorageRpcResponse &&rpcResp for (auto &resp : rpcResp.responses()) { if (resp.data_ref().has_value()) { nebula::DataSet &data = *resp.data_ref(); - // TODO : convert the column name to alias. + // TODO: convert the column name to alias. if (v.colNames.empty()) { v.colNames = data.colNames; } @@ -78,7 +78,6 @@ Status IndexScanExecutor::handleResp(storage::StorageRpcResponse &&rpcResp DCHECK_EQ(node()->colNames().size(), v.colNames.size()); v.colNames = node()->colNames(); } - VLOG(2) << "Dataset produced by IndexScan: \n" << v << "\n"; return finish( ResultBuilder().value(std::move(v)).iter(Iterator::Kind::kProp).state(state).build()); } diff --git a/src/graph/executor/test/CMakeLists.txt b/src/graph/executor/test/CMakeLists.txt index 4244415bb18..6e48b1e4ed4 100644 --- a/src/graph/executor/test/CMakeLists.txt +++ b/src/graph/executor/test/CMakeLists.txt @@ -21,6 +21,7 @@ SET(EXEC_QUERY_TEST_OBJS $ $ $ + $ $ $ $ diff --git a/src/graph/optimizer/test/CMakeLists.txt b/src/graph/optimizer/test/CMakeLists.txt index 1ff93fc2374..1d99b18f5cb 100644 --- a/src/graph/optimizer/test/CMakeLists.txt +++ b/src/graph/optimizer/test/CMakeLists.txt @@ -45,7 +45,7 @@ set(OPTIMIZER_TEST_LIB $ $ $ - $ + $ ) nebula_add_test( diff --git a/src/graph/planner/test/CMakeLists.txt b/src/graph/planner/test/CMakeLists.txt index e586d57febb..7fdd86e480d 100644 --- a/src/graph/planner/test/CMakeLists.txt +++ b/src/graph/planner/test/CMakeLists.txt @@ -49,7 +49,7 @@ nebula_add_test( $ $ $ - $ + $ LIBRARIES gtest ${PROXYGEN_LIBRARIES} diff --git a/src/graph/service/GraphFlags.cpp b/src/graph/service/GraphFlags.cpp index 6018bde637a..c6c744adb3d 100644 --- a/src/graph/service/GraphFlags.cpp +++ b/src/graph/service/GraphFlags.cpp @@ -59,8 +59,6 @@ DEFINE_uint32(ft_request_retry_times, 3, "Retry times if fulltext request failed DEFINE_bool(accept_partial_success, false, "Whether to accept partial success, default false"); -DEFINE_double(system_memory_high_watermark_ratio, 0.8, "high watermark ratio of system memory"); - DEFINE_bool(disable_octal_escape_char, false, "Octal escape character will be disabled" @@ -72,3 +70,5 @@ DEFINE_bool(enable_client_white_list, true, "Turn on/off the client white list." DEFINE_string(client_white_list, nebula::getOriginVersion() + ":2.5.0:2.5.1:2.6.0", "A white list for different client versions, seperate with colon."); + +DEFINE_int32(num_rows_to_check_memory, 1024, "number rows to check memory"); diff --git a/src/graph/service/GraphFlags.h b/src/graph/service/GraphFlags.h index 21b6a769e78..c1bcc6d3717 100644 --- a/src/graph/service/GraphFlags.h +++ b/src/graph/service/GraphFlags.h @@ -37,7 +37,6 @@ DECLARE_bool(enable_authorize); DECLARE_string(auth_type); DECLARE_string(cloud_http_url); DECLARE_uint32(max_allowed_statements); -DECLARE_double(system_memory_high_watermark_ratio); // optimizer DECLARE_bool(enable_optimizer); @@ -50,4 +49,7 @@ DECLARE_bool(enable_experimental_feature); DECLARE_bool(enable_client_white_list); DECLARE_string(client_white_list); + +DECLARE_int32(num_rows_to_check_memory); + #endif // GRAPH_GRAPHFLAGS_H_ diff --git a/src/graph/service/GraphService.cpp b/src/graph/service/GraphService.cpp index bad422047b6..3a539a8cf6b 100644 --- a/src/graph/service/GraphService.cpp +++ b/src/graph/service/GraphService.cpp @@ -32,7 +32,6 @@ Status GraphService::init(std::shared_ptr ioExecuto options.serviceName_ = "graph"; options.skipConfig_ = FLAGS_local_config; options.role_ = meta::cpp2::HostRole::GRAPH; - std::string localIP = network::NetworkUtils::getIPv4FromDevice(FLAGS_listen_netdev).value(); options.localHost_ = hostAddr; options.gitInfoSHA_ = gitInfoSha(); @@ -42,17 +41,16 @@ Status GraphService::init(std::shared_ptr ioExecuto bool loadDataOk = metaClient_->waitForMetadReady(3); if (!loadDataOk) { // Resort to retrying in the background - LOG(WARNING) << "Failed to synchronously wait for meta service ready"; + LOG(WARNING) << "Failed to wait for meta service ready synchronously."; } sessionManager_ = std::make_unique(metaClient_.get(), hostAddr); auto initSessionMgrStatus = sessionManager_->init(); if (!initSessionMgrStatus.ok()) { - LOG(WARNING) << "Init sessin manager failed: " << initSessionMgrStatus.toString(); + LOG(WARNING) << "Failed to initialize session manager: " << initSessionMgrStatus.toString(); } - queryEngine_ = std::make_unique(); - myAddr_ = hostAddr; + queryEngine_ = std::make_unique(); return queryEngine_->init(std::move(ioExecutor), metaClient_.get()); } diff --git a/src/graph/service/GraphService.h b/src/graph/service/GraphService.h index 634d821fb9a..dd000919e1a 100644 --- a/src/graph/service/GraphService.h +++ b/src/graph/service/GraphService.h @@ -48,7 +48,6 @@ class GraphService final : public cpp2::GraphServiceSvIf { std::unique_ptr sessionManager_; std::unique_ptr queryEngine_; std::unique_ptr metaClient_; - HostAddr myAddr_; }; } // namespace graph diff --git a/src/graph/service/QueryEngine.cpp b/src/graph/service/QueryEngine.cpp index d4cc509bd44..018d677344c 100644 --- a/src/graph/service/QueryEngine.cpp +++ b/src/graph/service/QueryEngine.cpp @@ -7,6 +7,7 @@ #include "graph/service/QueryEngine.h" #include "common/base/Base.h" +#include "common/memory/MemoryUtils.h" #include "common/meta/ServerBasedIndexManager.h" #include "common/meta/ServerBasedSchemaManager.h" #include "graph/context/QueryContext.h" @@ -19,6 +20,7 @@ DECLARE_bool(local_config); DECLARE_bool(enable_optimizer); DECLARE_string(meta_server_addrs); +DEFINE_int32(check_memory_interval_in_secs, 1, "Memory check interval in seconds"); namespace nebula { namespace graph { @@ -39,19 +41,40 @@ Status QueryEngine::init(std::shared_ptr ioExecutor } optimizer_ = std::make_unique(rulesets); - return Status::OK(); + return setupMemoryMonitorThread(); } void QueryEngine::execute(RequestContextPtr rctx) { - auto ectx = std::make_unique(std::move(rctx), + auto qctx = std::make_unique(std::move(rctx), schemaManager_.get(), indexManager_.get(), storage_.get(), metaClient_, charsetInfo_); - auto* instance = new QueryInstance(std::move(ectx), optimizer_.get()); + auto* instance = new QueryInstance(std::move(qctx), optimizer_.get()); instance->execute(); } +Status QueryEngine::setupMemoryMonitorThread() { + memoryMonitorThread_ = std::make_unique(); + if (!memoryMonitorThread_ || !memoryMonitorThread_->start("query-engine-bg")) { + return Status::Error("Fail to start query engine background thread."); + } + + auto updateMemoryWatermark = []() -> Status { + auto status = MemoryUtils::hitsHighWatermark(); + NG_RETURN_IF_ERROR(status); + MemoryUtils::kHitMemoryHighWatermark.store(std::move(status).value()); + return Status::OK(); + }; + + // Just to test whether to get the right memory info + NG_RETURN_IF_ERROR(updateMemoryWatermark()); + + memoryMonitorThread_->addRepeatTask(FLAGS_check_memory_interval_in_secs, updateMemoryWatermark); + + return Status::OK(); +} + } // namespace graph } // namespace nebula diff --git a/src/graph/service/QueryEngine.h b/src/graph/service/QueryEngine.h index e18ee813db8..66efd23cb5f 100644 --- a/src/graph/service/QueryEngine.h +++ b/src/graph/service/QueryEngine.h @@ -20,15 +20,14 @@ #include "graph/service/RequestContext.h" #include "interface/gen-cpp2/GraphService.h" +namespace nebula { +namespace graph { + /** * QueryEngine is responsible to create and manage ExecutionPlan. * For the time being, we don't have the execution plan cache support, * instead we create a plan for each query, and destroy it upon finish. */ - -namespace nebula { -namespace graph { - class QueryEngine final : public cpp::NonCopyable, public cpp::NonMovable { public: QueryEngine() = default; @@ -43,11 +42,14 @@ class QueryEngine final : public cpp::NonCopyable, public cpp::NonMovable { const meta::MetaClient* metaClient() const { return metaClient_; } private: + Status setupMemoryMonitorThread(); + std::unique_ptr schemaManager_; std::unique_ptr indexManager_; std::unique_ptr storage_; std::unique_ptr optimizer_; - meta::MetaClient* metaClient_; + std::unique_ptr memoryMonitorThread_; + meta::MetaClient* metaClient_{nullptr}; CharsetInfo* charsetInfo_{nullptr}; }; diff --git a/src/graph/util/Utils.h b/src/graph/util/Utils.h index 9cadaa4cbec..6d62e6a634e 100644 --- a/src/graph/util/Utils.h +++ b/src/graph/util/Utils.h @@ -13,8 +13,7 @@ #include #include -namespace nebula { -namespace util { +namespace nebula::graph::util { template std::string join(const Container& container, Fn fn, const std::string& delimiter = ",") { @@ -25,7 +24,6 @@ std::string join(const Container& container, Fn fn, const std::string& delimiter return folly::join(delimiter, strs); } -} // namespace util -} // namespace nebula +} // namespace nebula::graph::util #endif // GRAPH_UTIL_UTILS_H_ diff --git a/src/graph/util/test/CMakeLists.txt b/src/graph/util/test/CMakeLists.txt index a3f01a7aac7..c0d01f77833 100644 --- a/src/graph/util/test/CMakeLists.txt +++ b/src/graph/util/test/CMakeLists.txt @@ -43,6 +43,7 @@ nebula_add_test( $ $ $ + $ $ $ LIBRARIES diff --git a/src/graph/validator/test/CMakeLists.txt b/src/graph/validator/test/CMakeLists.txt index 60e004bfe03..f6032fb78d8 100644 --- a/src/graph/validator/test/CMakeLists.txt +++ b/src/graph/validator/test/CMakeLists.txt @@ -35,6 +35,7 @@ set(VALIDATOR_TEST_LIBS $ $ $ + $ $ $ $ diff --git a/src/graph/visitor/test/CMakeLists.txt b/src/graph/visitor/test/CMakeLists.txt index ec7c18f8775..cd27f31f110 100644 --- a/src/graph/visitor/test/CMakeLists.txt +++ b/src/graph/visitor/test/CMakeLists.txt @@ -58,6 +58,7 @@ nebula_add_test( $ $ $ + $ LIBRARIES gtest ${THRIFT_LIBRARIES} diff --git a/src/parser/test/CMakeLists.txt b/src/parser/test/CMakeLists.txt index 89d1f367c35..86016b7ba1e 100644 --- a/src/parser/test/CMakeLists.txt +++ b/src/parser/test/CMakeLists.txt @@ -44,7 +44,7 @@ set(PARSER_TEST_LIBS $ $ $ - $ + $ ) nebula_add_test( diff --git a/src/version/Version.cpp.in b/src/version/Version.cpp.in index f0e3b16e66b..f8675b605e0 100644 --- a/src/version/Version.cpp.in +++ b/src/version/Version.cpp.in @@ -4,7 +4,7 @@ * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ -#include +#include #include "version/Version.h" @@ -20,9 +20,9 @@ std::string versionString(bool verbose) { version = folly::stringPrintf("%s, ", "@NEBULA_BUILD_VERSION@"); #endif if (verbose) { - version += folly::stringPrintf("Git: %s, ", gitInfoSha().c_str()); + version += folly::sformat("Git: {}, ", gitInfoSha()); } - version += folly::stringPrintf("Build Time: %s %s", __DATE__, __TIME__); + version += folly::sformat("Build Time: {} {}", __DATE__, __TIME__); if (verbose) { version += "\nThis source code is licensed under Apache 2.0 License," diff --git a/tests/Makefile b/tests/Makefile index 79d25120ab7..776cb227fe9 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -8,7 +8,6 @@ PYPI_MIRROR = https://mirrors.aliyun.com/pypi/simple/ # PYPI_MIRROR = http://pypi.mirrors.ustc.edu.cn/simple --trusted-host pypi.mirrors.ustc.edu.cn CURR_DIR = $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) -gherkin_fmt = ~/.local/bin/reformat-gherkin PY_VERSION = $(shell python3 --version | cut -f2 -d' ' | cut -f2 -d'.') RM_DIR ?= true @@ -20,6 +19,13 @@ ENABLE_SSL ?= false ENABLE_GRAPH_SSL ?= false ENABLE_META_SSL ?= false CA_SIGNED ?= false +CONTAINERIZED ?= false + +# commands +gherkin_fmt = ~/.local/bin/reformat-gherkin +run_test = PYTHONPATH=$$PYTHONPATH:$(CURR_DIR)/.. $(CURR_DIR)/nebula-test-run.py +test_without_skip = python3 -m pytest -m "not skip" +test_j = $(test_without_skip) -n$(J) install-deps: pip3 install --user -U setuptools wheel -i $(PYPI_MIRROR) @@ -53,40 +59,52 @@ check: up: clean @mkdir -p $(CURR_DIR)/.pytest - PYTHONPATH=$$PYTHONPATH:$(CURR_DIR)/.. $(CURR_DIR)/nebula-test-run.py --cmd=start --rm_dir=$(RM_DIR) --build_dir=$(BUILD_DIR) --debug=$(DEBUG) --multi_graphd=true --enable_ssl=$(ENABLE_SSL) --enable_graph_ssl=$(ENABLE_GRAPH_SSL) --enable_meta_ssl=$(ENABLE_META_SSL) --ca_signed=$(CA_SIGNED) + $(run_test) --cmd=start \ + --build_dir=$(BUILD_DIR) \ + --debug=$(DEBUG) \ + --multi_graphd=true \ + --enable_ssl=$(ENABLE_SSL) \ + --enable_graph_ssl=$(ENABLE_GRAPH_SSL) \ + --enable_meta_ssl=$(ENABLE_META_SSL) \ + --ca_signed=$(CA_SIGNED) \ + --containerized=$(CONTAINERIZED) down: - PYTHONPATH=$$PYTHONPATH:$(CURR_DIR)/.. $(CURR_DIR)/nebula-test-run.py --cmd=stop --rm_dir=$(RM_DIR) + $(run_test) --cmd=stop --rm_dir=$(RM_DIR) currdir: cd $(CURR_DIR) sess: currdir - python3 -m pytest -m "not skip" -k "not tck" job/test_session.py + $(test_without_skip) -k "not tck" job/test_session.py jobs: currdir - python3 -m pytest -m "not skip" tck/steps/test_jobs.py + $(test_without_skip) tck/steps/test_jobs.py test: sess - python3 -m pytest -n$(J) --dist=loadfile -m "not skip" -k "not tck" $(TEST_DIR) + $(test_j) --dist=loadfile -k "not tck" $(TEST_DIR) slow-query: currdir - python3 -m pytest -n$(J) -m "not skip" tck/steps/test_kill_slow_query_via_same_service.py && \ - python3 -m pytest -n$(J) -m "not skip" tck/steps/test_kill_slow_query_via_different_service.py + $(test_j) tck/steps/test_kill_slow_query_via_same_service.py && \ + $(test_j) tck/steps/test_kill_slow_query_via_different_service.py tck: jobs slow-query - python3 -m pytest -n$(J) -m "not skip" tck/steps/test_tck.py + $(test_j) tck/steps/test_tck.py fail: currdir - python3 -m pytest --last-failed --gherkin-terminal-reporter --gherkin-terminal-reporter-expanded tck/steps/test_tck.py - -report: - @mv $(CURR_DIR)/tck-report.json $(CURR_DIR)/tck-report-bak.json - @jq . $(CURR_DIR)/tck-report-bak.json > tck-report.json - @rm -rf $(CURR_DIR)/tck-report-bak.json + python3 -m pytest \ + --last-failed \ + --gherkin-terminal-reporter \ + --gherkin-terminal-reporter-expanded \ + tck/steps/test_tck.py clean: - @rm -rf $(CURR_DIR)/nebula-python $(CURR_DIR)/reformat-gherkin $(CURR_DIR)/.pytest/* $(CURR_DIR)/.pytest_cache $(CURR_DIR)/tck-report.json $(CURR_DIR)/*.lock + @rm -rf $(CURR_DIR)/nebula-python \ + $(CURR_DIR)/reformat-gherkin \ + $(CURR_DIR)/.pytest/* \ + $(CURR_DIR)/.pytest_cache \ + $(CURR_DIR)/*.lock \ + $(BUILD_DIR)/server_* kill: ps -ef | grep -P '\sbin/nebula-' | grep "$$(whoami)" | sed 's/\s\s*/ /g' | cut -f2 -d' ' | xargs kill -9 diff --git a/tests/admin/test_configs.py b/tests/admin/test_configs.py index 14f60535572..e9089eba1e8 100644 --- a/tests/admin/test_configs.py +++ b/tests/admin/test_configs.py @@ -66,6 +66,7 @@ def test_configs(self): ['GRAPH', 'meta_client_retry_times', 'int', 'MUTABLE', 3], ['GRAPH', 'accept_partial_success', 'bool', 'MUTABLE', False], ['GRAPH', 'system_memory_high_watermark_ratio', 'float', 'MUTABLE', 0.95], + ['GRAPH', 'num_rows_to_check_memory', 'int', 'MUTABLE', 4], ['GRAPH', 'session_idle_timeout_secs', 'int', 'MUTABLE', 0], ['GRAPH', 'session_reclaim_interval_secs', 'int', 'MUTABLE', 2], ['GRAPH', 'max_allowed_connections', 'int', 'MUTABLE', 9223372036854775807], @@ -75,19 +76,21 @@ def test_configs(self): resp = self.client.execute('SHOW CONFIGS storage') self.check_resp_succeeded(resp) - expected_result = [['STORAGE', 'v', 'int', 'MUTABLE', 3], - ['STORAGE', 'wal_ttl', 'int', 'MUTABLE', 14400], - ['STORAGE', 'minloglevel', 'int', 'MUTABLE', 0], - ['STORAGE', 'custom_filter_interval_secs', 'int', 'MUTABLE', 86400], - ['STORAGE', 'slow_op_threshhold_ms', 'int', 'MUTABLE', 50], - ['STORAGE', 'heartbeat_interval_secs', 'int', 'MUTABLE', 1], - ['STORAGE', 'meta_client_retry_times', 'int', 'MUTABLE', 3], - ['STORAGE', 'rocksdb_db_options', 'map', 'MUTABLE', {}], - ['STORAGE', 'clean_wal_interval_secs', 'int', 'MUTABLE', 600], - ['STORAGE', 'rocksdb_column_family_options', 'map', 'MUTABLE', - {"write_buffer_size":"67108864","max_bytes_for_level_base":"268435456","max_write_buffer_number":"4"}], - ['STORAGE', 'rocksdb_block_based_table_options', 'map', 'MUTABLE', {"block_size":"8192"}], - ["STORAGE", "max_edge_returned_per_vertex", "int", "MUTABLE", 2147483647]] + expected_result = [ + ['STORAGE', 'v', 'int', 'MUTABLE', 3], + ['STORAGE', 'wal_ttl', 'int', 'MUTABLE', 14400], + ['STORAGE', 'minloglevel', 'int', 'MUTABLE', 0], + ['STORAGE', 'custom_filter_interval_secs', 'int', 'MUTABLE', 86400], + ['STORAGE', 'slow_op_threshhold_ms', 'int', 'MUTABLE', 50], + ['STORAGE', 'heartbeat_interval_secs', 'int', 'MUTABLE', 1], + ['STORAGE', 'meta_client_retry_times', 'int', 'MUTABLE', 3], + ['STORAGE', 'rocksdb_db_options', 'map', 'MUTABLE', {}], + ['STORAGE', 'clean_wal_interval_secs', 'int', 'MUTABLE', 600], + ['STORAGE', 'rocksdb_column_family_options', 'map', 'MUTABLE', {"write_buffer_size": "67108864", "max_bytes_for_level_base": "268435456", "max_write_buffer_number": "4"}], + ['STORAGE', 'rocksdb_block_based_table_options', 'map', 'MUTABLE', {"block_size": "8192"}], + ["STORAGE", "max_edge_returned_per_vertex", "int", "MUTABLE", 2147483647], + ['STORAGE', 'system_memory_high_watermark_ratio', 'float', 'MUTABLE', 0.8], + ] print(resp) self.check_out_of_order_result(resp, expected_result) diff --git a/tests/common/nebula_service.py b/tests/common/nebula_service.py index c13f494f99c..9d337258428 100644 --- a/tests/common/nebula_service.py +++ b/tests/common/nebula_service.py @@ -19,19 +19,11 @@ class NebulaService(object): - def __init__(self, build_dir, src_dir, cleanup=True): + def __init__(self, build_dir, src_dir): self.build_dir = str(build_dir) self.src_dir = str(src_dir) self.work_dir = os.path.join(self.build_dir, 'server_' + time.strftime("%Y-%m-%dT%H-%M-%S", time.localtime())) self.pids = {} - self._cleanup = cleanup - - def __enter__(self): - self.install() - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.stop(cleanup=self._cleanup) def set_work_dir(self, work_dir): self.work_dir = work_dir @@ -89,11 +81,12 @@ def _format_nebula_command(self, name, meta_port, ports, debug_log=True, ca_sign params.append('--cert_path=share/resources/test.ca.pem') params.append('--key_path=share/resources/test.ca.key') params.append('--password_path=share/resources/test.ca.password') - + if name == 'graphd': params.append('--local_config=false') params.append('--enable_authorize=true') params.append('--system_memory_high_watermark_ratio=0.95') + params.append('--num_rows_to_check_memory=4') params.append('--session_reclaim_interval_secs=2') if name == 'storaged': params.append('--local_config=false') @@ -172,12 +165,11 @@ def _check_servers_status(self, ports): time.sleep(1) return False - def start(self, debug_log=True, multi_graphd=False, enable_ssl=False, enable_graph_ssl=False, enable_meta_ssl=False, ca_signed=False): + def start(self, debug_log="true", multi_graphd=False, ca_signed="false", **kwargs): os.chdir(self.work_dir) metad_ports = self._find_free_port() all_ports = [metad_ports[0]] - command = '' graph_ports = [] server_ports = [] servers = [] @@ -192,29 +184,30 @@ def start(self, debug_log=True, multi_graphd=False, enable_ssl=False, enable_gra if server_name != 'metad': while True: ports = self._find_free_port() - if all((ports[0] + i) not in all_ports - for i in range(-2, 3)): + if all((ports[0] + i) not in all_ports for i in range(-2, 3)): all_ports += [ports[0]] break else: ports = metad_ports server_ports.append(ports[0]) - new_name = server_name + new_name = server_name if server_name != 'graphd1' else 'graphd' + command = [ + self._format_nebula_command(new_name, + metad_ports[0], + ports, + debug_log, + ca_signed=ca_signed) + ] if server_name == 'graphd1': - new_name = 'graphd' - command = self._format_nebula_command(new_name, - metad_ports[0], - ports, - debug_log, - ca_signed=ca_signed) - if server_name == 'graphd1': - command += ' --log_dir=logs1' - command += ' --pid_file=pids1/nebula-graphd.pid' - command += ' --enable_ssl={}'.format(enable_ssl) - command += ' --enable_graph_ssl={}'.format(enable_graph_ssl) - command += ' --enable_meta_ssl={}'.format(enable_meta_ssl) - print("exec: " + command) - p = subprocess.Popen([command], shell=True, stdout=subprocess.PIPE) + command.append('--log_dir=logs1') + command.append('--pid_file=pids1/nebula-graphd.pid') + + for k,v in kwargs.items(): + command.append("--{}={}".format(k, v)) + + cmd = " ".join(command) + print("exec: " + cmd) + p = subprocess.Popen([cmd], shell=True, stdout=subprocess.PIPE) p.wait() if p.returncode != 0: print("error: " + bytes.decode(p.communicate()[0])) @@ -241,7 +234,7 @@ def _collect_pids(self): with open(pf) as f: self.pids[f.name] = int(f.readline()) - def stop(self): + def stop(self, cleanup): print("try to stop nebula services...") self._collect_pids() self.kill_all(signal.SIGTERM) @@ -253,7 +246,7 @@ def stop(self): self.kill_all(signal.SIGKILL) - if self._cleanup: + if cleanup: shutil.rmtree(self.work_dir, ignore_errors=True) def kill_all(self, sig): diff --git a/tests/nebula-test-run.py b/tests/nebula-test-run.py index d2f7eabeac1..2f1fbc371fe 100755 --- a/tests/nebula-test-run.py +++ b/tests/nebula-test-run.py @@ -68,6 +68,10 @@ def init_parser(): dest='ca_signed', default='false', help='Whether enable CA signed SSL/TLS mode.') + opt_parser.add_option('--containerized', + dest='containerized', + default='false', + help='run this process inside container') return opt_parser @@ -85,12 +89,15 @@ def start_nebula(nb, configs): else: nb.install() address = "localhost" - debug = opt_is(configs.debug, "true") - enable_ssl = opt_is(configs.enable_ssl, "true") - enable_meta_ssl = opt_is(configs.enable_meta_ssl, "true") - enable_graph_ssl = opt_is(configs.enable_graph_ssl, "true") - ca_signed = opt_is(configs.ca_signed, "true") - ports = nb.start(debug_log=debug, multi_graphd=configs.multi_graphd, enable_ssl=enable_ssl, enable_graph_ssl=enable_graph_ssl, enable_meta_ssl=enable_meta_ssl, ca_signed=ca_signed) + ports = nb.start( + debug_log=opt_is(configs.debug, "true"), + multi_graphd=configs.multi_graphd, + ca_signed=opt_is(configs.ca_signed, "true"), + enable_ssl=opt_is(configs.enable_ssl, "true"), + enable_graph_ssl=opt_is(configs.enable_graph_ssl, "true"), + enable_meta_ssl=opt_is(configs.enable_meta_ssl, "true"), + containerized=opt_is(configs.containerized, "true") + ) # Load csv data pool = get_conn_pool(address, ports[0]) @@ -125,7 +132,10 @@ def stop_nebula(nb, configs=None): with open(NB_TMP_PATH, "r") as f: data = json.loads(f.readline()) nb.set_work_dir(data["work_dir"]) - nb.stop() + + cleanup = opt_is(configs.rm_dir, "true") + nb.stop(cleanup) + shutil.rmtree(TMP_DIR, ignore_errors=True) print('nebula services have been stopped.') @@ -136,8 +146,7 @@ def stop_nebula(nb, configs=None): (configs, opts) = parser.parse_args() # Setup nebula graph service - cleanup = opt_is(configs.rm_dir, "true") - nebula_svc = NebulaService(configs.build_dir, NEBULA_HOME, cleanup) + nebula_svc = NebulaService(configs.build_dir, NEBULA_HOME) if opt_is(configs.cmd, "start"): start_nebula(nebula_svc, configs) diff --git a/tests/tck/features/insert/Insert.IntVid.feature b/tests/tck/features/insert/Insert.IntVid.feature index 81631698894..2ae02c68c5f 100644 --- a/tests/tck/features/insert/Insert.IntVid.feature +++ b/tests/tck/features/insert/Insert.IntVid.feature @@ -32,7 +32,7 @@ Feature: Insert int vid of vertex and edge Scenario: insert vertex and edge test Given wait 3 seconds # insert vretex with default property names - When executing query: + When try to execute query: """ INSERT VERTEX person VALUES hash("Tom"):("Tom", 18); """ diff --git a/tests/tck/features/match/Base.feature b/tests/tck/features/match/Base.feature index d8dfe4a255d..98705f6b212 100644 --- a/tests/tck/features/match/Base.feature +++ b/tests/tck/features/match/Base.feature @@ -514,7 +514,7 @@ Feature: Basic match """ MATCH (v:player) where v.name return v """ - Then a ExecutionError should be raised at runtime: Internal Error: Wrong type result, the type should be NULL,EMPTY or BOOL + Then a ExecutionError should be raised at runtime: Wrong type result, the type should be NULL, EMPTY or BOOL Scenario: Unimplemented features When executing query: diff --git a/tests/tck/features/schema/Schema.feature b/tests/tck/features/schema/Schema.feature index 4678480aec8..e960e43c278 100644 --- a/tests/tck/features/schema/Schema.feature +++ b/tests/tck/features/schema/Schema.feature @@ -499,7 +499,7 @@ Feature: Insert string vid of vertex and edge Then the execution should be successful And wait 3 seconds # insert - When executing query: + When try to execute query: """ INSERT VERTEX t() VALUES "1":() """