Skip to content

Commit

Permalink
[native]Advance velox version and update memory usage
Browse files Browse the repository at this point in the history
  • Loading branch information
xiaoxmeng committed Jun 5, 2024
1 parent 7443f83 commit 36f7d62
Show file tree
Hide file tree
Showing 2 changed files with 3 additions and 3 deletions.
4 changes: 2 additions & 2 deletions presto-native-execution/presto_cpp/main/PrestoTask.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -370,7 +370,7 @@ protocol::TaskStatus PrestoTask::updateStatusLocked() {
}

const auto veloxTaskMemStats = task->pool()->stats();
info.taskStatus.memoryReservationInBytes = task->pool()->usedBytes();
info.taskStatus.memoryReservationInBytes = veloxTaskMemStats.usedBytes;
info.taskStatus.systemMemoryReservationInBytes = 0;
// NOTE: a presto worker may run multiple tasks from the same query.
// 'peakNodeTotalMemoryReservationInBytes' represents peak memory usage across
Expand Down Expand Up @@ -547,7 +547,7 @@ void PrestoTask::updateMemoryInfoLocked(
protocol::TaskStats& prestoTaskStats = info.stats;

const auto veloxTaskMemStats = task->pool()->stats();
const auto currentBytes = task->pool()->usedBytes();
const auto currentBytes = veloxTaskMemStats.usedBytes;
prestoTaskStats.userMemoryReservationInBytes = currentBytes;
prestoTaskStats.systemMemoryReservationInBytes = 0;
prestoTaskStats.peakUserMemoryInBytes = veloxTaskMemStats.peakBytes;
Expand Down
2 changes: 1 addition & 1 deletion presto-native-execution/velox
Submodule velox updated 97 files
+0 −17 .github/workflows/linux-build.yml
+0 −1 .github/workflows/scheduled.yml
+48 −12 CMake/Findre2.cmake
+36 −0 CMake/resolve_dependency_modules/absl.cmake
+44 −0 CMake/resolve_dependency_modules/google_cloud_cpp_storage.cmake
+57 −0 CMake/resolve_dependency_modules/grpc.cmake
+6 −40 CMake/resolve_dependency_modules/protobuf.cmake
+17 −18 CMakeLists.txt
+2 −2 velox/common/base/PeriodicStatsReporter.cpp
+0 −4 velox/common/base/tests/StatsReporterTest.cpp
+12 −7 velox/common/caching/CachedFactory.h
+9 −27 velox/common/caching/SsdCache.cpp
+0 −10 velox/common/caching/SsdCache.h
+10 −3 velox/common/caching/tests/CachedFactoryTest.cpp
+9 −9 velox/common/memory/MemoryPool.cpp
+5 −21 velox/common/memory/MemoryPool.h
+2 −2 velox/common/memory/tests/MemoryCapExceededTest.cpp
+43 −43 velox/common/memory/tests/MemoryPoolTest.cpp
+7 −2 velox/connectors/hive/FileHandle.cpp
+5 −1 velox/connectors/hive/FileHandle.h
+37 −0 velox/connectors/hive/FileProperties.h
+9 −2 velox/connectors/hive/HiveConnectorSplit.h
+4 −1 velox/connectors/hive/SplitReader.cpp
+8 −4 velox/connectors/hive/iceberg/IcebergSplit.cpp
+4 −2 velox/connectors/hive/iceberg/IcebergSplit.h
+15 −0 velox/connectors/hive/storage_adapters/abfs/tests/AbfsFileSystemTest.cpp
+26 −0 velox/connectors/hive/tests/FileHandleTest.cpp
+68 −1 velox/docs/functions/presto/decimal.rst
+23 −0 velox/docs/functions/presto/json.rst
+9 −0 velox/docs/functions/presto/map.rst
+7 −0 velox/docs/functions/presto/math.rst
+2 −3 velox/dwio/common/CMakeLists.txt
+39 −15 velox/dwio/common/ColumnVisitors.h
+11 −0 velox/dwio/common/Options.h
+18 −6 velox/dwio/common/SelectiveColumnReader.h
+0 −1 velox/dwio/common/SelectiveColumnReaderInternal.h
+2 −1 velox/dwio/common/tests/CMakeLists.txt
+22 −0 velox/dwio/common/tests/utils/DataSetBuilder.h
+6 −0 velox/dwio/common/tests/utils/E2EFilterTestBase.h
+1 −1 velox/dwio/dwrf/proto/CMakeLists.txt
+1 −1 velox/dwio/dwrf/reader/SelectiveIntegerDictionaryColumnReader.cpp
+2 −2 velox/dwio/dwrf/reader/SelectiveStringDictionaryColumnReader.cpp
+13 −1 velox/dwio/dwrf/reader/SelectiveTimestampColumnReader.cpp
+2 −0 velox/dwio/dwrf/reader/SelectiveTimestampColumnReader.h
+1 −0 velox/dwio/dwrf/test/TestColumnReader.cpp
+0 −4 velox/dwio/dwrf/test/WriterFlushTest.cpp
+1 −1 velox/exec/Operator.h
+3 −1 velox/exec/RowContainer.h
+2 −2 velox/exec/Task.cpp
+1 −1 velox/exec/fuzzer/AggregationFuzzerOptions.h
+2 −0 velox/exec/fuzzer/JoinFuzzer.cpp
+42 −54 velox/exec/fuzzer/WriterFuzzer.cpp
+52 −81 velox/exec/tests/HashJoinTest.cpp
+2 −2 velox/exec/tests/OrderByTest.cpp
+48 −0 velox/exec/tests/RowContainerTest.cpp
+2 −2 velox/exec/tests/SortBufferTest.cpp
+28 −0 velox/exec/tests/TableScanTest.cpp
+1 −0 velox/exec/tests/TaskTest.cpp
+2 −1 velox/exec/tests/utils/HiveConnectorTestBase.h
+0 −27 velox/expression/CastExpr-inl.h
+20 −13 velox/expression/CastExpr.cpp
+0 −8 velox/expression/CastExpr.h
+2 −2 velox/functions/lib/aggregates/tests/utils/AggregationTestBase.cpp
+5 −26 velox/functions/prestosql/DateTimeFunctions.h
+113 −0 velox/functions/prestosql/DecimalFunctions.cpp
+2 −0 velox/functions/prestosql/DecimalFunctions.h
+37 −0 velox/functions/prestosql/MapRemoveNullValues.h
+14 −0 velox/functions/prestosql/Probability.h
+117 −91 velox/functions/prestosql/SIMDJsonFunctions.h
+21 −16 velox/functions/prestosql/json/JsonPathTokenizer.cpp
+9 −2 velox/functions/prestosql/json/JsonPathTokenizer.h
+11 −1 velox/functions/prestosql/json/SIMDJsonExtractor.cpp
+1 −1 velox/functions/prestosql/json/tests/JsonExtractorTest.cpp
+11 −1 velox/functions/prestosql/json/tests/JsonPathTokenizerTest.cpp
+1 −1 velox/functions/prestosql/json/tests/SIMDJsonExtractorTest.cpp
+5 −0 velox/functions/prestosql/registration/JsonFunctionsRegistration.cpp
+10 −0 velox/functions/prestosql/registration/MapFunctionsRegistration.cpp
+1 −0 velox/functions/prestosql/registration/MathematicalOperatorsRegistration.cpp
+2 −0 velox/functions/prestosql/registration/ProbabilityTrigonometricFunctionsRegistration.cpp
+66 −0 velox/functions/prestosql/tests/DecimalArithmeticTest.cpp
+29 −0 velox/functions/prestosql/tests/JsonCastTest.cpp
+3 −2 velox/functions/prestosql/tests/JsonExtractScalarTest.cpp
+62 −10 velox/functions/prestosql/tests/JsonFunctionsTest.cpp
+61 −0 velox/functions/prestosql/tests/MapRemoveNullValuesTest.cpp
+49 −0 velox/functions/prestosql/tests/ProbabilityTest.cpp
+3 −0 velox/functions/prestosql/types/JsonType.cpp
+19 −14 velox/functions/sparksql/DateTimeFunctions.h
+9 −3 velox/functions/sparksql/tests/DateTimeFunctionsTest.cpp
+1 −1 velox/substrait/CMakeLists.txt
+7 −5 velox/type/Timestamp.h
+22 −0 velox/type/TimestampConversion.cpp
+4 −0 velox/type/TimestampConversion.h
+355 −0 website/blog/2024-05-31-optimize-try-more.mdx
+ website/static/img/optimize-try-more-numSilentThrow.png
+ website/static/img/optimize-try-more-profile1.png
+ website/static/img/optimize-try-more-profile2.png
+ website/static/img/optimize-try-more-profile3.png

0 comments on commit 36f7d62

Please sign in to comment.