From b435c2992dfe4167b875613754a5189a0a09ba07 Mon Sep 17 00:00:00 2001 From: xiaoxmeng Date: Tue, 5 Dec 2023 09:08:12 -0800 Subject: [PATCH] [native]Advance velox and update metric code --- .../presto_cpp/main/PeriodicTaskManager.cpp | 171 ++++++------- .../presto_cpp/main/PrestoExchangeSource.cpp | 6 +- .../presto_cpp/main/PrestoServer.cpp | 6 +- .../presto_cpp/main/TaskManager.cpp | 8 +- .../presto_cpp/main/common/Counters.cpp | 237 ++++++++---------- .../presto_cpp/main/common/Counters.h | 2 +- .../main/http/filters/StatsFilter.cpp | 6 +- presto-native-execution/velox | 2 +- 8 files changed, 195 insertions(+), 243 deletions(-) diff --git a/presto-native-execution/presto_cpp/main/PeriodicTaskManager.cpp b/presto-native-execution/presto_cpp/main/PeriodicTaskManager.cpp index 67176ce0b8ab..667494345563 100644 --- a/presto-native-execution/presto_cpp/main/PeriodicTaskManager.cpp +++ b/presto-native-execution/presto_cpp/main/PeriodicTaskManager.cpp @@ -31,9 +31,9 @@ #include namespace { -#define REPORT_IF_NOT_ZERO(name, counter) \ - if ((counter) != 0) { \ - REPORT_ADD_STAT_VALUE((name), (counter)); \ +#define REPORT_IF_NOT_ZERO(name, counter) \ + if ((counter) != 0) { \ + RECORD_METRIC_VALUE((name), (counter)); \ } } // namespace @@ -128,14 +128,14 @@ void PeriodicTaskManager::stop() { void PeriodicTaskManager::updateExecutorStats() { if (driverCPUExecutor_ != nullptr) { // Report the current queue size of the thread pool. - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterDriverCPUExecutorQueueSize, driverCPUExecutor_->getTaskQueueSize()); // Report driver execution latency. folly::stop_watch timer; driverCPUExecutor_->add([timer = timer]() { - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterDriverCPUExecutorLatencyMs, timer.elapsed().count()); }); } @@ -144,7 +144,7 @@ void PeriodicTaskManager::updateExecutorStats() { // Report the latency between scheduling the task and its execution. folly::stop_watch timer; httpExecutor_->add([timer = timer]() { - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterHTTPExecutorLatencyMs, timer.elapsed().count()); }); } @@ -161,25 +161,25 @@ void PeriodicTaskManager::updateTaskStats() { // Report the number of tasks and drivers in the system. size_t numTasks{0}; auto taskNumbers = taskManager_->getTaskNumbers(numTasks); - REPORT_ADD_STAT_VALUE(kCounterNumTasks, taskManager_->getNumTasks()); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE(kCounterNumTasks, taskManager_->getNumTasks()); + RECORD_METRIC_VALUE( kCounterNumTasksRunning, taskNumbers[velox::exec::TaskState::kRunning]); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterNumTasksFinished, taskNumbers[velox::exec::TaskState::kFinished]); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterNumTasksCancelled, taskNumbers[velox::exec::TaskState::kCanceled]); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterNumTasksAborted, taskNumbers[velox::exec::TaskState::kAborted]); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterNumTasksFailed, taskNumbers[velox::exec::TaskState::kFailed]); auto driverCountStats = taskManager_->getDriverCountStats(); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterNumRunningDrivers, driverCountStats.numRunningDrivers); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterNumBlockedDrivers, driverCountStats.numBlockedDrivers); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterTotalPartitionedOutputBuffer, velox::exec::OutputBufferManager::getInstance().lock()->numBuffers()); } @@ -206,20 +206,20 @@ void PeriodicTaskManager::addOldTaskCleanupTask() { } void PeriodicTaskManager::updateMemoryAllocatorStats() { - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMappedMemoryBytes, (velox::memory::AllocationTraits::pageBytes( memoryAllocator_->numMapped()))); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterAllocatedMemoryBytes, (velox::memory::AllocationTraits::pageBytes( memoryAllocator_->numAllocated()))); // TODO(jtan6): Remove condition after T150019700 is done if (auto* mmapAllocator = dynamic_cast(memoryAllocator_)) { - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMmapRawAllocBytesSmall, (mmapAllocator->numMallocBytes())); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMmapExternalMappedBytes, velox::memory::AllocationTraits::pageBytes( (mmapAllocator->numExternalMapped()))); @@ -240,7 +240,7 @@ void PeriodicTaskManager::updatePrestoExchangeSourceMemoryStats() { PrestoExchangeSource::getMemoryUsage( currQueuedMemoryBytes, peakQueuedMemoryBytes); PrestoExchangeSource::resetPeakMemoryUsage(); - REPORT_ADD_HISTOGRAM_VALUE( + RECORD_HISTOGRAM_METRIC_VALUE( kCounterExchangeSourcePeakQueuedBytes, peakQueuedMemoryBytes); } @@ -255,49 +255,49 @@ void PeriodicTaskManager::updateCacheStats() { const auto memoryCacheStats = asyncDataCache_->refreshStats(); // Snapshots. - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMemoryCacheNumEntries, memoryCacheStats.numEntries); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMemoryCacheNumEmptyEntries, memoryCacheStats.numEmptyEntries); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMemoryCacheNumSharedEntries, memoryCacheStats.numShared); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMemoryCacheNumExclusiveEntries, memoryCacheStats.numExclusive); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMemoryCacheNumPrefetchedEntries, memoryCacheStats.numPrefetch); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMemoryCacheTotalTinyBytes, memoryCacheStats.tinySize); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMemoryCacheTotalLargeBytes, memoryCacheStats.largeSize); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMemoryCacheTotalTinyPaddingBytes, memoryCacheStats.tinyPadding); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMemoryCacheTotalLargePaddingBytes, memoryCacheStats.largePadding); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMemoryCacheTotalPrefetchBytes, memoryCacheStats.prefetchBytes); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMemoryCacheSumEvictScore, memoryCacheStats.sumEvictScore); // Interval cumulatives. - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMemoryCacheNumHit, memoryCacheStats.numHit - lastMemoryCacheHits_); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMemoryCacheHitBytes, memoryCacheStats.hitBytes - lastMemoryCacheHitsBytes_); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMemoryCacheNumNew, memoryCacheStats.numNew - lastMemoryCacheInserts_); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMemoryCacheNumEvict, memoryCacheStats.numEvict - lastMemoryCacheEvictions_); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMemoryCacheNumEvictChecks, memoryCacheStats.numEvictChecks - lastMemoryCacheEvictionChecks_); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMemoryCacheNumWaitExclusive, memoryCacheStats.numWaitExclusive - lastMemoryCacheStalls_); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMemoryCacheNumAllocClocks, memoryCacheStats.allocClocks - lastMemoryCacheAllocClocks_); @@ -310,67 +310,67 @@ void PeriodicTaskManager::updateCacheStats() { lastMemoryCacheAllocClocks_ = memoryCacheStats.allocClocks; // All time cumulatives. - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMemoryCacheNumCumulativeHit, memoryCacheStats.numHit); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMemoryCacheCumulativeHitBytes, memoryCacheStats.hitBytes); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMemoryCacheNumCumulativeNew, memoryCacheStats.numNew); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMemoryCacheNumCumulativeEvict, memoryCacheStats.numEvict); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMemoryCacheNumCumulativeEvictChecks, memoryCacheStats.numEvictChecks); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMemoryCacheNumCumulativeWaitExclusive, memoryCacheStats.numWaitExclusive); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterMemoryCacheNumCumulativeAllocClocks, memoryCacheStats.allocClocks); if (memoryCacheStats.ssdStats != nullptr) { - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterSsdCacheCumulativeReadEntries, memoryCacheStats.ssdStats->entriesRead) - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterSsdCacheCumulativeReadBytes, memoryCacheStats.ssdStats->bytesRead); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterSsdCacheCumulativeWrittenEntries, memoryCacheStats.ssdStats->entriesWritten); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterSsdCacheCumulativeWrittenBytes, memoryCacheStats.ssdStats->bytesWritten); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterSsdCacheCumulativeCachedEntries, memoryCacheStats.ssdStats->entriesCached); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterSsdCacheCumulativeCachedBytes, memoryCacheStats.ssdStats->bytesCached); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterSsdCacheCumulativeOpenSsdErrors, memoryCacheStats.ssdStats->openFileErrors); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterSsdCacheCumulativeOpenCheckpointErrors, memoryCacheStats.ssdStats->openCheckpointErrors); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterSsdCacheCumulativeOpenLogErrors, memoryCacheStats.ssdStats->openLogErrors); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterSsdCacheCumulativeDeleteCheckpointErrors, memoryCacheStats.ssdStats->deleteCheckpointErrors); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterSsdCacheCumulativeGrowFileErrors, memoryCacheStats.ssdStats->growFileErrors); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterSsdCacheCumulativeWriteSsdErrors, memoryCacheStats.ssdStats->writeSsdErrors); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterSsdCacheCumulativeWriteCheckpointErrors, memoryCacheStats.ssdStats->writeCheckpointErrors); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterSsdCacheCumulativeReadSsdErrors, memoryCacheStats.ssdStats->readSsdErrors); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterSsdCacheCumulativeReadCheckpointErrors, memoryCacheStats.ssdStats->readCheckpointErrors); } @@ -411,20 +411,15 @@ void PeriodicTaskManager::addConnectorStatsTask() { oldValues[kNumLookupsMetricName] = 0; // Exporting metrics types here since the metrics key is dynamic - REPORT_ADD_STAT_EXPORT_TYPE( - kNumElementsMetricName, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( - kPinnedSizeMetricName, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( - kCurSizeMetricName, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC(kNumElementsMetricName, facebook::velox::StatType::AVG); + DEFINE_METRIC(kPinnedSizeMetricName, facebook::velox::StatType::AVG); + DEFINE_METRIC(kCurSizeMetricName, facebook::velox::StatType::AVG); + DEFINE_METRIC( kNumAccumulativeHitsMetricName, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kNumAccumulativeLookupsMetricName, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( - kNumHitsMetricName, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( - kNumLookupsMetricName, facebook::velox::StatType::AVG); + DEFINE_METRIC(kNumHitsMetricName, facebook::velox::StatType::AVG); + DEFINE_METRIC(kNumLookupsMetricName, facebook::velox::StatType::AVG); addTask( [hiveConnector, @@ -437,22 +432,22 @@ void PeriodicTaskManager::addConnectorStatsTask() { kNumHitsMetricName, kNumLookupsMetricName]() { auto fileHandleCacheStats = hiveConnector->fileHandleCacheStats(); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kNumElementsMetricName, fileHandleCacheStats.numElements); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kPinnedSizeMetricName, fileHandleCacheStats.pinnedSize); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCurSizeMetricName, fileHandleCacheStats.curSize); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kNumAccumulativeHitsMetricName, fileHandleCacheStats.numHits); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kNumAccumulativeLookupsMetricName, fileHandleCacheStats.numLookups); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kNumHitsMetricName, fileHandleCacheStats.numHits - oldValues[kNumHitsMetricName]); oldValues[kNumHitsMetricName] = fileHandleCacheStats.numHits; - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kNumLookupsMetricName, fileHandleCacheStats.numLookups - oldValues[kNumLookupsMetricName]); @@ -472,35 +467,35 @@ void PeriodicTaskManager::updateOperatingSystemStats() { const int64_t userCpuTimeUs{ (int64_t)usage.ru_utime.tv_sec * 1'000'000 + (int64_t)usage.ru_utime.tv_usec}; - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterOsUserCpuTimeMicros, userCpuTimeUs - lastUserCpuTimeUs_); lastUserCpuTimeUs_ = userCpuTimeUs; const int64_t systemCpuTimeUs{ (int64_t)usage.ru_stime.tv_sec * 1'000'000 + (int64_t)usage.ru_stime.tv_usec}; - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterOsSystemCpuTimeMicros, systemCpuTimeUs - lastSystemCpuTimeUs_); lastSystemCpuTimeUs_ = systemCpuTimeUs; const int64_t softPageFaults{usage.ru_minflt}; - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterOsNumSoftPageFaults, softPageFaults - lastSoftPageFaults_); lastSoftPageFaults_ = softPageFaults; const int64_t hardPageFaults{usage.ru_majflt}; - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterOsNumHardPageFaults, hardPageFaults - lastHardPageFaults_); lastHardPageFaults_ = hardPageFaults; const int64_t voluntaryContextSwitches{usage.ru_nvcsw}; - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterOsNumVoluntaryContextSwitches, voluntaryContextSwitches - lastVoluntaryContextSwitches_); lastVoluntaryContextSwitches_ = voluntaryContextSwitches; const int64_t forcedContextSwitches{usage.ru_nivcsw}; - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterOsNumForcedContextSwitches, forcedContextSwitches - lastForcedContextSwitches_); lastForcedContextSwitches_ = forcedContextSwitches; @@ -595,10 +590,8 @@ void PeriodicTaskManager::updateSpillStatsTask() { LOG(INFO) << "Spill memory usage: current[" << velox::succinctBytes(spillMemoryStats.currentBytes) << "] peak[" << velox::succinctBytes(spillMemoryStats.peakBytes) << "]"; - REPORT_ADD_STAT_VALUE( - kCounterSpillMemoryBytes, spillMemoryStats.currentBytes); - REPORT_ADD_STAT_VALUE( - kCounterSpillPeakMemoryBytes, spillMemoryStats.peakBytes); + RECORD_METRIC_VALUE(kCounterSpillMemoryBytes, spillMemoryStats.currentBytes); + RECORD_METRIC_VALUE(kCounterSpillPeakMemoryBytes, spillMemoryStats.peakBytes); lastSpillStats_ = updatedSpillStats; } diff --git a/presto-native-execution/presto_cpp/main/PrestoExchangeSource.cpp b/presto-native-execution/presto_cpp/main/PrestoExchangeSource.cpp index b670360ba012..ae98b4ee86d9 100644 --- a/presto-native-execution/presto_cpp/main/PrestoExchangeSource.cpp +++ b/presto-native-execution/presto_cpp/main/PrestoExchangeSource.cpp @@ -112,8 +112,8 @@ PrestoExchangeSource::PrestoExchangeSource( clientCertAndKeyPath_, ciphers_, [](size_t bufferBytes) { - REPORT_ADD_STAT_VALUE(kCounterHttpClientPrestoExchangeNumOnBody); - REPORT_ADD_HISTOGRAM_VALUE( + RECORD_METRIC_VALUE(kCounterHttpClientPrestoExchangeNumOnBody); + RECORD_HISTOGRAM_METRIC_VALUE( kCounterHttpClientPrestoExchangeOnBodyBytes, bufferBytes); }); } @@ -292,7 +292,7 @@ void PrestoExchangeSource::processDataResponse( const int64_t pageSize = empty ? 0 : page->size(); - REPORT_ADD_HISTOGRAM_VALUE( + RECORD_HISTOGRAM_METRIC_VALUE( kCounterPrestoExchangeSerializedPageSize, pageSize); { diff --git a/presto-native-execution/presto_cpp/main/PrestoServer.cpp b/presto-native-execution/presto_cpp/main/PrestoServer.cpp index 0301467bbe25..267e2788f775 100644 --- a/presto-native-execution/presto_cpp/main/PrestoServer.cpp +++ b/presto-native-execution/presto_cpp/main/PrestoServer.cpp @@ -901,8 +901,8 @@ void PrestoServer::registerMemoryArbitrators() { } void PrestoServer::registerStatsCounters() { - registerPrestoCppCounters(); - registerVeloxCounters(); + registerPrestoMetrics(); + registerVeloxMetrics(); } std::string PrestoServer::getLocalIp() const { @@ -956,7 +956,7 @@ void PrestoServer::populateMemAndCPUInfo() { {queryId, {protocol::MemoryAllocation{"total", bytes}}}); ++numContexts; }); - REPORT_ADD_STAT_VALUE(kCounterNumQueryContexts, numContexts); + RECORD_METRIC_VALUE(kCounterNumQueryContexts, numContexts); cpuMon_.update(); **memoryInfo_.wlock() = std::move(memoryInfo); } diff --git a/presto-native-execution/presto_cpp/main/TaskManager.cpp b/presto-native-execution/presto_cpp/main/TaskManager.cpp index 80dd42650c5e..e972f23b341a 100644 --- a/presto-native-execution/presto_cpp/main/TaskManager.cpp +++ b/presto-native-execution/presto_cpp/main/TaskManager.cpp @@ -156,7 +156,7 @@ void getData( promiseHolder->promise.setValue(std::move(result)); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterPartitionedOutputBufferGetDataLatencyMs, getCurrentTimeMs() - startMs); }); @@ -709,11 +709,11 @@ size_t TaskManager::cleanOldTasks() { if (zombiePrestoTaskCounts.numTotal > 0) { zombiePrestoTaskCounts.logZombieTaskStatus("PrestoTask"); } - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterNumZombieVeloxTasks, zombieVeloxTaskCounts.numTotal); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterNumZombiePrestoTasks, zombiePrestoTaskCounts.numTotal); - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterNumTasksWithStuckOperator, numTasksWithStuckOperator); return taskIdsToClean.size(); } diff --git a/presto-native-execution/presto_cpp/main/common/Counters.cpp b/presto-native-execution/presto_cpp/main/common/Counters.cpp index a7b5ad79ae74..d06ecb28672f 100644 --- a/presto-native-execution/presto_cpp/main/common/Counters.cpp +++ b/presto-native-execution/presto_cpp/main/common/Counters.cpp @@ -17,23 +17,19 @@ namespace facebook::presto { -void registerPrestoCppCounters() { - REPORT_ADD_STAT_EXPORT_TYPE( +void registerPrestoMetrics() { + DEFINE_METRIC( kCounterDriverCPUExecutorQueueSize, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterDriverCPUExecutorLatencyMs, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterHTTPExecutorLatencyMs, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterNumHTTPRequest, facebook::velox::StatType::COUNT); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterNumHTTPRequestError, facebook::velox::StatType::COUNT); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterHTTPRequestLatencyMs, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC(kCounterHTTPExecutorLatencyMs, facebook::velox::StatType::AVG); + DEFINE_METRIC(kCounterNumHTTPRequest, facebook::velox::StatType::COUNT); + DEFINE_METRIC(kCounterNumHTTPRequestError, facebook::velox::StatType::COUNT); + DEFINE_METRIC(kCounterHTTPRequestLatencyMs, facebook::velox::StatType::AVG); + DEFINE_METRIC( kCounterHttpClientPrestoExchangeNumOnBody, facebook::velox::StatType::COUNT); - REPORT_ADD_HISTOGRAM_EXPORT_PERCENTILE( + DEFINE_HISTOGRAM_METRIC( kCounterHttpClientPrestoExchangeOnBodyBytes, 1000, 0, @@ -43,7 +39,7 @@ void registerPrestoCppCounters() { 95, 99, 100); - REPORT_ADD_HISTOGRAM_EXPORT_PERCENTILE( + DEFINE_HISTOGRAM_METRIC( kCounterPrestoExchangeSerializedPageSize, 10000, 0, @@ -53,199 +49,162 @@ void registerPrestoCppCounters() { 95, 99, 100); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterNumQueryContexts, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE(kCounterNumTasks, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterNumTasksRunning, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterNumTasksFinished, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterNumTasksCancelled, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterNumTasksAborted, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterNumTasksFailed, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterNumZombieVeloxTasks, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterNumZombiePrestoTasks, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC(kCounterNumQueryContexts, facebook::velox::StatType::AVG); + DEFINE_METRIC(kCounterNumTasks, facebook::velox::StatType::AVG); + DEFINE_METRIC(kCounterNumTasksRunning, facebook::velox::StatType::AVG); + DEFINE_METRIC(kCounterNumTasksFinished, facebook::velox::StatType::AVG); + DEFINE_METRIC(kCounterNumTasksCancelled, facebook::velox::StatType::AVG); + DEFINE_METRIC(kCounterNumTasksAborted, facebook::velox::StatType::AVG); + DEFINE_METRIC(kCounterNumTasksFailed, facebook::velox::StatType::AVG); + DEFINE_METRIC(kCounterNumZombieVeloxTasks, facebook::velox::StatType::AVG); + DEFINE_METRIC(kCounterNumZombiePrestoTasks, facebook::velox::StatType::AVG); + DEFINE_METRIC( kCounterNumTasksWithStuckOperator, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterNumRunningDrivers, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterNumBlockedDrivers, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterMappedMemoryBytes, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterAllocatedMemoryBytes, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterMmapRawAllocBytesSmall, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC(kCounterNumRunningDrivers, facebook::velox::StatType::AVG); + DEFINE_METRIC(kCounterNumBlockedDrivers, facebook::velox::StatType::AVG); + DEFINE_METRIC(kCounterMappedMemoryBytes, facebook::velox::StatType::AVG); + DEFINE_METRIC(kCounterAllocatedMemoryBytes, facebook::velox::StatType::AVG); + DEFINE_METRIC(kCounterMmapRawAllocBytesSmall, facebook::velox::StatType::AVG); + DEFINE_METRIC( kCounterMmapExternalMappedBytes, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterTotalPartitionedOutputBuffer, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterPartitionedOutputBufferGetDataLatencyMs, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterOsUserCpuTimeMicros, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterOsSystemCpuTimeMicros, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterOsNumSoftPageFaults, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterOsNumHardPageFaults, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC(kCounterOsUserCpuTimeMicros, facebook::velox::StatType::AVG); + DEFINE_METRIC(kCounterOsSystemCpuTimeMicros, facebook::velox::StatType::AVG); + DEFINE_METRIC(kCounterOsNumSoftPageFaults, facebook::velox::StatType::AVG); + DEFINE_METRIC(kCounterOsNumHardPageFaults, facebook::velox::StatType::AVG); + DEFINE_METRIC( kCounterOsNumVoluntaryContextSwitches, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterOsNumForcedContextSwitches, facebook::velox::StatType::AVG); - REPORT_ADD_HISTOGRAM_EXPORT_PERCENTILE( + DEFINE_HISTOGRAM_METRIC( kCounterExchangeSourcePeakQueuedBytes, 1l * 1024 * 1024 * 1024, 0, 62l * 1024 * 1024 * 1024, // max bucket value: 62GB 100); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterMemoryCacheNumEntries, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC(kCounterMemoryCacheNumEntries, facebook::velox::StatType::AVG); + DEFINE_METRIC( kCounterMemoryCacheNumEmptyEntries, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterMemoryCacheNumSharedEntries, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterMemoryCacheNumExclusiveEntries, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterMemoryCacheNumPrefetchedEntries, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterMemoryCacheTotalTinyBytes, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterMemoryCacheTotalLargeBytes, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterMemoryCacheTotalTinyPaddingBytes, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterMemoryCacheTotalLargePaddingBytes, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterMemoryCacheTotalPrefetchBytes, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterMemoryCacheTotalTinyPaddingBytes, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterMemoryCacheSumEvictScore, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterMemoryCacheNumCumulativeHit, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterMemoryCacheNumHit, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC(kCounterMemoryCacheNumHit, facebook::velox::StatType::AVG); + DEFINE_METRIC( kCounterMemoryCacheCumulativeHitBytes, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterMemoryCacheHitBytes, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC(kCounterMemoryCacheHitBytes, facebook::velox::StatType::AVG); + DEFINE_METRIC( kCounterMemoryCacheNumCumulativeNew, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterMemoryCacheNumNew, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC(kCounterMemoryCacheNumNew, facebook::velox::StatType::AVG); + DEFINE_METRIC( kCounterMemoryCacheNumCumulativeEvict, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterMemoryCacheNumEvict, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC(kCounterMemoryCacheNumEvict, facebook::velox::StatType::AVG); + DEFINE_METRIC( kCounterMemoryCacheNumCumulativeEvictChecks, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterMemoryCacheNumEvictChecks, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterMemoryCacheNumCumulativeWaitExclusive, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterMemoryCacheNumWaitExclusive, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterMemoryCacheNumCumulativeAllocClocks, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterMemoryCacheNumAllocClocks, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterSsdCacheCumulativeReadEntries, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterSsdCacheCumulativeReadBytes, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterSsdCacheCumulativeWrittenEntries, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterSsdCacheCumulativeWrittenBytes, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterSsdCacheCumulativeCachedEntries, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterSsdCacheCumulativeCachedBytes, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterSsdCacheCumulativeOpenSsdErrors, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterSsdCacheCumulativeOpenCheckpointErrors, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterSsdCacheCumulativeOpenLogErrors, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterSsdCacheCumulativeDeleteCheckpointErrors, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterSsdCacheCumulativeGrowFileErrors, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterSsdCacheCumulativeWriteSsdErrors, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterSsdCacheCumulativeWriteCheckpointErrors, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterSsdCacheCumulativeReadSsdErrors, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterSsdCacheCumulativeReadCheckpointErrors, facebook::velox::StatType::AVG); // Disk spilling stats. - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterSpillRuns, facebook::velox::StatType::SUM); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterSpilledFiles, facebook::velox::StatType::SUM); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterSpilledRows, facebook::velox::StatType::SUM); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterSpilledBytes, facebook::velox::StatType::SUM); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterSpillFillTimeUs, facebook::velox::StatType::SUM); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterSpillSortTimeUs, facebook::velox::StatType::SUM); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC(kCounterSpillRuns, facebook::velox::StatType::SUM); + DEFINE_METRIC(kCounterSpilledFiles, facebook::velox::StatType::SUM); + DEFINE_METRIC(kCounterSpilledRows, facebook::velox::StatType::SUM); + DEFINE_METRIC(kCounterSpilledBytes, facebook::velox::StatType::SUM); + DEFINE_METRIC(kCounterSpillFillTimeUs, facebook::velox::StatType::SUM); + DEFINE_METRIC(kCounterSpillSortTimeUs, facebook::velox::StatType::SUM); + DEFINE_METRIC( kCounterSpillSerializationTimeUs, facebook::velox::StatType::SUM); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterSpillDiskWrites, facebook::velox::StatType::SUM); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterSpillFlushTimeUs, facebook::velox::StatType::SUM); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterSpillWriteTimeUs, facebook::velox::StatType::SUM); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterSpillMemoryBytes, facebook::velox::StatType::AVG); - REPORT_ADD_HISTOGRAM_EXPORT_PERCENTILE( + DEFINE_METRIC(kCounterSpillDiskWrites, facebook::velox::StatType::SUM); + DEFINE_METRIC(kCounterSpillFlushTimeUs, facebook::velox::StatType::SUM); + DEFINE_METRIC(kCounterSpillWriteTimeUs, facebook::velox::StatType::SUM); + DEFINE_METRIC(kCounterSpillMemoryBytes, facebook::velox::StatType::AVG); + DEFINE_HISTOGRAM_METRIC( kCounterSpillPeakMemoryBytes, 1l * 512 * 1024 * 1024, 0, 20l * 1024 * 1024 * 1024, // max bucket value: 20GB 100); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterSpillMaxLevelExceeded, facebook::velox::StatType::SUM); + DEFINE_METRIC(kCounterSpillMaxLevelExceeded, facebook::velox::StatType::SUM); // Memory arbitrator stats. - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterArbitratorNumRequests, facebook::velox::StatType::SUM); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterArbitratorNumAborted, facebook::velox::StatType::SUM); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterArbitratorNumFailures, facebook::velox::StatType::SUM); - REPORT_ADD_STAT_EXPORT_TYPE( - kCounterArbitratorQueueTimeUs, facebook::velox::StatType::SUM); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC(kCounterArbitratorNumRequests, facebook::velox::StatType::SUM); + DEFINE_METRIC(kCounterArbitratorNumAborted, facebook::velox::StatType::SUM); + DEFINE_METRIC(kCounterArbitratorNumFailures, facebook::velox::StatType::SUM); + DEFINE_METRIC(kCounterArbitratorQueueTimeUs, facebook::velox::StatType::SUM); + DEFINE_METRIC( kCounterArbitratorArbitrationTimeUs, facebook::velox::StatType::SUM); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterArbitratorNumShrunkBytes, facebook::velox::StatType::SUM); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterArbitratorNumReclaimedBytes, facebook::velox::StatType::SUM); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterArbitratorFreeCapacityBytes, facebook::velox::StatType::AVG); - REPORT_ADD_STAT_EXPORT_TYPE( + DEFINE_METRIC( kCounterArbitratorNonReclaimableAttempts, facebook::velox::StatType::SUM); // NOTE: Metrics type exporting for file handle cache counters are in diff --git a/presto-native-execution/presto_cpp/main/common/Counters.h b/presto-native-execution/presto_cpp/main/common/Counters.h index 54d2f62bd983..b1cab2768092 100644 --- a/presto-native-execution/presto_cpp/main/common/Counters.h +++ b/presto-native-execution/presto_cpp/main/common/Counters.h @@ -20,7 +20,7 @@ namespace facebook::presto { // Sets up all the counters in the presto cpp, but specifying their types. // See velox/common/base/StatsReporter.h for the interface. -void registerPrestoCppCounters(); +void registerPrestoMetrics(); constexpr folly::StringPiece kCounterDriverCPUExecutorQueueSize{ "presto_cpp.driver_cpu_executor_queue_size"}; diff --git a/presto-native-execution/presto_cpp/main/http/filters/StatsFilter.cpp b/presto-native-execution/presto_cpp/main/http/filters/StatsFilter.cpp index 52819bee4e76..4669d82421a2 100644 --- a/presto-native-execution/presto_cpp/main/http/filters/StatsFilter.cpp +++ b/presto-native-execution/presto_cpp/main/http/filters/StatsFilter.cpp @@ -24,12 +24,12 @@ StatsFilter::StatsFilter(proxygen::RequestHandler* upstream) void StatsFilter::onRequest( std::unique_ptr msg) noexcept { startTime_ = std::chrono::steady_clock::now(); - REPORT_ADD_STAT_VALUE(kCounterNumHTTPRequest, 1); + RECORD_METRIC_VALUE(kCounterNumHTTPRequest, 1); Filter::onRequest(std::move(msg)); } void StatsFilter::requestComplete() noexcept { - REPORT_ADD_STAT_VALUE( + RECORD_METRIC_VALUE( kCounterHTTPRequestLatencyMs, std::chrono::duration_cast( std::chrono::steady_clock::now() - startTime_) @@ -38,7 +38,7 @@ void StatsFilter::requestComplete() noexcept { } void StatsFilter::onError(proxygen::ProxygenError err) noexcept { - REPORT_ADD_STAT_VALUE(kCounterNumHTTPRequestError, 1); + RECORD_METRIC_VALUE(kCounterNumHTTPRequestError, 1); delete this; } diff --git a/presto-native-execution/velox b/presto-native-execution/velox index aa60e3d6401b..975ca3ac9424 160000 --- a/presto-native-execution/velox +++ b/presto-native-execution/velox @@ -1 +1 @@ -Subproject commit aa60e3d6401bae4d8eef1bcd3dfb42c0ad7933e3 +Subproject commit 975ca3ac942472d6312b54280d8543a7b870d6b0