From 3fe419eafaf25780f50a1b119d01e376d77a12ef Mon Sep 17 00:00:00 2001 From: Jibing-Li <64681310+Jibing-Li@users.noreply.github.com> Date: Thu, 17 Aug 2023 15:20:02 +0800 Subject: [PATCH] [Fix](statistics)Fix update cached column stats bug (#23049) `show column cached stats` sometimes show wrong min/max value: ``` mysql> show column cached stats hive.tpch100.region; +-------------+-------+------+----------+-----------+---------------+------+------+--------------+ | column_name | count | ndv | num_null | data_size | avg_size_byte | min | max | updated_time | +-------------+-------+------+----------+-----------+---------------+------+------+--------------+ | r_regionkey | 5.0 | 5.0 | 0.0 | 24.0 | 4.0 | N/A | N/A | null | | r_comment | 5.0 | 5.0 | 0.0 | 396.0 | 66.0 | N/A | N/A | null | | r_name | 5.0 | 5.0 | 0.0 | 40.8 | 6.8 | N/A | N/A | null | +-------------+-------+------+----------+-----------+---------------+------+------+--------------+ ``` This pr is to fix this bug. It is because while transferring ColumnStatistic object to JSON, it doesn't contain the minExpr and maxExpr attribute. --- .../org/apache/doris/service/FrontendServiceImpl.java | 10 +++++++--- .../org/apache/doris/statistics/StatisticsCache.java | 4 ++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 2e3562e49a6575..e7b82039d0be98 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -87,7 +87,6 @@ import org.apache.doris.qe.QueryState; import org.apache.doris.qe.StmtExecutor; import org.apache.doris.qe.VariableMgr; -import org.apache.doris.statistics.ColumnStatistic; import org.apache.doris.statistics.StatisticsCacheKey; import org.apache.doris.statistics.query.QueryStats; import org.apache.doris.system.Backend; @@ -2943,8 +2942,13 @@ private TGetBinlogLagResult getBinlogLagImpl(TGetBinlogRequest request, String c @Override public TStatus updateStatsCache(TUpdateFollowerStatsCacheRequest request) throws TException { StatisticsCacheKey key = GsonUtils.GSON.fromJson(request.key, StatisticsCacheKey.class); - ColumnStatistic columnStatistic = GsonUtils.GSON.fromJson(request.colStats, ColumnStatistic.class); - Env.getCurrentEnv().getStatisticsCache().putCache(key, columnStatistic); + /* + TODO: Need to handle minExpr and maxExpr, so that we can generate the columnStatistic + here and use putCache to update cached directly. + ColumnStatistic columnStatistic = GsonUtils.GSON.fromJson(request.colStats, ColumnStatistic.class); + Env.getCurrentEnv().getStatisticsCache().putCache(key, columnStatistic); + */ + Env.getCurrentEnv().getStatisticsCache().refreshColStatsSync(key.tableId, key.idxId, key.colName); return new TStatus(TStatusCode.OK); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java index b405647ad83a72..0b720037cbe9f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java @@ -257,6 +257,10 @@ public void syncLoadColStats(long tableId, long idxId, String colName) { updateFollowerStatsCacheRequest.key = GsonUtils.GSON.toJson(k); updateFollowerStatsCacheRequest.colStats = GsonUtils.GSON.toJson(c); for (Frontend frontend : Env.getCurrentEnv().getFrontends(FrontendNodeType.FOLLOWER)) { + if (frontend.getHost().equals(Env.getCurrentEnv().getSelfNode().getHost())) { + // Doesn't need to send request to current node. + continue; + } TNetworkAddress address = new TNetworkAddress(frontend.getHost(), frontend.getRpcPort()); FrontendService.Client client = null;