Merge remote-tracking branch 'upstream/master' into help-doc-for-redi…

…s-cluster
apache · Jul 5, 2024 · 959bffc · 959bffc
2 parents 2ad2993 + 336eaff
commit 959bffc
Show file tree

Hide file tree

Showing 639 changed files with 34,603 additions and 11,347 deletions.
diff --git a/.all-contributorsrc b/.all-contributorsrc
@@ -1889,6 +1889,51 @@
       "contributions": [
         "doc"
       ]
+    },
+    {
+      "login": "yykaue",
+      "name": "Limbo",
+      "avatar_url": "https://avatars.githubusercontent.com/u/22905143?v=4",
+      "profile": "https://github.com/yykaue",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "irenhongyan",
+      "name": "哈哈哈哈哈哈哈哈哈",
+      "avatar_url": "https://avatars.githubusercontent.com/u/53438321?v=4",
+      "profile": "https://github.com/irenhongyan",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "ileonli",
+      "name": "Leon Li",
+      "avatar_url": "https://avatars.githubusercontent.com/u/45332412?v=4",
+      "profile": "https://github.com/ileonli",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "killme2008",
+      "name": "dennis zhuang",
+      "avatar_url": "https://avatars.githubusercontent.com/u/14142?v=4",
+      "profile": "http://fnil.net/",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "kerwin612",
+      "name": "Kerwin Bryant",
+      "avatar_url": "https://avatars.githubusercontent.com/u/3371163?v=4",
+      "profile": "https://github.com/kerwin612",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

diff --git a/.github/workflows/backend-build-test.yml b/.github/workflows/backend-build-test.yml
@@ -49,6 +49,10 @@ jobs:
         java-version: 17
     - name: Build with Maven
       run: mvn clean -B package -Prelease --file pom.xml
+    - name: Upload coverage reports to Codecov
+      uses: codecov/[email protected]
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
     - name: Build Image
       env:
         IMAGE_PUSH: false

diff --git a/README.md b/README.md
@@ -14,7 +14,8 @@
 [![Reddit](https://img.shields.io/badge/Reddit-Community-7289DA?logo=reddit)](https://www.reddit.com/r/hertzbeat/)
 [![Twitter](https://img.shields.io/twitter/follow/hertzbeat1024?logo=twitter)](https://twitter.com/hertzbeat1024)
 [![OpenSSF Best Practices](https://www.bestpractices.dev/projects/8139/badge)](https://www.bestpractices.dev/projects/8139)
-[![Docker Pulls](https://img.shields.io/docker/pulls/apache/hertzbeat?style=%20for-the-badge&logo=docker&label=DockerHub%20Download)](https://hub.docker.com/repository/docker/apache/hertzbeat/general)
+[![codecov](https://codecov.io/gh/apache/HertzBeat/branch/master/graph/badge.svg)](https://app.codecov.io/gh/apache/hertzbeat)
+[![Docker Pulls](https://img.shields.io/docker/pulls/apache/hertzbeat?style=%20for-the-badge&logo=docker&label=DockerHub%20Download)](https://hub.docker.com/r/apache/hertzbeat)
 [![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/hertzbeat)](https://artifacthub.io/packages/search?repo=hertzbeat)
 [![YouTube Channel Subscribers](https://img.shields.io/youtube/channel/subscribers/UCri75zfWX0GHqJFPENEbLow?logo=youtube&label=YouTube%20Channel)](https://www.youtube.com/channel/UCri75zfWX0GHqJFPENEbLow)
 
@@ -160,13 +161,13 @@ Detailed config refer to [Install HertzBeat via Package](https://hertzbeat.apach
 
 Detailed steps refer to [CONTRIBUTING](CONTRIBUTING.md)        
 
-##### 4：Install All(hertzbeat+mysql+tsdb) via Docker-compose  
+##### 4：Install All(hertzbeat+postgresql+tsdb) via Docker-compose  
 
-Install the mysql/postgresql database, iotdb/tdengine/victoria-metrics database and hertzbeat at one time through [docker-compose deployment script](script/docker-compose).
+Install the postgresql/mysql database, victoria-metrics/iotdb/tdengine database and hertzbeat at one time through [docker-compose deployment script](script/docker-compose).
 
 Detailed steps refer to [Install via Docker-Compose](script/docker-compose/README.md)      
 
-##### 5. Install All(hertzbeat+collector+mysql+tsdb) via kubernetes helm charts
+##### 5. Install All(hertzbeat+collector+postgresql+tsdb) via kubernetes helm charts
 
 Install HertzBeat cluster in a Kubernetes cluster by Helm chart.
 
@@ -446,6 +447,13 @@ Thanks to these wonderful people, welcome to join us:
       <td align="center" valign="top" width="14.28%"><a href="https://www.yitianyigexiangfa.com/"><img src="https://avatars.githubusercontent.com/u/3973419?v=4?s=100" width="100px;" alt="Bill Lau"/><br /><sub><b>Bill Lau</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=JavaProgrammerLB" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/lw-yang"><img src="https://avatars.githubusercontent.com/u/23456873?v=4?s=100" width="100px;" alt="lwyang"/><br /><sub><b>lwyang</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=lw-yang" title="Documentation">📖</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/xfl12345"><img src="https://avatars.githubusercontent.com/u/17960863?v=4?s=100" width="100px;" alt="xfl12345"/><br /><sub><b>xfl12345</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=xfl12345" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/yykaue"><img src="https://avatars.githubusercontent.com/u/22905143?v=4?s=100" width="100px;" alt="Limbo"/><br /><sub><b>Limbo</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=yykaue" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/irenhongyan"><img src="https://avatars.githubusercontent.com/u/53438321?v=4?s=100" width="100px;" alt="哈哈哈哈哈哈哈哈哈"/><br /><sub><b>哈哈哈哈哈哈哈哈哈</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=irenhongyan" title="Code">💻</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/ileonli"><img src="https://avatars.githubusercontent.com/u/45332412?v=4?s=100" width="100px;" alt="Leon Li"/><br /><sub><b>Leon Li</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=ileonli" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="http://fnil.net/"><img src="https://avatars.githubusercontent.com/u/14142?v=4?s=100" width="100px;" alt="dennis zhuang"/><br /><sub><b>dennis zhuang</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=killme2008" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/kerwin612"><img src="https://avatars.githubusercontent.com/u/3371163?v=4?s=100" width="100px;" alt="Kerwin Bryant"/><br /><sub><b>Kerwin Bryant</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=kerwin612" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

diff --git a/README_CN.md b/README_CN.md
@@ -14,7 +14,8 @@
 [![Reddit](https://img.shields.io/badge/Reddit-Community-7289DA?logo=reddit)](https://www.reddit.com/r/hertzbeat/)
 [![Twitter](https://img.shields.io/twitter/follow/hertzbeat1024?logo=twitter)](https://twitter.com/hertzbeat1024)
 [![OpenSSF Best Practices](https://www.bestpractices.dev/projects/8139/badge)](https://www.bestpractices.dev/projects/8139)
-[![Docker Pulls](https://img.shields.io/docker/pulls/apache/hertzbeat?style=%20for-the-badge&logo=docker&label=DockerHub%20Download)](https://hub.docker.com/repository/docker/apache/hertzbeat/general)
+[![codecov](https://codecov.io/gh/apache/HertzBeat/branch/master/graph/badge.svg)](https://app.codecov.io/gh/apache/hertzbeat)
+[![Docker Pulls](https://img.shields.io/docker/pulls/apache/hertzbeat?style=%20for-the-badge&logo=docker&label=DockerHub%20Download)](https://hub.docker.com/r/apache/hertzbeat)
 [![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/hertzbeat)](https://artifacthub.io/packages/search?repo=hertzbeat)
 [![YouTube Channel Subscribers](https://img.shields.io/youtube/channel/subscribers/UCri75zfWX0GHqJFPENEbLow?logo=youtube&label=YouTube%20Channel)](https://www.youtube.com/channel/UCri75zfWX0GHqJFPENEbLow)
 
@@ -159,13 +160,13 @@ docker run -d -e IDENTITY=custom-collector-name -e MANAGER_HOST=127.0.0.1 -e MAN
 
 详细步骤参考 [参与贡献之本地代码启动](CONTRIBUTING.md)
 
-##### 方式四：Docker-Compose 统一安装 hertzbeat+mysql+tsdb
+##### 方式四：Docker-Compose 统一安装 hertzbeat+postgresql+tsdb
 
-通过 [Docker-Compose 部署脚本](script/docker-compose) 一次性把 mysql/postgresql 数据库, iotdb/tdengine/victoria-metrics 时序数据库和 hertzbeat 安装部署。
+通过 [Docker-Compose 部署脚本](script/docker-compose) 一次性把 postgresql/mysql 数据库, victoria-metrics/iotdb/tdengine 时序数据库和 hertzbeat 安装部署。
 
 详细步骤参考 [通过Docker-Compose安装HertzBeat](script/docker-compose/README.md)  
 
-##### 方式五：Kubernetes Helm Charts 部署 hertzbeat+collector+mysql+tsdb
+##### 方式五：Kubernetes Helm Charts 部署 hertzbeat+collector+postgresql+tsdb
 
 通过 Helm Chart 一次性将 HertzBeat 集群组件部署到 Kubernetes 集群中。
 
@@ -445,6 +446,13 @@ Thanks these wonderful people, welcome to join us:
       <td align="center" valign="top" width="14.28%"><a href="https://www.yitianyigexiangfa.com/"><img src="https://avatars.githubusercontent.com/u/3973419?v=4?s=100" width="100px;" alt="Bill Lau"/><br /><sub><b>Bill Lau</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=JavaProgrammerLB" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/lw-yang"><img src="https://avatars.githubusercontent.com/u/23456873?v=4?s=100" width="100px;" alt="lwyang"/><br /><sub><b>lwyang</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=lw-yang" title="Documentation">📖</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/xfl12345"><img src="https://avatars.githubusercontent.com/u/17960863?v=4?s=100" width="100px;" alt="xfl12345"/><br /><sub><b>xfl12345</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=xfl12345" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/yykaue"><img src="https://avatars.githubusercontent.com/u/22905143?v=4?s=100" width="100px;" alt="Limbo"/><br /><sub><b>Limbo</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=yykaue" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/irenhongyan"><img src="https://avatars.githubusercontent.com/u/53438321?v=4?s=100" width="100px;" alt="哈哈哈哈哈哈哈哈哈"/><br /><sub><b>哈哈哈哈哈哈哈哈哈</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=irenhongyan" title="Code">💻</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/ileonli"><img src="https://avatars.githubusercontent.com/u/45332412?v=4?s=100" width="100px;" alt="Leon Li"/><br /><sub><b>Leon Li</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=ileonli" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="http://fnil.net/"><img src="https://avatars.githubusercontent.com/u/14142?v=4?s=100" width="100px;" alt="dennis zhuang"/><br /><sub><b>dennis zhuang</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=killme2008" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/kerwin612"><img src="https://avatars.githubusercontent.com/u/3371163?v=4?s=100" width="100px;" alt="Kerwin Bryant"/><br /><sub><b>Kerwin Bryant</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=kerwin612" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

diff --git a/alerter/src/main/java/org/apache/hertzbeat/alert/AlerterWorkerPool.java b/alerter/src/main/java/org/apache/hertzbeat/alert/AlerterWorkerPool.java
@@ -34,9 +34,11 @@
 public class AlerterWorkerPool {
 
     private ThreadPoolExecutor workerExecutor;
+    private ThreadPoolExecutor notifyExecutor;
 
     public AlerterWorkerPool() {
         initWorkExecutor();
+        initNotifyExecutor();
     }
 
     private void initWorkExecutor() {
@@ -57,6 +59,24 @@ private void initWorkExecutor() {
                 new ThreadPoolExecutor.AbortPolicy());
     }
 
+    private void initNotifyExecutor() {
+        ThreadFactory threadFactory = new ThreadFactoryBuilder()
+                .setUncaughtExceptionHandler((thread, throwable) -> {
+                    log.error("notifyExecutor has uncaughtException.");
+                    log.error(throwable.getMessage(), throwable);
+                })
+                .setDaemon(true)
+                .setNameFormat("notify-worker-%d")
+                .build();
+        notifyExecutor = new ThreadPoolExecutor(6,
+                10,
+                10,
+                TimeUnit.SECONDS,
+                new SynchronousQueue<>(),
+                threadFactory,
+                new ThreadPoolExecutor.AbortPolicy());
+    }
+
     /**
      * Run the alerter task
      * @param runnable task
@@ -65,4 +85,15 @@ private void initWorkExecutor() {
     public void executeJob(Runnable runnable) throws RejectedExecutionException {
         workerExecutor.execute(runnable);
     }
+
+    /**
+     * Executes the given runnable task using the notifyExecutor.
+     *
+     * @param runnable the task to be executed
+     * @throws RejectedExecutionException if the task cannot be accepted for execution
+     */
+    public void executeNotify(Runnable runnable) throws RejectedExecutionException {
+        notifyExecutor.execute(runnable);
+    }
+
 }
diff --git a/alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmConvergeReduce.java b/alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmConvergeReduce.java
@@ -55,25 +55,33 @@ public AlarmConvergeReduce(AlertConvergeDao alertConvergeDao) {
     @SuppressWarnings("unchecked")
     public boolean filterConverge(Alert currentAlert) {
         // ignore monitor status auto recover notice
-        if (currentAlert.getTags() != null && currentAlert.getTags().containsKey(CommonConstants.IGNORE)) {
-            return true;
-        }
-        if (currentAlert.getStatus() == CommonConstants.ALERT_STATUS_CODE_RESTORED) {
+        if ((currentAlert.getTags() != null && currentAlert.getTags().containsKey(CommonConstants.IGNORE))
+                || currentAlert.getStatus() == CommonConstants.ALERT_STATUS_CODE_RESTORED) {
             // restored alert
+            boolean isHasIgnore = false;
+            Map<String, String> tags = currentAlert.getTags();
+            if (tags.containsKey(CommonConstants.IGNORE)) {
+                isHasIgnore = true;
+                tags.remove(CommonConstants.IGNORE);
+            }
             int alertHash = Objects.hash(CommonConstants.ALERT_PRIORITY_CODE_CRITICAL)
-                    + Arrays.hashCode(currentAlert.getTags().keySet().toArray(new String[0]))
-                    + Arrays.hashCode(currentAlert.getTags().values().toArray(new String[0]));
+                    + Arrays.hashCode(tags.keySet().toArray(new String[0]))
+                    + Arrays.hashCode(tags.values().toArray(new String[0]));
             converageAlertMap.remove(alertHash);
             alertHash = Objects.hash(CommonConstants.ALERT_PRIORITY_CODE_EMERGENCY)
-                    + Arrays.hashCode(currentAlert.getTags().keySet().toArray(new String[0]))
-                    + Arrays.hashCode(currentAlert.getTags().values().toArray(new String[0]));
+                    + Arrays.hashCode(tags.keySet().toArray(new String[0]))
+                    + Arrays.hashCode(tags.values().toArray(new String[0]));
             converageAlertMap.remove(alertHash);
             alertHash = Objects.hash(CommonConstants.ALERT_PRIORITY_CODE_WARNING)
-                    + Arrays.hashCode(currentAlert.getTags().keySet().toArray(new String[0]))
-                    + Arrays.hashCode(currentAlert.getTags().values().toArray(new String[0]));
+                    + Arrays.hashCode(tags.keySet().toArray(new String[0]))
+                    + Arrays.hashCode(tags.values().toArray(new String[0]));
             converageAlertMap.remove(alertHash);
+            if (isHasIgnore) {
+                tags.put(CommonConstants.IGNORE, CommonConstants.IGNORE);
+            }
             return true;
         }
+
         CommonCacheService<String, Object> convergeCache = CacheFactory.getAlertConvergeCache();
         List<AlertConverge> alertConvergeList = (List<AlertConverge>) convergeCache.get(CommonConstants.CACHE_ALERT_CONVERGE);
         if (alertConvergeList == null) {

diff --git a/alerter/src/main/java/org/apache/hertzbeat/alert/service/impl/AlertDefineServiceImpl.java b/alerter/src/main/java/org/apache/hertzbeat/alert/service/impl/AlertDefineServiceImpl.java
@@ -26,14 +26,17 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.hertzbeat.alert.dao.AlertDefineBindDao;
 import org.apache.hertzbeat.alert.dao.AlertDefineDao;
+import org.apache.hertzbeat.alert.dao.AlertMonitorDao;
 import org.apache.hertzbeat.alert.service.AlertDefineImExportService;
 import org.apache.hertzbeat.alert.service.AlertDefineService;
 import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
 import org.apache.hertzbeat.common.entity.alerter.AlertDefineMonitorBind;
+import org.apache.hertzbeat.common.entity.manager.Monitor;
 import org.apache.hertzbeat.common.util.JexlExpressionRunner;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.domain.Page;
@@ -59,6 +62,9 @@ public class AlertDefineServiceImpl implements AlertDefineService {
     @Autowired
     private AlertDefineBindDao alertDefineBindDao;
 
+    @Autowired
+    private AlertMonitorDao alertMonitorDao;
+
     private final Map<String, AlertDefineImExportService> alertDefineImExportServiceMap = new HashMap<>();
 
     public AlertDefineServiceImpl(List<AlertDefineImExportService> alertDefineImExportServiceList) {
@@ -147,7 +153,24 @@ public Page<AlertDefine> getAlertDefines(Specification<AlertDefine> specificatio
 
     @Override
     public List<AlertDefineMonitorBind> getBindAlertDefineMonitors(long alertDefineId) {
-        return alertDefineBindDao.getAlertDefineBindsByAlertDefineIdEquals(alertDefineId);
+        List<AlertDefineMonitorBind> defineMonitorBinds = alertDefineBindDao.getAlertDefineBindsByAlertDefineIdEquals(alertDefineId);
+        if (defineMonitorBinds == null || defineMonitorBinds.isEmpty()) {
+            return defineMonitorBinds;
+        }
+        List<Long> needLoadMonitorIds = defineMonitorBinds.stream()
+                .filter(bind -> bind.getMonitor() == null)
+                .map(AlertDefineMonitorBind::getMonitorId).toList();
+        if (needLoadMonitorIds.isEmpty()) {
+            return defineMonitorBinds;
+        }
+        Map<Long, Monitor> monitorMap = alertMonitorDao.findAllById(needLoadMonitorIds)
+                .stream().collect(Collectors.toMap(Monitor::getId, Function.identity()));
+        for (AlertDefineMonitorBind bind : defineMonitorBinds) {
+            if (bind.getMonitor() == null) {
+                bind.setMonitor(monitorMap.get(bind.getMonitorId()));
+            }
+        }
+        return defineMonitorBinds;
     }
 
     @Override

diff --git a/collector/src/main/java/org/apache/hertzbeat/collector/collect/dns/DnsCollectImpl.java b/collector/src/main/java/org/apache/hertzbeat/collector/collect/dns/DnsCollectImpl.java
@@ -158,7 +158,7 @@ private DnsResolveResult dig(DnsProtocol dns) throws IOException {
         responseTimeStopWatch.start();
 
         Name name = Name.fromString(dns.getAddress(), Name.root);
-        Message query = Message.newQuery(Record.newRecord(name, Type.ANY, DClass.ANY));
+        Message query = Message.newQuery(Record.newRecord(name, Type.ANY, DClass.value(dns.getQueryClass())));
         Resolver res = new SimpleResolver(dns.getDnsServerIP());
         res.setTimeout(Duration.of(Long.parseLong(dns.getTimeout()), ChronoUnit.MILLIS));
         res.setTCP(Boolean.parseBoolean(dns.getTcp()));
@@ -195,7 +195,7 @@ private Map<String, String> getHeaderInfo(Message message, Long responseTime) {
 
     private List<String> getSectionInfo(Message message, int section) {
         List<RRset> currentSetList = message.getSectionRRsets(section);
-        
+
         if (CollectionUtils.isEmpty(currentSetList)) {
             return Lists.newArrayList();
         }