[feature] Implement IMAP protocol support (#2059)

Co-authored-by: tomsun28 <[email protected]>

collector/src/main/java/org/apache/hertzbeat/collector/collect/imap/ImapCollectImpl.java

@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hertzbeat.collector.collect.imap;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.net.imap.IMAPClient;
+import org.apache.commons.net.imap.IMAPSClient;
+import org.apache.hertzbeat.collector.collect.AbstractCollect;
+import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
+import org.apache.hertzbeat.collector.util.CollectUtil;
+import org.apache.hertzbeat.common.constants.CommonConstants;
+import org.apache.hertzbeat.common.entity.job.Metrics;
+import org.apache.hertzbeat.common.entity.job.protocol.ImapProtocol;
+import org.apache.hertzbeat.common.entity.message.CollectRep;
+import org.apache.hertzbeat.common.util.CommonUtil;
+import org.springframework.util.Assert;
+
+/**
+ * imap collect
+ */
+@Slf4j
+public class ImapCollectImpl extends AbstractCollect {
+
+    private static final String UTF_7_X = "X-MODIFIED-UTF-7";
+    private static final String STATUS = "STATUS";
+    private static final String STATUS_COMMAND = "(MESSAGES RECENT UNSEEN)";
+    private static final String MESSAGES = "MESSAGES";
+    private static final String RECENT = "RECENT";
+    private static final String UNSEEN = "UNSEEN";
+    private static final String RESPONSETIME = "responseTime";
+    private static final String totalMessageCount = "TotalMessageCount";
+    private static final String recentMessageCount = "RecentMessageCount";
+    private static final String unseenMessageCount = "UnseenMessageCount";
+
+    @Override
+    public void preCheck(Metrics metrics) throws IllegalArgumentException {
+        ImapProtocol imapProtocol = metrics.getImap();
+        Assert.notNull(metrics, "IMAP collect must has Imap params");
+        Assert.notNull(metrics.getImap(), "IMAP collect must has Imap params");
+        Assert.hasText(imapProtocol.getHost(), "IMAP host is required");
+        Assert.hasText(imapProtocol.getPort(), "IMAP port is required");
+        Assert.hasText(imapProtocol.getEmail(), "IMAP email is required");
+        Assert.hasText(imapProtocol.getAuthorize(), "IMAP authorize code is required");
+        Assert.hasText(imapProtocol.getFolderName(), "IMAP folder name is required");
+    }
+
+    @Override
+    public void collect(CollectRep.MetricsData.Builder builder, long monitorId, String app, Metrics metrics) {
+        long startTime = System.currentTimeMillis();
+        ImapProtocol imapProtocol = metrics.getImap();
+        IMAPClient imapClient = null;
+        boolean ssl = Boolean.parseBoolean(imapProtocol.getSsl());
+
+        try {
+            imapClient = createImapClient(imapProtocol, ssl);
+            // if Connected, then collect metrics
+            if (imapClient.isConnected()) {
+                long responseTime = System.currentTimeMillis() - startTime;
+                String folderName = imapProtocol.getFolderName();
+                collectImapMetrics(builder, imapClient, metrics.getAliasFields(), folderName, responseTime);
+            } else {
+                builder.setCode(CollectRep.Code.UN_CONNECTABLE);
+                builder.setMsg("Peer connect failed，Timeout " + imapProtocol.getTimeout() + "ms");
+            }
+        } catch (Exception e) {
+            String errorMsg = CommonUtil.getMessageFromThrowable(e);
+            log.error(errorMsg);
+            builder.setCode(CollectRep.Code.FAIL);
+            builder.setMsg(errorMsg);
+        } finally {
+            if (imapClient != null) {
+                try {
+                    imapClient.logout();
+                    imapClient.disconnect();
+                } catch (IOException e) {
+                    String errorMsg = CommonUtil.getMessageFromThrowable(e);
+                    log.error(errorMsg);
+                    builder.setCode(CollectRep.Code.FAIL);
+                    builder.setMsg(errorMsg);
+                }
+            }
+        }
+    }
+
+    @Override
+    public String supportProtocol() {
+        return DispatchConstants.PROTOCOL_IMAP;
+    }
+
+    private IMAPClient createImapClient(ImapProtocol imapProtocol, boolean ssl) throws Exception {
+        IMAPClient imapClient = null;
+        // determine whether to use SSL-encrypted connections
+        imapClient = new IMAPSClient(true);
+        if (!ssl) {
+            imapClient = new IMAPClient();
+        }
+        // set timeout
+        int timeout = Integer.parseInt(imapProtocol.getTimeout());
+        if (timeout > 0) {
+            imapClient.setConnectTimeout(timeout);
+        }
+        //set Charset
+        imapClient.setCharset(StandardCharsets.US_ASCII);
+        // connect to the IMAP server
+        String host = imapProtocol.getHost();
+        int port = Integer.parseInt(imapProtocol.getPort());
+        imapClient.connect(host, port);
+        // validate credentials
+        String email = imapProtocol.getEmail();
+        String authorize = imapProtocol.getAuthorize();
+        boolean isAuthenticated = imapClient.login(email, authorize);
+        if (!isAuthenticated) {
+            throw new Exception("IMAP client authentication failed");
+        }
+        return imapClient;
+
+    }
+
+    private void collectImapMetrics(CollectRep.MetricsData.Builder builder, IMAPClient imapClient, List<String> aliasFields,
+                                    String folderName, long responseTime) throws Exception {
+        Map<String, String> resultsMap = new HashMap<>();
+        resultsMap.put(RESPONSETIME, String.valueOf(responseTime));
+        imapClient.sendCommand(STATUS + " \"" + CollectUtil.stringEncodeUtf7String(folderName, UTF_7_X) + "\" " + STATUS_COMMAND);
+        String[] response = imapClient.getReplyString().split("\\s+|\\(|\\)");
+        for (int i = 0; i < response.length; i++) {
+            switch (response[i]) {
+                case MESSAGES:
+                    resultsMap.put(folderName + totalMessageCount, response[i + 1]);
+                    break;
+                case RECENT:
+                    resultsMap.put(folderName + recentMessageCount, response[i + 1]);
+                    break;
+                case UNSEEN:
+                    resultsMap.put(folderName + unseenMessageCount, response[i + 1]);
+                    break;
+                default:
+                    break;
+            }
+        }
+
+        CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
+        for (String field : aliasFields) {
+            String fieldValue = resultsMap.get(field);
+            valueRowBuilder.addColumns(Objects.requireNonNullElse(fieldValue, CommonConstants.NULL_VALUE));
+        }
+        builder.addValues(valueRowBuilder.build());
+    }
+}

collector/src/main/java/org/apache/hertzbeat/collector/dispatch/DispatchConstants.java

@@ -127,6 +127,10 @@ public interface DispatchConstants {
      * protocol redfish
      */
     String PROTOCOL_REDFISH = "redfish";
+    /**
+     * protocol imap
+     */
+    String PROTOCOL_IMAP = "imap";
 
     // Protocol type related - end
 

collector/src/main/java/org/apache/hertzbeat/collector/util/CollectUtil.java

@@ -17,12 +17,14 @@
 
 package org.apache.hertzbeat.collector.util;
 
+import com.beetstra.jutf7.CharsetProvider;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.google.gson.JsonArray;
 import com.google.gson.JsonElement;
 import com.google.gson.JsonNull;
 import com.google.gson.JsonObject;
 import com.google.gson.JsonPrimitive;
+import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
@@ -455,4 +457,24 @@ public static byte[] fromHexString(String hexString) {
         }
         return bytes;
     }
+
+    /**
+     * convert original string to UTF-7 String
+     * @param original original text
+     * @param charset encode charset
+     * @return String
+     */
+    public  static  String stringEncodeUtf7String(String original, String charset) {
+        return new String(original.getBytes(new CharsetProvider().charsetForName(charset)), StandardCharsets.US_ASCII);
+    }
+
+    /**
+     * convert UTF-7 string to original String
+     * @param encoded encoded String
+     * @param charset encode charset
+     * @return String
+     */
+    public  static  String utf7StringDecodeString(String encoded, String charset) {
+        return new String(encoded.getBytes(StandardCharsets.US_ASCII), new CharsetProvider().charsetForName(charset));
+    }
 }

collector/src/main/resources/META-INF/services/org.apache.hertzbeat.collector.collect.AbstractCollect

@@ -23,3 +23,4 @@ org.apache.hertzbeat.collector.collect.pop3.Pop3CollectImpl
 org.apache.hertzbeat.collector.collect.httpsd.HttpsdImpl
 org.apache.hertzbeat.collector.collect.redfish.RedfishCollectImpl
 org.apache.hertzbeat.collector.collect.nebulagraph.NgqlCollectImpl
+org.apache.hertzbeat.collector.collect.imap.ImapCollectImpl

collector/src/test/java/org/apache/hertzbeat/collector/collect/imap/ImapCollectImplTest.java

@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hertzbeat.collector.collect.imap;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import java.util.List;
+import org.apache.commons.net.imap.IMAPClient;
+import org.apache.commons.net.imap.IMAPSClient;
+import org.apache.hertzbeat.common.entity.job.Metrics;
+import org.apache.hertzbeat.common.entity.job.protocol.ImapProtocol;
+import org.apache.hertzbeat.common.entity.message.CollectRep;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.MockedConstruction;
+import org.mockito.Mockito;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+/**
+ * Test case for {@link ImapCollectImpl}
+ */
+@ExtendWith(MockitoExtension.class)
+public class ImapCollectImplTest {
+    private Metrics metrics;
+    private CollectRep.MetricsData.Builder builder;
+    @Mock
+    private ImapProtocol imapProtocol;
+    @InjectMocks
+    private ImapCollectImpl imapCollect;
+
+    @BeforeEach
+    void setUp() {
+        imapProtocol = ImapProtocol.builder()
+                .host("0.0.0.0")
+                .port("993")
+                .ssl("true")
+                .email("[email protected]")
+                .authorize("test")
+                .folderName("testFolder")
+                .timeout("6000")
+                .build();
+        metrics = new Metrics();
+        metrics.setName("testMailboxInfo");
+        metrics.setImap(imapProtocol);
+        metrics.setAliasFields(List.of("responseTime", "testFolderTotalMessageCount", "testFolderRecentMessageCount", "testFolderUnseenMessageCount"));
+        builder = CollectRep.MetricsData.newBuilder();
+    }
+
+    @Test
+    void preCheck() {
+        assertDoesNotThrow(() -> {
+            imapCollect.preCheck(metrics);
+        });
+        assertThrows(NullPointerException.class, () -> {
+            imapCollect.preCheck(null);
+        });
+        metrics.setImap(null);
+        assertThrows(NullPointerException.class, () -> {
+            imapCollect.preCheck(null);
+        });
+    }
+
+    @Test
+    void enableSslCollect() {
+        String response = "* STATUS \"testFolder\" (MESSAGES 3 RECENT 2 UNSEEN 1)";
+        MockedConstruction<IMAPSClient> mocked = Mockito.mockConstruction(IMAPSClient.class,
+            (imapsClient, context) -> {
+                Mockito.doNothing().when(imapsClient).connect(Mockito.anyString(), Mockito.anyInt());
+                Mockito.doAnswer(invocationOnMock -> true).when(imapsClient).login(Mockito.anyString(), Mockito.anyString());
+                Mockito.doAnswer(invocationOnMock -> true).when(imapsClient).isConnected();
+                Mockito.when(imapsClient.sendCommand(Mockito.anyString())).thenReturn(0);
+                Mockito.when(imapsClient.getReplyString()).thenReturn(response);
+                Mockito.doAnswer(invocationOnMock -> true).when(imapsClient).logout();
+                Mockito.doNothing().when(imapsClient).disconnect();
+            });
+
+        imapCollect.preCheck(metrics);
+        imapCollect.collect(builder, 1L, "testIMAP", metrics);
+        assertEquals(1, builder.getValuesCount());
+        for (CollectRep.ValueRow valueRow : builder.getValuesList()) {
+            assertNotNull(valueRow.getColumns(0));
+            assertEquals("3", valueRow.getColumns(1));
+            assertEquals("2", valueRow.getColumns(2));
+            assertEquals("1", valueRow.getColumns(3));
+
+        }
+        mocked.close();
+    }
+
+    @Test
+    void disableSslCollect() {
+        metrics.getImap().setSsl("false");
+        String response = "* STATUS \"testFolder\" (MESSAGES 3 RECENT 2 UNSEEN 1)";
+        MockedConstruction<IMAPClient> mocked = Mockito.mockConstruction(IMAPClient.class,
+                (imapClient, context) -> {
+                    Mockito.doNothing().when(imapClient).connect(Mockito.anyString(), Mockito.anyInt());
+                    Mockito.doAnswer(invocationOnMock -> true).when(imapClient).login(Mockito.anyString(), Mockito.anyString());
+                    Mockito.doAnswer(invocationOnMock -> true).when(imapClient).isConnected();
+                    Mockito.when(imapClient.sendCommand(Mockito.anyString())).thenReturn(0);
+                    Mockito.when(imapClient.getReplyString()).thenReturn(response);
+                    Mockito.doAnswer(invocationOnMock -> true).when(imapClient).logout();
+                    Mockito.doNothing().when(imapClient).disconnect();
+                });
+
+        imapCollect.preCheck(metrics);
+        imapCollect.collect(builder, 1L, "testIMAP", metrics);
+        assertEquals(1, builder.getValuesCount());
+        for (CollectRep.ValueRow valueRow : builder.getValuesList()) {
+            assertNotNull(valueRow.getColumns(0));
+            assertEquals("3", valueRow.getColumns(1));
+            assertEquals("2", valueRow.getColumns(2));
+            assertEquals("1", valueRow.getColumns(3));
+
+        }
+        mocked.close();
+    }
+
+    @Test
+    void supportProtocol() {
+        assertEquals("imap", imapCollect.supportProtocol());
+    }
+}

common/pom.xml

@@ -116,5 +116,12 @@
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-collections4</artifactId>
         </dependency>
+
+        <!-- https://mvnrepository.com/artifact/com.beetstra.jutf7/jutf7 -->
+        <dependency>
+            <groupId>com.beetstra.jutf7</groupId>
+            <artifactId>jutf7</artifactId>
+            <version>1.0.0</version>
+        </dependency>
     </dependencies>
 </project>