From 13298d0ffe7530c66a7f95772150f103b3cf0152 Mon Sep 17 00:00:00 2001 From: yukon Date: Wed, 13 Dec 2017 21:51:32 +0800 Subject: [PATCH 01/73] [maven-release-plugin] prepare for next development iteration --- broker/pom.xml | 2 +- client/pom.xml | 2 +- common/pom.xml | 2 +- distribution/pom.xml | 2 +- example/pom.xml | 4 ++-- filter/pom.xml | 2 +- filtersrv/pom.xml | 2 +- logappender/pom.xml | 2 +- namesrv/pom.xml | 2 +- openmessaging/pom.xml | 2 +- pom.xml | 4 ++-- remoting/pom.xml | 2 +- srvutil/pom.xml | 2 +- store/pom.xml | 2 +- test/pom.xml | 2 +- tools/pom.xml | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/broker/pom.xml b/broker/pom.xml index d655959a9a8..2afb80c8c84 100644 --- a/broker/pom.xml +++ b/broker/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.2.0 + 4.2.1-SNAPSHOT 4.0.0 diff --git a/client/pom.xml b/client/pom.xml index a8034c2165d..d2ffb784d80 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.2.0 + 4.2.1-SNAPSHOT 4.0.0 diff --git a/common/pom.xml b/common/pom.xml index 05c2e221fe5..406570066d8 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.2.0 + 4.2.1-SNAPSHOT 4.0.0 diff --git a/distribution/pom.xml b/distribution/pom.xml index fccb91e5c7a..1dbb73b2201 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -20,7 +20,7 @@ org.apache.rocketmq rocketmq-all - 4.2.0 + 4.2.1-SNAPSHOT rocketmq-distribution rocketmq-distribution ${project.version} diff --git a/example/pom.xml b/example/pom.xml index d246de429d4..3c4c7d5777c 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.2.0 + 4.2.1-SNAPSHOT 4.0.0 @@ -51,7 +51,7 @@ org.apache.rocketmq rocketmq-openmessaging - 4.2.0 + 4.2.1-SNAPSHOT diff --git a/filter/pom.xml b/filter/pom.xml index c024effdb6e..6d592835ccd 100644 --- a/filter/pom.xml +++ b/filter/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.2.0 + 4.2.1-SNAPSHOT 4.0.0 diff --git a/filtersrv/pom.xml b/filtersrv/pom.xml index 96ce62a5d6c..b7ece1f4dee 100644 --- a/filtersrv/pom.xml +++ b/filtersrv/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.2.0 + 4.2.1-SNAPSHOT 4.0.0 diff --git a/logappender/pom.xml b/logappender/pom.xml index 0e0ac44911b..857f9d5c915 100644 --- a/logappender/pom.xml +++ b/logappender/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.2.0 + 4.2.1-SNAPSHOT 4.0.0 rocketmq-logappender diff --git a/namesrv/pom.xml b/namesrv/pom.xml index c2134c1cd2c..74cef87a0d3 100644 --- a/namesrv/pom.xml +++ b/namesrv/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.2.0 + 4.2.1-SNAPSHOT 4.0.0 diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml index 56246f8665b..e1a8337767b 100644 --- a/openmessaging/pom.xml +++ b/openmessaging/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.2.0 + 4.2.1-SNAPSHOT 4.0.0 diff --git a/pom.xml b/pom.xml index 80db29f5414..02e173be38f 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ 2012 org.apache.rocketmq rocketmq-all - 4.2.0 + 4.2.1-SNAPSHOT pom Apache RocketMQ ${project.version} http://rocketmq.apache.org/ @@ -42,7 +42,7 @@ git@github.com:apache/rocketmq.git scm:git:git@github.com:apache/rocketmq.git scm:git:git@github.com:apache/rocketmq.git - rocketmq-all-4.2.0 + HEAD diff --git a/remoting/pom.xml b/remoting/pom.xml index 4dc0e7390ab..9ede84c31db 100644 --- a/remoting/pom.xml +++ b/remoting/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.2.0 + 4.2.1-SNAPSHOT 4.0.0 diff --git a/srvutil/pom.xml b/srvutil/pom.xml index 7965966c6d0..6e2a60ee5d6 100644 --- a/srvutil/pom.xml +++ b/srvutil/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.2.0 + 4.2.1-SNAPSHOT 4.0.0 diff --git a/store/pom.xml b/store/pom.xml index ca2eb1adfe1..342617d4bf4 100644 --- a/store/pom.xml +++ b/store/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.2.0 + 4.2.1-SNAPSHOT 4.0.0 diff --git a/test/pom.xml b/test/pom.xml index b7713aed385..b6e87308071 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.2.0 + 4.2.1-SNAPSHOT 4.0.0 diff --git a/tools/pom.xml b/tools/pom.xml index 21852934af6..1cac4728f1d 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.2.0 + 4.2.1-SNAPSHOT 4.0.0 From 4678272cba5ebb06b723fdf53dd2722cc7178371 Mon Sep 17 00:00:00 2001 From: yukon Date: Mon, 18 Dec 2017 13:54:59 +0800 Subject: [PATCH 02/73] [HOTFIX] Remove imported dirty file frrm pull request --- 1 | 105 -------------------------------------------------------------- 1 file changed, 105 deletions(-) delete mode 100644 1 diff --git a/1 b/1 deleted file mode 100644 index 5f9f277ed4b..00000000000 --- a/1 +++ /dev/null @@ -1,105 +0,0 @@ -# This is a combination of 3 commits. -# The first commit's message is: -remove spaces before and after the properties - -# This is the 2nd commit message: - -remove spaces before and after the properties - -remove spaces before and after the properties - -remove spaces before and after the properties - -remove spaces before and after the properties - -Remove spaces before and after the properties - -Remove spaces before and after the properties - -Remove spaces before and after the properties - -Remove spaces before and after the properties - -Remove the code that is not useful in the loop - -Remove the code that is not useful in the loop - -Remove the code that is not useful in the loop - -Remove the code that is not useful in the loop - -Remove the code that is not useful in the loop - -Remove spaces before and after the properties - -Remove spaces before and after the properties - -Remove the code that is not useful in the loop - -Remove the code that is not useful in the loop - -Remove the code that is not useful in the loop - -Remove the code that is not useful in the loop - -Remove the code that is not useful in the loop - -remove spaces before and after the properties - -remove spaces before and after the properties - -Remove spaces before and after the properties - -Remove spaces before and after the properties - -Remove spaces before and after the properties - -Remove spaces before and after the properties - -Remove the code that is not useful in the loop - -Remove the code that is not useful in the loop - -Remove the code that is not useful in the loop - -Remove the code that is not useful in the loop - -Remove the code that is not useful in the loop - -Remove spaces before and after the properties - -Remove spaces before and after the properties - -Remove the code that is not useful in the loop - -Remove the code that is not useful in the loop - -Remove the code that is not useful in the loop - -Remove the code that is not useful in the loop - -Remove the code that is not useful in the loop - -# This is the 3rd commit message: - -remove spaces before and after the properties - -# Please enter the commit message for your changes. Lines starting -# with '#' will be ignored, and an empty message aborts the commit. -# -# Date: Wed May 17 16:11:07 2017 +0800 -# -# interactive rebase in progress; onto a2894b6 -# Last commands done (3 commands done): -# squash 76b35ab remove spaces before and after the properties -# squash 92f83bc remove spaces before and after the properties -# Next commands to do (12 remaining commands): -# squash d142037 remove spaces before and after the properties -# squash 9df0fcb remove spaces before and after the properties -# You are currently rebasing branch 'Branch_0616' on 'a2894b6'. -# -# Changes to be committed: -# modified: client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java -# modified: common/src/main/java/org/apache/rocketmq/common/MixAll.java -# new file: "\357\200\272" -# From c606c3c22c122d54472dc246e29860655c52ce80 Mon Sep 17 00:00:00 2001 From: yukon Date: Mon, 18 Dec 2017 14:18:09 +0800 Subject: [PATCH 03/73] [HOTFIX] Exclude test cert files from rat check --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 02e173be38f..7904e998620 100644 --- a/pom.xml +++ b/pom.xml @@ -253,6 +253,7 @@ CONTRIBUTING.md bin/README.md PULL_REQUEST_TEMPLATE.md + src/test/resources/certs/* From 7ae0d4e105bf906cd22a9b0c03fb57d6fd71eca2 Mon Sep 17 00:00:00 2001 From: yukon Date: Mon, 18 Dec 2017 14:19:33 +0800 Subject: [PATCH 04/73] [HOTFIX] Move pull request template to .github --- PULL_REQUEST_TEMPLATE.md => .github/PULL_REQUEST_TEMPLATE.md | 0 pom.xml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename PULL_REQUEST_TEMPLATE.md => .github/PULL_REQUEST_TEMPLATE.md (100%) diff --git a/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md similarity index 100% rename from PULL_REQUEST_TEMPLATE.md rename to .github/PULL_REQUEST_TEMPLATE.md diff --git a/pom.xml b/pom.xml index 7904e998620..4a8e463fc0f 100644 --- a/pom.xml +++ b/pom.xml @@ -252,7 +252,7 @@ .travis.yml CONTRIBUTING.md bin/README.md - PULL_REQUEST_TEMPLATE.md + .github/* src/test/resources/certs/* From 69043c0d28707c2ef2d95535c5be6036765ab4e0 Mon Sep 17 00:00:00 2001 From: yukon Date: Mon, 18 Dec 2017 14:23:23 +0800 Subject: [PATCH 05/73] 4.2.0 has been released, go to next development iteration 4.3.0-SNAPSHOT --- broker/pom.xml | 2 +- client/pom.xml | 2 +- common/pom.xml | 2 +- distribution/pom.xml | 2 +- example/pom.xml | 4 ++-- filter/pom.xml | 2 +- filtersrv/pom.xml | 2 +- logappender/pom.xml | 2 +- namesrv/pom.xml | 2 +- openmessaging/pom.xml | 2 +- pom.xml | 2 +- remoting/pom.xml | 2 +- srvutil/pom.xml | 2 +- store/pom.xml | 2 +- test/pom.xml | 2 +- tools/pom.xml | 2 +- 16 files changed, 17 insertions(+), 17 deletions(-) diff --git a/broker/pom.xml b/broker/pom.xml index 2afb80c8c84..3da70fa3368 100644 --- a/broker/pom.xml +++ b/broker/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.2.1-SNAPSHOT + 4.3.0-SNAPSHOT 4.0.0 diff --git a/client/pom.xml b/client/pom.xml index d2ffb784d80..56dea38b941 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.2.1-SNAPSHOT + 4.3.0-SNAPSHOT 4.0.0 diff --git a/common/pom.xml b/common/pom.xml index 406570066d8..ce16df4dd2a 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.2.1-SNAPSHOT + 4.3.0-SNAPSHOT 4.0.0 diff --git a/distribution/pom.xml b/distribution/pom.xml index 1dbb73b2201..fbfbd402837 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -20,7 +20,7 @@ org.apache.rocketmq rocketmq-all - 4.2.1-SNAPSHOT + 4.3.0-SNAPSHOT rocketmq-distribution rocketmq-distribution ${project.version} diff --git a/example/pom.xml b/example/pom.xml index 3c4c7d5777c..b9d97df3d9a 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.2.1-SNAPSHOT + 4.3.0-SNAPSHOT 4.0.0 @@ -51,7 +51,7 @@ org.apache.rocketmq rocketmq-openmessaging - 4.2.1-SNAPSHOT + 4.3.0-SNAPSHOT diff --git a/filter/pom.xml b/filter/pom.xml index 6d592835ccd..eebfa30af49 100644 --- a/filter/pom.xml +++ b/filter/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.2.1-SNAPSHOT + 4.3.0-SNAPSHOT 4.0.0 diff --git a/filtersrv/pom.xml b/filtersrv/pom.xml index b7ece1f4dee..b6202b2908c 100644 --- a/filtersrv/pom.xml +++ b/filtersrv/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.2.1-SNAPSHOT + 4.3.0-SNAPSHOT 4.0.0 diff --git a/logappender/pom.xml b/logappender/pom.xml index 857f9d5c915..ff5f7e2bced 100644 --- a/logappender/pom.xml +++ b/logappender/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.2.1-SNAPSHOT + 4.3.0-SNAPSHOT 4.0.0 rocketmq-logappender diff --git a/namesrv/pom.xml b/namesrv/pom.xml index 74cef87a0d3..02ed402bf2d 100644 --- a/namesrv/pom.xml +++ b/namesrv/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.2.1-SNAPSHOT + 4.3.0-SNAPSHOT 4.0.0 diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml index e1a8337767b..857a70eeb3e 100644 --- a/openmessaging/pom.xml +++ b/openmessaging/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.2.1-SNAPSHOT + 4.3.0-SNAPSHOT 4.0.0 diff --git a/pom.xml b/pom.xml index 4a8e463fc0f..4326b4669d3 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ 2012 org.apache.rocketmq rocketmq-all - 4.2.1-SNAPSHOT + 4.3.0-SNAPSHOT pom Apache RocketMQ ${project.version} http://rocketmq.apache.org/ diff --git a/remoting/pom.xml b/remoting/pom.xml index 9ede84c31db..c788202cbb5 100644 --- a/remoting/pom.xml +++ b/remoting/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.2.1-SNAPSHOT + 4.3.0-SNAPSHOT 4.0.0 diff --git a/srvutil/pom.xml b/srvutil/pom.xml index 6e2a60ee5d6..b22e45f9b5c 100644 --- a/srvutil/pom.xml +++ b/srvutil/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.2.1-SNAPSHOT + 4.3.0-SNAPSHOT 4.0.0 diff --git a/store/pom.xml b/store/pom.xml index 342617d4bf4..07feb444ae1 100644 --- a/store/pom.xml +++ b/store/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.2.1-SNAPSHOT + 4.3.0-SNAPSHOT 4.0.0 diff --git a/test/pom.xml b/test/pom.xml index b6e87308071..fc01d2c75e8 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.2.1-SNAPSHOT + 4.3.0-SNAPSHOT 4.0.0 diff --git a/tools/pom.xml b/tools/pom.xml index 1cac4728f1d..97e5e9c5d00 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.2.1-SNAPSHOT + 4.3.0-SNAPSHOT 4.0.0 From c7fea667b01566c6375b2ecc21118f9786c955dd Mon Sep 17 00:00:00 2001 From: yukon Date: Wed, 20 Dec 2017 15:46:24 +0800 Subject: [PATCH 06/73] [ROCKETMQ-335] Reload server certificate, private key and root ca when these are changed (#207) --- .../rocketmq/broker/BrokerController.java | 51 ++++++ .../rocketmq/namesrv/NamesrvController.java | 50 ++++++ .../remoting/netty/NettyRemotingAbstract.java | 2 +- .../remoting/netty/NettyRemotingServer.java | 4 + .../org/apache/rocketmq/remoting/TlsTest.java | 28 ++++ .../rocketmq/srvutil/FileWatchService.java | 106 ++++++++++++ .../srvutil/FileWatchServiceTest.java | 151 ++++++++++++++++++ 7 files changed, 391 insertions(+), 1 deletion(-) create mode 100644 srvutil/src/main/java/org/apache/rocketmq/srvutil/FileWatchService.java create mode 100644 srvutil/src/main/test/org/apache/rocketmq/srvutil/FileWatchServiceTest.java diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 0a6f0b45e9d..8891bd322e3 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -73,11 +73,14 @@ import org.apache.rocketmq.common.stats.MomentStatsItem; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.RemotingServer; +import org.apache.rocketmq.remoting.common.TlsMode; import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyRemotingServer; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.netty.NettyServerConfig; import org.apache.rocketmq.remoting.netty.RequestTask; +import org.apache.rocketmq.remoting.netty.TlsSystemConfig; +import org.apache.rocketmq.srvutil.FileWatchService; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.MessageArrivingListener; import org.apache.rocketmq.store.MessageStore; @@ -136,6 +139,7 @@ public class BrokerController { private InetSocketAddress storeHost; private BrokerFastFailure brokerFastFailure; private Configuration configuration; + private FileWatchService fileWatchService; public BrokerController( final BrokerConfig brokerConfig, @@ -387,6 +391,45 @@ public void run() { } }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS); } + + if (TlsSystemConfig.tlsMode != TlsMode.DISABLED) { + // Register a listener to reload SslContext + try { + fileWatchService = new FileWatchService( + new String[] { + TlsSystemConfig.tlsServerCertPath, + TlsSystemConfig.tlsServerKeyPath, + TlsSystemConfig.tlsServerTrustCertPath + }, + new FileWatchService.Listener() { + boolean certChanged, keyChanged = false; + @Override + public void onChanged(String path) { + if (path.equals(TlsSystemConfig.tlsServerTrustCertPath)) { + log.info("The trust certificate changed, reload the ssl context"); + reloadServerSslContext(); + } + if (path.equals(TlsSystemConfig.tlsServerCertPath)) { + certChanged = true; + } + if (path.equals(TlsSystemConfig.tlsServerKeyPath)) { + keyChanged = true; + } + if (certChanged && keyChanged) { + log.info("The certificate and private key changed, reload the ssl context"); + certChanged = keyChanged = false; + reloadServerSslContext(); + } + } + private void reloadServerSslContext() { + ((NettyRemotingServer) remotingServer).loadSslContext(); + ((NettyRemotingServer) fastRemotingServer).loadSslContext(); + } + }); + } catch (Exception e) { + log.warn("FileWatchService created error, can't load the certificate dynamically"); + } + } } return result; @@ -594,6 +637,10 @@ public void shutdown() { this.fastRemotingServer.shutdown(); } + if (this.fileWatchService != null) { + this.fileWatchService.shutdown(); + } + if (this.messageStore != null) { this.messageStore.shutdown(); } @@ -662,6 +709,10 @@ public void start() throws Exception { this.fastRemotingServer.start(); } + if (this.fileWatchService != null) { + this.fileWatchService.start(); + } + if (this.brokerOuterAPI != null) { this.brokerOuterAPI.start(); } diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java index 51b20b416dd..2ed599c11c4 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java @@ -30,8 +30,11 @@ import org.apache.rocketmq.namesrv.routeinfo.BrokerHousekeepingService; import org.apache.rocketmq.namesrv.routeinfo.RouteInfoManager; import org.apache.rocketmq.remoting.RemotingServer; +import org.apache.rocketmq.remoting.common.TlsMode; import org.apache.rocketmq.remoting.netty.NettyRemotingServer; import org.apache.rocketmq.remoting.netty.NettyServerConfig; +import org.apache.rocketmq.remoting.netty.TlsSystemConfig; +import org.apache.rocketmq.srvutil.FileWatchService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,6 +57,7 @@ public class NamesrvController { private ExecutorService remotingExecutor; private Configuration configuration; + private FileWatchService fileWatchService; public NamesrvController(NamesrvConfig namesrvConfig, NettyServerConfig nettyServerConfig) { this.namesrvConfig = namesrvConfig; @@ -95,6 +99,44 @@ public void run() { } }, 1, 10, TimeUnit.MINUTES); + if (TlsSystemConfig.tlsMode != TlsMode.DISABLED) { + // Register a listener to reload SslContext + try { + fileWatchService = new FileWatchService( + new String[] { + TlsSystemConfig.tlsServerCertPath, + TlsSystemConfig.tlsServerKeyPath, + TlsSystemConfig.tlsServerTrustCertPath + }, + new FileWatchService.Listener() { + boolean certChanged, keyChanged = false; + @Override + public void onChanged(String path) { + if (path.equals(TlsSystemConfig.tlsServerTrustCertPath)) { + log.info("The trust certificate changed, reload the ssl context"); + reloadServerSslContext(); + } + if (path.equals(TlsSystemConfig.tlsServerCertPath)) { + certChanged = true; + } + if (path.equals(TlsSystemConfig.tlsServerKeyPath)) { + keyChanged = true; + } + if (certChanged && keyChanged) { + log.info("The certificate and private key changed, reload the ssl context"); + certChanged = keyChanged = false; + reloadServerSslContext(); + } + } + private void reloadServerSslContext() { + ((NettyRemotingServer) remotingServer).loadSslContext(); + } + }); + } catch (Exception e) { + log.warn("FileWatchService created error, can't load the certificate dynamically"); + } + } + return true; } @@ -111,12 +153,20 @@ private void registerProcessor() { public void start() throws Exception { this.remotingServer.start(); + + if (this.fileWatchService != null) { + this.fileWatchService.start(); + } } public void shutdown() { this.remotingServer.shutdown(); this.remotingExecutor.shutdown(); this.scheduledExecutorService.shutdown(); + + if (this.fileWatchService != null) { + this.fileWatchService.shutdown(); + } } public NamesrvConfig getNamesrvConfig() { diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java index 76752529af2..557ad5602b0 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java @@ -93,7 +93,7 @@ public abstract class NettyRemotingAbstract { /** * SSL context via which to create {@link SslHandler}. */ - protected SslContext sslContext; + protected volatile SslContext sslContext; /** * Constructor, specifying capacity of one-way and asynchronous semaphores. diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java index cd6ed4704f4..c8709a50149 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java @@ -139,6 +139,10 @@ public Thread newThread(Runnable r) { }); } + loadSslContext(); + } + + public void loadSslContext() { TlsMode tlsMode = TlsSystemConfig.tlsMode; log.info("Server is running in TLS {} mode", tlsMode.getName()); diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/TlsTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/TlsTest.java index 5e516dd74bd..13bb17282e3 100644 --- a/remoting/src/test/java/org/apache/rocketmq/remoting/TlsTest.java +++ b/remoting/src/test/java/org/apache/rocketmq/remoting/TlsTest.java @@ -25,6 +25,7 @@ import org.apache.rocketmq.remoting.common.TlsMode; import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.netty.NettyRemotingServer; import org.apache.rocketmq.remoting.netty.TlsHelper; import org.apache.rocketmq.remoting.protocol.LanguageCode; import org.apache.rocketmq.remoting.protocol.RemotingCommand; @@ -134,6 +135,9 @@ else if ("noClientAuthFailure".equals(name.getMethodName())) { clientConfig.setUseTLS(false); } else if ("serverRejectsSSLClient".equals(name.getMethodName())) { tlsMode = TlsMode.DISABLED; + } else if ("reloadSslContextForServer".equals(name.getMethodName())) { + tlsClientAuthServer = false; + tlsServerNeedClientAuth = "none"; } remotingServer = RemotingServerTest.createRemotingServer(); @@ -156,6 +160,26 @@ public void basicClientServerIntegrationTest() throws Exception { requestThenAssertResponse(); } + @Test + public void reloadSslContextForServer() throws Exception { + requestThenAssertResponse(); + + //Use new cert and private key + tlsClientKeyPath = getCertsPath("badClient.key"); + tlsClientCertPath = getCertsPath("badClient.pem"); + + ((NettyRemotingServer) remotingServer).loadSslContext(); + + //Request Again + requestThenAssertResponse(); + + //Start another client + NettyClientConfig clientConfig = new NettyClientConfig(); + clientConfig.setUseTLS(true); + RemotingClient remotingClient = RemotingServerTest.createRemotingClient(clientConfig); + requestThenAssertResponse(remotingClient); + } + @Test public void serverNotNeedClientAuth() throws Exception { requestThenAssertResponse(); @@ -289,6 +313,10 @@ private static RemotingCommand createRequest() { } private void requestThenAssertResponse() throws Exception { + requestThenAssertResponse(remotingClient); + } + + private void requestThenAssertResponse(RemotingClient remotingClient) throws Exception { RemotingCommand response = remotingClient.invokeSync("localhost:8888", createRequest(), 1000 * 3); assertTrue(response != null); assertThat(response.getLanguage()).isEqualTo(LanguageCode.JAVA); diff --git a/srvutil/src/main/java/org/apache/rocketmq/srvutil/FileWatchService.java b/srvutil/src/main/java/org/apache/rocketmq/srvutil/FileWatchService.java new file mode 100644 index 00000000000..bc68d6a3ca6 --- /dev/null +++ b/srvutil/src/main/java/org/apache/rocketmq/srvutil/FileWatchService.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.srvutil; + +import com.google.common.base.Strings; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.List; +import org.apache.rocketmq.common.ServiceThread; +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.constant.LoggerName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FileWatchService extends ServiceThread { + private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); + + private final List watchFiles; + private final List fileCurrentHash; + private final Listener listener; + private static final int WATCH_INTERVAL = 500; + private MessageDigest md = MessageDigest.getInstance("MD5"); + + public FileWatchService(final String[] watchFiles, + final Listener listener) throws Exception { + this.listener = listener; + this.watchFiles = new ArrayList<>(); + this.fileCurrentHash = new ArrayList<>(); + + for (int i = 0; i < watchFiles.length; i++) { + if (!Strings.isNullOrEmpty(watchFiles[i]) && new File(watchFiles[i]).exists()) { + this.watchFiles.add(watchFiles[i]); + this.fileCurrentHash.add(hash(watchFiles[i])); + } + } + } + + @Override + public String getServiceName() { + return "FileWatchService"; + } + + @Override + public void run() { + log.info(this.getServiceName() + " service started"); + + while (!this.isStopped()) { + try { + this.waitForRunning(WATCH_INTERVAL); + + for (int i = 0; i < watchFiles.size(); i++) { + String newHash; + try { + newHash = hash(watchFiles.get(i)); + } catch (Exception ignored) { + log.warn(this.getServiceName() + " service has exception when calculate the file hash. ", ignored); + continue; + } + if (!newHash.equals(fileCurrentHash.get(i))) { + fileCurrentHash.set(i, newHash); + listener.onChanged(watchFiles.get(i)); + } + } + } catch (Exception e) { + log.warn(this.getServiceName() + " service has exception. ", e); + } + } + log.info(this.getServiceName() + " service end"); + } + + private String hash(String filePath) throws IOException, NoSuchAlgorithmException { + Path path = Paths.get(filePath); + md.update(Files.readAllBytes(path)); + byte[] hash = md.digest(); + return UtilAll.bytes2string(hash); + } + + public interface Listener { + /** + * Will be called when the target files are changed + * @param path the changed file path + */ + void onChanged(String path); + } +} diff --git a/srvutil/src/main/test/org/apache/rocketmq/srvutil/FileWatchServiceTest.java b/srvutil/src/main/test/org/apache/rocketmq/srvutil/FileWatchServiceTest.java new file mode 100644 index 00000000000..791abcf0c95 --- /dev/null +++ b/srvutil/src/main/test/org/apache/rocketmq/srvutil/FileWatchServiceTest.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.srvutil; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.assertj.core.api.Assertions.assertThat; + +@RunWith(MockitoJUnitRunner.class) +public class FileWatchServiceTest { + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + @Test + public void watchSingleFile() throws Exception { + final File file = tempFolder.newFile(); + final Semaphore waitSemaphore = new Semaphore(0); + FileWatchService fileWatchService = new FileWatchService(new String[] {file.getAbsolutePath()}, new FileWatchService.Listener() { + @Override + public void onChanged(String path) { + assertThat(file.getAbsolutePath()).isEqualTo(path); + waitSemaphore.release(); + } + }); + fileWatchService.start(); + modifyFile(file); + boolean result = waitSemaphore.tryAcquire(1, 1000, TimeUnit.MILLISECONDS); + assertThat(result).isTrue(); + } + + @Test + public void watchSingleFile_FileDeleted() throws Exception { + File file = tempFolder.newFile(); + final Semaphore waitSemaphore = new Semaphore(0); + FileWatchService fileWatchService = new FileWatchService(new String[] {file.getAbsolutePath()}, + new FileWatchService.Listener() { + @Override + public void onChanged(String path) { + waitSemaphore.release(); + } + }); + fileWatchService.start(); + file.delete(); + boolean result = waitSemaphore.tryAcquire(1, 1000, TimeUnit.MILLISECONDS); + assertThat(result).isFalse(); + file.createNewFile(); + modifyFile(file); + result = waitSemaphore.tryAcquire(1, 2000, TimeUnit.MILLISECONDS); + assertThat(result).isTrue(); + } + + @Test + public void watchTwoFile_FileDeleted() throws Exception { + File fileA = tempFolder.newFile(); + File fileB = tempFolder.newFile(); + final Semaphore waitSemaphore = new Semaphore(0); + FileWatchService fileWatchService = new FileWatchService( + new String[] {fileA.getAbsolutePath(), fileB.getAbsolutePath()}, + new FileWatchService.Listener() { + @Override + public void onChanged(String path) { + waitSemaphore.release(); + } + }); + fileWatchService.start(); + fileA.delete(); + boolean result = waitSemaphore.tryAcquire(1, 1000, TimeUnit.MILLISECONDS); + assertThat(result).isFalse(); + modifyFile(fileB); + result = waitSemaphore.tryAcquire(1, 1000, TimeUnit.MILLISECONDS); + assertThat(result).isTrue(); + fileA.createNewFile(); + modifyFile(fileA); + result = waitSemaphore.tryAcquire(1, 1000, TimeUnit.MILLISECONDS); + assertThat(result).isTrue(); + } + + @Test + public void watchTwoFiles_ModifyOne() throws Exception { + final File fileA = tempFolder.newFile(); + File fileB = tempFolder.newFile(); + final Semaphore waitSemaphore = new Semaphore(0); + FileWatchService fileWatchService = new FileWatchService( + new String[] {fileA.getAbsolutePath(), fileB.getAbsolutePath()}, + new FileWatchService.Listener() { + @Override + public void onChanged(String path) { + assertThat(path).isEqualTo(fileA.getAbsolutePath()); + waitSemaphore.release(); + } + }); + fileWatchService.start(); + modifyFile(fileA); + boolean result = waitSemaphore.tryAcquire(1, 1000, TimeUnit.MILLISECONDS); + assertThat(result).isTrue(); + } + + @Test + public void watchTwoFiles() throws Exception { + File fileA = tempFolder.newFile(); + File fileB = tempFolder.newFile(); + final Semaphore waitSemaphore = new Semaphore(0); + FileWatchService fileWatchService = new FileWatchService( + new String[] {fileA.getAbsolutePath(), fileB.getAbsolutePath()}, + new FileWatchService.Listener() { + @Override + public void onChanged(String path) { + waitSemaphore.release(); + } + }); + fileWatchService.start(); + modifyFile(fileA); + modifyFile(fileB); + boolean result = waitSemaphore.tryAcquire(2, 1000, TimeUnit.MILLISECONDS); + assertThat(result).isTrue(); + } + + private static void modifyFile(File file) { + try { + PrintWriter out = new PrintWriter(file); + out.println(System.nanoTime()); + out.flush(); + out.close(); + } catch (IOException ignore) { + } + } +} \ No newline at end of file From cc0b3d091814a390107b8e733dd8a5427ff49296 Mon Sep 17 00:00:00 2001 From: lindzh Date: Wed, 20 Dec 2017 15:48:33 +0800 Subject: [PATCH 07/73] [ROCKETMQ-336] Wrong format System.out.printf used in mqadmin (#208) --- .../main/java/org/apache/rocketmq/broker/BrokerStartup.java | 3 +-- .../main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java | 4 ++-- .../tools/command/cluster/CLusterSendMsgRTCommand.java | 2 +- .../rocketmq/tools/command/consumer/ConsumerSubCommand.java | 2 +- .../tools/command/message/PrintMessageSubCommand.java | 4 ++-- .../rocketmq/tools/command/topic/UpdateTopicSubCommand.java | 2 +- 6 files changed, 8 insertions(+), 9 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java index e768c7f95eb..40d20863747 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java @@ -134,8 +134,7 @@ public static BrokerController createBrokerController(String[] args) { MixAll.properties2Object(ServerUtil.commandLine2Properties(commandLine), brokerConfig); if (null == brokerConfig.getRocketmqHome()) { - System.out.printf("Please set the " + MixAll.ROCKETMQ_HOME_ENV - + " variable in your environment to match the location of the RocketMQ installation"); + System.out.printf("Please set the %s variable in your environment to match the location of the RocketMQ installation", MixAll.ROCKETMQ_HOME_ENV); System.exit(-2); } diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java index 0a0c6562d6f..d7a2b2b644d 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java @@ -72,7 +72,7 @@ public static NamesrvController main0(String[] args) { namesrvConfig.setConfigStorePath(file); - System.out.printf("load config properties file OK, " + file + "%n"); + System.out.printf("load config properties file OK, %s%n", file); in.close(); } } @@ -123,7 +123,7 @@ public Void call() throws Exception { String tip = "The Name Server boot success. serializeType=" + RemotingCommand.getSerializeTypeConfigInThisServer(); log.info(tip); - System.out.printf(tip + "%n"); + System.out.printf("%s%n", tip); return controller; } catch (Throwable e) { diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java index 37d66e4c76c..ecbe1f38a54 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java @@ -171,7 +171,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t failCount ); } else { - System.out.printf(String.format("%s|%s|%s|%s|%s%n", getCurTime(), + System.out.printf("%s", String.format("%s|%s|%s|%s|%s%n", getCurTime(), machineRoom, clusterName, brokerName, new BigDecimal(rt).setScale(0, BigDecimal.ROUND_HALF_UP))); } diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerSubCommand.java index a6df8ea8ecf..f3b1fccf32d 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerSubCommand.java @@ -126,7 +126,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t ConsumerRunningInfo consumerRunningInfo = defaultMQAdminExt.getConsumerRunningInfo(group, clientId, jstack); if (consumerRunningInfo != null) { - System.out.printf(consumerRunningInfo.formatString()); + System.out.printf("%s", consumerRunningInfo.formatString()); } } } catch (Exception e) { diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java index c8fced3b16e..b204f0abfb0 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java @@ -146,11 +146,11 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t printMessage(pullResult.getMsgFoundList(), charsetName, printBody); break; case NO_MATCHED_MSG: - System.out.printf(mq + " no matched msg. status=%s, offset=%s", pullResult.getPullStatus(), offset); + System.out.printf("%s no matched msg. status=%s, offset=%s", mq, pullResult.getPullStatus(), offset); break; case NO_NEW_MSG: case OFFSET_ILLEGAL: - System.out.printf(mq + " print msg finished. status=%s, offset=%s", pullResult.getPullStatus(), offset); + System.out.printf("%s print msg finished. status=%s, offset=%s", mq, pullResult.getPullStatus(), offset); break READQ; } } catch (Exception e) { diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java index 8fdb9d3dbf8..c9c8a88fd92 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java @@ -138,7 +138,7 @@ public void execute(final CommandLine commandLine, final Options options, String brokerName = CommandUtil.fetchBrokerNameByAddr(defaultMQAdminExt, addr); String orderConf = brokerName + ":" + topicConfig.getWriteQueueNums(); defaultMQAdminExt.createOrUpdateOrderConf(topicConfig.getTopicName(), orderConf, false); - System.out.printf(String.format("set broker orderConf. isOrder=%s, orderConf=[%s]", + System.out.printf("%s", String.format("set broker orderConf. isOrder=%s, orderConf=[%s]", isOrder, orderConf.toString())); } System.out.printf("create topic to %s success.%n", addr); From d849e0ad989dc89e5e439d48700824d138d09ae9 Mon Sep 17 00:00:00 2001 From: yukon Date: Tue, 16 Jan 2018 14:28:20 +0800 Subject: [PATCH 08/73] Update the out of date test certificates --- remoting/src/test/resources/certs/client.pem | 29 ++++++++++---------- remoting/src/test/resources/certs/server.pem | 29 ++++++++++---------- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/remoting/src/test/resources/certs/client.pem b/remoting/src/test/resources/certs/client.pem index 31412fea8b0..cb6580d9003 100644 --- a/remoting/src/test/resources/certs/client.pem +++ b/remoting/src/test/resources/certs/client.pem @@ -1,18 +1,19 @@ -----BEGIN CERTIFICATE----- -MIIC8zCCAdsCAQIwDQYJKoZIhvcNAQEFBQAwfDELMAkGA1UEBhMCemgxCzAJBgNV +MIIDATCCAekCAQIwDQYJKoZIhvcNAQEFBQAwfDELMAkGA1UEBhMCemgxCzAJBgNV BAgMAnpqMQswCQYDVQQHDAJoejEPMA0GA1UECgwGYXBhY2hlMREwDwYDVQQLDAhy b2NrZXRtcTEOMAwGA1UEAwwFeXVrb24xHzAdBgkqhkiG9w0BCQEWEHl1a29uQGFw -YWNoZS5vcmcwHhcNMTcxMjExMDkyOTUyWhcNMTgwMTEwMDkyOTUyWjCBhjELMAkG -A1UEBhMCemgxCzAJBgNVBAgMAnpqMQswCQYDVQQHDAJoejEPMA0GA1UECgwGYXBh -Y2hlMREwDwYDVQQLDAhyb2NrZXRtcTEWMBQGA1UEAwwNZm9vYmFyLmNsaWVudDEh -MB8GCSqGSIb3DQEJARYSZm9vQGJhci5jbGllbnQuY29tMIGfMA0GCSqGSIb3DQEB -AQUAA4GNADCBiQKBgQDoz5Uo2ZN+1ywSQXORv6kDbDXVQ72ZxBp7a6ltFLh4xdk/ -yz7bBjmryz+cAh759s8DEdngl2cCnSiM0r5NC91zox/12Di4EWt3IPdJVe1s5/VD -Bqt5zoxbYyDgz47c95cGALgLdTB/itBPgobghQYBanWPVBNLLltw19DLf1gd6QID -AQABMA0GCSqGSIb3DQEBBQUAA4IBAQDEpVFFcFILdnEXjyDSbgJ8rxXaUYV2aK+a -lgrYfoHBv83MlEuHbydmHBoTL7BmPIL7JCIfufvRnnyBwcECi0E6qFbvMYNoMy6b -OUiTq3nAnPSSoOgi2MxFxpGaOE0s2dp4K9U5nV6bXKLIwIZbJAiZT6aPVenNYJbv -4arzFDe0Yjs/l3VYn2k9TjiiU2fxaW/8Ikx6o9nGWLTKeX/WtXfBNISqOPIL5dPF -eaf0YKCVzvBQ3dIJiUyanRP1BCJJFrCsrPpyu4xFprbjRmDTnOpYB6CdIas5TMC8 -6HzB1fSFoltNEiCjlnLlfjpb5ueSLSbs6h1A7VH7NUEmLmncSlHf +YWNoZS5vcmcwIBcNMTgwMTE2MDYxNjQ0WhgPMjExNzEyMjMwNjE2NDRaMIGSMQsw +CQYDVQQGEwJDTjERMA8GA1UECAwIWmhlamlhbmcxETAPBgNVBAcMCEhhbmd6aG91 +MQ8wDQYDVQQKDAZhcGFjaGUxETAPBgNVBAsMCHJvY2tldG1xMRgwFgYDVQQDDA9h +cGFjaGUgcm9ja2V0bXExHzAdBgkqhkiG9w0BCQEWEHl1a29uQGFwYWNoZS5vcmcw +gZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAOjPlSjZk37XLBJBc5G/qQNsNdVD +vZnEGntrqW0UuHjF2T/LPtsGOavLP5wCHvn2zwMR2eCXZwKdKIzSvk0L3XOjH/XY +OLgRa3cg90lV7Wzn9UMGq3nOjFtjIODPjtz3lwYAuAt1MH+K0E+ChuCFBgFqdY9U +E0suW3DX0Mt/WB3pAgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAFGPaZKyCZzQihKj +n/7I1J0wKl1HrU7N4sOie8E+ntcpKeX9zKYAou/4Iy0qwgxgRsnucB1rDous560a ++8DFDU8+FnikK9cQtKfQqu4F266IkkXolviZMSfkmB+NIsByIl95eMJlQHVlAvnX +vnpGdhD/Jhs+acE1VHhO6K+8omKLA6Og8MmYGRwmnBLcxIvqoSNDlEShfQyjaECg +I4bEi4ZhH3lSHE46FybJdoxDbj9IjHWqpOnjM23EOyfd1zcwOZJA7a54kfOpiTjz +wrtes5yoQznun5WtGcLM8ZmyaQ+Jr3j6NyZhOwULzK1+A8YUsW6Ww39xTxQoIHEQ +7eirb54= -----END CERTIFICATE----- diff --git a/remoting/src/test/resources/certs/server.pem b/remoting/src/test/resources/certs/server.pem index b73e33ef869..0187247af28 100644 --- a/remoting/src/test/resources/certs/server.pem +++ b/remoting/src/test/resources/certs/server.pem @@ -1,18 +1,19 @@ -----BEGIN CERTIFICATE----- -MIIC5DCCAcwCAQEwDQYJKoZIhvcNAQEFBQAwfDELMAkGA1UEBhMCemgxCzAJBgNV +MIIDATCCAekCAQIwDQYJKoZIhvcNAQEFBQAwfDELMAkGA1UEBhMCemgxCzAJBgNV BAgMAnpqMQswCQYDVQQHDAJoejEPMA0GA1UECgwGYXBhY2hlMREwDwYDVQQLDAhy b2NrZXRtcTEOMAwGA1UEAwwFeXVrb24xHzAdBgkqhkiG9w0BCQEWEHl1a29uQGFw -YWNoZS5vcmcwHhcNMTcxMjExMDkyNjIwWhcNMTgwMTEwMDkyNjIwWjB4MQswCQYD -VQQGEwJ6aDELMAkGA1UECAwCemoxCzAJBgNVBAcMAmh6MQ8wDQYDVQQKDAZhcGFj -aGUxETAPBgNVBAsMCHJvY2tldG1xMQ8wDQYDVQQDDAZmb29iYXIxGjAYBgkqhkiG -9w0BCQEWC2Zvb0BiYXIuY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDr -JqeGLayEbAXQUC8jyJmnwkcp0yZbNzTReIOp2a/XtHnwWPN9IMPZAXm3JutN7HWT -ZfeX80a9vrK14uRSWxuxZqtfhdMSzSv5CloqxrTTE9g7m5oVXSsl2CUryJ6BkZK0 -M2zhtDHQMsxOkDujo4zTfsRpJmnVsgsjRekvGKjCUwIDAQABMA0GCSqGSIb3DQEB -BQUAA4IBAQCmhSgxU5PRhBD2qahj2eWKcmz3FCevXgfyN/EUrwI2dZTU5fXPP+m9 -YBLAYUINI0eYGWt0wlGJ6UFyEgt1fcXP3gqsye9fjECmWoae1kVjvYdaxYGsEXrM -bxSum1D1bz6yRA+eSOaT5aesfw1ZL74AkIq5aRKQ4cgLGzlbIYeoa62XcAj6GrBo -V2s/mvKCc1FPrqnpUlTTYFM9eRbEyC7HkOm9c+NAy6FqoLFr3tegH+q8ZxENDw4k -z9gojQ6t1LDPOAmLGHwvMshHa841CwfOduSvzldtxzjnLVUvYB9cyXS1JXvuC9jj -Q6BOXIYI+0HVgkJbcPOIYDlgC+g6QJqf +YWNoZS5vcmcwIBcNMTgwMTE2MDYxMzQ5WhgPMjExNzEyMjMwNjEzNDlaMIGSMQsw +CQYDVQQGEwJDTjERMA8GA1UECAwIWmhlamlhbmcxETAPBgNVBAcMCEhhbmd6aG91 +MQ8wDQYDVQQKDAZhcGFjaGUxETAPBgNVBAsMCHJvY2tldG1xMRgwFgYDVQQDDA9h +cGFjaGUgcm9ja2V0bXExHzAdBgkqhkiG9w0BCQEWEHl1a29uQGFwYWNoZS5vcmcw +gZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAOsmp4YtrIRsBdBQLyPImafCRynT +Jls3NNF4g6nZr9e0efBY830gw9kBebcm603sdZNl95fzRr2+srXi5FJbG7Fmq1+F +0xLNK/kKWirGtNMT2DubmhVdKyXYJSvInoGRkrQzbOG0MdAyzE6QO6OjjNN+xGkm +adWyCyNF6S8YqMJTAgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAAzbwXyAULmXitiU ++8/2vbUZQlzB/nXY52OIq7qu3F55hE5qlHkcVxG2JZjO3p5UETwOyNUpU4dpu3uT +7WSdygH4Iagl87ILpGsob9pAf0joAbaXAY4sGDhg+WjR5JInAxbmT+QWZ+4NTuLQ +fSudUSJrv+HmUlmcVOvLiNStgt9rbtcgJAvpVwY+iCv0HQziFuQxmOkDv09ZLzu/ +lxCMqnbgkEFYkwdntN6MVk38K3MovszedGO/n19hNOFss7nn5XDEeEnc6BqKGdck +YDoy6amohY0Ds0o0gJ2rq0Y8Gjl9spQ3oeXpoNUoz84OF4KIBRTzSMv8CrmqPdFY +Zd2MGjw= -----END CERTIFICATE----- From 04e97de8a74b346b8d9e386b18c34e28ea2514fe Mon Sep 17 00:00:00 2001 From: yukon Date: Tue, 16 Jan 2018 21:33:12 +0800 Subject: [PATCH 09/73] [ROCKETMQ-332] Fix concurrent bug in MappedFileQueue#findMappedFileByOffset, which may cause message loss --- .../rocketmq/store/MappedFileQueue.java | 38 ++++++++++++------- .../rocketmq/store/MappedFileQueueTest.java | 18 +++++++++ 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java index 9eb3b3ab063..bdb851bab11 100644 --- a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java +++ b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java @@ -461,26 +461,38 @@ public boolean commit(final int commitLeastPages) { */ public MappedFile findMappedFileByOffset(final long offset, final boolean returnFirstOnNotFound) { try { - MappedFile mappedFile = this.getFirstMappedFile(); - if (mappedFile != null) { - int index = (int) ((offset / this.mappedFileSize) - (mappedFile.getFileFromOffset() / this.mappedFileSize)); + MappedFile firstMappedFile = this.getFirstMappedFile(); + if (firstMappedFile != null) { + int index = (int) ((offset / this.mappedFileSize) - (firstMappedFile.getFileFromOffset() / this.mappedFileSize)); if (index < 0 || index >= this.mappedFiles.size()) { - LOG_ERROR.warn("Offset for {} not matched. Request offset: {}, index: {}, " + - "mappedFileSize: {}, mappedFiles count: {}", - mappedFile, + LOG_ERROR.warn("Offset for {} not matched. Request offset: {}, index: {}, mappedFileSize: {}, mappedFiles count: {}", + firstMappedFile, offset, index, this.mappedFileSize, this.mappedFiles.size()); - } + } else { + MappedFile targetFile = null; + try { + targetFile = this.mappedFiles.get(index); + } catch (Exception ignored) { + } - try { - return this.mappedFiles.get(index); - } catch (Exception e) { - if (returnFirstOnNotFound) { - return mappedFile; + if (targetFile != null && offset >= targetFile.getFileFromOffset() + && offset < targetFile.getFileFromOffset() + this.mappedFileSize) { + return targetFile; } - LOG_ERROR.warn("findMappedFileByOffset failure. ", e); + + for (MappedFile tmpMappedFile : this.mappedFiles) { + if (offset >= tmpMappedFile.getFileFromOffset() + && offset < tmpMappedFile.getFileFromOffset() + this.mappedFileSize) { + return tmpMappedFile; + } + } + } + + if (returnFirstOnNotFound) { + return firstMappedFile; } } } catch (Exception e) { diff --git a/store/src/test/java/org/apache/rocketmq/store/MappedFileQueueTest.java b/store/src/test/java/org/apache/rocketmq/store/MappedFileQueueTest.java index 92f1876b2f0..8f76051d1f8 100644 --- a/store/src/test/java/org/apache/rocketmq/store/MappedFileQueueTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/MappedFileQueueTest.java @@ -229,6 +229,24 @@ public void testDeleteExpiredFileByTime() throws Exception { assertThat(mappedFileQueue.getMappedFiles().size()).isEqualTo(45); } + @Test + public void testFindMappedFile_ByIteration() { + MappedFileQueue mappedFileQueue = + new MappedFileQueue("target/unit_test_store/g/", 1024, null); + for (int i =0 ; i < 3; i++) { + MappedFile mappedFile = mappedFileQueue.getLastMappedFile(1024 * i); + mappedFile.wrotePosition.set(1024); + } + + assertThat(mappedFileQueue.findMappedFileByOffset(1028).getFileFromOffset()).isEqualTo(1024); + + // Switch two MappedFiles and verify findMappedFileByOffset method + MappedFile tmpFile = mappedFileQueue.getMappedFiles().get(1); + mappedFileQueue.getMappedFiles().set(1, mappedFileQueue.getMappedFiles().get(2)); + mappedFileQueue.getMappedFiles().set(2, tmpFile); + assertThat(mappedFileQueue.findMappedFileByOffset(1028).getFileFromOffset()).isEqualTo(1024); + } + @After public void destory() { File file = new File("target/unit_test_store"); From 9151a32b2e62c4263604a2e4c2593da03f022d24 Mon Sep 17 00:00:00 2001 From: yukon Date: Wed, 17 Jan 2018 21:27:22 +0800 Subject: [PATCH 10/73] [ROCKETMQ-332] Verify the offset by firstMappedFile and lastMappedFile --- .../org/apache/rocketmq/store/MappedFileQueue.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java index bdb851bab11..c30316f1904 100644 --- a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java +++ b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java @@ -462,16 +462,17 @@ public boolean commit(final int commitLeastPages) { public MappedFile findMappedFileByOffset(final long offset, final boolean returnFirstOnNotFound) { try { MappedFile firstMappedFile = this.getFirstMappedFile(); - if (firstMappedFile != null) { - int index = (int) ((offset / this.mappedFileSize) - (firstMappedFile.getFileFromOffset() / this.mappedFileSize)); - if (index < 0 || index >= this.mappedFiles.size()) { - LOG_ERROR.warn("Offset for {} not matched. Request offset: {}, index: {}, mappedFileSize: {}, mappedFiles count: {}", - firstMappedFile, + MappedFile lastMappedFile = this.getLastMappedFile(); + if (firstMappedFile != null && lastMappedFile != null) { + if (offset < firstMappedFile.getFileFromOffset() || offset >= lastMappedFile.getFileFromOffset() + this.mappedFileSize) { + LOG_ERROR.warn("Offset not matched. Request offset: {}, firstOffset: {}, lastOffset: {}, mappedFileSize: {}, mappedFiles count: {}", offset, - index, + firstMappedFile.getFileFromOffset(), + lastMappedFile.getFileFromOffset() + this.mappedFileSize, this.mappedFileSize, this.mappedFiles.size()); } else { + int index = (int) ((offset / this.mappedFileSize) - (firstMappedFile.getFileFromOffset() / this.mappedFileSize)); MappedFile targetFile = null; try { targetFile = this.mappedFiles.get(index); From a096580f3152e12fc9f9876c9e9721eb0109a90a Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Tue, 23 Jan 2018 14:12:33 +0800 Subject: [PATCH 11/73] [ROCKETMQ-351] Add a default value to expressionType and polish the judgement. (#214) --- .../org/apache/rocketmq/common/filter/ExpressionType.java | 2 +- .../rocketmq/common/protocol/heartbeat/SubscriptionData.java | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/org/apache/rocketmq/common/filter/ExpressionType.java b/common/src/main/java/org/apache/rocketmq/common/filter/ExpressionType.java index 3b7940abc7e..bc37733ed8c 100644 --- a/common/src/main/java/org/apache/rocketmq/common/filter/ExpressionType.java +++ b/common/src/main/java/org/apache/rocketmq/common/filter/ExpressionType.java @@ -59,7 +59,7 @@ public class ExpressionType { public static final String TAG = "TAG"; public static boolean isTagType(String type) { - if (type == null || TAG.equals(type)) { + if (type == null || "".equals(type) || TAG.equals(type)) { return true; } return false; diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/heartbeat/SubscriptionData.java b/common/src/main/java/org/apache/rocketmq/common/protocol/heartbeat/SubscriptionData.java index e456b7e606a..83e254f22ed 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/heartbeat/SubscriptionData.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/heartbeat/SubscriptionData.java @@ -21,6 +21,8 @@ package org.apache.rocketmq.common.protocol.heartbeat; import com.alibaba.fastjson.annotation.JSONField; +import org.apache.rocketmq.common.filter.ExpressionType; + import java.util.HashSet; import java.util.Set; @@ -32,7 +34,7 @@ public class SubscriptionData implements Comparable { private Set tagsSet = new HashSet(); private Set codeSet = new HashSet(); private long subVersion = System.currentTimeMillis(); - private String expressionType; + private String expressionType = ExpressionType.TAG; @JSONField(serialize = false) private String filterClassSource; From 305f5717c6fe4ad6596433673d27c4831b382d1e Mon Sep 17 00:00:00 2001 From: lindzh Date: Fri, 2 Mar 2018 16:07:41 +0800 Subject: [PATCH 12/73] [ROCKETMQ-367] Add logging component. (#224) --- logging/pom.xml | 50 + .../rocketmq/logging/InnerLoggerFactory.java | 478 +++++++ .../rocketmq/logging/InternalLogger.java | 63 + .../logging/InternalLoggerFactory.java | 93 ++ .../rocketmq/logging/Slf4jLoggerFactory.java | 153 ++ .../rocketmq/logging/inner/Appender.java | 228 +++ .../apache/rocketmq/logging/inner/Layout.java | 39 + .../apache/rocketmq/logging/inner/Level.java | 156 +++ .../apache/rocketmq/logging/inner/Logger.java | 467 +++++++ .../logging/inner/LoggingBuilder.java | 1230 +++++++++++++++++ .../rocketmq/logging/inner/LoggingEvent.java | 121 ++ .../rocketmq/logging/inner/SysLogger.java | 89 ++ .../apache/rocketmq/logging/package-info.java | 35 + .../rocketmq/logging/BasicloggerTest.java | 70 + .../logging/InnerLoggerFactoryTest.java | 92 ++ .../rocketmq/logging/InternalLoggerTest.java | 72 + .../logging/Slf4jLoggerFactoryTest.java | 80 ++ .../rocketmq/logging/inner/AppenderTest.java | 160 +++ .../rocketmq/logging/inner/LayoutTest.java | 54 + .../rocketmq/logging/inner/LevelTest.java | 37 + .../logging/inner/LoggerRepositoryTest.java | 49 + .../rocketmq/logging/inner/LoggerTest.java | 115 ++ .../logging/inner/LoggingBuilderTest.java | 113 ++ .../logging/inner/MessageFormatterTest.java | 40 + logging/src/test/resources/logback_test.xml | 46 + pom.xml | 6 + 26 files changed, 4136 insertions(+) create mode 100644 logging/pom.xml create mode 100644 logging/src/main/java/org/apache/rocketmq/logging/InnerLoggerFactory.java create mode 100644 logging/src/main/java/org/apache/rocketmq/logging/InternalLogger.java create mode 100644 logging/src/main/java/org/apache/rocketmq/logging/InternalLoggerFactory.java create mode 100644 logging/src/main/java/org/apache/rocketmq/logging/Slf4jLoggerFactory.java create mode 100755 logging/src/main/java/org/apache/rocketmq/logging/inner/Appender.java create mode 100644 logging/src/main/java/org/apache/rocketmq/logging/inner/Layout.java create mode 100755 logging/src/main/java/org/apache/rocketmq/logging/inner/Level.java create mode 100755 logging/src/main/java/org/apache/rocketmq/logging/inner/Logger.java create mode 100644 logging/src/main/java/org/apache/rocketmq/logging/inner/LoggingBuilder.java create mode 100644 logging/src/main/java/org/apache/rocketmq/logging/inner/LoggingEvent.java create mode 100755 logging/src/main/java/org/apache/rocketmq/logging/inner/SysLogger.java create mode 100644 logging/src/main/java/org/apache/rocketmq/logging/package-info.java create mode 100644 logging/src/test/java/org/apache/rocketmq/logging/BasicloggerTest.java create mode 100644 logging/src/test/java/org/apache/rocketmq/logging/InnerLoggerFactoryTest.java create mode 100644 logging/src/test/java/org/apache/rocketmq/logging/InternalLoggerTest.java create mode 100644 logging/src/test/java/org/apache/rocketmq/logging/Slf4jLoggerFactoryTest.java create mode 100644 logging/src/test/java/org/apache/rocketmq/logging/inner/AppenderTest.java create mode 100644 logging/src/test/java/org/apache/rocketmq/logging/inner/LayoutTest.java create mode 100644 logging/src/test/java/org/apache/rocketmq/logging/inner/LevelTest.java create mode 100644 logging/src/test/java/org/apache/rocketmq/logging/inner/LoggerRepositoryTest.java create mode 100644 logging/src/test/java/org/apache/rocketmq/logging/inner/LoggerTest.java create mode 100644 logging/src/test/java/org/apache/rocketmq/logging/inner/LoggingBuilderTest.java create mode 100644 logging/src/test/java/org/apache/rocketmq/logging/inner/MessageFormatterTest.java create mode 100644 logging/src/test/resources/logback_test.xml diff --git a/logging/pom.xml b/logging/pom.xml new file mode 100644 index 00000000000..baad6ad0c49 --- /dev/null +++ b/logging/pom.xml @@ -0,0 +1,50 @@ + + + + + org.apache.rocketmq + rocketmq-all + 4.3.0-SNAPSHOT + + + 4.0.0 + jar + rocketmq-logging + rocketmq-logging ${project.version} + + + 1.6 + 1.6 + + + + + org.slf4j + slf4j-api + true + + + ch.qos.logback + logback-classic + test + + + + \ No newline at end of file diff --git a/logging/src/main/java/org/apache/rocketmq/logging/InnerLoggerFactory.java b/logging/src/main/java/org/apache/rocketmq/logging/InnerLoggerFactory.java new file mode 100644 index 00000000000..7714640ff8d --- /dev/null +++ b/logging/src/main/java/org/apache/rocketmq/logging/InnerLoggerFactory.java @@ -0,0 +1,478 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.logging; + +import org.apache.rocketmq.logging.inner.Logger; + +import java.util.HashMap; +import java.util.Map; + +public class InnerLoggerFactory extends InternalLoggerFactory { + + @Override + protected InternalLogger getLoggerInstance(String name) { + return new InnerLogger(name); + } + + @Override + protected String getLoggerType() { + return LOGGER_INNER; + } + + @Override + protected void shutdown() { + Logger.getRepository().shutdown(); + } + + public static class InnerLogger implements InternalLogger { + + private Logger logger; + + public InnerLogger(String name) { + logger = Logger.getLogger(name); + } + + @Override + public String getName() { + return logger.getName(); + } + + @Override + public void debug(String var1) { + logger.debug(var1); + } + + @Override + public void debug(String var1, Throwable var2) { + logger.debug(var1, var2); + } + + @Override + public void info(String var1) { + logger.info(var1); + } + + @Override + public void info(String var1, Throwable var2) { + logger.info(var1, var2); + } + + @Override + public void warn(String var1) { + logger.warn(var1); + } + + @Override + public void warn(String var1, Throwable var2) { + logger.warn(var1, var2); + } + + @Override + public void error(String var1) { + logger.error(var1); + } + + @Override + public void error(String var1, Throwable var2) { + logger.error(var1, var2); + } + + @Override + public void debug(String var1, Object var2) { + FormattingTuple format = MessageFormatter.format(var1, var2); + logger.debug(format.getMessage(), format.getThrowable()); + } + + @Override + public void debug(String var1, Object var2, Object var3) { + FormattingTuple format = MessageFormatter.format(var1, var2, var3); + logger.debug(format.getMessage(), format.getThrowable()); + } + + @Override + public void debug(String var1, Object... var2) { + FormattingTuple format = MessageFormatter.arrayFormat(var1, var2); + logger.debug(format.getMessage(), format.getThrowable()); + } + + @Override + public void info(String var1, Object var2) { + FormattingTuple format = MessageFormatter.format(var1, var2); + logger.info(format.getMessage(), format.getThrowable()); + } + + @Override + public void info(String var1, Object var2, Object var3) { + FormattingTuple format = MessageFormatter.format(var1, var2, var3); + logger.info(format.getMessage(), format.getThrowable()); + } + + @Override + public void info(String var1, Object... var2) { + FormattingTuple format = MessageFormatter.arrayFormat(var1, var2); + logger.info(format.getMessage(), format.getThrowable()); + } + + @Override + public void warn(String var1, Object var2) { + FormattingTuple format = MessageFormatter.format(var1, var2); + logger.warn(format.getMessage(), format.getThrowable()); + } + + @Override + public void warn(String var1, Object... var2) { + FormattingTuple format = MessageFormatter.arrayFormat(var1, var2); + logger.warn(format.getMessage(), format.getThrowable()); + } + + @Override + public void warn(String var1, Object var2, Object var3) { + FormattingTuple format = MessageFormatter.format(var1, var2, var3); + logger.warn(format.getMessage(), format.getThrowable()); + } + + @Override + public void error(String var1, Object var2) { + FormattingTuple format = MessageFormatter.format(var1, var2); + logger.warn(format.getMessage(), format.getThrowable()); + } + + @Override + public void error(String var1, Object var2, Object var3) { + FormattingTuple format = MessageFormatter.format(var1, var2, var3); + logger.warn(format.getMessage(), format.getThrowable()); + } + + @Override + public void error(String var1, Object... var2) { + FormattingTuple format = MessageFormatter.arrayFormat(var1, var2); + logger.warn(format.getMessage(), format.getThrowable()); + } + + public Logger getLogger() { + return logger; + } + } + + + public static class FormattingTuple { + private String message; + private Throwable throwable; + private Object[] argArray; + + public FormattingTuple(String message) { + this(message, null, null); + } + + public FormattingTuple(String message, Object[] argArray, Throwable throwable) { + this.message = message; + this.throwable = throwable; + if (throwable == null) { + this.argArray = argArray; + } else { + this.argArray = trimmedCopy(argArray); + } + + } + + static Object[] trimmedCopy(Object[] argArray) { + if (argArray != null && argArray.length != 0) { + int trimemdLen = argArray.length - 1; + Object[] trimmed = new Object[trimemdLen]; + System.arraycopy(argArray, 0, trimmed, 0, trimemdLen); + return trimmed; + } else { + throw new IllegalStateException("non-sensical empty or null argument array"); + } + } + + public String getMessage() { + return this.message; + } + + public Object[] getArgArray() { + return this.argArray; + } + + public Throwable getThrowable() { + return this.throwable; + } + } + + public static class MessageFormatter { + + public MessageFormatter() { + } + + public static FormattingTuple format(String messagePattern, Object arg) { + return arrayFormat(messagePattern, new Object[]{arg}); + } + + public static FormattingTuple format(String messagePattern, Object arg1, Object arg2) { + return arrayFormat(messagePattern, new Object[]{arg1, arg2}); + } + + static Throwable getThrowableCandidate(Object[] argArray) { + if (argArray != null && argArray.length != 0) { + Object lastEntry = argArray[argArray.length - 1]; + return lastEntry instanceof Throwable ? (Throwable) lastEntry : null; + } else { + return null; + } + } + + public static FormattingTuple arrayFormat(String messagePattern, Object[] argArray) { + Throwable throwableCandidate = getThrowableCandidate(argArray); + if (messagePattern == null) { + return new FormattingTuple(null, argArray, throwableCandidate); + } else if (argArray == null) { + return new FormattingTuple(messagePattern); + } else { + int i = 0; + StringBuilder sbuf = new StringBuilder(messagePattern.length() + 50); + + int len; + for (len = 0; len < argArray.length; ++len) { + int j = messagePattern.indexOf("{}", i); + if (j == -1) { + if (i == 0) { + return new FormattingTuple(messagePattern, argArray, throwableCandidate); + } + + sbuf.append(messagePattern.substring(i, messagePattern.length())); + return new FormattingTuple(sbuf.toString(), argArray, throwableCandidate); + } + + if (isEscapeDelimeter(messagePattern, j)) { + if (!isDoubleEscaped(messagePattern, j)) { + --len; + sbuf.append(messagePattern.substring(i, j - 1)); + sbuf.append('{'); + i = j + 1; + } else { + sbuf.append(messagePattern.substring(i, j - 1)); + deeplyAppendParameter(sbuf, argArray[len], null); + i = j + 2; + } + } else { + sbuf.append(messagePattern.substring(i, j)); + deeplyAppendParameter(sbuf, argArray[len], null); + i = j + 2; + } + } + + sbuf.append(messagePattern.substring(i, messagePattern.length())); + if (len < argArray.length - 1) { + return new FormattingTuple(sbuf.toString(), argArray, throwableCandidate); + } else { + return new FormattingTuple(sbuf.toString(), argArray, null); + } + } + } + + static boolean isEscapeDelimeter(String messagePattern, int delimeterStartIndex) { + if (delimeterStartIndex == 0) { + return false; + } else { + char potentialEscape = messagePattern.charAt(delimeterStartIndex - 1); + return potentialEscape == 92; + } + } + + static boolean isDoubleEscaped(String messagePattern, int delimeterStartIndex) { + return delimeterStartIndex >= 2 && messagePattern.charAt(delimeterStartIndex - 2) == 92; + } + + private static void deeplyAppendParameter(StringBuilder sbuf, Object o, Map seenMap) { + if (o == null) { + sbuf.append("null"); + } else { + if (!o.getClass().isArray()) { + safeObjectAppend(sbuf, o); + } else if (o instanceof boolean[]) { + booleanArrayAppend(sbuf, (boolean[]) o); + } else if (o instanceof byte[]) { + byteArrayAppend(sbuf, (byte[]) o); + } else if (o instanceof char[]) { + charArrayAppend(sbuf, (char[]) o); + } else if (o instanceof short[]) { + shortArrayAppend(sbuf, (short[]) o); + } else if (o instanceof int[]) { + intArrayAppend(sbuf, (int[]) o); + } else if (o instanceof long[]) { + longArrayAppend(sbuf, (long[]) o); + } else if (o instanceof float[]) { + floatArrayAppend(sbuf, (float[]) o); + } else if (o instanceof double[]) { + doubleArrayAppend(sbuf, (double[]) o); + } else { + objectArrayAppend(sbuf, (Object[]) o, seenMap); + } + + } + } + + private static void safeObjectAppend(StringBuilder sbuf, Object o) { + try { + String t = o.toString(); + sbuf.append(t); + } catch (Throwable var3) { + System.err.println("RocketMQ InnerLogger: Failed toString() invocation on an object of type [" + o.getClass().getName() + "]"); + var3.printStackTrace(); + sbuf.append("[FAILED toString()]"); + } + + } + + private static void objectArrayAppend(StringBuilder sbuf, Object[] a, Map seenMap) { + if (seenMap == null) { + seenMap = new HashMap(); + } + sbuf.append('['); + if (!seenMap.containsKey(a)) { + seenMap.put(a, null); + int len = a.length; + + for (int i = 0; i < len; ++i) { + deeplyAppendParameter(sbuf, a[i], seenMap); + if (i != len - 1) { + sbuf.append(", "); + } + } + + seenMap.remove(a); + } else { + sbuf.append("..."); + } + + sbuf.append(']'); + } + + private static void booleanArrayAppend(StringBuilder sbuf, boolean[] a) { + sbuf.append('['); + int len = a.length; + + for (int i = 0; i < len; ++i) { + sbuf.append(a[i]); + if (i != len - 1) { + sbuf.append(", "); + } + } + + sbuf.append(']'); + } + + private static void byteArrayAppend(StringBuilder sbuf, byte[] a) { + sbuf.append('['); + int len = a.length; + + for (int i = 0; i < len; ++i) { + sbuf.append(a[i]); + if (i != len - 1) { + sbuf.append(", "); + } + } + + sbuf.append(']'); + } + + private static void charArrayAppend(StringBuilder sbuf, char[] a) { + sbuf.append('['); + int len = a.length; + + for (int i = 0; i < len; ++i) { + sbuf.append(a[i]); + if (i != len - 1) { + sbuf.append(", "); + } + } + + sbuf.append(']'); + } + + private static void shortArrayAppend(StringBuilder sbuf, short[] a) { + sbuf.append('['); + int len = a.length; + + for (int i = 0; i < len; ++i) { + sbuf.append(a[i]); + if (i != len - 1) { + sbuf.append(", "); + } + } + + sbuf.append(']'); + } + + private static void intArrayAppend(StringBuilder sbuf, int[] a) { + sbuf.append('['); + int len = a.length; + + for (int i = 0; i < len; ++i) { + sbuf.append(a[i]); + if (i != len - 1) { + sbuf.append(", "); + } + } + + sbuf.append(']'); + } + + private static void longArrayAppend(StringBuilder sbuf, long[] a) { + sbuf.append('['); + int len = a.length; + + for (int i = 0; i < len; ++i) { + sbuf.append(a[i]); + if (i != len - 1) { + sbuf.append(", "); + } + } + + sbuf.append(']'); + } + + private static void floatArrayAppend(StringBuilder sbuf, float[] a) { + sbuf.append('['); + int len = a.length; + + for (int i = 0; i < len; ++i) { + sbuf.append(a[i]); + if (i != len - 1) { + sbuf.append(", "); + } + } + + sbuf.append(']'); + } + + private static void doubleArrayAppend(StringBuilder sbuf, double[] a) { + sbuf.append('['); + int len = a.length; + + for (int i = 0; i < len; ++i) { + sbuf.append(a[i]); + if (i != len - 1) { + sbuf.append(", "); + } + } + + sbuf.append(']'); + } + } +} diff --git a/logging/src/main/java/org/apache/rocketmq/logging/InternalLogger.java b/logging/src/main/java/org/apache/rocketmq/logging/InternalLogger.java new file mode 100644 index 00000000000..fae69dda6c5 --- /dev/null +++ b/logging/src/main/java/org/apache/rocketmq/logging/InternalLogger.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.logging; + +public interface InternalLogger { + + String getName(); + + void debug(String var1); + + void debug(String var1, Object var2); + + void debug(String var1, Object var2, Object var3); + + void debug(String var1, Object... var2); + + void debug(String var1, Throwable var2); + + void info(String var1); + + void info(String var1, Object var2); + + void info(String var1, Object var2, Object var3); + + void info(String var1, Object... var2); + + void info(String var1, Throwable var2); + + void warn(String var1); + + void warn(String var1, Object var2); + + void warn(String var1, Object... var2); + + void warn(String var1, Object var2, Object var3); + + void warn(String var1, Throwable var2); + + void error(String var1); + + void error(String var1, Object var2); + + void error(String var1, Object var2, Object var3); + + void error(String var1, Object... var2); + + void error(String var1, Throwable var2); +} diff --git a/logging/src/main/java/org/apache/rocketmq/logging/InternalLoggerFactory.java b/logging/src/main/java/org/apache/rocketmq/logging/InternalLoggerFactory.java new file mode 100644 index 00000000000..ec176ce6b19 --- /dev/null +++ b/logging/src/main/java/org/apache/rocketmq/logging/InternalLoggerFactory.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.logging; + +import java.util.concurrent.ConcurrentHashMap; + +public abstract class InternalLoggerFactory { + + public static final String LOGGER_SLF4J = "slf4j"; + + public static final String LOGGER_INNER = "inner"; + + public static final String DEFAULT_LOGGER = LOGGER_SLF4J; + + private static String loggerType = null; + + private static ConcurrentHashMap loggerFactoryCache = new ConcurrentHashMap(); + + public static InternalLogger getLogger(Class clazz) { + return getLogger(clazz.getName()); + } + + public static InternalLogger getLogger(String name) { + return getLoggerFactory().getLoggerInstance(name); + } + + private static InternalLoggerFactory getLoggerFactory() { + InternalLoggerFactory internalLoggerFactory = null; + if (loggerType != null) { + internalLoggerFactory = loggerFactoryCache.get(loggerType); + } + if (internalLoggerFactory == null) { + internalLoggerFactory = loggerFactoryCache.get(DEFAULT_LOGGER); + } + if (internalLoggerFactory == null) { + internalLoggerFactory = loggerFactoryCache.get(LOGGER_INNER); + } + if (internalLoggerFactory == null) { + throw new RuntimeException("[RocketMQ] Logger init failed, please check logger"); + } + return internalLoggerFactory; + } + + public static void setCurrentLoggerType(String type) { + loggerType = type; + } + + static { + try { + new Slf4jLoggerFactory(); + } catch (Throwable e) { + //ignore + } + try { + new InnerLoggerFactory(); + } catch (Throwable e) { + //ignore + } + } + + public InternalLoggerFactory() { + doRegister(); + } + + protected void doRegister() { + String loggerType = getLoggerType(); + if (loggerFactoryCache.get(loggerType) != null) { + return; + } + loggerFactoryCache.put(loggerType, this); + } + + protected abstract void shutdown(); + + protected abstract InternalLogger getLoggerInstance(String name); + + protected abstract String getLoggerType(); +} diff --git a/logging/src/main/java/org/apache/rocketmq/logging/Slf4jLoggerFactory.java b/logging/src/main/java/org/apache/rocketmq/logging/Slf4jLoggerFactory.java new file mode 100644 index 00000000000..1a246847388 --- /dev/null +++ b/logging/src/main/java/org/apache/rocketmq/logging/Slf4jLoggerFactory.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.logging; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Slf4jLoggerFactory extends InternalLoggerFactory { + + @Override + protected String getLoggerType() { + return InternalLoggerFactory.LOGGER_SLF4J; + } + + @Override + protected InternalLogger getLoggerInstance(String name) { + return new Slf4jLogger(name); + } + + @Override + protected void shutdown() { + + } + + public static class Slf4jLogger implements InternalLogger { + + private Logger logger = null; + + public Slf4jLogger(String name) { + logger = LoggerFactory.getLogger(name); + } + + @Override + public String getName() { + return logger.getName(); + } + + @Override + public void debug(String s) { + logger.debug(s); + } + + @Override + public void debug(String s, Object o) { + logger.debug(s, o); + } + + @Override + public void debug(String s, Object o, Object o1) { + logger.debug(s, o, o1); + } + + @Override + public void debug(String s, Object... objects) { + logger.debug(s, objects); + } + + @Override + public void debug(String s, Throwable throwable) { + logger.debug(s, throwable); + } + + @Override + public void info(String s) { + logger.info(s); + } + + @Override + public void info(String s, Object o) { + logger.info(s, o); + } + + @Override + public void info(String s, Object o, Object o1) { + logger.info(s, o, o1); + } + + @Override + public void info(String s, Object... objects) { + logger.info(s, objects); + } + + @Override + public void info(String s, Throwable throwable) { + logger.info(s, throwable); + } + + @Override + public void warn(String s) { + logger.warn(s); + } + + @Override + public void warn(String s, Object o) { + logger.warn(s, o); + } + + @Override + public void warn(String s, Object... objects) { + logger.warn(s, objects); + } + + @Override + public void warn(String s, Object o, Object o1) { + logger.warn(s, o, o1); + } + + @Override + public void warn(String s, Throwable throwable) { + logger.warn(s, throwable); + } + + @Override + public void error(String s) { + logger.error(s); + } + + @Override + public void error(String s, Object o) { + logger.error(s, o); + } + + @Override + public void error(String s, Object o, Object o1) { + logger.error(s, o, o1); + } + + @Override + public void error(String s, Object... objects) { + logger.error(s, objects); + } + + @Override + public void error(String s, Throwable throwable) { + logger.error(s, throwable); + } + } +} diff --git a/logging/src/main/java/org/apache/rocketmq/logging/inner/Appender.java b/logging/src/main/java/org/apache/rocketmq/logging/inner/Appender.java new file mode 100755 index 00000000000..c06156310f8 --- /dev/null +++ b/logging/src/main/java/org/apache/rocketmq/logging/inner/Appender.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.logging.inner; + + +import java.io.InterruptedIOException; +import java.util.Enumeration; +import java.util.Vector; + +public abstract class Appender { + + public static final int CODE_WRITE_FAILURE = 1; + public static final int CODE_FLUSH_FAILURE = 2; + public static final int CODE_CLOSE_FAILURE = 3; + public static final int CODE_FILE_OPEN_FAILURE = 4; + + public final static String LINE_SEP = System.getProperty("line.separator"); + + boolean firstTime = true; + + protected Layout layout; + + protected String name; + + protected boolean closed = false; + + public void activateOptions() { + } + + abstract protected void append(LoggingEvent event); + + public void finalize() { + try { + super.finalize(); + } catch (Throwable throwable) { + SysLogger.error("Finalizing appender named [" + name + "]. error", throwable); + } + if (this.closed) { + return; + } + + SysLogger.debug("Finalizing appender named [" + name + "]."); + close(); + } + + public Layout getLayout() { + return layout; + } + + public final String getName() { + return this.name; + } + + public synchronized void doAppend(LoggingEvent event) { + if (closed) { + SysLogger.error("Attempted to append to closed appender named [" + name + "]."); + return; + } + this.append(event); + } + + public void setLayout(Layout layout) { + this.layout = layout; + } + + public void setName(String name) { + this.name = name; + } + + public abstract void close(); + + public void handleError(String message, Exception e, int errorCode) { + if (e instanceof InterruptedIOException || e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + if (firstTime) { + SysLogger.error(message + " code:" + errorCode, e); + firstTime = false; + } + } + + public void handleError(String message) { + if (firstTime) { + SysLogger.error(message); + firstTime = false; + } + } + + + public interface AppenderPipeline { + + void addAppender(Appender newAppender); + + Enumeration getAllAppenders(); + + Appender getAppender(String name); + + boolean isAttached(Appender appender); + + void removeAllAppenders(); + + void removeAppender(Appender appender); + + void removeAppender(String name); + } + + + public static class AppenderPipelineImpl implements AppenderPipeline { + + + protected Vector appenderList; + + public void addAppender(Appender newAppender) { + if (newAppender == null) { + return; + } + + if (appenderList == null) { + appenderList = new Vector(1); + } + if (!appenderList.contains(newAppender)) { + appenderList.addElement(newAppender); + } + } + + public int appendLoopOnAppenders(LoggingEvent event) { + int size = 0; + Appender appender; + + if (appenderList != null) { + size = appenderList.size(); + for (int i = 0; i < size; i++) { + appender = appenderList.elementAt(i); + appender.doAppend(event); + } + } + return size; + } + + public Enumeration getAllAppenders() { + if (appenderList == null) { + return null; + } else { + return appenderList.elements(); + } + } + + public Appender getAppender(String name) { + if (appenderList == null || name == null) { + return null; + } + + int size = appenderList.size(); + Appender appender; + for (int i = 0; i < size; i++) { + appender = appenderList.elementAt(i); + if (name.equals(appender.getName())) { + return appender; + } + } + return null; + } + + public boolean isAttached(Appender appender) { + if (appenderList == null || appender == null) { + return false; + } + + int size = appenderList.size(); + Appender a; + for (int i = 0; i < size; i++) { + a = appenderList.elementAt(i); + if (a == appender) { + return true; + } + } + return false; + } + + public void removeAllAppenders() { + if (appenderList != null) { + int len = appenderList.size(); + for (int i = 0; i < len; i++) { + Appender a = appenderList.elementAt(i); + a.close(); + } + appenderList.removeAllElements(); + appenderList = null; + } + } + + public void removeAppender(Appender appender) { + if (appender == null || appenderList == null) { + return; + } + appenderList.removeElement(appender); + } + + public void removeAppender(String name) { + if (name == null || appenderList == null) { + return; + } + int size = appenderList.size(); + for (int i = 0; i < size; i++) { + if (name.equals((appenderList.elementAt(i)).getName())) { + appenderList.removeElementAt(i); + break; + } + } + } + + } +} diff --git a/logging/src/main/java/org/apache/rocketmq/logging/inner/Layout.java b/logging/src/main/java/org/apache/rocketmq/logging/inner/Layout.java new file mode 100644 index 00000000000..7ea3561df35 --- /dev/null +++ b/logging/src/main/java/org/apache/rocketmq/logging/inner/Layout.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.logging.inner; + +public abstract class Layout { + + public abstract String format(LoggingEvent event); + + public String getContentType() { + return "text/plain"; + } + + public String getHeader() { + return null; + } + + public String getFooter() { + return null; + } + + + abstract public boolean ignoresThrowable(); + +} \ No newline at end of file diff --git a/logging/src/main/java/org/apache/rocketmq/logging/inner/Level.java b/logging/src/main/java/org/apache/rocketmq/logging/inner/Level.java new file mode 100755 index 00000000000..487682cd44b --- /dev/null +++ b/logging/src/main/java/org/apache/rocketmq/logging/inner/Level.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.logging.inner; + +import java.io.Serializable; + +public class Level implements Serializable { + + transient int level; + transient String levelStr; + transient int syslogEquivalent; + + public final static int OFF_INT = Integer.MAX_VALUE; + public final static int ERROR_INT = 40000; + public final static int WARN_INT = 30000; + public final static int INFO_INT = 20000; + public final static int DEBUG_INT = 10000; + public final static int ALL_INT = Integer.MIN_VALUE; + + + private static final String ALL_NAME = "ALL"; + + private static final String DEBUG_NAME = "DEBUG"; + + private static final String INFO_NAME = "INFO"; + + private static final String WARN_NAME = "WARN"; + + private static final String ERROR_NAME = "ERROR"; + + private static final String OFF_NAME = "OFF"; + + final static public Level OFF = new Level(OFF_INT, OFF_NAME, 0); + + final static public Level ERROR = new Level(ERROR_INT, ERROR_NAME, 3); + + final static public Level WARN = new Level(WARN_INT, WARN_NAME, 4); + + final static public Level INFO = new Level(INFO_INT, INFO_NAME, 6); + + final static public Level DEBUG = new Level(DEBUG_INT, DEBUG_NAME, 7); + + final static public Level ALL = new Level(ALL_INT, ALL_NAME, 7); + + static final long serialVersionUID = 3491141966387921974L; + + protected Level(int level, String levelStr, int syslogEquivalent) { + this.level = level; + this.levelStr = levelStr; + this.syslogEquivalent = syslogEquivalent; + } + + public static Level toLevel(String sArg) { + return toLevel(sArg, Level.DEBUG); + } + + public static Level toLevel(int val) { + return toLevel(val, Level.DEBUG); + } + + public static Level toLevel(int val, Level defaultLevel) { + switch (val) { + case ALL_INT: + return ALL; + case DEBUG_INT: + return Level.DEBUG; + case INFO_INT: + return Level.INFO; + case WARN_INT: + return Level.WARN; + case ERROR_INT: + return Level.ERROR; + case OFF_INT: + return OFF; + default: + return defaultLevel; + } + } + + public static Level toLevel(String sArg, Level defaultLevel) { + if (sArg == null) { + return defaultLevel; + } + String s = sArg.toUpperCase(); + + if (s.equals(ALL_NAME)) { + return Level.ALL; + } + if (s.equals(DEBUG_NAME)) { + return Level.DEBUG; + } + if (s.equals(INFO_NAME)) { + return Level.INFO; + } + if (s.equals(WARN_NAME)) { + return Level.WARN; + } + if (s.equals(ERROR_NAME)) { + return Level.ERROR; + } + if (s.equals(OFF_NAME)) { + return Level.OFF; + } + + if (s.equals(INFO_NAME)) { + return Level.INFO; + } + return defaultLevel; + } + + + public boolean equals(Object o) { + if (o instanceof Level) { + Level r = (Level) o; + return this.level == r.level; + } else { + return false; + } + } + + @Override + public int hashCode() { + int result = level; + result = 31 * result + (levelStr != null ? levelStr.hashCode() : 0); + result = 31 * result + syslogEquivalent; + return result; + } + + public boolean isGreaterOrEqual(Level r) { + return level >= r.level; + } + + final public String toString() { + return levelStr; + } + + public final int toInt() { + return level; + } + +} diff --git a/logging/src/main/java/org/apache/rocketmq/logging/inner/Logger.java b/logging/src/main/java/org/apache/rocketmq/logging/inner/Logger.java new file mode 100755 index 00000000000..470ed41daf9 --- /dev/null +++ b/logging/src/main/java/org/apache/rocketmq/logging/inner/Logger.java @@ -0,0 +1,467 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.logging.inner; + +import java.util.Enumeration; +import java.util.Hashtable; +import java.util.Vector; + + +public class Logger implements Appender.AppenderPipeline { + + private static final String FQCN = Logger.class.getName(); + + private static final DefaultLoggerRepository REPOSITORY = new DefaultLoggerRepository(new RootLogger(Level.DEBUG)); + + public static LoggerRepository getRepository() { + return REPOSITORY; + } + + private String name; + + volatile private Level level; + + volatile private Logger parent; + + Appender.AppenderPipelineImpl appenderPipeline; + + private boolean additive = true; + + private Logger(String name) { + this.name = name; + } + + static public Logger getLogger(String name) { + return getRepository().getLogger(name); + } + + static public Logger getLogger(Class clazz) { + return getRepository().getLogger(clazz.getName()); + } + + public static Logger getRootLogger() { + return getRepository().getRootLogger(); + } + + synchronized public void addAppender(Appender newAppender) { + if (appenderPipeline == null) { + appenderPipeline = new Appender.AppenderPipelineImpl(); + } + appenderPipeline.addAppender(newAppender); + } + + public void callAppenders(LoggingEvent event) { + int writes = 0; + + for (Logger logger = this; logger != null; logger = logger.parent) { + synchronized (logger) { + if (logger.appenderPipeline != null) { + writes += logger.appenderPipeline.appendLoopOnAppenders(event); + } + if (!logger.additive) { + break; + } + } + } + + if (writes == 0) { + getRepository().emitNoAppenderWarning(this); + } + } + + synchronized void closeNestedAppenders() { + Enumeration enumeration = this.getAllAppenders(); + if (enumeration != null) { + while (enumeration.hasMoreElements()) { + Appender a = (Appender) enumeration.nextElement(); + if (a instanceof Appender.AppenderPipeline) { + a.close(); + } + } + } + } + + public void debug(Object message) { + if (getRepository().isDisabled(Level.DEBUG_INT)) { + return; + } + if (Level.DEBUG.isGreaterOrEqual(this.getEffectiveLevel())) { + forcedLog(FQCN, Level.DEBUG, message, null); + } + } + + + public void debug(Object message, Throwable t) { + if (getRepository().isDisabled(Level.DEBUG_INT)) { + return; + } + if (Level.DEBUG.isGreaterOrEqual(this.getEffectiveLevel())) { + forcedLog(FQCN, Level.DEBUG, message, t); + } + } + + + public void error(Object message) { + if (getRepository().isDisabled(Level.ERROR_INT)) { + return; + } + if (Level.ERROR.isGreaterOrEqual(this.getEffectiveLevel())) { + forcedLog(FQCN, Level.ERROR, message, null); + } + } + + public void error(Object message, Throwable t) { + if (getRepository().isDisabled(Level.ERROR_INT)) { + return; + } + if (Level.ERROR.isGreaterOrEqual(this.getEffectiveLevel())) { + forcedLog(FQCN, Level.ERROR, message, t); + } + + } + + + protected void forcedLog(String fqcn, Level level, Object message, Throwable t) { + callAppenders(new LoggingEvent(fqcn, this, level, message, t)); + } + + + synchronized public Enumeration getAllAppenders() { + if (appenderPipeline == null) { + return null; + } else { + return appenderPipeline.getAllAppenders(); + } + } + + synchronized public Appender getAppender(String name) { + if (appenderPipeline == null || name == null) { + return null; + } + + return appenderPipeline.getAppender(name); + } + + public Level getEffectiveLevel() { + for (Logger c = this; c != null; c = c.parent) { + if (c.level != null) { + return c.level; + } + } + return null; + } + + public final String getName() { + return name; + } + + final public Level getLevel() { + return this.level; + } + + + public void info(Object message) { + if (getRepository().isDisabled(Level.INFO_INT)) { + return; + } + if (Level.INFO.isGreaterOrEqual(this.getEffectiveLevel())) { + forcedLog(FQCN, Level.INFO, message, null); + } + } + + public void info(Object message, Throwable t) { + if (getRepository().isDisabled(Level.INFO_INT)) { + return; + } + if (Level.INFO.isGreaterOrEqual(this.getEffectiveLevel())) { + forcedLog(FQCN, Level.INFO, message, t); + } + } + + public boolean isAttached(Appender appender) { + return appender != null && appenderPipeline != null && appenderPipeline.isAttached(appender); + } + + synchronized public void removeAllAppenders() { + if (appenderPipeline != null) { + appenderPipeline.removeAllAppenders(); + appenderPipeline = null; + } + } + + synchronized public void removeAppender(Appender appender) { + if (appender == null || appenderPipeline == null) { + return; + } + appenderPipeline.removeAppender(appender); + } + + synchronized public void removeAppender(String name) { + if (name == null || appenderPipeline == null) { + return; + } + appenderPipeline.removeAppender(name); + } + + public void setAdditivity(boolean additive) { + this.additive = additive; + } + + public void setLevel(Level level) { + this.level = level; + } + + public void warn(Object message) { + if (getRepository().isDisabled(Level.WARN_INT)) { + return; + } + + if (Level.WARN.isGreaterOrEqual(this.getEffectiveLevel())) { + forcedLog(FQCN, Level.WARN, message, null); + } + } + + public void warn(Object message, Throwable t) { + if (getRepository().isDisabled(Level.WARN_INT)) { + return; + } + if (Level.WARN.isGreaterOrEqual(this.getEffectiveLevel())) { + forcedLog(FQCN, Level.WARN, message, t); + } + } + + public interface LoggerRepository { + + boolean isDisabled(int level); + + void setLogLevel(Level level); + + void emitNoAppenderWarning(Logger cat); + + Level getLogLevel(); + + Logger getLogger(String name); + + Logger getRootLogger(); + + Logger exists(String name); + + void shutdown(); + + Enumeration getCurrentLoggers(); + } + + public static class ProvisionNode extends Vector { + + ProvisionNode(Logger logger) { + super(); + addElement(logger); + } + } + + public static class DefaultLoggerRepository implements LoggerRepository { + + final Hashtable ht = new Hashtable(); + Logger root; + + int logLevelInt; + Level logLevel; + + boolean emittedNoAppenderWarning = false; + + public DefaultLoggerRepository(Logger root) { + this.root = root; + setLogLevel(Level.ALL); + } + + public void emitNoAppenderWarning(Logger cat) { + if (!this.emittedNoAppenderWarning) { + SysLogger.warn("No appenders could be found for logger (" + cat.getName() + ")."); + SysLogger.warn("Please initialize the logger system properly."); + this.emittedNoAppenderWarning = true; + } + } + + public Logger exists(String name) { + Object o = ht.get(new CategoryKey(name)); + if (o instanceof Logger) { + return (Logger) o; + } else { + return null; + } + } + + public void setLogLevel(Level l) { + if (l != null) { + logLevelInt = l.level; + logLevel = l; + } + } + + public Level getLogLevel() { + return logLevel; + } + + + public Logger getLogger(String name) { + CategoryKey key = new CategoryKey(name); + Logger logger; + + synchronized (ht) { + Object o = ht.get(key); + if (o == null) { + logger = makeNewLoggerInstance(name); + ht.put(key, logger); + updateParents(logger); + return logger; + } else if (o instanceof Logger) { + return (Logger) o; + } else if (o instanceof ProvisionNode) { + logger = makeNewLoggerInstance(name); + ht.put(key, logger); + updateChildren((ProvisionNode) o, logger); + updateParents(logger); + return logger; + } else { + return null; + } + } + } + + public Logger makeNewLoggerInstance(String name) { + return new Logger(name); + } + + public Enumeration getCurrentLoggers() { + Vector loggers = new Vector(ht.size()); + + Enumeration elems = ht.elements(); + while (elems.hasMoreElements()) { + Object o = elems.nextElement(); + if (o instanceof Logger) { + Logger logger = (Logger)o; + loggers.addElement(logger); + } + } + return loggers.elements(); + } + + + public Logger getRootLogger() { + return root; + } + + public boolean isDisabled(int level) { + return logLevelInt > level; + } + + + public void shutdown() { + Logger root = getRootLogger(); + root.closeNestedAppenders(); + + synchronized (ht) { + Enumeration cats = this.getCurrentLoggers(); + while (cats.hasMoreElements()) { + Logger c = (Logger) cats.nextElement(); + c.closeNestedAppenders(); + } + root.removeAllAppenders(); + } + } + + + private void updateParents(Logger cat) { + String name = cat.name; + int length = name.length(); + boolean parentFound = false; + + for (int i = name.lastIndexOf('.', length - 1); i >= 0; + i = name.lastIndexOf('.', i - 1)) { + String substr = name.substring(0, i); + + CategoryKey key = new CategoryKey(substr); + Object o = ht.get(key); + if (o == null) { + ht.put(key, new ProvisionNode(cat)); + } else if (o instanceof Logger) { + parentFound = true; + cat.parent = (Logger) o; + break; + } else if (o instanceof ProvisionNode) { + ((ProvisionNode) o).addElement(cat); + } else { + Exception e = new IllegalStateException("unexpected object type " + o.getClass() + " in ht."); + e.printStackTrace(); + } + } + if (!parentFound) { + cat.parent = root; + } + } + + private void updateChildren(ProvisionNode pn, Logger logger) { + final int last = pn.size(); + + for (int i = 0; i < last; i++) { + Logger l = pn.elementAt(i); + if (!l.parent.name.startsWith(logger.name)) { + logger.parent = l.parent; + l.parent = logger; + } + } + } + + private class CategoryKey { + + String name; + int hashCache; + + CategoryKey(String name) { + this.name = name; + hashCache = name.hashCode(); + } + + final public int hashCode() { + return hashCache; + } + + final public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o != null && o instanceof CategoryKey) { + CategoryKey cc = (CategoryKey) o; + return name.equals(cc.name); + } else { + return false; + } + } + } + + } + + public static class RootLogger extends Logger { + + public RootLogger(Level level) { + super("root"); + setLevel(level); + } + } +} diff --git a/logging/src/main/java/org/apache/rocketmq/logging/inner/LoggingBuilder.java b/logging/src/main/java/org/apache/rocketmq/logging/inner/LoggingBuilder.java new file mode 100644 index 00000000000..469cb52b290 --- /dev/null +++ b/logging/src/main/java/org/apache/rocketmq/logging/inner/LoggingBuilder.java @@ -0,0 +1,1230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.logging.inner; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FilterWriter; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.text.MessageFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collection; +import java.util.Date; +import java.util.Enumeration; +import java.util.GregorianCalendar; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.TimeZone; + +public class LoggingBuilder { + + public static final String SYSTEM_OUT = "System.out"; + public static final String SYSTEM_ERR = "System.err"; + + public static final String LOGGING_ENCODING = "rocketmq.logging.inner.encoding"; + public static final String ENCODING = System.getProperty(LOGGING_ENCODING, "UTF-8"); + + public static AppenderBuilder newAppenderBuilder() { + return new AppenderBuilder(); + } + + public static class AppenderBuilder { + private AsyncAppender asyncAppender; + + private Appender appender = null; + + private AppenderBuilder() { + + } + + public AppenderBuilder withLayout(Layout layout) { + appender.setLayout(layout); + return this; + } + + public AppenderBuilder withName(String name) { + appender.setName(name); + return this; + } + + public AppenderBuilder withConsoleAppender(String target) { + ConsoleAppender consoleAppender = new ConsoleAppender(); + consoleAppender.setTarget(target); + consoleAppender.activateOptions(); + this.appender = consoleAppender; + return this; + } + + public AppenderBuilder withFileAppender(String file) { + FileAppender appender = new FileAppender(); + appender.setFile(file); + appender.setAppend(true); + appender.setBufferedIO(false); + appender.setEncoding(ENCODING); + appender.setImmediateFlush(true); + appender.activateOptions(); + this.appender = appender; + return this; + } + + public AppenderBuilder withRollingFileAppender(String file, String maxFileSize, int maxFileIndex) { + RollingFileAppender appender = new RollingFileAppender(); + appender.setFile(file); + appender.setAppend(true); + appender.setBufferedIO(false); + appender.setEncoding(ENCODING); + appender.setImmediateFlush(true); + appender.setMaximumFileSize(Integer.parseInt(maxFileSize)); + appender.setMaxBackupIndex(maxFileIndex); + appender.activateOptions(); + this.appender = appender; + return this; + } + + public AppenderBuilder withDailyFileRollingAppender(String file, String datePattern) { + DailyRollingFileAppender appender = new DailyRollingFileAppender(); + appender.setFile(file); + appender.setAppend(true); + appender.setBufferedIO(false); + appender.setEncoding(ENCODING); + appender.setImmediateFlush(true); + appender.setDatePattern(datePattern); + appender.activateOptions(); + this.appender = appender; + return this; + } + + public AppenderBuilder withAsync(boolean blocking, int buffSize) { + AsyncAppender asyncAppender = new AsyncAppender(); + asyncAppender.setBlocking(blocking); + asyncAppender.setBufferSize(buffSize); + this.asyncAppender = asyncAppender; + return this; + } + + public Appender build() { + if (appender == null) { + throw new RuntimeException("please specify appender first"); + } + if (asyncAppender != null) { + asyncAppender.addAppender(appender); + return asyncAppender; + } else { + return appender; + } + } + } + + public static class AsyncAppender extends Appender implements Appender.AppenderPipeline { + + public static final int DEFAULT_BUFFER_SIZE = 128; + + private final List buffer = new ArrayList(); + + private final Map discardMap = new HashMap(); + + private int bufferSize = DEFAULT_BUFFER_SIZE; + + private final AppenderPipelineImpl appenderPipeline; + + private final Thread dispatcher; + + private boolean blocking = true; + + public AsyncAppender() { + appenderPipeline = new AppenderPipelineImpl(); + + dispatcher = new Thread(new Dispatcher(this, buffer, discardMap, appenderPipeline)); + + dispatcher.setDaemon(true); + + dispatcher.setName("AsyncAppender-Dispatcher-" + dispatcher.getName()); + dispatcher.start(); + } + + public void addAppender(final Appender newAppender) { + synchronized (appenderPipeline) { + appenderPipeline.addAppender(newAppender); + } + } + + public void append(final LoggingEvent event) { + if ((dispatcher == null) || !dispatcher.isAlive() || (bufferSize <= 0)) { + synchronized (appenderPipeline) { + appenderPipeline.appendLoopOnAppenders(event); + } + + return; + } + + event.getThreadName(); + event.getRenderedMessage(); + + synchronized (buffer) { + while (true) { + int previousSize = buffer.size(); + + if (previousSize < bufferSize) { + buffer.add(event); + + if (previousSize == 0) { + buffer.notifyAll(); + } + + break; + } + + boolean discard = true; + if (blocking + && !Thread.interrupted() + && Thread.currentThread() != dispatcher) { + try { + buffer.wait(); + discard = false; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + if (discard) { + String loggerName = event.getLoggerName(); + DiscardSummary summary = discardMap.get(loggerName); + + if (summary == null) { + summary = new DiscardSummary(event); + discardMap.put(loggerName, summary); + } else { + summary.add(event); + } + + break; + } + } + } + } + + public void close() { + + synchronized (buffer) { + closed = true; + buffer.notifyAll(); + } + + try { + dispatcher.join(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + SysLogger.error( + "Got an InterruptedException while waiting for the " + + "dispatcher to finish.", e); + } + + synchronized (appenderPipeline) { + Enumeration iter = appenderPipeline.getAllAppenders(); + if (iter != null) { + while (iter.hasMoreElements()) { + Object next = iter.nextElement(); + if (next instanceof Appender) { + ((Appender) next).close(); + } + } + } + } + } + + public Enumeration getAllAppenders() { + synchronized (appenderPipeline) { + return appenderPipeline.getAllAppenders(); + } + } + + public Appender getAppender(final String name) { + synchronized (appenderPipeline) { + return appenderPipeline.getAppender(name); + } + } + + public boolean isAttached(final Appender appender) { + synchronized (appenderPipeline) { + return appenderPipeline.isAttached(appender); + } + } + + public void removeAllAppenders() { + synchronized (appenderPipeline) { + appenderPipeline.removeAllAppenders(); + } + } + + public void removeAppender(final Appender appender) { + synchronized (appenderPipeline) { + appenderPipeline.removeAppender(appender); + } + } + + public void removeAppender(final String name) { + synchronized (appenderPipeline) { + appenderPipeline.removeAppender(name); + } + } + + public void setBufferSize(final int size) { + if (size < 0) { + throw new NegativeArraySizeException("size"); + } + + synchronized (buffer) { + bufferSize = (size < 1) ? 1 : size; + buffer.notifyAll(); + } + } + + public int getBufferSize() { + return bufferSize; + } + + public void setBlocking(final boolean value) { + synchronized (buffer) { + blocking = value; + buffer.notifyAll(); + } + } + + public boolean getBlocking() { + return blocking; + } + + private final class DiscardSummary { + + private LoggingEvent maxEvent; + + private int count; + + public DiscardSummary(final LoggingEvent event) { + maxEvent = event; + count = 1; + } + + public void add(final LoggingEvent event) { + if (event.getLevel().toInt() > maxEvent.getLevel().toInt()) { + maxEvent = event; + } + count++; + } + + public LoggingEvent createEvent() { + String msg = + MessageFormat.format( + "Discarded {0} messages due to full event buffer including: {1}", + count, maxEvent.getMessage()); + + return new LoggingEvent( + "AsyncAppender.DONT_REPORT_LOCATION", + Logger.getLogger(maxEvent.getLoggerName()), + maxEvent.getLevel(), + msg, + null); + } + } + + private class Dispatcher implements Runnable { + + private final AsyncAppender parent; + + private final List buffer; + + private final Map discardMap; + + private final AppenderPipelineImpl appenderPipeline; + + public Dispatcher( + final AsyncAppender parent, final List buffer, final Map discardMap, + final AppenderPipelineImpl appenderPipeline) { + + this.parent = parent; + this.buffer = buffer; + this.appenderPipeline = appenderPipeline; + this.discardMap = discardMap; + } + + public void run() { + boolean isActive = true; + + try { + while (isActive) { + LoggingEvent[] events = null; + + synchronized (buffer) { + int bufferSize = buffer.size(); + isActive = !parent.closed; + + while ((bufferSize == 0) && isActive) { + buffer.wait(); + bufferSize = buffer.size(); + isActive = !parent.closed; + } + + if (bufferSize > 0) { + events = new LoggingEvent[bufferSize + discardMap.size()]; + buffer.toArray(events); + + int index = bufferSize; + Collection values = discardMap.values(); + for (DiscardSummary value : values) { + events[index++] = value.createEvent(); + } + + buffer.clear(); + discardMap.clear(); + + buffer.notifyAll(); + } + } + if (events != null) { + for (LoggingEvent event : events) { + synchronized (appenderPipeline) { + appenderPipeline.appendLoopOnAppenders(event); + } + } + } + } + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } + } + } + } + + private static class QuietWriter extends FilterWriter { + + protected Appender appender; + + public QuietWriter(Writer writer, Appender appender) { + super(writer); + this.appender = appender; + } + + public void write(String string) { + if (string != null) { + try { + out.write(string); + } catch (Exception e) { + appender.handleError("Failed to write [" + string + "].", e, + Appender.CODE_WRITE_FAILURE); + } + } + } + + public void flush() { + try { + out.flush(); + } catch (Exception e) { + appender.handleError("Failed to flush writer,", e, + Appender.CODE_FLUSH_FAILURE); + } + } + } + + public static class WriterAppender extends Appender { + + + protected boolean immediateFlush = true; + + protected String encoding; + + + protected QuietWriter qw; + + public WriterAppender() { + + } + + public void setImmediateFlush(boolean value) { + immediateFlush = value; + } + + + public boolean getImmediateFlush() { + return immediateFlush; + } + + public void activateOptions() { + } + + + public void append(LoggingEvent event) { + if (!checkEntryConditions()) { + return; + } + subAppend(event); + } + + protected boolean checkEntryConditions() { + if (this.closed) { + SysLogger.warn("Not allowed to write to a closed appender."); + return false; + } + + if (this.qw == null) { + handleError("No output stream or file set for the appender named [" + + name + "]."); + return false; + } + + if (this.layout == null) { + handleError("No layout set for the appender named [" + name + "]."); + return false; + } + return true; + } + + public synchronized void close() { + if (this.closed) { + return; + } + this.closed = true; + writeFooter(); + reset(); + } + + protected void closeWriter() { + if (qw != null) { + try { + qw.close(); + } catch (IOException e) { + handleError("Could not close " + qw, e, CODE_CLOSE_FAILURE); + } + } + } + + protected OutputStreamWriter createWriter(OutputStream os) { + OutputStreamWriter retval = null; + + String enc = getEncoding(); + if (enc != null) { + try { + retval = new OutputStreamWriter(os, enc); + } catch (IOException e) { + SysLogger.warn("Error initializing output writer."); + SysLogger.warn("Unsupported encoding?"); + } + } + if (retval == null) { + retval = new OutputStreamWriter(os); + } + return retval; + } + + public String getEncoding() { + return encoding; + } + + public void setEncoding(String value) { + encoding = value; + } + + + public synchronized void setWriter(Writer writer) { + reset(); + this.qw = new QuietWriter(writer, this); + writeHeader(); + } + + protected void subAppend(LoggingEvent event) { + this.qw.write(this.layout.format(event)); + + if (layout.ignoresThrowable()) { + String[] s = event.getThrowableStr(); + if (s != null) { + for (String s1 : s) { + this.qw.write(s1); + this.qw.write(LINE_SEP); + } + } + } + + if (shouldFlush(event)) { + this.qw.flush(); + } + } + + protected void reset() { + closeWriter(); + this.qw = null; + } + + protected void writeFooter() { + if (layout != null) { + String f = layout.getFooter(); + if (f != null && this.qw != null) { + this.qw.write(f); + this.qw.flush(); + } + } + } + + protected void writeHeader() { + if (layout != null) { + String h = layout.getHeader(); + if (h != null && this.qw != null) { + this.qw.write(h); + } + } + } + + protected boolean shouldFlush(final LoggingEvent event) { + return event != null && immediateFlush; + } + } + + + public static class FileAppender extends WriterAppender { + + protected boolean fileAppend = true; + + protected String fileName = null; + + protected boolean bufferedIO = false; + + protected int bufferSize = 8 * 1024; + + public FileAppender() { + } + + public FileAppender(Layout layout, String filename, boolean append) + throws IOException { + this.layout = layout; + this.setFile(filename, append, false, bufferSize); + } + + public void setFile(String file) { + fileName = file.trim(); + } + + public boolean getAppend() { + return fileAppend; + } + + public String getFile() { + return fileName; + } + + public void activateOptions() { + if (fileName != null) { + try { + setFile(fileName, fileAppend, bufferedIO, bufferSize); + } catch (IOException e) { + handleError("setFile(" + fileName + "," + fileAppend + ") call failed.", + e, CODE_FILE_OPEN_FAILURE); + } + } else { + SysLogger.warn("File option not set for appender [" + name + "]."); + SysLogger.warn("Are you using FileAppender instead of ConsoleAppender?"); + } + } + + protected void closeFile() { + if (this.qw != null) { + try { + this.qw.close(); + } catch (IOException e) { + if (e instanceof InterruptedIOException) { + Thread.currentThread().interrupt(); + } + SysLogger.error("Could not close " + qw, e); + } + } + } + + public boolean getBufferedIO() { + return this.bufferedIO; + } + + public int getBufferSize() { + return this.bufferSize; + } + + public void setAppend(boolean flag) { + fileAppend = flag; + } + + public void setBufferedIO(boolean bufferedIO) { + this.bufferedIO = bufferedIO; + if (bufferedIO) { + immediateFlush = false; + } + } + + public void setBufferSize(int bufferSize) { + this.bufferSize = bufferSize; + } + + public synchronized void setFile(String fileName, boolean append, boolean bufferedIO, int bufferSize) + throws IOException { + SysLogger.debug("setFile called: " + fileName + ", " + append); + + if (bufferedIO) { + setImmediateFlush(false); + } + + reset(); + FileOutputStream ostream; + try { + ostream = new FileOutputStream(fileName, append); + } catch (FileNotFoundException ex) { + String parentName = new File(fileName).getParent(); + if (parentName != null) { + File parentDir = new File(parentName); + if (!parentDir.exists() && parentDir.mkdirs()) { + ostream = new FileOutputStream(fileName, append); + } else { + throw ex; + } + } else { + throw ex; + } + } + Writer fw = createWriter(ostream); + if (bufferedIO) { + fw = new BufferedWriter(fw, bufferSize); + } + this.setQWForFiles(fw); + this.fileName = fileName; + this.fileAppend = append; + this.bufferedIO = bufferedIO; + this.bufferSize = bufferSize; + writeHeader(); + SysLogger.debug("setFile ended"); + } + + protected void setQWForFiles(Writer writer) { + this.qw = new QuietWriter(writer, this); + } + + protected void reset() { + closeFile(); + this.fileName = null; + super.reset(); + } + } + + + public static class RollingFileAppender extends FileAppender { + + protected long maxFileSize = 10 * 1024 * 1024; + + protected int maxBackupIndex = 1; + + private long nextRollover = 0; + + public RollingFileAppender() { + super(); + } + + public int getMaxBackupIndex() { + return maxBackupIndex; + } + + public long getMaximumFileSize() { + return maxFileSize; + } + + public void rollOver() { + File target; + File file; + + if (qw != null) { + long size = ((CountingQuietWriter) qw).getCount(); + SysLogger.debug("rolling over count=" + size); + nextRollover = size + maxFileSize; + } + SysLogger.debug("maxBackupIndex=" + maxBackupIndex); + + boolean renameSucceeded = true; + if (maxBackupIndex > 0) { + file = new File(fileName + '.' + maxBackupIndex); + if (file.exists()) { + renameSucceeded = file.delete(); + } + + for (int i = maxBackupIndex - 1; i >= 1 && renameSucceeded; i--) { + file = new File(fileName + "." + i); + if (file.exists()) { + target = new File(fileName + '.' + (i + 1)); + SysLogger.debug("Renaming file " + file + " to " + target); + renameSucceeded = file.renameTo(target); + } + } + + if (renameSucceeded) { + target = new File(fileName + "." + 1); + + this.closeFile(); // keep windows happy. + + file = new File(fileName); + SysLogger.debug("Renaming file " + file + " to " + target); + renameSucceeded = file.renameTo(target); + + if (!renameSucceeded) { + try { + this.setFile(fileName, true, bufferedIO, bufferSize); + } catch (IOException e) { + if (e instanceof InterruptedIOException) { + Thread.currentThread().interrupt(); + } + SysLogger.error("setFile(" + fileName + ", true) call failed.", e); + } + } + } + } + + if (renameSucceeded) { + try { + this.setFile(fileName, false, bufferedIO, bufferSize); + nextRollover = 0; + } catch (IOException e) { + if (e instanceof InterruptedIOException) { + Thread.currentThread().interrupt(); + } + SysLogger.error("setFile(" + fileName + ", false) call failed.", e); + } + } + } + + public synchronized void setFile(String fileName, boolean append, boolean bufferedIO, int bufferSize) + throws IOException { + super.setFile(fileName, append, this.bufferedIO, this.bufferSize); + if (append) { + File f = new File(fileName); + ((CountingQuietWriter) qw).setCount(f.length()); + } + } + + public void setMaxBackupIndex(int maxBackups) { + this.maxBackupIndex = maxBackups; + } + + public void setMaximumFileSize(long maxFileSize) { + this.maxFileSize = maxFileSize; + } + + protected void setQWForFiles(Writer writer) { + this.qw = new CountingQuietWriter(writer, this); + } + + protected void subAppend(LoggingEvent event) { + super.subAppend(event); + if (fileName != null && qw != null) { + long size = ((CountingQuietWriter) qw).getCount(); + if (size >= maxFileSize && size >= nextRollover) { + rollOver(); + } + } + } + + protected class CountingQuietWriter extends QuietWriter { + + protected long count; + + public CountingQuietWriter(Writer writer, Appender appender) { + super(writer, appender); + } + + public void write(String string) { + try { + out.write(string); + count += string.length(); + } catch (IOException e) { + appender.handleError("Write failure.", e, Appender.CODE_WRITE_FAILURE); + } + } + + public long getCount() { + return count; + } + + public void setCount(long count) { + this.count = count; + } + + } + } + + + public static class DailyRollingFileAppender extends FileAppender { + + static final int TOP_OF_TROUBLE = -1; + static final int TOP_OF_MINUTE = 0; + static final int TOP_OF_HOUR = 1; + static final int HALF_DAY = 2; + static final int TOP_OF_DAY = 3; + static final int TOP_OF_WEEK = 4; + static final int TOP_OF_MONTH = 5; + + + /** + * The date pattern. By default, the pattern is set to + * "'.'yyyy-MM-dd" meaning daily rollover. + */ + private String datePattern = "'.'yyyy-MM-dd"; + + private String scheduledFilename; + + private long nextCheck = System.currentTimeMillis() - 1; + + Date now = new Date(); + + SimpleDateFormat sdf; + + RollingCalendar rc = new RollingCalendar(); + + final TimeZone gmtTimeZone = TimeZone.getTimeZone("GMT"); + + + public void setDatePattern(String pattern) { + datePattern = pattern; + } + + public String getDatePattern() { + return datePattern; + } + + public void activateOptions() { + super.activateOptions(); + if (datePattern != null && fileName != null) { + now.setTime(System.currentTimeMillis()); + sdf = new SimpleDateFormat(datePattern); + int type = computeCheckPeriod(); + printPeriodicity(type); + rc.setType(type); + File file = new File(fileName); + scheduledFilename = fileName + sdf.format(new Date(file.lastModified())); + + } else { + SysLogger.error("Either File or DatePattern options are not set for appender [" + name + "]."); + } + } + + void printPeriodicity(int type) { + switch (type) { + case TOP_OF_MINUTE: + SysLogger.debug("Appender [" + name + "] to be rolled every minute."); + break; + case TOP_OF_HOUR: + SysLogger.debug("Appender [" + name + "] to be rolled on top of every hour."); + break; + case HALF_DAY: + SysLogger.debug("Appender [" + name + "] to be rolled at midday and midnight."); + break; + case TOP_OF_DAY: + SysLogger.debug("Appender [" + name + "] to be rolled at midnight."); + break; + case TOP_OF_WEEK: + SysLogger.debug("Appender [" + name + "] to be rolled at start of week."); + break; + case TOP_OF_MONTH: + SysLogger.debug("Appender [" + name + "] to be rolled at start of every month."); + break; + default: + SysLogger.warn("Unknown periodicity for appender [" + name + "]."); + } + } + + int computeCheckPeriod() { + RollingCalendar rollingCalendar = new RollingCalendar(gmtTimeZone, Locale.getDefault()); + // set sate to 1970-01-01 00:00:00 GMT + Date epoch = new Date(0); + if (datePattern != null) { + for (int i = TOP_OF_MINUTE; i <= TOP_OF_MONTH; i++) { + SimpleDateFormat simpleDateFormat = new SimpleDateFormat(datePattern); + simpleDateFormat.setTimeZone(gmtTimeZone); + String r0 = simpleDateFormat.format(epoch); + rollingCalendar.setType(i); + Date next = new Date(rollingCalendar.getNextCheckMillis(epoch)); + String r1 = simpleDateFormat.format(next); + if (r0.equals(r1)) { + return i; + } + } + } + return TOP_OF_TROUBLE; + } + + void rollOver() throws IOException { + + if (datePattern == null) { + handleError("Missing DatePattern option in rollOver()."); + return; + } + + String datedFilename = fileName + sdf.format(now); + + if (scheduledFilename.equals(datedFilename)) { + return; + } + this.closeFile(); + + File target = new File(scheduledFilename); + if (target.exists() && !target.delete()) { + SysLogger.error("Failed to delete [" + scheduledFilename + "]."); + } + + File file = new File(fileName); + boolean result = file.renameTo(target); + if (result) { + SysLogger.debug(fileName + " -> " + scheduledFilename); + } else { + SysLogger.error("Failed to rename [" + fileName + "] to [" + scheduledFilename + "]."); + } + + try { + this.setFile(fileName, true, this.bufferedIO, this.bufferSize); + } catch (IOException e) { + handleError("setFile(" + fileName + ", true) call failed."); + } + scheduledFilename = datedFilename; + } + + protected void subAppend(LoggingEvent event) { + long n = System.currentTimeMillis(); + if (n >= nextCheck) { + now.setTime(n); + nextCheck = rc.getNextCheckMillis(now); + try { + rollOver(); + } catch (IOException ioe) { + if (ioe instanceof InterruptedIOException) { + Thread.currentThread().interrupt(); + } + SysLogger.error("rollOver() failed.", ioe); + } + } + super.subAppend(event); + } + } + + private static class RollingCalendar extends GregorianCalendar { + private static final long serialVersionUID = -3560331770601814177L; + + int type = DailyRollingFileAppender.TOP_OF_TROUBLE; + + RollingCalendar() { + super(); + } + + RollingCalendar(TimeZone tz, Locale locale) { + super(tz, locale); + } + + void setType(int type) { + this.type = type; + } + + public long getNextCheckMillis(Date now) { + return getNextCheckDate(now).getTime(); + } + + public Date getNextCheckDate(Date now) { + this.setTime(now); + + switch (type) { + case DailyRollingFileAppender.TOP_OF_MINUTE: + this.set(Calendar.SECOND, 0); + this.set(Calendar.MILLISECOND, 0); + this.add(Calendar.MINUTE, 1); + break; + case DailyRollingFileAppender.TOP_OF_HOUR: + this.set(Calendar.MINUTE, 0); + this.set(Calendar.SECOND, 0); + this.set(Calendar.MILLISECOND, 0); + this.add(Calendar.HOUR_OF_DAY, 1); + break; + case DailyRollingFileAppender.HALF_DAY: + this.set(Calendar.MINUTE, 0); + this.set(Calendar.SECOND, 0); + this.set(Calendar.MILLISECOND, 0); + int hour = get(Calendar.HOUR_OF_DAY); + if (hour < 12) { + this.set(Calendar.HOUR_OF_DAY, 12); + } else { + this.set(Calendar.HOUR_OF_DAY, 0); + this.add(Calendar.DAY_OF_MONTH, 1); + } + break; + case DailyRollingFileAppender.TOP_OF_DAY: + this.set(Calendar.HOUR_OF_DAY, 0); + this.set(Calendar.MINUTE, 0); + this.set(Calendar.SECOND, 0); + this.set(Calendar.MILLISECOND, 0); + this.add(Calendar.DATE, 1); + break; + case DailyRollingFileAppender.TOP_OF_WEEK: + this.set(Calendar.DAY_OF_WEEK, getFirstDayOfWeek()); + this.set(Calendar.HOUR_OF_DAY, 0); + this.set(Calendar.MINUTE, 0); + this.set(Calendar.SECOND, 0); + this.set(Calendar.MILLISECOND, 0); + this.add(Calendar.WEEK_OF_YEAR, 1); + break; + case DailyRollingFileAppender.TOP_OF_MONTH: + this.set(Calendar.DATE, 1); + this.set(Calendar.HOUR_OF_DAY, 0); + this.set(Calendar.MINUTE, 0); + this.set(Calendar.SECOND, 0); + this.set(Calendar.MILLISECOND, 0); + this.add(Calendar.MONTH, 1); + break; + default: + throw new IllegalStateException("Unknown periodicity type."); + } + return getTime(); + } + } + + public static class ConsoleAppender extends WriterAppender { + + protected String target = SYSTEM_OUT; + + public ConsoleAppender() { + } + + public void setTarget(String value) { + String v = value.trim(); + + if (SYSTEM_OUT.equalsIgnoreCase(v)) { + target = SYSTEM_OUT; + } else if (SYSTEM_ERR.equalsIgnoreCase(v)) { + target = SYSTEM_ERR; + } else { + targetWarn(value); + } + } + + public String getTarget() { + return target; + } + + void targetWarn(String val) { + SysLogger.warn("[" + val + "] should be System.out or System.err."); + SysLogger.warn("Using previously set target, System.out by default."); + } + + public void activateOptions() { + if (target.equals(SYSTEM_ERR)) { + setWriter(createWriter(System.err)); + } else { + setWriter(createWriter(System.out)); + } + super.activateOptions(); + } + + protected final void closeWriter() { + + } + } + + public static LayoutBuilder newLayoutBuilder() { + return new LayoutBuilder(); + } + + public static class LayoutBuilder { + + private Layout layout; + + public LayoutBuilder withSimpleLayout() { + layout = new SimpleLayout(); + return this; + } + + public LayoutBuilder withDefaultLayout() { + layout = new DefaultLayout(); + return this; + } + + public Layout build() { + if (layout == null) { + layout = new SimpleLayout(); + } + return layout; + } + } + + public static class SimpleLayout extends Layout { + + @Override + public String format(LoggingEvent event) { + + StringBuilder sb = new StringBuilder(); + sb.append(event.getLevel().toString()); + sb.append(" - "); + sb.append(event.getRenderedMessage()); + sb.append("\r\n"); + return sb.toString(); + } + + @Override + public boolean ignoresThrowable() { + return false; + } + } + + + /** + * %d{yyy-MM-dd HH:mm:ss,SSS} %p %c{1}%L - %m%n + */ + public static class DefaultLayout extends Layout { + @Override + public String format(LoggingEvent event) { + + StringBuilder sb = new StringBuilder(); + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,sss"); + String format = simpleDateFormat.format(new Date(event.timeStamp)); + sb.append(format); + sb.append(" "); + sb.append(event.getLevel()); + sb.append(" "); + sb.append(event.getLoggerName()); + sb.append(" - "); + sb.append(event.getMessage()); + String[] throwableStr = event.getThrowableStr(); + if (throwableStr != null) { + sb.append("\r\n"); + for (String s : throwableStr) { + sb.append(s); + sb.append("\r\n"); + } + } + sb.append("\r\n"); + return sb.toString(); + } + + @Override + public boolean ignoresThrowable() { + return false; + } + } +} diff --git a/logging/src/main/java/org/apache/rocketmq/logging/inner/LoggingEvent.java b/logging/src/main/java/org/apache/rocketmq/logging/inner/LoggingEvent.java new file mode 100644 index 00000000000..1b3e9553933 --- /dev/null +++ b/logging/src/main/java/org/apache/rocketmq/logging/inner/LoggingEvent.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.logging.inner; + +import java.io.IOException; +import java.io.InterruptedIOException; +import java.io.LineNumberReader; +import java.io.PrintWriter; +import java.io.StringReader; +import java.io.StringWriter; +import java.util.ArrayList; + +public class LoggingEvent implements java.io.Serializable { + + transient public final String fqnOfCategoryClass; + + transient private Object message; + + transient private Level level; + + transient private Logger logger; + + private String renderedMessage; + + private String threadName; + + public final long timeStamp; + + private Throwable throwable; + + public LoggingEvent(String fqnOfCategoryClass, Logger logger, + Level level, Object message, Throwable throwable) { + this.fqnOfCategoryClass = fqnOfCategoryClass; + this.message = message; + this.logger = logger; + this.throwable = throwable; + this.level = level; + timeStamp = System.currentTimeMillis(); + } + + public Object getMessage() { + if (message != null) { + return message; + } else { + return getRenderedMessage(); + } + } + + public String getRenderedMessage() { + if (renderedMessage == null && message != null) { + if (message instanceof String) { + renderedMessage = (String) message; + } else { + renderedMessage = message.toString(); + } + } + return renderedMessage; + } + + public String getThreadName() { + if (threadName == null) { + threadName = (Thread.currentThread()).getName(); + } + return threadName; + } + + public Level getLevel() { + return level; + } + + public String getLoggerName() { + return logger.getName(); + } + + public String[] getThrowableStr() { + if (throwable == null) { + return null; + } + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + try { + throwable.printStackTrace(pw); + } catch (RuntimeException ex) { + SysLogger.warn("InnerLogger print stack trace error", ex); + } + pw.flush(); + LineNumberReader reader = new LineNumberReader( + new StringReader(sw.toString())); + ArrayList lines = new ArrayList(); + try { + String line = reader.readLine(); + while (line != null) { + lines.add(line); + line = reader.readLine(); + } + } catch (IOException ex) { + if (ex instanceof InterruptedIOException) { + Thread.currentThread().interrupt(); + } + lines.add(ex.toString()); + } + String[] tempRep = new String[lines.size()]; + lines.toArray(tempRep); + return tempRep; + } +} diff --git a/logging/src/main/java/org/apache/rocketmq/logging/inner/SysLogger.java b/logging/src/main/java/org/apache/rocketmq/logging/inner/SysLogger.java new file mode 100755 index 00000000000..b6d10497782 --- /dev/null +++ b/logging/src/main/java/org/apache/rocketmq/logging/inner/SysLogger.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.logging.inner; + +public class SysLogger { + + protected static boolean debugEnabled = false; + + private static boolean quietMode = false; + + private static final String PREFIX = "RocketMQLog: "; + private static final String ERR_PREFIX = "RocketMQLog:ERROR "; + private static final String WARN_PREFIX = "RocketMQLog:WARN "; + + public static void setInternalDebugging(boolean enabled) { + debugEnabled = enabled; + } + + public static void debug(String msg) { + if (debugEnabled && !quietMode) { + System.out.printf("%s", PREFIX + msg); + } + } + + public static void debug(String msg, Throwable t) { + if (debugEnabled && !quietMode) { + System.out.printf("%s", PREFIX + msg); + if (t != null) { + t.printStackTrace(System.out); + } + } + } + + public static void error(String msg) { + if (quietMode) { + return; + } + System.err.println(ERR_PREFIX + msg); + } + + public static void error(String msg, Throwable t) { + if (quietMode) { + return; + } + + System.err.println(ERR_PREFIX + msg); + if (t != null) { + t.printStackTrace(); + } + } + + public static void setQuietMode(boolean quietMode) { + SysLogger.quietMode = quietMode; + } + + public static void warn(String msg) { + if (quietMode) { + return; + } + + System.err.println(WARN_PREFIX + msg); + } + + public static void warn(String msg, Throwable t) { + if (quietMode) { + return; + } + + System.err.println(WARN_PREFIX + msg); + if (t != null) { + t.printStackTrace(); + } + } +} diff --git a/logging/src/main/java/org/apache/rocketmq/logging/package-info.java b/logging/src/main/java/org/apache/rocketmq/logging/package-info.java new file mode 100644 index 00000000000..7cb0645de0c --- /dev/null +++ b/logging/src/main/java/org/apache/rocketmq/logging/package-info.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.logging; + +/* + This package is a minimal logger on the basis of Apache Log4j without + file configuration and pattern layout configuration. Main forked files are + followed as below: + 1. LoggingEvent + 2. Logger + 3. Layout + 4. Level + 5. AsyncAppender + 6. FileAppender + 7. RollingFileAppender + 8. DailyRollingFileAppender + 9. ConsoleAppender + + For more information about Apache Log4j, please go to https://github.com/apache/log4j. + */ \ No newline at end of file diff --git a/logging/src/test/java/org/apache/rocketmq/logging/BasicloggerTest.java b/logging/src/test/java/org/apache/rocketmq/logging/BasicloggerTest.java new file mode 100644 index 00000000000..28496dd5674 --- /dev/null +++ b/logging/src/test/java/org/apache/rocketmq/logging/BasicloggerTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.logging; + +import org.apache.rocketmq.logging.inner.Level; +import org.apache.rocketmq.logging.inner.Logger; +import org.apache.rocketmq.logging.inner.LoggingEvent; +import org.junit.After; +import org.junit.Before; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; + +public class BasicLoggerTest { + + protected Logger logger = Logger.getLogger("test"); + + protected LoggingEvent loggingEvent; + + protected String loggingDir = System.getProperty("user.home") + "/logs/rocketmq-test"; + + @Before + public void createLoggingEvent() { + loggingEvent = new LoggingEvent(Logger.class.getName(), logger, Level.INFO, + "junit test error", new RuntimeException("createLogging error")); + } + + public String readFile(String file) throws IOException { + StringBuilder stringBuilder = new StringBuilder(); + FileInputStream fileInputStream = new FileInputStream(file); + BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(fileInputStream)); + String line = bufferedReader.readLine(); + while (line != null) { + stringBuilder.append(line); + stringBuilder.append("\r\n"); + line = bufferedReader.readLine(); + } + bufferedReader.close(); + return stringBuilder.toString(); + } + + @After + public void clean() { + File file = new File(loggingDir); + if (file.exists()) { + File[] files = file.listFiles(); + for (File file1 : files) { + file1.delete(); + } + } + } +} diff --git a/logging/src/test/java/org/apache/rocketmq/logging/InnerLoggerFactoryTest.java b/logging/src/test/java/org/apache/rocketmq/logging/InnerLoggerFactoryTest.java new file mode 100644 index 00000000000..c47dba6840a --- /dev/null +++ b/logging/src/test/java/org/apache/rocketmq/logging/InnerLoggerFactoryTest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.logging; + +import org.apache.rocketmq.logging.inner.Appender; +import org.apache.rocketmq.logging.inner.Level; +import org.apache.rocketmq.logging.inner.Logger; +import org.apache.rocketmq.logging.inner.LoggingBuilder; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +public class InnerLoggerFactoryTest extends BasicLoggerTest { + + private ByteArrayOutputStream byteArrayOutputStream; + + public static final String LOGGER = "ConsoleLogger"; + + private PrintStream out; + + @Before + public void initLogger() { + out = System.out; + byteArrayOutputStream = new ByteArrayOutputStream(); + System.setOut(new PrintStream(byteArrayOutputStream)); + + Appender consoleAppender = LoggingBuilder.newAppenderBuilder() + .withConsoleAppender(LoggingBuilder.SYSTEM_OUT) + .withLayout(LoggingBuilder.newLayoutBuilder().withDefaultLayout().build()).build(); + + Logger consoleLogger = Logger.getLogger("ConsoleLogger"); + consoleLogger.setAdditivity(false); + consoleLogger.addAppender(consoleAppender); + consoleLogger.setLevel(Level.INFO); + } + + @After + public void fixConsole() { + System.setOut(out); + } + + @Test + public void testInnerLoggerFactory() { + InternalLoggerFactory.setCurrentLoggerType(InternalLoggerFactory.LOGGER_INNER); + + InternalLogger logger1 = InnerLoggerFactory.getLogger(LOGGER); + InternalLogger logger = InternalLoggerFactory.getLogger(LOGGER); + + Assert.assertTrue(logger.getName().equals(logger1.getName())); + + InternalLogger logger2 = InnerLoggerFactory.getLogger(InnerLoggerFactoryTest.class); + InnerLoggerFactory.InnerLogger logger3 = (InnerLoggerFactory.InnerLogger) logger2; + + logger.info("innerLogger inner info Message"); + logger.error("innerLogger inner error Message", new RuntimeException()); + logger.debug("innerLogger inner debug message"); + logger3.info("innerLogger info message"); + logger3.error("logback error message"); + logger3.info("info {}", "hahahah"); + logger3.warn("warn {}", "hahahah"); + logger3.warn("logger3 warn"); + logger3.error("error {}", "hahahah"); + logger3.debug("debug {}", "hahahah"); + + String content = new String(byteArrayOutputStream.toByteArray()); + System.out.println(content); + + Assert.assertTrue(content.contains("InnerLoggerFactoryTest")); + Assert.assertTrue(content.contains("info")); + Assert.assertTrue(content.contains("RuntimeException")); + Assert.assertTrue(!content.contains("debug")); + } +} diff --git a/logging/src/test/java/org/apache/rocketmq/logging/InternalLoggerTest.java b/logging/src/test/java/org/apache/rocketmq/logging/InternalLoggerTest.java new file mode 100644 index 00000000000..04b9f06e880 --- /dev/null +++ b/logging/src/test/java/org/apache/rocketmq/logging/InternalLoggerTest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.logging; + +import org.apache.rocketmq.logging.inner.Appender; +import org.apache.rocketmq.logging.inner.Level; +import org.apache.rocketmq.logging.inner.Logger; +import org.apache.rocketmq.logging.inner.LoggingBuilder; +import org.apache.rocketmq.logging.inner.SysLogger; +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + + +public class InternalLoggerTest { + + + @Test + public void testInternalLogger() { + SysLogger.setQuietMode(false); + SysLogger.setInternalDebugging(true); + PrintStream out = System.out; + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + System.setOut(new PrintStream(byteArrayOutputStream)); + + Appender consoleAppender = LoggingBuilder.newAppenderBuilder() + .withConsoleAppender(LoggingBuilder.SYSTEM_OUT) + .withLayout(LoggingBuilder.newLayoutBuilder().withDefaultLayout().build()).build(); + + + Logger consoleLogger = Logger.getLogger("ConsoleLogger"); + consoleLogger.setAdditivity(false); + consoleLogger.addAppender(consoleAppender); + consoleLogger.setLevel(Level.INFO); + + Logger.getRootLogger().addAppender(consoleAppender); + + InternalLoggerFactory.setCurrentLoggerType(InternalLoggerFactory.LOGGER_INNER); + InternalLogger logger = InternalLoggerFactory.getLogger(InternalLoggerTest.class); + InternalLogger consoleLogger1 = InternalLoggerFactory.getLogger("ConsoleLogger"); + + consoleLogger1.warn("simple warn {}", 14555); + + logger.info("testInternalLogger"); + consoleLogger1.info("consoleLogger1"); + + System.setOut(out); + consoleAppender.close(); + + String result = new String(byteArrayOutputStream.toByteArray()); + Assert.assertTrue(result.contains("consoleLogger1")); + Assert.assertTrue(result.contains("testInternalLogger")); + } + +} diff --git a/logging/src/test/java/org/apache/rocketmq/logging/Slf4jLoggerFactoryTest.java b/logging/src/test/java/org/apache/rocketmq/logging/Slf4jLoggerFactoryTest.java new file mode 100644 index 00000000000..ba6ec3ba68c --- /dev/null +++ b/logging/src/test/java/org/apache/rocketmq/logging/Slf4jLoggerFactoryTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.logging; + +import ch.qos.logback.classic.joran.JoranConfigurator; +import ch.qos.logback.core.Context; +import ch.qos.logback.core.joran.spi.JoranException; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URL; + +public class Slf4jLoggerFactoryTest extends BasicLoggerTest { + + public static final String LOGGER = "Slf4jTestLogger"; + + @Before + public void initLogback() throws JoranException { + InternalLoggerFactory.setCurrentLoggerType(InternalLoggerFactory.LOGGER_SLF4J); + System.setProperty("loggingDir", loggingDir); + ILoggerFactory iLoggerFactory = LoggerFactory.getILoggerFactory(); + JoranConfigurator joranConfigurator = new JoranConfigurator(); + joranConfigurator.setContext((Context) iLoggerFactory); + URL logbackConfigFile = Slf4jLoggerFactoryTest.class.getClassLoader().getResource("logback_test.xml"); + if (logbackConfigFile == null) { + throw new RuntimeException("can't find logback_test.xml"); + } else { + joranConfigurator.doConfigure(logbackConfigFile); + } + } + + @Test + public void testSlf4j() throws IOException { + InternalLogger logger1 = Slf4jLoggerFactory.getLogger(LOGGER); + InternalLogger logger = InternalLoggerFactory.getLogger(LOGGER); + Assert.assertTrue(logger.getName().equals(logger1.getName())); + InternalLogger logger2 = Slf4jLoggerFactory.getLogger(Slf4jLoggerFactoryTest.class); + Slf4jLoggerFactory.Slf4jLogger logger3 = (Slf4jLoggerFactory.Slf4jLogger) logger2; + + String file = loggingDir + "/logback_test.log"; + + logger.info("logback slf4j info Message"); + logger.error("logback slf4j error Message", new RuntimeException()); + logger.debug("logback slf4j debug message"); + logger3.info("logback info message"); + logger3.error("logback error message"); + logger3.info("info {}", "hahahah"); + logger3.warn("warn {}", "hahahah"); + logger3.warn("logger3 warn"); + logger3.error("error {}", "hahahah"); + logger3.debug("debug {}", "hahahah"); + String content = readFile(file); + System.out.println(content); + + Assert.assertTrue(content.contains("Slf4jLoggerFactoryTest")); + Assert.assertTrue(content.contains("info")); + Assert.assertTrue(content.contains("RuntimeException")); + Assert.assertTrue(!content.contains("debug")); + } + +} diff --git a/logging/src/test/java/org/apache/rocketmq/logging/inner/AppenderTest.java b/logging/src/test/java/org/apache/rocketmq/logging/inner/AppenderTest.java new file mode 100644 index 00000000000..37ff8bd47f6 --- /dev/null +++ b/logging/src/test/java/org/apache/rocketmq/logging/inner/AppenderTest.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.logging.inner; + +import org.apache.rocketmq.logging.BasicLoggerTest; +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; + +public class AppenderTest extends BasicLoggerTest { + + @Test + public void testConsole() { + SysLogger.setQuietMode(false); + SysLogger.setInternalDebugging(true); + PrintStream out = System.out; + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + System.setOut(new PrintStream(byteArrayOutputStream)); + + Appender consoleAppender = LoggingBuilder.newAppenderBuilder() + .withConsoleAppender(LoggingBuilder.SYSTEM_OUT) + .withLayout(LoggingBuilder.newLayoutBuilder().withDefaultLayout().build()).build(); + + LoggingBuilder.ConsoleAppender consoleAppender1 = (LoggingBuilder.ConsoleAppender) consoleAppender; + String target = consoleAppender1.getTarget(); + Assert.assertTrue(target.equals(LoggingBuilder.SYSTEM_OUT)); + + Layout layout = consoleAppender.getLayout(); + Assert.assertTrue(layout instanceof LoggingBuilder.DefaultLayout); + + Logger consoleLogger = Logger.getLogger("ConsoleLogger"); + consoleLogger.setAdditivity(false); + consoleLogger.addAppender(consoleAppender); + consoleLogger.setLevel(Level.INFO); + + Logger.getRootLogger().addAppender(consoleAppender); + Logger.getLogger(AppenderTest.class).info("this is a AppenderTest log"); + + Logger.getLogger("ConsoleLogger").info("console info Message"); + Logger.getLogger("ConsoleLogger").error("console error Message", new RuntimeException()); + Logger.getLogger("ConsoleLogger").debug("console debug message"); + System.setOut(out); + consoleAppender.close(); + + String result = new String(byteArrayOutputStream.toByteArray()); + + Assert.assertTrue(result.contains("info")); + Assert.assertTrue(result.contains("RuntimeException")); + Assert.assertTrue(!result.contains("debug")); + Assert.assertTrue(result.contains("AppenderTest")); + } + + @Test + public void testInnerFile() throws IOException { + String file = loggingDir + "/logger.log"; + + Logger fileLogger = Logger.getLogger("fileLogger"); + + Appender myappender = LoggingBuilder.newAppenderBuilder() + .withDailyFileRollingAppender(file, "'.'yyyy-MM-dd") + .withName("myappender") + .withLayout(LoggingBuilder.newLayoutBuilder().withDefaultLayout().build()).build(); + + fileLogger.addAppender(myappender); + + Logger.getLogger("fileLogger").setLevel(Level.INFO); + + Logger.getLogger("fileLogger").info("fileLogger info Message"); + Logger.getLogger("fileLogger").error("fileLogger error Message", new RuntimeException()); + Logger.getLogger("fileLogger").debug("fileLogger debug message"); + + myappender.close(); + + String content = readFile(file); + + System.out.println(content); + + Assert.assertTrue(content.contains("info")); + Assert.assertTrue(content.contains("RuntimeException")); + Assert.assertTrue(!content.contains("debug")); + } + + + + @Test + public void asyncAppenderTest() { + Appender appender = LoggingBuilder.newAppenderBuilder().withAsync(false, 1024) + .withConsoleAppender(LoggingBuilder.SYSTEM_OUT) + .withLayout(LoggingBuilder.newLayoutBuilder().withDefaultLayout().build()).build(); + Assert.assertTrue(appender instanceof LoggingBuilder.AsyncAppender); + LoggingBuilder.AsyncAppender asyncAppender = (LoggingBuilder.AsyncAppender) appender; + Assert.assertTrue(!asyncAppender.getBlocking()); + Assert.assertTrue(asyncAppender.getBufferSize() > 0); + } + + @Test + public void testWriteAppender() { + LoggingBuilder.WriterAppender writerAppender = new LoggingBuilder.WriterAppender(); + writerAppender.setImmediateFlush(true); + Assert.assertTrue(writerAppender.getImmediateFlush()); + } + + @Test + public void testFileAppender() throws IOException { + LoggingBuilder.FileAppender fileAppender = new LoggingBuilder.FileAppender( + new LoggingBuilder.SimpleLayout(), loggingDir + "/simple.log", true); + fileAppender.setBufferSize(1024); + int bufferSize = fileAppender.getBufferSize(); + boolean bufferedIO = fileAppender.getBufferedIO(); + Assert.assertTrue(!bufferedIO); + Assert.assertTrue(bufferSize > 0); + Assert.assertTrue(fileAppender.getAppend()); + + LoggingBuilder.RollingFileAppender rollingFileAppender = new LoggingBuilder.RollingFileAppender(); + rollingFileAppender.setImmediateFlush(true); + rollingFileAppender.setMaximumFileSize(1024 * 1024); + rollingFileAppender.setMaxBackupIndex(10); + rollingFileAppender.setAppend(true); + rollingFileAppender.setFile(loggingDir + "/rolling_file.log"); + rollingFileAppender.setName("myRollingFileAppender"); + + rollingFileAppender.activateOptions(); + + Assert.assertTrue(rollingFileAppender.getMaximumFileSize() > 0); + Assert.assertTrue(rollingFileAppender.getMaxBackupIndex() == 10); + } + + @Test + public void testDailyRollingAppender() { + LoggingBuilder.DailyRollingFileAppender dailyRollingFileAppender = new LoggingBuilder.DailyRollingFileAppender(); + dailyRollingFileAppender.setFile(loggingDir + "/daily.log"); + dailyRollingFileAppender.setName("dailyAppender"); + dailyRollingFileAppender.setAppend(true); + dailyRollingFileAppender.setDatePattern("'.'yyyy-mm-dd"); + String datePattern = dailyRollingFileAppender.getDatePattern(); + Assert.assertTrue(datePattern != null); + dailyRollingFileAppender.activateOptions(); + } + +} + + diff --git a/logging/src/test/java/org/apache/rocketmq/logging/inner/LayoutTest.java b/logging/src/test/java/org/apache/rocketmq/logging/inner/LayoutTest.java new file mode 100644 index 00000000000..66ef18eaeb6 --- /dev/null +++ b/logging/src/test/java/org/apache/rocketmq/logging/inner/LayoutTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.logging.inner; + +import org.apache.rocketmq.logging.BasicLoggerTest; +import org.junit.Assert; +import org.junit.Test; + +public class LayoutTest extends BasicLoggerTest { + + @Test + public void testSimpleLayout() { + Layout layout = LoggingBuilder.newLayoutBuilder().withSimpleLayout().build(); + String format = layout.format(loggingEvent); + Assert.assertTrue(format.contains("junit")); + } + + @Test + public void testDefaultLayout() { + Layout layout = LoggingBuilder.newLayoutBuilder().withDefaultLayout().build(); + String format = layout.format(loggingEvent); + String contentType = layout.getContentType(); + Assert.assertTrue(contentType.contains("text")); + Assert.assertTrue(format.contains("createLoggingEvent")); + Assert.assertTrue(format.contains("createLogging error")); + Assert.assertTrue(format.contains(Thread.currentThread().getName())); + } + + @Test + public void testLogFormat() { + Layout innerLayout = LoggingBuilder.newLayoutBuilder().withDefaultLayout().build(); + + LoggingEvent loggingEvent = new LoggingEvent(Logger.class.getName(), logger, org.apache.rocketmq.logging.inner.Level.INFO, + "junit test error", null); + String format = innerLayout.format(loggingEvent); + + System.out.println(format); + } +} diff --git a/logging/src/test/java/org/apache/rocketmq/logging/inner/LevelTest.java b/logging/src/test/java/org/apache/rocketmq/logging/inner/LevelTest.java new file mode 100644 index 00000000000..21667e1480f --- /dev/null +++ b/logging/src/test/java/org/apache/rocketmq/logging/inner/LevelTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.logging.inner; + +import org.junit.Assert; +import org.junit.Test; + +public class LevelTest { + + @Test + public void levelTest() { + Level info = Level.toLevel("info"); + Level error = Level.toLevel(3); + Assert.assertTrue(error != null && info != null); + } + + @Test + public void loggerLevel(){ + Level level = Logger.getRootLogger().getLevel(); + Assert.assertTrue(level!=null); + } +} diff --git a/logging/src/test/java/org/apache/rocketmq/logging/inner/LoggerRepositoryTest.java b/logging/src/test/java/org/apache/rocketmq/logging/inner/LoggerRepositoryTest.java new file mode 100644 index 00000000000..6a56c20ff7a --- /dev/null +++ b/logging/src/test/java/org/apache/rocketmq/logging/inner/LoggerRepositoryTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.logging.inner; + +import org.apache.rocketmq.logging.BasicLoggerTest; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Enumeration; + +public class LoggerRepositoryTest extends BasicLoggerTest { + + @Test + public void testLoggerRepository() { + Logger.getRepository().setLogLevel(Level.INFO); + + String file = loggingDir + "/repo.log"; + Logger fileLogger = Logger.getLogger("repoLogger"); + + Appender myappender = LoggingBuilder.newAppenderBuilder() + .withDailyFileRollingAppender(file, "'.'yyyy-MM-dd") + .withName("repoAppender") + .withLayout(LoggingBuilder.newLayoutBuilder().withDefaultLayout().build()).build(); + + fileLogger.addAppender(myappender); + Logger.getLogger("repoLogger").setLevel(Level.INFO); + Logger repoLogger = Logger.getRepository().exists("repoLogger"); + Assert.assertTrue(repoLogger != null); + Enumeration currentLoggers = Logger.getRepository().getCurrentLoggers(); + Level logLevel = Logger.getRepository().getLogLevel(); + Assert.assertTrue(logLevel.equals(Level.INFO)); +// Logger.getRepository().shutdown(); + } +} diff --git a/logging/src/test/java/org/apache/rocketmq/logging/inner/LoggerTest.java b/logging/src/test/java/org/apache/rocketmq/logging/inner/LoggerTest.java new file mode 100644 index 00000000000..4e738e230cd --- /dev/null +++ b/logging/src/test/java/org/apache/rocketmq/logging/inner/LoggerTest.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.logging.inner; + +import org.apache.rocketmq.logging.BasicLoggerTest; +import org.apache.rocketmq.logging.InnerLoggerFactory; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; + +public class LoggerTest extends BasicLoggerTest { + + + @Before + public void init() { + InternalLoggerFactory.setCurrentLoggerType(InnerLoggerFactory.LOGGER_INNER); + } + + @Test + public void testInnerConsoleLogger() throws IOException { + PrintStream out = System.out; + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + System.setOut(new PrintStream(byteArrayOutputStream)); + + Appender consoleAppender = LoggingBuilder.newAppenderBuilder() + .withConsoleAppender(LoggingBuilder.SYSTEM_OUT) + .withLayout(LoggingBuilder.newLayoutBuilder().withDefaultLayout().build()).build(); + + Logger.getLogger("ConsoleLogger").addAppender(consoleAppender); + Logger.getLogger("ConsoleLogger").setLevel(Level.INFO); + + InternalLogger consoleLogger1 = InternalLoggerFactory.getLogger("ConsoleLogger"); + consoleLogger1.info("console info Message"); + consoleLogger1.error("console error Message", new RuntimeException()); + consoleLogger1.debug("console debug message"); + + consoleLogger1.info("console {} test", "simple"); + consoleLogger1.info("[WATERMARK] Send Queue Size: {} SlowTimeMills: {}", 1, 300); + consoleLogger1.info("new consumer connected, group: {} {} {} channel: {}", "mygroup", "orderly", + "broudcast", new RuntimeException("simple object")); + + System.setOut(out); + consoleAppender.close(); + + String result = new String(byteArrayOutputStream.toByteArray()); + + System.out.println(result); + + Assert.assertTrue(result.contains("info")); + Assert.assertTrue(result.contains("RuntimeException")); + Assert.assertTrue(result.contains("WATERMARK")); + Assert.assertTrue(result.contains("consumer")); + Assert.assertTrue(result.contains("broudcast")); + Assert.assertTrue(result.contains("simple test")); + Assert.assertTrue(!result.contains("debug")); + } + + @Test + public void testInnerFileLogger() throws IOException { + String file = loggingDir + "/inner.log"; + + Logger fileLogger = Logger.getLogger("innerLogger"); + + Appender myappender = LoggingBuilder.newAppenderBuilder() + .withDailyFileRollingAppender(file, "'.'yyyy-MM-dd") + .withName("innerAppender") + .withLayout(LoggingBuilder.newLayoutBuilder().withDefaultLayout().build()).build(); + + fileLogger.addAppender(myappender); + fileLogger.setLevel(Level.INFO); + + InternalLogger innerLogger = InternalLoggerFactory.getLogger("innerLogger"); + + innerLogger.info("fileLogger info Message"); + innerLogger.error("fileLogger error Message", new RuntimeException()); + innerLogger.debug("fileLogger debug message"); + + myappender.close(); + + String content = readFile(file); + + System.out.println(content); + + Assert.assertTrue(content.contains("info")); + Assert.assertTrue(content.contains("RuntimeException")); + Assert.assertTrue(!content.contains("debug")); + } + + @After + public void close() { + InternalLoggerFactory.setCurrentLoggerType(null); + } +} diff --git a/logging/src/test/java/org/apache/rocketmq/logging/inner/LoggingBuilderTest.java b/logging/src/test/java/org/apache/rocketmq/logging/inner/LoggingBuilderTest.java new file mode 100644 index 00000000000..977e553b70b --- /dev/null +++ b/logging/src/test/java/org/apache/rocketmq/logging/inner/LoggingBuilderTest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.logging.inner; + +import org.apache.rocketmq.logging.BasicLoggerTest; +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FilenameFilter; +import java.io.PrintStream; + +public class LoggingBuilderTest extends BasicLoggerTest { + + @Test + public void testConsole() { + PrintStream out = System.out; + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + System.setOut(new PrintStream(byteArrayOutputStream)); + + Appender consoleAppender = LoggingBuilder.newAppenderBuilder() + .withConsoleAppender(LoggingBuilder.SYSTEM_OUT) + .withLayout(LoggingBuilder.newLayoutBuilder().withDefaultLayout().build()).build(); + consoleAppender.doAppend(loggingEvent); + String result = new String(byteArrayOutputStream.toByteArray()); + System.setOut(out); + + Assert.assertTrue(result.contains(loggingEvent.getMessage().toString())); + + } + + @Test + public void testFileAppender() throws InterruptedException { + String logFile = loggingDir + "/file.log"; + Appender rollingFileAppender = LoggingBuilder.newAppenderBuilder().withAsync(false, 102400) + .withFileAppender(logFile).withLayout(LoggingBuilder.newLayoutBuilder().withDefaultLayout().build()).build(); + + for (int i = 0; i < 10; i++) { + rollingFileAppender.doAppend(loggingEvent); + } + rollingFileAppender.close(); + + File file = new File(logFile); + Assert.assertTrue(file.length() > 0); + } + + @Test + public void testRollingFileAppender() throws InterruptedException { + + String rollingFile = loggingDir + "/rolling.log"; + Appender rollingFileAppender = LoggingBuilder.newAppenderBuilder().withAsync(false, 1024) + .withRollingFileAppender(rollingFile, "1024", 5) + .withLayout(LoggingBuilder.newLayoutBuilder().withDefaultLayout().build()).build(); + + for (int i = 0; i < 100; i++) { + rollingFileAppender.doAppend(loggingEvent); + } + rollingFileAppender.close(); + + int cc = 0; + for (int i = 0; i < 5; i++) { + File file; + if (i == 0) { + file = new File(rollingFile); + } else { + file = new File(rollingFile + "." + i); + } + if (file.exists() && file.length() > 0) { + cc += 1; + } + } + Assert.assertTrue(cc >= 2); + } + + @Test + public void testDailyRollingFileAppender() throws InterruptedException { + String rollingFile = loggingDir + "/daily-rolling--222.log"; + Appender rollingFileAppender = LoggingBuilder.newAppenderBuilder().withAsync(false, 1024) + .withDailyFileRollingAppender(rollingFile, "'.'yyyy-MM-dd_HH-mm-ss-SSS") + .withLayout(LoggingBuilder.newLayoutBuilder().withDefaultLayout().build()).build(); + + for (int i = 0; i < 100; i++) { + rollingFileAppender.doAppend(loggingEvent); + } + + rollingFileAppender.close(); + + File file = new File(loggingDir); + String[] list = file.list(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.startsWith("daily-rolling--222.log"); + } + }); + Assert.assertTrue(list.length > 0); + } +} diff --git a/logging/src/test/java/org/apache/rocketmq/logging/inner/MessageFormatterTest.java b/logging/src/test/java/org/apache/rocketmq/logging/inner/MessageFormatterTest.java new file mode 100644 index 00000000000..5fa80ad56e8 --- /dev/null +++ b/logging/src/test/java/org/apache/rocketmq/logging/inner/MessageFormatterTest.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.logging.inner; + + +import org.apache.rocketmq.logging.InnerLoggerFactory; +import org.junit.Assert; +import org.junit.Test; + +public class MessageFormatterTest { + + @Test + public void formatTest(){ + InnerLoggerFactory.FormattingTuple logging = InnerLoggerFactory.MessageFormatter.format("this is {},and {}", "logging", 6546); + String message = logging.getMessage(); + Assert.assertTrue(message.contains("logging")); + + InnerLoggerFactory.FormattingTuple format = InnerLoggerFactory.MessageFormatter.format("cause exception {}", 143545, new RuntimeException()); + String message1 = format.getMessage(); + Throwable throwable = format.getThrowable(); + System.out.println(message1); + Assert.assertTrue(throwable != null); + } + +} diff --git a/logging/src/test/resources/logback_test.xml b/logging/src/test/resources/logback_test.xml new file mode 100644 index 00000000000..c1ab200449c --- /dev/null +++ b/logging/src/test/resources/logback_test.xml @@ -0,0 +1,46 @@ + + + + + + ${loggingDir}/logback_test.log + true + + ${loggingDir}/logback_test.%i.log + + 1 + 5 + + + 10MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + + + diff --git a/pom.xml b/pom.xml index 4326b4669d3..4be64a6f692 100644 --- a/pom.xml +++ b/pom.xml @@ -125,6 +125,7 @@ test distribution openmessaging + logging @@ -528,6 +529,11 @@ rocketmq-example ${project.version} + + ${project.groupId} + rocketmq-logging + ${project.version} + org.slf4j slf4j-api From c19fa25c87d6f6a7e5af7eab99910c271173225f Mon Sep 17 00:00:00 2001 From: yukon Date: Fri, 2 Mar 2018 16:18:11 +0800 Subject: [PATCH 13/73] [ROCKETMQ-367] Rename BasicLoggerTest to pass the compile process --- .../logging/{BasicloggerTest.java => BasicLoggerTest.java} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename logging/src/test/java/org/apache/rocketmq/logging/{BasicloggerTest.java => BasicLoggerTest.java} (100%) diff --git a/logging/src/test/java/org/apache/rocketmq/logging/BasicloggerTest.java b/logging/src/test/java/org/apache/rocketmq/logging/BasicLoggerTest.java similarity index 100% rename from logging/src/test/java/org/apache/rocketmq/logging/BasicloggerTest.java rename to logging/src/test/java/org/apache/rocketmq/logging/BasicLoggerTest.java From bf3f87c524f33d7c1bbd3b6b8d8fd976e11d4327 Mon Sep 17 00:00:00 2001 From: yukon Date: Tue, 6 Mar 2018 14:01:58 +0800 Subject: [PATCH 14/73] [ROCKETMQ-356] Change MQVersion to 4.3.0-SNAPSHOT --- common/src/main/java/org/apache/rocketmq/common/MQVersion.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java index 404f28df519..22a49a1cb55 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java +++ b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java @@ -18,7 +18,7 @@ public class MQVersion { - public static final int CURRENT_VERSION = Version.V4_2_0_SNAPSHOT.ordinal(); + public static final int CURRENT_VERSION = Version.V4_3_0_SNAPSHOT.ordinal(); public static String getVersionDesc(int value) { int length = Version.values().length; From 46f91479d95186e171b7ac43c9639972354e7ff2 Mon Sep 17 00:00:00 2001 From: lindzh Date: Wed, 7 Mar 2018 14:29:09 +0800 Subject: [PATCH 15/73] [ROCKETMQ-359] Replace slf4j api used in RocketMQ with InternalLogger (#221) --- .../rocketmq/broker/BrokerController.java | 10 +- .../apache/rocketmq/broker/BrokerStartup.java | 11 +- .../client/ClientHousekeepingService.java | 6 +- .../broker/client/ConsumerGroupInfo.java | 6 +- .../broker/client/ConsumerManager.java | 6 +- .../broker/client/ProducerManager.java | 6 +- .../broker/client/net/Broker2Client.java | 6 +- .../rebalance/RebalanceLockManager.java | 6 +- .../filter/CommitLogDispatcherCalcBitMap.java | 6 +- .../broker/filter/ConsumerFilterManager.java | 6 +- .../filter/ExpressionMessageFilter.java | 6 +- .../broker/filtersrv/FilterServerManager.java | 6 +- .../broker/filtersrv/FilterServerUtil.java | 5 +- .../broker/latency/BrokerFastFailure.java | 6 +- .../longpolling/PullRequestHoldService.java | 6 +- .../broker/offset/ConsumerOffsetManager.java | 6 +- .../rocketmq/broker/out/BrokerOuterAPI.java | 6 +- .../AbstractSendMessageProcessor.java | 6 +- .../processor/AdminBrokerProcessor.java | 6 +- .../processor/ClientManageProcessor.java | 6 +- .../processor/ConsumerManageProcessor.java | 6 +- .../processor/EndTransactionProcessor.java | 6 +- .../processor/ForwardRequestProcessor.java | 6 +- .../processor/PullMessageProcessor.java | 6 +- .../processor/QueryMessageProcessor.java | 6 +- .../broker/slave/SlaveSynchronize.java | 6 +- .../SubscriptionGroupManager.java | 6 +- .../broker/topic/TopicConfigManager.java | 6 +- .../jdbc/JDBCTransactionStore.java | 6 +- client/pom.xml | 1 + .../org/apache/rocketmq/client/MQHelper.java | 4 +- .../MQPullConsumerScheduleService.java | 4 +- .../AllocateMessageQueueAveragely.java | 4 +- ...AllocateMessageQueueAveragelyByCircle.java | 4 +- .../AllocateMessageQueueConsistentHash.java | 4 +- .../consumer/store/LocalFileOffsetStore.java | 4 +- .../store/RemoteBrokerOffsetStore.java | 4 +- .../client/impl/ClientRemotingProcessor.java | 5 +- .../rocketmq/client/impl/MQAdminImpl.java | 4 +- .../rocketmq/client/impl/MQClientAPIImpl.java | 4 +- .../rocketmq/client/impl/MQClientManager.java | 4 +- .../ConsumeMessageConcurrentlyService.java | 4 +- .../ConsumeMessageOrderlyService.java | 4 +- .../consumer/DefaultMQPullConsumerImpl.java | 4 +- .../consumer/DefaultMQPushConsumerImpl.java | 4 +- .../client/impl/consumer/ProcessQueue.java | 4 +- .../client/impl/consumer/PullAPIWrapper.java | 4 +- .../impl/consumer/PullMessageService.java | 4 +- .../client/impl/consumer/RebalanceImpl.java | 4 +- .../impl/consumer/RebalanceService.java | 4 +- .../client/impl/factory/MQClientInstance.java | 4 +- .../impl/producer/DefaultMQProducerImpl.java | 5 +- .../client/latency/MQFaultStrategy.java | 4 +- .../rocketmq/client/log/ClientLogger.java | 142 ++++---- .../client/stat/ConsumerStatsManager.java | 8 +- .../main/resources/log4j2_rocketmq_client.xml | 48 --- .../main/resources/log4j_rocketmq_client.xml | 58 ---- .../resources/logback_rocketmq_client.xml | 58 ---- .../consumer/DefaultMQPushConsumerTest.java | 2 + ...AllocateMessageQueueConsitentHashTest.java | 1 - .../rocketmq/client/log/ClientLoggerTest.java | 34 +- .../apache/rocketmq/common/BrokerConfig.java | 6 +- .../apache/rocketmq/common/ConfigManager.java | 16 +- .../apache/rocketmq/common/Configuration.java | 11 +- .../org/apache/rocketmq/common/MixAll.java | 10 +- .../apache/rocketmq/common/ServiceThread.java | 6 +- .../org/apache/rocketmq/common/UtilAll.java | 7 +- .../common/namesrv/NamesrvConfig.java | 6 +- .../common/namesrv/TopAddressing.java | 8 +- .../common/protocol/MQProtosHelper.java | 6 +- .../common/queue/ConcurrentTreeMap.java | 8 +- .../common/stats/MomentStatsItem.java | 6 +- .../common/stats/MomentStatsItemSet.java | 6 +- .../rocketmq/common/stats/StatsItem.java | 6 +- .../rocketmq/common/stats/StatsItemSet.java | 6 +- .../rocketmq/common/utils/ThreadUtils.java | 6 +- .../rocketmq/example/benchmark/Producer.java | 4 +- .../filtersrv/FiltersrvController.java | 6 +- .../rocketmq/filtersrv/FiltersrvStartup.java | 7 +- .../rocketmq/filtersrv/filter/DynaCode.java | 7 +- .../filtersrv/filter/FilterClassManager.java | 6 +- .../filter/HttpFilterClassFetchMethod.java | 6 +- .../processor/DefaultRequestProcessor.java | 8 +- .../stats/FilterServerStatsManager.java | 6 +- .../rocketmq/logging/InnerLoggerFactory.java | 4 + .../logging/InternalLoggerFactory.java | 4 - .../rocketmq/logging/Slf4jLoggerFactory.java | 5 + .../rocketmq/namesrv/NamesrvController.java | 7 +- .../rocketmq/namesrv/NamesrvStartup.java | 5 +- .../namesrv/kvconfig/KVConfigManager.java | 7 +- .../ClusterTestRequestProcessor.java | 6 +- .../processor/DefaultRequestProcessor.java | 10 +- .../routeinfo/BrokerHousekeepingService.java | 6 +- .../namesrv/routeinfo/RouteInfoManager.java | 10 +- .../DefaultRequestProcessorTest.java | 7 +- .../rocketmq/consumer/LocalMessageCache.java | 4 +- .../rocketmq/consumer/PullConsumerImpl.java | 4 +- .../producer/AbstractOMSProducer.java | 4 +- .../rocketmq/promise/DefaultPromise.java | 7 +- .../rocketmq/utils/BeanUtils.java | 4 +- pom.xml | 5 + remoting/pom.xml | 5 +- .../remoting/common/RemotingHelper.java | 6 +- .../remoting/common/RemotingUtil.java | 7 +- .../remoting/common/ServiceThread.java | 7 +- .../rocketmq/remoting/netty/NettyDecoder.java | 6 +- .../rocketmq/remoting/netty/NettyEncoder.java | 6 +- .../rocketmq/remoting/netty/NettyLogger.java | 315 ++++++++++++++++++ .../remoting/netty/NettyRemotingAbstract.java | 10 +- .../remoting/netty/NettyRemotingClient.java | 6 +- .../remoting/netty/NettyRemotingServer.java | 6 +- .../rocketmq/remoting/netty/TlsHelper.java | 6 +- .../remoting/protocol/RemotingCommand.java | 6 +- .../rocketmq/srvutil/FileWatchService.java | 6 +- .../rocketmq/srvutil/ShutdownHookThread.java | 7 +- .../store/AllocateMappedFileService.java | 6 +- .../org/apache/rocketmq/store/CommitLog.java | 10 +- .../apache/rocketmq/store/ConsumeQueue.java | 8 +- .../rocketmq/store/ConsumeQueueExt.java | 6 +- .../rocketmq/store/DefaultMessageStore.java | 6 +- .../org/apache/rocketmq/store/MappedFile.java | 6 +- .../rocketmq/store/MappedFileQueue.java | 8 +- .../rocketmq/store/StoreCheckpoint.java | 6 +- .../rocketmq/store/StoreStatsService.java | 6 +- .../rocketmq/store/TransientStorePool.java | 6 +- .../rocketmq/store/ha/HAConnection.java | 6 +- .../apache/rocketmq/store/ha/HAService.java | 6 +- .../rocketmq/store/ha/WaitNotifyObject.java | 6 +- .../rocketmq/store/index/IndexFile.java | 6 +- .../rocketmq/store/index/IndexService.java | 6 +- .../schedule/ScheduleMessageService.java | 6 +- .../rocketmq/store/stats/BrokerStats.java | 6 +- .../store/stats/BrokerStatsManager.java | 8 +- .../test/base/IntegrationTestBase.java | 6 +- .../tools/admin/DefaultMQAdminExtImpl.java | 4 +- .../consumer/ConsumerProgressSubCommand.java | 4 +- .../consumer/StartMonitoringSubCommand.java | 4 +- .../tools/monitor/DefaultMonitorListener.java | 4 +- .../tools/monitor/MonitorService.java | 4 +- 139 files changed, 799 insertions(+), 635 deletions(-) delete mode 100644 client/src/main/resources/log4j2_rocketmq_client.xml delete mode 100644 client/src/main/resources/log4j_rocketmq_client.xml delete mode 100644 client/src/main/resources/logback_rocketmq_client.xml create mode 100644 remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyLogger.java diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 8891bd322e3..409b1d06b64 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -67,6 +67,8 @@ import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.PermName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.namesrv.RegisterBrokerResult; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; @@ -88,13 +90,11 @@ import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.stats.BrokerStats; import org.apache.rocketmq.store.stats.BrokerStatsManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class BrokerController { - private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); - private static final Logger LOG_PROTECTION = LoggerFactory.getLogger(LoggerName.PROTECTION_LOGGER_NAME); - private static final Logger LOG_WATER_MARK = LoggerFactory.getLogger(LoggerName.WATER_MARK_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final InternalLogger LOG_PROTECTION = InternalLoggerFactory.getLogger(LoggerName.PROTECTION_LOGGER_NAME); + private static final InternalLogger LOG_WATER_MARK = InternalLoggerFactory.getLogger(LoggerName.WATER_MARK_LOGGER_NAME); private final BrokerConfig brokerConfig; private final NettyServerConfig nettyServerConfig; private final NettyClientConfig nettyClientConfig; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java index 40d20863747..7627bb967aa 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java @@ -31,6 +31,8 @@ import org.apache.rocketmq.common.MQVersion; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.common.RemotingUtil; import org.apache.rocketmq.remoting.common.TlsMode; import org.apache.rocketmq.remoting.netty.NettyClientConfig; @@ -41,7 +43,6 @@ import org.apache.rocketmq.srvutil.ServerUtil; import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.config.MessageStoreConfig; -import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_ENABLE; @@ -50,7 +51,7 @@ public class BrokerStartup { public static Properties properties = null; public static CommandLine commandLine = null; public static String configFile = null; - public static Logger log; + public static InternalLogger log; public static void main(String[] args) { start(createBrokerController(args)); @@ -177,14 +178,14 @@ public static BrokerController createBrokerController(String[] args) { configurator.doConfigure(brokerConfig.getRocketmqHome() + "/conf/logback_broker.xml"); if (commandLine.hasOption('p')) { - Logger console = LoggerFactory.getLogger(LoggerName.BROKER_CONSOLE_NAME); + InternalLogger console = InternalLoggerFactory.getLogger(LoggerName.BROKER_CONSOLE_NAME); MixAll.printObjectProperties(console, brokerConfig); MixAll.printObjectProperties(console, nettyServerConfig); MixAll.printObjectProperties(console, nettyClientConfig); MixAll.printObjectProperties(console, messageStoreConfig); System.exit(0); } else if (commandLine.hasOption('m')) { - Logger console = LoggerFactory.getLogger(LoggerName.BROKER_CONSOLE_NAME); + InternalLogger console = InternalLoggerFactory.getLogger(LoggerName.BROKER_CONSOLE_NAME); MixAll.printObjectProperties(console, brokerConfig, true); MixAll.printObjectProperties(console, nettyServerConfig, true); MixAll.printObjectProperties(console, nettyClientConfig, true); @@ -192,7 +193,7 @@ public static BrokerController createBrokerController(String[] args) { System.exit(0); } - log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); MixAll.printObjectProperties(log, brokerConfig); MixAll.printObjectProperties(log, nettyServerConfig); MixAll.printObjectProperties(log, nettyClientConfig); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ClientHousekeepingService.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ClientHousekeepingService.java index c61531c201d..d536db5055e 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/ClientHousekeepingService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ClientHousekeepingService.java @@ -23,12 +23,12 @@ import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.ChannelEventListener; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class ClientHousekeepingService implements ChannelEventListener { - private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private final BrokerController brokerController; private ScheduledExecutorService scheduledExecutorService = Executors diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java index 91b6c8181ec..c90d4942ac2 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java @@ -26,14 +26,14 @@ import java.util.concurrent.ConcurrentMap; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class ConsumerGroupInfo { - private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private final String groupName; private final ConcurrentMap subscriptionTable = new ConcurrentHashMap(); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java index 32632fc5149..cb6065540af 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java @@ -25,16 +25,16 @@ import java.util.concurrent.ConcurrentMap; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.common.RemotingUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class ConsumerManager { - private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final long CHANNEL_EXPIRED_TIMEOUT = 1000 * 120; private final ConcurrentMap consumerTable = new ConcurrentHashMap(1024); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java index 010c1aef6f3..88f1fdeaf73 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java @@ -25,13 +25,13 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.common.RemotingUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class ProducerManager { - private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final long LOCK_TIMEOUT_MILLIS = 3000; private static final long CHANNEL_EXPIRED_TIMEOUT = 1000 * 120; private final Lock groupChannelLock = new ReentrantLock(); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java b/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java index 65b444e6e4b..2a104457805 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java @@ -34,6 +34,8 @@ import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.message.MessageQueueForC; import org.apache.rocketmq.common.protocol.RequestCode; @@ -50,11 +52,9 @@ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.store.SelectMappedBufferResult; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class Broker2Client { - private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private final BrokerController brokerController; public Broker2Client(BrokerController brokerController) { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManager.java index 519745e0919..678b1f54a15 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManager.java @@ -23,12 +23,12 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.message.MessageQueue; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class RebalanceLockManager { - private static final Logger log = LoggerFactory.getLogger(LoggerName.REBALANCE_LOCK_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.REBALANCE_LOCK_LOGGER_NAME); private final static long REBALANCE_LOCK_MAX_LIVE_TIME = Long.parseLong(System.getProperty( "rocketmq.broker.rebalance.lockMaxLiveTime", "60000")); private final Lock lock = new ReentrantLock(); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/filter/CommitLogDispatcherCalcBitMap.java b/broker/src/main/java/org/apache/rocketmq/broker/filter/CommitLogDispatcherCalcBitMap.java index 85415d6297c..288351fb807 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/filter/CommitLogDispatcherCalcBitMap.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/filter/CommitLogDispatcherCalcBitMap.java @@ -19,11 +19,11 @@ import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.filter.util.BitsArray; import org.apache.rocketmq.store.CommitLogDispatcher; import org.apache.rocketmq.store.DispatchRequest; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Collection; import java.util.Iterator; @@ -33,7 +33,7 @@ */ public class CommitLogDispatcherCalcBitMap implements CommitLogDispatcher { - private static final Logger log = LoggerFactory.getLogger(LoggerName.FILTER_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.FILTER_LOGGER_NAME); protected final BrokerConfig brokerConfig; protected final ConsumerFilterManager consumerFilterManager; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/filter/ConsumerFilterManager.java b/broker/src/main/java/org/apache/rocketmq/broker/filter/ConsumerFilterManager.java index 482893f7cee..e9c5286bb1b 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/filter/ConsumerFilterManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/filter/ConsumerFilterManager.java @@ -22,14 +22,14 @@ import org.apache.rocketmq.broker.BrokerPathConfigHelper; import org.apache.rocketmq.common.ConfigManager; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.filter.FilterFactory; import org.apache.rocketmq.common.filter.ExpressionType; import org.apache.rocketmq.filter.util.BloomFilter; import org.apache.rocketmq.filter.util.BloomFilterData; import org.apache.rocketmq.remoting.protocol.RemotingSerializable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Collection; import java.util.HashSet; @@ -42,7 +42,7 @@ */ public class ConsumerFilterManager extends ConfigManager { - private static final Logger log = LoggerFactory.getLogger(LoggerName.FILTER_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.FILTER_LOGGER_NAME); private static final long MS_24_HOUR = 24 * 3600 * 1000; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/filter/ExpressionMessageFilter.java b/broker/src/main/java/org/apache/rocketmq/broker/filter/ExpressionMessageFilter.java index 64c28ece3e6..0c90880b0f2 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/filter/ExpressionMessageFilter.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/filter/ExpressionMessageFilter.java @@ -19,21 +19,21 @@ import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.filter.ExpressionType; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.filter.util.BitsArray; import org.apache.rocketmq.filter.util.BloomFilter; import org.apache.rocketmq.store.ConsumeQueueExt; import org.apache.rocketmq.store.MessageFilter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.nio.ByteBuffer; import java.util.Map; public class ExpressionMessageFilter implements MessageFilter { - protected static final Logger log = LoggerFactory.getLogger(LoggerName.FILTER_LOGGER_NAME); + protected static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.FILTER_LOGGER_NAME); protected final SubscriptionData subscriptionData; protected final ConsumerFilterData consumerFilterData; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/filtersrv/FilterServerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/filtersrv/FilterServerManager.java index f8f9943e826..c1a860a912a 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/filtersrv/FilterServerManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/filtersrv/FilterServerManager.java @@ -31,14 +31,14 @@ import org.apache.rocketmq.broker.BrokerStartup; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.common.RemotingUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class FilterServerManager { public static final long FILTER_SERVER_MAX_IDLE_TIME_MILLS = 30000; - private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private final ConcurrentMap filterServerTable = new ConcurrentHashMap(16); private final BrokerController brokerController; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/filtersrv/FilterServerUtil.java b/broker/src/main/java/org/apache/rocketmq/broker/filtersrv/FilterServerUtil.java index 5b142c11eca..3f4d24d0624 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/filtersrv/FilterServerUtil.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/filtersrv/FilterServerUtil.java @@ -17,10 +17,11 @@ package org.apache.rocketmq.broker.filtersrv; -import org.slf4j.Logger; + +import org.apache.rocketmq.logging.InternalLogger; public class FilterServerUtil { - public static void callShell(final String shellString, final Logger log) { + public static void callShell(final String shellString, final InternalLogger log) { Process process = null; try { String[] cmdArray = splitShellString(shellString); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java index 6aefe81a113..a5967a0f6b6 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java @@ -23,17 +23,17 @@ import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.netty.RequestTask; import org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * BrokerFastFailure will cover {@link BrokerController#sendThreadPoolQueue} and * {@link BrokerController#pullThreadPoolQueue} */ public class BrokerFastFailure { - private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl( "BrokerFastFailureScheduledThread")); private final BrokerController brokerController; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java index d0668cb45b0..417ec0dfd67 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java @@ -25,12 +25,12 @@ import org.apache.rocketmq.common.ServiceThread; import org.apache.rocketmq.common.SystemClock; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.store.ConsumeQueueExt; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class PullRequestHoldService extends ServiceThread { - private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final String TOPIC_QUEUEID_SEPARATOR = "@"; private final BrokerController brokerController; private final SystemClock systemClock = new SystemClock(); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java index 0257f94b737..ebc9dd8acce 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java @@ -29,12 +29,12 @@ import org.apache.rocketmq.common.ConfigManager; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.protocol.RemotingSerializable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class ConsumerOffsetManager extends ConfigManager { - private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final String TOPIC_GROUP_SEPARATOR = "@"; private ConcurrentMap> offsetTable = diff --git a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java index cba70a0ee8a..87c00a3f33d 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java @@ -22,6 +22,8 @@ import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.namesrv.RegisterBrokerResult; import org.apache.rocketmq.common.namesrv.TopAddressing; import org.apache.rocketmq.common.protocol.RequestCode; @@ -44,11 +46,9 @@ import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyRemotingClient; import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class BrokerOuterAPI { - private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private final RemotingClient remotingClient; private final TopAddressing topAddressing = new TopAddressing(MixAll.getWSAddr()); private String nameSrvAddr = null; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java index 410192f3b05..aa072e8b6a5 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java @@ -27,6 +27,8 @@ import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; @@ -43,8 +45,6 @@ import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.store.MessageExtBrokerInner; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.net.InetSocketAddress; import java.net.SocketAddress; @@ -53,7 +53,7 @@ import java.util.Random; public abstract class AbstractSendMessageProcessor implements NettyRequestProcessor { - protected static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + protected static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); protected final static int DLQ_NUMS_PER_GROUP = 1; protected final BrokerController brokerController; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index d69a78700c9..e8be2d4f4b5 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -44,6 +44,8 @@ import org.apache.rocketmq.common.admin.TopicOffset; import org.apache.rocketmq.common.admin.TopicStatsTable; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageId; import org.apache.rocketmq.common.message.MessageQueue; @@ -113,11 +115,9 @@ import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.MessageFilter; import org.apache.rocketmq.store.SelectMappedBufferResult; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class AdminBrokerProcessor implements NettyRequestProcessor { - private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private final BrokerController brokerController; public AdminBrokerProcessor(final BrokerController brokerController) { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ClientManageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ClientManageProcessor.java index 67807a863da..b5e608558cb 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ClientManageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ClientManageProcessor.java @@ -23,6 +23,7 @@ import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.filter.ExpressionType; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.body.CheckClientRequestBody; @@ -35,15 +36,14 @@ import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.common.sysflag.TopicSysFlag; import org.apache.rocketmq.filter.FilterFactory; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class ClientManageProcessor implements NettyRequestProcessor { - private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private final BrokerController brokerController; public ClientManageProcessor(final BrokerController brokerController) { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java index bb427050d9c..028d21bf976 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java @@ -21,6 +21,7 @@ import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.client.ConsumerGroupInfo; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupRequestHeader; @@ -30,15 +31,14 @@ import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetResponseHeader; import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader; import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetResponseHeader; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class ConsumerManageProcessor implements NettyRequestProcessor { - private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private final BrokerController brokerController; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java index fee1420a9e7..6801f759ea5 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java @@ -20,6 +20,8 @@ import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.TopicFilterType; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; @@ -34,11 +36,9 @@ import org.apache.rocketmq.store.MessageExtBrokerInner; import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.PutMessageResult; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class EndTransactionProcessor implements NettyRequestProcessor { - private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); + private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); private final BrokerController brokerController; public EndTransactionProcessor(final BrokerController brokerController) { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ForwardRequestProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ForwardRequestProcessor.java index 199aa940d69..b0f0a054552 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ForwardRequestProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ForwardRequestProcessor.java @@ -19,13 +19,13 @@ import io.netty.channel.ChannelHandlerContext; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class ForwardRequestProcessor implements NettyRequestProcessor { - private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private final BrokerController brokerController; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java index a46cbff2ebe..10c0112feb4 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java @@ -41,6 +41,8 @@ import org.apache.rocketmq.common.filter.ExpressionType; import org.apache.rocketmq.common.filter.FilterAPI; import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.ResponseCode; @@ -63,11 +65,9 @@ import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.stats.BrokerStatsManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class PullMessageProcessor implements NettyRequestProcessor { - private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private final BrokerController brokerController; private List consumeMessageHookList; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/QueryMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/QueryMessageProcessor.java index e8f97d0afbb..a5ca872a861 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/QueryMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/QueryMessageProcessor.java @@ -25,6 +25,8 @@ import org.apache.rocketmq.broker.pagecache.QueryMessageTransfer; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.QueryMessageRequestHeader; @@ -35,11 +37,9 @@ import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.store.QueryMessageResult; import org.apache.rocketmq.store.SelectMappedBufferResult; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class QueryMessageProcessor implements NettyRequestProcessor { - private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private final BrokerController brokerController; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java b/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java index 44c8264f30d..643a812fa02 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java @@ -21,15 +21,15 @@ import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.protocol.body.ConsumerOffsetSerializeWrapper; import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.store.config.StorePathConfigHelper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class SlaveSynchronize { - private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private final BrokerController brokerController; private volatile String masterAddr = null; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java index 0cbb76172ca..41f7a8a4714 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java @@ -26,13 +26,13 @@ import org.apache.rocketmq.common.DataVersion; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.remoting.protocol.RemotingSerializable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class SubscriptionGroupManager extends ConfigManager { - private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private final ConcurrentMap subscriptionGroupTable = new ConcurrentHashMap(1024); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java index cd30a089b7a..29e22808c5a 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java @@ -34,14 +34,14 @@ import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.PermName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.protocol.body.KVTable; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.common.sysflag.TopicSysFlag; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class TopicConfigManager extends ConfigManager { - private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final long LOCK_TIMEOUT_MILLIS = 3000; private transient final Lock lockTopicConfigTable = new ReentrantLock(); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/jdbc/JDBCTransactionStore.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/jdbc/JDBCTransactionStore.java index 056d93902fc..da4958defaf 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/jdbc/JDBCTransactionStore.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/jdbc/JDBCTransactionStore.java @@ -31,11 +31,11 @@ import org.apache.rocketmq.broker.transaction.TransactionStore; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.constant.LoggerName; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; public class JDBCTransactionStore implements TransactionStore { - private static final Logger log = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); private final JDBCTransactionStoreConfig jdbcTransactionStoreConfig; private Connection connection; private AtomicLong totalRecordsValue = new AtomicLong(0); diff --git a/client/pom.xml b/client/pom.xml index 56dea38b941..30b0e18d4a1 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -46,6 +46,7 @@ org.slf4j slf4j-api + test org.apache.commons diff --git a/client/src/main/java/org/apache/rocketmq/client/MQHelper.java b/client/src/main/java/org/apache/rocketmq/client/MQHelper.java index 7f0cef31b4d..c3378e4b309 100644 --- a/client/src/main/java/org/apache/rocketmq/client/MQHelper.java +++ b/client/src/main/java/org/apache/rocketmq/client/MQHelper.java @@ -20,9 +20,9 @@ import java.util.TreeSet; import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; -import org.slf4j.Logger; public class MQHelper { public static void resetOffsetByTimestamp( @@ -48,7 +48,7 @@ public static void resetOffsetByTimestamp( final String consumerGroup, final String topic, final long timestamp) throws Exception { - final Logger log = ClientLogger.getLog(); + final InternalLogger log = ClientLogger.getLog(); DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(consumerGroup); consumer.setInstanceName(instanceName); diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumerScheduleService.java b/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumerScheduleService.java index e0b546d247e..44b864e3487 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumerScheduleService.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumerScheduleService.java @@ -26,15 +26,15 @@ import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.common.ThreadFactoryImpl; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; -import org.slf4j.Logger; /** * Schedule service for pull consumer */ public class MQPullConsumerScheduleService { - private final Logger log = ClientLogger.getLog(); + private final InternalLogger log = ClientLogger.getLog(); private final MessageQueueListener messageQueueListener = new MessageQueueListenerImpl(); private final ConcurrentMap taskTable = new ConcurrentHashMap(); diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java index 35edbe0400f..155e692ad0b 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java @@ -20,14 +20,14 @@ import java.util.List; import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy; import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageQueue; -import org.slf4j.Logger; /** * Average Hashing queue algorithm */ public class AllocateMessageQueueAveragely implements AllocateMessageQueueStrategy { - private final Logger log = ClientLogger.getLog(); + private final InternalLogger log = ClientLogger.getLog(); @Override public List allocate(String consumerGroup, String currentCID, List mqAll, diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyByCircle.java b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyByCircle.java index d715ea1f9d0..fe78f0a6bbf 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyByCircle.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyByCircle.java @@ -20,14 +20,14 @@ import java.util.List; import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy; import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageQueue; -import org.slf4j.Logger; /** * Cycle average Hashing queue algorithm */ public class AllocateMessageQueueAveragelyByCircle implements AllocateMessageQueueStrategy { - private final Logger log = ClientLogger.getLog(); + private final InternalLogger log = ClientLogger.getLog(); @Override public List allocate(String consumerGroup, String currentCID, List mqAll, diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsistentHash.java b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsistentHash.java index b00326e6fef..65dcf799271 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsistentHash.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsistentHash.java @@ -24,14 +24,14 @@ import org.apache.rocketmq.common.consistenthash.ConsistentHashRouter; import org.apache.rocketmq.common.consistenthash.HashFunction; import org.apache.rocketmq.common.consistenthash.Node; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageQueue; -import org.slf4j.Logger; /** * Consistent Hashing queue algorithm */ public class AllocateMessageQueueConsistentHash implements AllocateMessageQueueStrategy { - private final Logger log = ClientLogger.getLog(); + private final InternalLogger log = ClientLogger.getLog(); private final int virtualNodeCnt; private final HashFunction customHashFunction; diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/store/LocalFileOffsetStore.java b/client/src/main/java/org/apache/rocketmq/client/consumer/store/LocalFileOffsetStore.java index 22ec674288c..b758ac70609 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/store/LocalFileOffsetStore.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/store/LocalFileOffsetStore.java @@ -31,9 +31,9 @@ import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.remoting.exception.RemotingException; -import org.slf4j.Logger; /** * Local storage implementation @@ -42,7 +42,7 @@ public class LocalFileOffsetStore implements OffsetStore { public final static String LOCAL_OFFSET_STORE_DIR = System.getProperty( "rocketmq.client.localOffsetStoreDir", System.getProperty("user.home") + File.separator + ".rocketmq_offsets"); - private final static Logger log = ClientLogger.getLog(); + private final static InternalLogger log = ClientLogger.getLog(); private final MQClientInstance mQClientFactory; private final String groupName; private final String storePath; diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java b/client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java index b82e992875e..c1524e10777 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java @@ -30,17 +30,17 @@ import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetRequestHeader; import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader; import org.apache.rocketmq.remoting.exception.RemotingException; -import org.slf4j.Logger; /** * Remote storage implementation */ public class RemoteBrokerOffsetStore implements OffsetStore { - private final static Logger log = ClientLogger.getLog(); + private final static InternalLogger log = ClientLogger.getLog(); private final MQClientInstance mQClientFactory; private final String groupName; private ConcurrentMap offsetTable = diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java b/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java index 69478cf3287..9771f145fb5 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java @@ -24,6 +24,7 @@ import org.apache.rocketmq.client.impl.producer.MQProducerInner; import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; @@ -44,10 +45,10 @@ import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import org.slf4j.Logger; + public class ClientRemotingProcessor implements NettyRequestProcessor { - private final Logger log = ClientLogger.getLog(); + private final InternalLogger log = ClientLogger.getLog(); private final MQClientInstance mqClientFactory; public ClientRemotingProcessor(final MQClientInstance mqClientFactory) { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java index b582b81e00d..3d2df0f5dd1 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java @@ -35,6 +35,7 @@ import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageClientIDSetter; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; @@ -52,11 +53,10 @@ import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.netty.ResponseFuture; import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import org.slf4j.Logger; public class MQAdminImpl { - private final Logger log = ClientLogger.getLog(); + private final InternalLogger log = ClientLogger.getLog(); private final MQClientInstance mQClientFactory; private long timeoutMillis = 6000; diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index c5abc36bf67..b07778499b7 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -48,6 +48,7 @@ import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.admin.ConsumeStats; import org.apache.rocketmq.common.admin.TopicStatsTable; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageBatch; import org.apache.rocketmq.common.message.MessageClientIDSetter; @@ -150,11 +151,10 @@ import org.apache.rocketmq.remoting.protocol.LanguageCode; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RemotingSerializable; -import org.slf4j.Logger; public class MQClientAPIImpl { - private final static Logger log = ClientLogger.getLog(); + private final static InternalLogger log = ClientLogger.getLog(); public static boolean sendSmartMsg = Boolean.parseBoolean(System.getProperty("org.apache.rocketmq.client.sendSmartMsg", "true")); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientManager.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientManager.java index 25877d7386d..31553a67ae5 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientManager.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientManager.java @@ -22,11 +22,11 @@ import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.RPCHook; -import org.slf4j.Logger; public class MQClientManager { - private final static Logger log = ClientLogger.getLog(); + private final static InternalLogger log = ClientLogger.getLog(); private static MQClientManager instance = new MQClientManager(); private AtomicInteger factoryIndexGenerator = new AtomicInteger(); private ConcurrentMap factoryTable = diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java index 87120178368..58b985f1011 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java @@ -39,6 +39,7 @@ import org.apache.rocketmq.client.stat.ConsumerStatsManager; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.ThreadFactoryImpl; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; @@ -46,10 +47,9 @@ import org.apache.rocketmq.common.protocol.body.CMResult; import org.apache.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; import org.apache.rocketmq.remoting.common.RemotingHelper; -import org.slf4j.Logger; public class ConsumeMessageConcurrentlyService implements ConsumeMessageService { - private static final Logger log = ClientLogger.getLog(); + private static final InternalLogger log = ClientLogger.getLog(); private final DefaultMQPushConsumerImpl defaultMQPushConsumerImpl; private final DefaultMQPushConsumer defaultMQPushConsumer; private final MessageListenerConcurrently messageListener; diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java index 43199e5b28c..d4a9953e783 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java @@ -37,6 +37,7 @@ import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; @@ -46,10 +47,9 @@ import org.apache.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.remoting.common.RemotingHelper; -import org.slf4j.Logger; public class ConsumeMessageOrderlyService implements ConsumeMessageService { - private static final Logger log = ClientLogger.getLog(); + private static final InternalLogger log = ClientLogger.getLog(); private final static long MAX_TIME_CONSUME_CONTINUOUSLY = Long.parseLong(System.getProperty("rocketmq.client.maxTimeConsumeContinuously", "60000")); private final DefaultMQPushConsumerImpl defaultMQPushConsumerImpl; diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java index 6eca38184e3..420d89b2fd0 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java @@ -48,6 +48,7 @@ import org.apache.rocketmq.common.consumer.ConsumeFromWhere; import org.apache.rocketmq.common.filter.FilterAPI; import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; @@ -61,10 +62,9 @@ import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingException; -import org.slf4j.Logger; public class DefaultMQPullConsumerImpl implements MQConsumerInner { - private final Logger log = ClientLogger.getLog(); + private final InternalLogger log = ClientLogger.getLog(); private final DefaultMQPullConsumer defaultMQPullConsumer; private final long consumerStartTimestamp = System.currentTimeMillis(); private final RPCHook rpcHook; diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java index f560376c6ad..85a40a9ddce 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java @@ -56,6 +56,7 @@ import org.apache.rocketmq.common.consumer.ConsumeFromWhere; import org.apache.rocketmq.common.filter.FilterAPI; import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; @@ -74,7 +75,6 @@ import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingException; -import org.slf4j.Logger; public class DefaultMQPushConsumerImpl implements MQConsumerInner { /** @@ -91,7 +91,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { private static final long PULL_TIME_DELAY_MILLS_WHEN_SUSPEND = 1000; private static final long BROKER_SUSPEND_MAX_TIME_MILLIS = 1000 * 15; private static final long CONSUMER_TIMEOUT_MILLIS_WHEN_SUSPEND = 1000 * 30; - private final Logger log = ClientLogger.getLog(); + private final InternalLogger log = ClientLogger.getLog(); private final DefaultMQPushConsumer defaultMQPushConsumer; private final RebalanceImpl rebalanceImpl = new RebalancePushImpl(this); private final ArrayList filterMessageHookList = new ArrayList(); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java index 0cea1aea890..0a52817ced7 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java @@ -28,11 +28,11 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.protocol.body.ProcessQueueInfo; -import org.slf4j.Logger; /** * Queue consumption snapshot @@ -42,7 +42,7 @@ public class ProcessQueue { Long.parseLong(System.getProperty("rocketmq.client.rebalance.lockMaxLiveTime", "30000")); public final static long REBALANCE_LOCK_INTERVAL = Long.parseLong(System.getProperty("rocketmq.client.rebalance.lockInterval", "20000")); private final static long PULL_MAX_IDLE_TIME = Long.parseLong(System.getProperty("rocketmq.client.pull.pullMaxIdleTime", "120000")); - private final Logger log = ClientLogger.getLog(); + private final InternalLogger log = ClientLogger.getLog(); private final ReadWriteLock lockTreeMap = new ReentrantReadWriteLock(); private final TreeMap msgTreeMap = new TreeMap(); private final AtomicLong msgCount = new AtomicLong(); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java index bbdf27d7a55..42c324fe2d3 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java @@ -37,6 +37,7 @@ import org.apache.rocketmq.common.MQVersion; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.filter.ExpressionType; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; @@ -47,10 +48,9 @@ import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.common.sysflag.PullSysFlag; import org.apache.rocketmq.remoting.exception.RemotingException; -import org.slf4j.Logger; public class PullAPIWrapper { - private final Logger log = ClientLogger.getLog(); + private final InternalLogger log = ClientLogger.getLog(); private final MQClientInstance mQClientFactory; private final String consumerGroup; private final boolean unitMode; diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullMessageService.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullMessageService.java index ed4b8371820..5f22f041eb2 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullMessageService.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullMessageService.java @@ -24,11 +24,11 @@ import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.common.ServiceThread; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.utils.ThreadUtils; -import org.slf4j.Logger; public class PullMessageService extends ServiceThread { - private final Logger log = ClientLogger.getLog(); + private final InternalLogger log = ClientLogger.getLog(); private final LinkedBlockingQueue pullRequestQueue = new LinkedBlockingQueue(); private final MQClientInstance mQClientFactory; private final ScheduledExecutorService scheduledExecutorService = Executors diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java index b555771c141..9ad07c7e443 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java @@ -32,19 +32,19 @@ import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.body.LockBatchRequestBody; import org.apache.rocketmq.common.protocol.body.UnlockBatchRequestBody; import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; -import org.slf4j.Logger; /** * Base class for rebalance algorithm */ public abstract class RebalanceImpl { - protected static final Logger log = ClientLogger.getLog(); + protected static final InternalLogger log = ClientLogger.getLog(); protected final ConcurrentMap processQueueTable = new ConcurrentHashMap(64); protected final ConcurrentMap> topicSubscribeInfoTable = new ConcurrentHashMap>(); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceService.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceService.java index fab07cb36e0..c8f8ab14079 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceService.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceService.java @@ -19,13 +19,13 @@ import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.common.ServiceThread; -import org.slf4j.Logger; +import org.apache.rocketmq.logging.InternalLogger; public class RebalanceService extends ServiceThread { private static long waitInterval = Long.parseLong(System.getProperty( "rocketmq.client.rebalance.waitInterval", "20000")); - private final Logger log = ClientLogger.getLog(); + private final InternalLogger log = ClientLogger.getLog(); private final MQClientInstance mqClientFactory; public RebalanceService(MQClientInstance mqClientFactory) { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java index 83b9ee7671c..e5400d71e2c 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java @@ -63,6 +63,7 @@ import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.filter.ExpressionType; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; @@ -80,11 +81,10 @@ import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import org.slf4j.Logger; public class MQClientInstance { private final static long LOCK_TIMEOUT_MILLIS = 3000; - private final Logger log = ClientLogger.getLog(); + private final InternalLogger log = ClientLogger.getLog(); private final ClientConfig clientConfig; private final int instanceIndex; private final String clientId; diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index 7c169796741..9dd8ee3ce0a 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -31,6 +31,8 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; + +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageClientIDSetter; import org.apache.rocketmq.common.message.MessageExt; @@ -79,10 +81,9 @@ import org.apache.rocketmq.remoting.exception.RemotingConnectException; import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; -import org.slf4j.Logger; public class DefaultMQProducerImpl implements MQProducerInner { - private final Logger log = ClientLogger.getLog(); + private final InternalLogger log = ClientLogger.getLog(); private final Random random = new Random(); private final DefaultMQProducer defaultMQProducer; private final ConcurrentMap topicPublishInfoTable = diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java b/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java index 235aa20af8e..7854fcb7d10 100644 --- a/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java +++ b/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java @@ -19,11 +19,11 @@ import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageQueue; -import org.slf4j.Logger; public class MQFaultStrategy { - private final static Logger log = ClientLogger.getLog(); + private final static InternalLogger log = ClientLogger.getLog(); private final LatencyFaultTolerance latencyFaultTolerance = new LatencyFaultToleranceImpl(); private boolean sendLatencyFaultEnable = false; diff --git a/client/src/main/java/org/apache/rocketmq/client/log/ClientLogger.java b/client/src/main/java/org/apache/rocketmq/client/log/ClientLogger.java index c3df9a63947..c0dbf324824 100644 --- a/client/src/main/java/org/apache/rocketmq/client/log/ClientLogger.java +++ b/client/src/main/java/org/apache/rocketmq/client/log/ClientLogger.java @@ -17,104 +17,82 @@ package org.apache.rocketmq.client.log; import org.apache.rocketmq.common.constant.LoggerName; -import org.slf4j.ILoggerFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.lang.reflect.Method; -import java.net.URL; +import org.apache.rocketmq.logging.InnerLoggerFactory; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.logging.inner.Appender; +import org.apache.rocketmq.logging.inner.Layout; +import org.apache.rocketmq.logging.inner.Level; +import org.apache.rocketmq.logging.inner.Logger; +import org.apache.rocketmq.logging.inner.LoggingBuilder; +import org.apache.rocketmq.remoting.common.RemotingHelper; public class ClientLogger { + + public static final String CLIENT_LOG_USESLF4J = "rocketmq.client.logUseSlf4j"; public static final String CLIENT_LOG_ROOT = "rocketmq.client.logRoot"; public static final String CLIENT_LOG_MAXINDEX = "rocketmq.client.logFileMaxIndex"; + public static final String CLIENT_LOG_FILESIZE = "rocketmq.client.logFileMaxSize"; public static final String CLIENT_LOG_LEVEL = "rocketmq.client.logLevel"; + public static final String CLIENT_LOG_FILENAME = "rocketmq.client.logFileName"; + public static final String CLIENT_LOG_ASYNC_QUEUESIZE = "rocketmq.client.logAsyncQueueSize"; + public static final String ROCKETMQ_CLIENT_APPENDER_NAME = "RocketmqClientAppender"; - private static Logger log; - - private static Logger createLogger(final String loggerName) { - String logConfigFilePath = System.getProperty("rocketmq.client.log.configFile", System.getenv("ROCKETMQ_CLIENT_LOG_CONFIGFILE")); - Boolean isloadconfig = - Boolean.parseBoolean(System.getProperty("rocketmq.client.log.loadconfig", "true")); + private static final InternalLogger CLIENT_LOGGER; - final String log4JResourceFile = - System.getProperty("rocketmq.client.log4j.resource.fileName", "log4j_rocketmq_client.xml"); + private static final boolean CLIENT_USE_SLF4J; - final String logbackResourceFile = - System.getProperty("rocketmq.client.logback.resource.fileName", "logback_rocketmq_client.xml"); + private static Appender rocketmqClientAppender = null; - final String log4J2ResourceFile = - System.getProperty("rocketmq.client.log4j2.resource.fileName", "log4j2_rocketmq_client.xml"); + static { + CLIENT_USE_SLF4J = Boolean.parseBoolean(System.getProperty(CLIENT_LOG_USESLF4J, "false")); + if (!CLIENT_USE_SLF4J) { + InternalLoggerFactory.setCurrentLoggerType(InnerLoggerFactory.LOGGER_INNER); + CLIENT_LOGGER = createLogger(LoggerName.CLIENT_LOGGER_NAME); + createLogger(LoggerName.COMMON_LOGGER_NAME); + createLogger(RemotingHelper.ROCKETMQ_REMOTING); + } else { + CLIENT_LOGGER = InternalLoggerFactory.getLogger(LoggerName.CLIENT_LOGGER_NAME); + } + } + private static synchronized void createClientAppender() { String clientLogRoot = System.getProperty(CLIENT_LOG_ROOT, System.getProperty("user.home") + "/logs/rocketmqlogs"); - System.setProperty("client.logRoot", clientLogRoot); - String clientLogLevel = System.getProperty(CLIENT_LOG_LEVEL, "INFO"); - System.setProperty("client.logLevel", clientLogLevel); String clientLogMaxIndex = System.getProperty(CLIENT_LOG_MAXINDEX, "10"); - System.setProperty("client.logFileMaxIndex", clientLogMaxIndex); - - if (isloadconfig) { - try { - ILoggerFactory iLoggerFactory = LoggerFactory.getILoggerFactory(); - Class classType = iLoggerFactory.getClass(); - if (classType.getName().equals("org.slf4j.impl.Log4jLoggerFactory")) { - Class domconfigurator; - Object domconfiguratorobj; - domconfigurator = Class.forName("org.apache.log4j.xml.DOMConfigurator"); - domconfiguratorobj = domconfigurator.newInstance(); - if (null == logConfigFilePath) { - Method configure = domconfiguratorobj.getClass().getMethod("configure", URL.class); - URL url = ClientLogger.class.getClassLoader().getResource(log4JResourceFile); - configure.invoke(domconfiguratorobj, url); - } else { - Method configure = domconfiguratorobj.getClass().getMethod("configure", String.class); - configure.invoke(domconfiguratorobj, logConfigFilePath); - } - - } else if (classType.getName().equals("ch.qos.logback.classic.LoggerContext")) { - Class joranConfigurator; - Class context = Class.forName("ch.qos.logback.core.Context"); - Object joranConfiguratoroObj; - joranConfigurator = Class.forName("ch.qos.logback.classic.joran.JoranConfigurator"); - joranConfiguratoroObj = joranConfigurator.newInstance(); - Method setContext = joranConfiguratoroObj.getClass().getMethod("setContext", context); - setContext.invoke(joranConfiguratoroObj, iLoggerFactory); - if (null == logConfigFilePath) { - URL url = ClientLogger.class.getClassLoader().getResource(logbackResourceFile); - Method doConfigure = - joranConfiguratoroObj.getClass().getMethod("doConfigure", URL.class); - doConfigure.invoke(joranConfiguratoroObj, url); - } else { - Method doConfigure = - joranConfiguratoroObj.getClass().getMethod("doConfigure", String.class); - doConfigure.invoke(joranConfiguratoroObj, logConfigFilePath); - } - - } else if (classType.getName().equals("org.apache.logging.slf4j.Log4jLoggerFactory")) { - Class joranConfigurator = Class.forName("org.apache.logging.log4j.core.config.Configurator"); - Method initialize = joranConfigurator.getDeclaredMethod("initialize", String.class, String.class); - if (null == logConfigFilePath) { - initialize.invoke(joranConfigurator, "log4j2", log4J2ResourceFile); - } else { - initialize.invoke(joranConfigurator, "log4j2", logConfigFilePath); - } - } - } catch (Exception e) { - System.err.println(e); - } - } - return LoggerFactory.getLogger(LoggerName.CLIENT_LOGGER_NAME); + String clientLogFileName = System.getProperty(CLIENT_LOG_FILENAME, "rocketmq_client.log"); + String maxFileSize = System.getProperty(CLIENT_LOG_FILESIZE, "1073741824"); + String asyncQueueSize = System.getProperty(CLIENT_LOG_ASYNC_QUEUESIZE, "1024"); + + String logFileName = clientLogRoot + "/" + clientLogFileName; + + int maxFileIndex = Integer.parseInt(clientLogMaxIndex); + int queueSize = Integer.parseInt(asyncQueueSize); + + Layout layout = LoggingBuilder.newLayoutBuilder().withDefaultLayout().build(); + + rocketmqClientAppender = LoggingBuilder.newAppenderBuilder() + .withRollingFileAppender(logFileName, maxFileSize, maxFileIndex) + .withAsync(false, queueSize).withName(ROCKETMQ_CLIENT_APPENDER_NAME).withLayout(layout).build(); + + Logger.getRootLogger().addAppender(rocketmqClientAppender); } - public static Logger getLog() { - if (log == null) { - log = createLogger(LoggerName.CLIENT_LOGGER_NAME); - return log; - } else { - return log; + private static InternalLogger createLogger(final String loggerName) { + String clientLogLevel = System.getProperty(CLIENT_LOG_LEVEL, "INFO"); + InternalLogger logger = InternalLoggerFactory.getLogger(loggerName); + InnerLoggerFactory.InnerLogger innerLogger = (InnerLoggerFactory.InnerLogger) logger; + Logger realLogger = innerLogger.getLogger(); + + if (rocketmqClientAppender == null) { + createClientAppender(); } + + realLogger.addAppender(rocketmqClientAppender); + realLogger.setLevel(Level.toLevel(clientLogLevel)); + return logger; } - public static void setLog(Logger log) { - ClientLogger.log = log; + public static InternalLogger getLog() { + return CLIENT_LOGGER; } } diff --git a/client/src/main/java/org/apache/rocketmq/client/stat/ConsumerStatsManager.java b/client/src/main/java/org/apache/rocketmq/client/stat/ConsumerStatsManager.java index 350e1dd7015..cf347b4dd7e 100644 --- a/client/src/main/java/org/apache/rocketmq/client/stat/ConsumerStatsManager.java +++ b/client/src/main/java/org/apache/rocketmq/client/stat/ConsumerStatsManager.java @@ -18,15 +18,15 @@ package org.apache.rocketmq.client.stat; import java.util.concurrent.ScheduledExecutorService; -import org.apache.rocketmq.common.constant.LoggerName; + +import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.protocol.body.ConsumeStatus; import org.apache.rocketmq.common.stats.StatsItemSet; import org.apache.rocketmq.common.stats.StatsSnapshot; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class ConsumerStatsManager { - private static final Logger log = LoggerFactory.getLogger(LoggerName.CLIENT_LOGGER_NAME); + private static final InternalLogger log = ClientLogger.getLog(); private static final String TOPIC_AND_GROUP_CONSUME_OK_TPS = "CONSUME_OK_TPS"; private static final String TOPIC_AND_GROUP_CONSUME_FAILED_TPS = "CONSUME_FAILED_TPS"; diff --git a/client/src/main/resources/log4j2_rocketmq_client.xml b/client/src/main/resources/log4j2_rocketmq_client.xml deleted file mode 100644 index 651553e0b09..00000000000 --- a/client/src/main/resources/log4j2_rocketmq_client.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/client/src/main/resources/log4j_rocketmq_client.xml b/client/src/main/resources/log4j_rocketmq_client.xml deleted file mode 100644 index cc86b71736b..00000000000 --- a/client/src/main/resources/log4j_rocketmq_client.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/client/src/main/resources/logback_rocketmq_client.xml b/client/src/main/resources/logback_rocketmq_client.xml deleted file mode 100644 index c045b42072e..00000000000 --- a/client/src/main/resources/logback_rocketmq_client.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - - - - ${client.logRoot}/rocketmq_client.log - true - - ${client.logRoot}/otherdays/rocketmq_client.%i.log - - 1 - ${client.logFileMaxIndex} - - - 100MB - - - %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n - UTF-8 - - - - - - - - - - - - - - - - - - - - - - diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java index b21edc9197c..5a612c64f77 100644 --- a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java @@ -58,6 +58,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; +import org.mockito.Spy; import org.mockito.invocation.InvocationOnMock; import org.mockito.junit.MockitoJUnitRunner; import org.mockito.stubbing.Answer; @@ -79,6 +80,7 @@ public class DefaultMQPushConsumerTest { private String topic = "FooBar"; private String brokerName = "BrokerA"; private MQClientInstance mQClientFactory; + @Mock private MQClientAPIImpl mQClientAPIImpl; private PullAPIWrapper pullAPIWrapper; diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsitentHashTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsitentHashTest.java index 3a581e13100..98ce7b6eb27 100644 --- a/client/src/test/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsitentHashTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsitentHashTest.java @@ -23,7 +23,6 @@ import java.util.Random; import java.util.TreeMap; import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy; -import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageQueue; import org.junit.Assert; import org.junit.Before; diff --git a/client/src/test/java/org/apache/rocketmq/client/log/ClientLoggerTest.java b/client/src/test/java/org/apache/rocketmq/client/log/ClientLoggerTest.java index d3f3be79b2a..9fe0d8b9c45 100644 --- a/client/src/test/java/org/apache/rocketmq/client/log/ClientLoggerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/log/ClientLoggerTest.java @@ -17,14 +17,17 @@ package org.apache.rocketmq.client.log; +import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.junit.After; +import org.junit.Assert; import org.junit.Test; -import org.slf4j.Logger; import java.io.File; +import java.io.IOException; -import static org.junit.Assert.assertEquals; public class ClientLoggerTest { @@ -35,18 +38,29 @@ public class ClientLoggerTest { LOG_DIR = System.getProperty(CLIENT_LOG_ROOT, System.getProperty("user.home") + "/logs/rocketmqlogs"); } + @Test + public void testClientlog() throws IOException { + InternalLogger logger = ClientLogger.getLog(); + InternalLogger rocketmqCommon = InternalLoggerFactory.getLogger("RocketmqCommon"); + InternalLogger rocketmqRemoting = InternalLoggerFactory.getLogger("RocketmqRemoting"); + + for (int i = 0; i < 10; i++) { + logger.info("testClientlog test {}", i); + rocketmqCommon.info("common message {}", i, new RuntimeException()); + rocketmqRemoting.info("remoting message {}", i, new RuntimeException()); + } + + String content = MixAll.file2String(LOG_DIR + "/rocketmq_client.log"); + Assert.assertTrue(content.contains("testClientlog")); + Assert.assertTrue(content.contains("RocketmqClient")); + + Assert.assertTrue(content.contains("RocketmqCommon")); + Assert.assertTrue(content.contains("RocketmqRemoting")); + } @After public void cleanFiles() { UtilAll.deleteFile(new File(LOG_DIR)); } - // FIXME: Workaround for concrete implementation for slf4j, is there any better solution for all slf4j implementations in one class ? 2017/8/1 - @Test - public void testLog4j2() throws Exception { - Logger logger = ClientLogger.getLog(); - - System.out.println(logger); - assertEquals("org.apache.logging.slf4j.Log4jLogger", logger.getClass().getName()); - } } diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index efb36b50a7e..4d7eb469ead 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -19,15 +19,15 @@ import org.apache.rocketmq.common.annotation.ImportantField; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.PermName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.common.RemotingUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.net.InetAddress; import java.net.UnknownHostException; public class BrokerConfig { - private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); private String rocketmqHome = System.getProperty(MixAll.ROCKETMQ_HOME_PROPERTY, System.getenv(MixAll.ROCKETMQ_HOME_ENV)); @ImportantField diff --git a/common/src/main/java/org/apache/rocketmq/common/ConfigManager.java b/common/src/main/java/org/apache/rocketmq/common/ConfigManager.java index c33ebdf324e..99b5f0c667b 100644 --- a/common/src/main/java/org/apache/rocketmq/common/ConfigManager.java +++ b/common/src/main/java/org/apache/rocketmq/common/ConfigManager.java @@ -18,11 +18,11 @@ import java.io.IOException; import org.apache.rocketmq.common.constant.LoggerName; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; public abstract class ConfigManager { - private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); public abstract String encode(); @@ -36,11 +36,11 @@ public boolean load() { return this.loadBak(); } else { this.decode(jsonString); - log.info("load {} OK", fileName); + log.info("load " + fileName + " OK"); return true; } } catch (Exception e) { - log.error("load [{}] failed, and try to load backup file", fileName, e); + log.error("load " + fileName + " failed, and try to load backup file", e); return this.loadBak(); } } @@ -54,11 +54,11 @@ private boolean loadBak() { String jsonString = MixAll.file2String(fileName + ".bak"); if (jsonString != null && jsonString.length() > 0) { this.decode(jsonString); - log.info("load [{}] OK", fileName); + log.info("load " + fileName + " OK"); return true; } } catch (Exception e) { - log.error("load [{}] Failed", fileName, e); + log.error("load " + fileName + " Failed", e); return false; } @@ -74,7 +74,7 @@ public synchronized void persist() { try { MixAll.string2File(jsonString, fileName); } catch (IOException e) { - log.error("persist file [{}] exception", fileName, e); + log.error("persist file " + fileName + " exception", e); } } } diff --git a/common/src/main/java/org/apache/rocketmq/common/Configuration.java b/common/src/main/java/org/apache/rocketmq/common/Configuration.java index 802f6eea2bd..b9eb46d3802 100644 --- a/common/src/main/java/org/apache/rocketmq/common/Configuration.java +++ b/common/src/main/java/org/apache/rocketmq/common/Configuration.java @@ -17,6 +17,8 @@ package org.apache.rocketmq.common; +import org.apache.rocketmq.logging.InternalLogger; + import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.Modifier; @@ -25,11 +27,10 @@ import java.util.Properties; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import org.slf4j.Logger; public class Configuration { - private final Logger log; + private final InternalLogger log; private List configObjectList = new ArrayList(4); private String storePath; @@ -43,11 +44,11 @@ public class Configuration { */ private Properties allConfigs = new Properties(); - public Configuration(Logger log) { + public Configuration(InternalLogger log) { this.log = log; } - public Configuration(Logger log, Object... configObjects) { + public Configuration(InternalLogger log, Object... configObjects) { this.log = log; if (configObjects == null || configObjects.length == 0) { return; @@ -57,7 +58,7 @@ public Configuration(Logger log, Object... configObjects) { } } - public Configuration(Logger log, String storePath, Object... configObjects) { + public Configuration(InternalLogger log, String storePath, Object... configObjects) { this(log, configObjects); this.storePath = storePath; } diff --git a/common/src/main/java/org/apache/rocketmq/common/MixAll.java b/common/src/main/java/org/apache/rocketmq/common/MixAll.java index 3a005e6a545..49085c3f1e1 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MixAll.java +++ b/common/src/main/java/org/apache/rocketmq/common/MixAll.java @@ -41,11 +41,11 @@ import org.apache.rocketmq.common.annotation.ImportantField; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.help.FAQUrl; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; public class MixAll { - private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); public static final String ROCKETMQ_HOME_ENV = "ROCKETMQ_HOME"; public static final String ROCKETMQ_HOME_PROPERTY = "rocketmq.home.dir"; @@ -227,11 +227,11 @@ public static String file2String(final URL url) { return null; } - public static void printObjectProperties(final Logger logger, final Object object) { + public static void printObjectProperties(final InternalLogger logger, final Object object) { printObjectProperties(logger, object, false); } - public static void printObjectProperties(final Logger logger, final Object object, + public static void printObjectProperties(final InternalLogger logger, final Object object, final boolean onlyImportantField) { Field[] fields = object.getClass().getDeclaredFields(); for (Field field : fields) { diff --git a/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java b/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java index bf7e724986a..53ca3290826 100644 --- a/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java +++ b/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java @@ -19,11 +19,11 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.rocketmq.common.constant.LoggerName; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; public abstract class ServiceThread implements Runnable { - private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); private static final long JOIN_TIME = 90 * 1000; diff --git a/common/src/main/java/org/apache/rocketmq/common/UtilAll.java b/common/src/main/java/org/apache/rocketmq/common/UtilAll.java index 9ed8ab881b7..a846755d8db 100644 --- a/common/src/main/java/org/apache/rocketmq/common/UtilAll.java +++ b/common/src/main/java/org/apache/rocketmq/common/UtilAll.java @@ -38,13 +38,12 @@ import java.util.zip.InflaterInputStream; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.common.RemotingHelper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - public class UtilAll { - private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); public static final String YYYY_MM_DD_HH_MM_SS = "yyyy-MM-dd HH:mm:ss"; public static final String YYYY_MM_DD_HH_MM_SS_SSS = "yyyy-MM-dd#HH:mm:ss:SSS"; diff --git a/common/src/main/java/org/apache/rocketmq/common/namesrv/NamesrvConfig.java b/common/src/main/java/org/apache/rocketmq/common/namesrv/NamesrvConfig.java index 6f740f7fd2a..f687d2c2437 100644 --- a/common/src/main/java/org/apache/rocketmq/common/namesrv/NamesrvConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/namesrv/NamesrvConfig.java @@ -23,11 +23,11 @@ import java.io.File; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.constant.LoggerName; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; public class NamesrvConfig { - private static final Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); private String rocketmqHome = System.getProperty(MixAll.ROCKETMQ_HOME_PROPERTY, System.getenv(MixAll.ROCKETMQ_HOME_ENV)); private String kvConfigPath = System.getProperty("user.home") + File.separator + "namesrv" + File.separator + "kvConfig.json"; diff --git a/common/src/main/java/org/apache/rocketmq/common/namesrv/TopAddressing.java b/common/src/main/java/org/apache/rocketmq/common/namesrv/TopAddressing.java index 57af0e709bd..88a4122b6d2 100644 --- a/common/src/main/java/org/apache/rocketmq/common/namesrv/TopAddressing.java +++ b/common/src/main/java/org/apache/rocketmq/common/namesrv/TopAddressing.java @@ -25,12 +25,12 @@ import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.utils.HttpTinyClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class TopAddressing { - private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); private String nsAddr; private String wsAddr; @@ -79,7 +79,7 @@ public final String fetchNSAddr(boolean verbose, long timeoutMills) { log.error("fetch nameserver address is null"); } } else { - log.error("fetch nameserver address failed. statusCode={}", result.code); + log.error("fetch nameserver address failed. statusCode=" + result.code); } } catch (IOException e) { if (verbose) { diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/MQProtosHelper.java b/common/src/main/java/org/apache/rocketmq/common/protocol/MQProtosHelper.java index c1cd69cd6ee..d8c1cedef14 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/MQProtosHelper.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/MQProtosHelper.java @@ -18,14 +18,14 @@ package org.apache.rocketmq.common.protocol; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerRequestHeader; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class MQProtosHelper { - private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); public static boolean registerBrokerToNameServer(final String nsaddr, final String brokerAddr, final long timeoutMillis) { diff --git a/common/src/main/java/org/apache/rocketmq/common/queue/ConcurrentTreeMap.java b/common/src/main/java/org/apache/rocketmq/common/queue/ConcurrentTreeMap.java index aab95150438..791c72c18a5 100644 --- a/common/src/main/java/org/apache/rocketmq/common/queue/ConcurrentTreeMap.java +++ b/common/src/main/java/org/apache/rocketmq/common/queue/ConcurrentTreeMap.java @@ -22,14 +22,14 @@ import java.util.TreeMap; import java.util.concurrent.locks.ReentrantLock; import org.apache.rocketmq.common.constant.LoggerName; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; /** * thread safe */ public class ConcurrentTreeMap { - private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private final ReentrantLock lock; private TreeMap tree; private RoundQueue roundQueue; @@ -58,7 +58,7 @@ public V putIfAbsentAndRetExsit(K key, V value) { tree.put(key, value); exsit = value; } - log.warn("putIfAbsentAndRetExsit success. {}", key); + log.warn("putIfAbsentAndRetExsit success. " + key); return exsit; } else { V exsit = tree.get(key); diff --git a/common/src/main/java/org/apache/rocketmq/common/stats/MomentStatsItem.java b/common/src/main/java/org/apache/rocketmq/common/stats/MomentStatsItem.java index 5f3229b38ba..99f16f54a09 100644 --- a/common/src/main/java/org/apache/rocketmq/common/stats/MomentStatsItem.java +++ b/common/src/main/java/org/apache/rocketmq/common/stats/MomentStatsItem.java @@ -21,7 +21,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import org.apache.rocketmq.common.UtilAll; -import org.slf4j.Logger; +import org.apache.rocketmq.logging.InternalLogger; public class MomentStatsItem { @@ -30,10 +30,10 @@ public class MomentStatsItem { private final String statsName; private final String statsKey; private final ScheduledExecutorService scheduledExecutorService; - private final Logger log; + private final InternalLogger log; public MomentStatsItem(String statsName, String statsKey, - ScheduledExecutorService scheduledExecutorService, Logger log) { + ScheduledExecutorService scheduledExecutorService, InternalLogger log) { this.statsName = statsName; this.statsKey = statsKey; this.scheduledExecutorService = scheduledExecutorService; diff --git a/common/src/main/java/org/apache/rocketmq/common/stats/MomentStatsItemSet.java b/common/src/main/java/org/apache/rocketmq/common/stats/MomentStatsItemSet.java index 57dfc386f4f..d3b5596611e 100644 --- a/common/src/main/java/org/apache/rocketmq/common/stats/MomentStatsItemSet.java +++ b/common/src/main/java/org/apache/rocketmq/common/stats/MomentStatsItemSet.java @@ -24,16 +24,16 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import org.apache.rocketmq.common.UtilAll; -import org.slf4j.Logger; +import org.apache.rocketmq.logging.InternalLogger; public class MomentStatsItemSet { private final ConcurrentMap statsItemTable = new ConcurrentHashMap(128); private final String statsName; private final ScheduledExecutorService scheduledExecutorService; - private final Logger log; + private final InternalLogger log; - public MomentStatsItemSet(String statsName, ScheduledExecutorService scheduledExecutorService, Logger log) { + public MomentStatsItemSet(String statsName, ScheduledExecutorService scheduledExecutorService, InternalLogger log) { this.statsName = statsName; this.scheduledExecutorService = scheduledExecutorService; this.log = log; diff --git a/common/src/main/java/org/apache/rocketmq/common/stats/StatsItem.java b/common/src/main/java/org/apache/rocketmq/common/stats/StatsItem.java index 9b37f800d90..210e57823db 100644 --- a/common/src/main/java/org/apache/rocketmq/common/stats/StatsItem.java +++ b/common/src/main/java/org/apache/rocketmq/common/stats/StatsItem.java @@ -22,7 +22,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import org.apache.rocketmq.common.UtilAll; -import org.slf4j.Logger; +import org.apache.rocketmq.logging.InternalLogger; public class StatsItem { @@ -39,10 +39,10 @@ public class StatsItem { private final String statsName; private final String statsKey; private final ScheduledExecutorService scheduledExecutorService; - private final Logger log; + private final InternalLogger log; public StatsItem(String statsName, String statsKey, ScheduledExecutorService scheduledExecutorService, - Logger log) { + InternalLogger log) { this.statsName = statsName; this.statsKey = statsKey; this.scheduledExecutorService = scheduledExecutorService; diff --git a/common/src/main/java/org/apache/rocketmq/common/stats/StatsItemSet.java b/common/src/main/java/org/apache/rocketmq/common/stats/StatsItemSet.java index 17dbf0d2a83..9a0caaa112d 100644 --- a/common/src/main/java/org/apache/rocketmq/common/stats/StatsItemSet.java +++ b/common/src/main/java/org/apache/rocketmq/common/stats/StatsItemSet.java @@ -24,7 +24,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import org.apache.rocketmq.common.UtilAll; -import org.slf4j.Logger; +import org.apache.rocketmq.logging.InternalLogger; public class StatsItemSet { private final ConcurrentMap statsItemTable = @@ -32,9 +32,9 @@ public class StatsItemSet { private final String statsName; private final ScheduledExecutorService scheduledExecutorService; - private final Logger log; + private final InternalLogger log; - public StatsItemSet(String statsName, ScheduledExecutorService scheduledExecutorService, Logger log) { + public StatsItemSet(String statsName, ScheduledExecutorService scheduledExecutorService, InternalLogger log) { this.statsName = statsName; this.scheduledExecutorService = scheduledExecutorService; this.log = log; diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java b/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java index 8c28d70027e..13c029396bd 100644 --- a/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java +++ b/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java @@ -26,11 +26,11 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import org.apache.rocketmq.common.constant.LoggerName; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; public final class ThreadUtils { - private static final Logger log = LoggerFactory.getLogger(LoggerName.TOOLS_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.TOOLS_LOGGER_NAME); public static ExecutorService newThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, BlockingQueue workQueue, String processName, boolean isDaemon) { diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java index 9bd52662830..ce2b83f9775 100644 --- a/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java +++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java @@ -33,11 +33,11 @@ import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.srvutil.ServerUtil; -import org.slf4j.Logger; public class Producer { public static void main(String[] args) throws MQClientException, UnsupportedEncodingException { @@ -56,7 +56,7 @@ public static void main(String[] args) throws MQClientException, UnsupportedEnco System.out.printf("topic %s threadCount %d messageSize %d keyEnable %s%n", topic, threadCount, messageSize, keyEnable); - final Logger log = ClientLogger.getLog(); + final InternalLogger log = ClientLogger.getLog(); final ExecutorService sendThreadPool = Executors.newFixedThreadPool(threadCount); diff --git a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FiltersrvController.java b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FiltersrvController.java index f6485bada59..0a41d8b2d36 100644 --- a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FiltersrvController.java +++ b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FiltersrvController.java @@ -25,6 +25,8 @@ import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.protocol.header.filtersrv.RegisterFilterServerResponseHeader; import org.apache.rocketmq.filtersrv.filter.FilterClassManager; import org.apache.rocketmq.filtersrv.processor.DefaultRequestProcessor; @@ -32,11 +34,9 @@ import org.apache.rocketmq.remoting.RemotingServer; import org.apache.rocketmq.remoting.netty.NettyRemotingServer; import org.apache.rocketmq.remoting.netty.NettyServerConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class FiltersrvController { - private static final Logger log = LoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); private final FiltersrvConfig filtersrvConfig; diff --git a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FiltersrvStartup.java b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FiltersrvStartup.java index 585024d4a77..9fa04b758f2 100644 --- a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FiltersrvStartup.java +++ b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FiltersrvStartup.java @@ -30,16 +30,17 @@ import org.apache.rocketmq.common.MQVersion; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.netty.NettyServerConfig; import org.apache.rocketmq.remoting.netty.NettySystemConfig; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.srvutil.ServerUtil; import org.apache.rocketmq.srvutil.ShutdownHookThread; -import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class FiltersrvStartup { - public static Logger log; + public static InternalLogger log; public static void main(String[] args) { start(createController(args)); @@ -125,7 +126,7 @@ public static FiltersrvController createController(String[] args) { configurator.setContext(lc); lc.reset(); configurator.doConfigure(filtersrvConfig.getRocketmqHome() + "/conf/logback_filtersrv.xml"); - log = LoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); + log = InternalLoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); final FiltersrvController controller = new FiltersrvController(filtersrvConfig, nettyServerConfig); diff --git a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/DynaCode.java b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/DynaCode.java index 92bbf8dad47..bde9961f6ea 100644 --- a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/DynaCode.java +++ b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/DynaCode.java @@ -39,12 +39,11 @@ import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.filter.FilterAPI; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.common.RemotingHelper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - public class DynaCode { - private static final Logger log = LoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); private static final String FILE_SP = System.getProperty("file.separator"); diff --git a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/FilterClassManager.java b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/FilterClassManager.java index 490c5821ab4..360341c8247 100644 --- a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/FilterClassManager.java +++ b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/FilterClassManager.java @@ -29,12 +29,12 @@ import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.filter.MessageFilter; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.filtersrv.FiltersrvController; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class FilterClassManager { - private static final Logger log = LoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); private final Object compileLock = new Object(); private final FiltersrvController filtersrvController; diff --git a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/HttpFilterClassFetchMethod.java b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/HttpFilterClassFetchMethod.java index 8e7d695f6d2..ebd59cd86cf 100644 --- a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/HttpFilterClassFetchMethod.java +++ b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/HttpFilterClassFetchMethod.java @@ -18,13 +18,13 @@ package org.apache.rocketmq.filtersrv.filter; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.utils.HttpTinyClient; import org.apache.rocketmq.common.utils.HttpTinyClient.HttpResult; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class HttpFilterClassFetchMethod implements FilterClassFetchMethod { - private static final Logger log = LoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); private final String url; public HttpFilterClassFetchMethod(String url) { diff --git a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/processor/DefaultRequestProcessor.java b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/processor/DefaultRequestProcessor.java index e459b1aeb6d..d5335bb6625 100644 --- a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/processor/DefaultRequestProcessor.java +++ b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/processor/DefaultRequestProcessor.java @@ -31,6 +31,8 @@ import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.filter.FilterContext; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; @@ -47,11 +49,9 @@ import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.store.CommitLog; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class DefaultRequestProcessor implements NettyRequestProcessor { - private static final Logger log = LoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); private final FiltersrvController filtersrvController; @@ -61,7 +61,7 @@ public DefaultRequestProcessor(FiltersrvController filtersrvController) { @Override public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws Exception { - if (log.isDebugEnabled()) { + if (ctx != null) { log.debug("receive request, {} {} {}", request.getCode(), RemotingHelper.parseChannelRemoteAddr(ctx.channel()), diff --git a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/stats/FilterServerStatsManager.java b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/stats/FilterServerStatsManager.java index 67e722c67ea..13bc834ff5a 100644 --- a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/stats/FilterServerStatsManager.java +++ b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/stats/FilterServerStatsManager.java @@ -21,12 +21,12 @@ import java.util.concurrent.ScheduledExecutorService; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.stats.StatsItemSet; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class FilterServerStatsManager { - private static final Logger log = LoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); private final ScheduledExecutorService scheduledExecutorService = Executors .newSingleThreadScheduledExecutor(new ThreadFactoryImpl("FSStatsThread")); diff --git a/logging/src/main/java/org/apache/rocketmq/logging/InnerLoggerFactory.java b/logging/src/main/java/org/apache/rocketmq/logging/InnerLoggerFactory.java index 7714640ff8d..d95245356ef 100644 --- a/logging/src/main/java/org/apache/rocketmq/logging/InnerLoggerFactory.java +++ b/logging/src/main/java/org/apache/rocketmq/logging/InnerLoggerFactory.java @@ -24,6 +24,10 @@ public class InnerLoggerFactory extends InternalLoggerFactory { + public InnerLoggerFactory() { + doRegister(); + } + @Override protected InternalLogger getLoggerInstance(String name) { return new InnerLogger(name); diff --git a/logging/src/main/java/org/apache/rocketmq/logging/InternalLoggerFactory.java b/logging/src/main/java/org/apache/rocketmq/logging/InternalLoggerFactory.java index ec176ce6b19..17c56bdd110 100644 --- a/logging/src/main/java/org/apache/rocketmq/logging/InternalLoggerFactory.java +++ b/logging/src/main/java/org/apache/rocketmq/logging/InternalLoggerFactory.java @@ -73,10 +73,6 @@ public static void setCurrentLoggerType(String type) { } } - public InternalLoggerFactory() { - doRegister(); - } - protected void doRegister() { String loggerType = getLoggerType(); if (loggerFactoryCache.get(loggerType) != null) { diff --git a/logging/src/main/java/org/apache/rocketmq/logging/Slf4jLoggerFactory.java b/logging/src/main/java/org/apache/rocketmq/logging/Slf4jLoggerFactory.java index 1a246847388..53dbc948d1d 100644 --- a/logging/src/main/java/org/apache/rocketmq/logging/Slf4jLoggerFactory.java +++ b/logging/src/main/java/org/apache/rocketmq/logging/Slf4jLoggerFactory.java @@ -22,6 +22,11 @@ public class Slf4jLoggerFactory extends InternalLoggerFactory { + public Slf4jLoggerFactory() { + LoggerFactory.getILoggerFactory(); + doRegister(); + } + @Override protected String getLoggerType() { return InternalLoggerFactory.LOGGER_SLF4J; diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java index 2ed599c11c4..a6654f271c3 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java @@ -23,6 +23,8 @@ import org.apache.rocketmq.common.Configuration; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.namesrv.NamesrvConfig; import org.apache.rocketmq.namesrv.kvconfig.KVConfigManager; import org.apache.rocketmq.namesrv.processor.ClusterTestRequestProcessor; @@ -35,11 +37,10 @@ import org.apache.rocketmq.remoting.netty.NettyServerConfig; import org.apache.rocketmq.remoting.netty.TlsSystemConfig; import org.apache.rocketmq.srvutil.FileWatchService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + public class NamesrvController { - private static final Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); private final NamesrvConfig namesrvConfig; diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java index d7a2b2b644d..a55735a9b88 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java @@ -30,12 +30,13 @@ import org.apache.rocketmq.common.MQVersion; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.namesrv.NamesrvConfig; import org.apache.rocketmq.remoting.netty.NettyServerConfig; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.srvutil.ServerUtil; import org.apache.rocketmq.srvutil.ShutdownHookThread; -import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class NamesrvStartup { @@ -95,7 +96,7 @@ public static NamesrvController main0(String[] args) { configurator.setContext(lc); lc.reset(); configurator.doConfigure(namesrvConfig.getRocketmqHome() + "/conf/logback_namesrv.xml"); - final Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); + final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); MixAll.printObjectProperties(log, namesrvConfig); MixAll.printObjectProperties(log, nettyServerConfig); diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/kvconfig/KVConfigManager.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/kvconfig/KVConfigManager.java index 376a8143094..f35115922d0 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/kvconfig/KVConfigManager.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/kvconfig/KVConfigManager.java @@ -24,13 +24,12 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.protocol.body.KVTable; import org.apache.rocketmq.namesrv.NamesrvController; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - public class KVConfigManager { - private static final Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); private final NamesrvController namesrvController; diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/ClusterTestRequestProcessor.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/ClusterTestRequestProcessor.java index f6611b6837a..a58a3b97fc2 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/ClusterTestRequestProcessor.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/ClusterTestRequestProcessor.java @@ -20,6 +20,8 @@ import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.namesrv.NamesrvUtil; import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.namesrv.GetRouteInfoRequestHeader; @@ -28,11 +30,9 @@ import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class ClusterTestRequestProcessor extends DefaultRequestProcessor { - private static final Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); private final DefaultMQAdminExt adminExt; private final String productEnvName; diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java index ed5b20b1667..490688601b9 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java @@ -25,6 +25,8 @@ import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.namesrv.NamesrvUtil; import org.apache.rocketmq.common.namesrv.RegisterBrokerResult; import org.apache.rocketmq.common.protocol.RequestCode; @@ -50,11 +52,9 @@ import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class DefaultRequestProcessor implements NettyRequestProcessor { - private static final Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); + private static InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); protected final NamesrvController namesrvController; @@ -65,13 +65,15 @@ public DefaultRequestProcessor(NamesrvController namesrvController) { @Override public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { - if (log.isDebugEnabled()) { + + if (ctx != null) { log.debug("receive request, {} {} {}", request.getCode(), RemotingHelper.parseChannelRemoteAddr(ctx.channel()), request); } + switch (request.getCode()) { case RequestCode.PUT_KV_CONFIG: return this.putKVConfig(ctx, request); diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/BrokerHousekeepingService.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/BrokerHousekeepingService.java index d3a320776ba..fedb4ae610f 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/BrokerHousekeepingService.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/BrokerHousekeepingService.java @@ -18,13 +18,13 @@ import io.netty.channel.Channel; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.namesrv.NamesrvController; import org.apache.rocketmq.remoting.ChannelEventListener; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class BrokerHousekeepingService implements ChannelEventListener { - private static final Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); private final NamesrvController namesrvController; public BrokerHousekeepingService(NamesrvController namesrvController) { diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java index 109d3e810ec..ef02dd0087f 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java @@ -33,6 +33,8 @@ import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.PermName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.namesrv.RegisterBrokerResult; import org.apache.rocketmq.common.protocol.body.ClusterInfo; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; @@ -42,11 +44,9 @@ import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.common.sysflag.TopicSysFlag; import org.apache.rocketmq.remoting.common.RemotingUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class RouteInfoManager { - private static final Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); private final static long BROKER_CHANNEL_EXPIRED_TIME = 1000 * 60 * 2; private final ReadWriteLock lock = new ReentrantReadWriteLock(); private final HashMap> topicQueueTable; @@ -395,9 +395,7 @@ public TopicRouteData pickupTopicRouteData(final String topic) { log.error("pickupTopicRouteData Exception", e); } - if (log.isDebugEnabled()) { - log.debug("pickupTopicRouteData {} {}", topic, topicRouteData); - } + log.debug("pickupTopicRouteData {} {}", topic, topicRouteData); if (foundBrokerData && foundQueueData) { return topicRouteData; diff --git a/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessorTest.java b/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessorTest.java index 97aa9ac016c..d4a2f66f99f 100644 --- a/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessorTest.java +++ b/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessorTest.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.namesrv.NamesrvConfig; import org.apache.rocketmq.common.namesrv.RegisterBrokerResult; import org.apache.rocketmq.common.protocol.RequestCode; @@ -44,7 +45,6 @@ import org.assertj.core.util.Maps; import org.junit.Before; import org.junit.Test; -import org.slf4j.Logger; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; @@ -61,7 +61,7 @@ public class DefaultRequestProcessorTest { private RouteInfoManager routeInfoManager; - private Logger logger; + private InternalLogger logger; @Before public void init() throws Exception { @@ -78,8 +78,7 @@ public void init() throws Exception { registerRouteInfoManager(); - logger = mock(Logger.class); - when(logger.isInfoEnabled()).thenReturn(false); + logger = mock(InternalLogger.class); setFinalStatic(DefaultRequestProcessor.class.getDeclaredField("log"), logger); } diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/LocalMessageCache.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/LocalMessageCache.java index 90f9e03ed3d..cc1a5157152 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/LocalMessageCache.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/LocalMessageCache.java @@ -37,11 +37,11 @@ import org.apache.rocketmq.client.impl.consumer.ProcessQueue; import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.common.ThreadFactoryImpl; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.utils.ThreadUtils; -import org.slf4j.Logger; class LocalMessageCache implements ServiceLifecycle { private final BlockingQueue consumeRequestCache; @@ -51,7 +51,7 @@ class LocalMessageCache implements ServiceLifecycle { private final ClientConfig clientConfig; private final ScheduledExecutorService cleanExpireMsgExecutors; - private final static Logger log = ClientLogger.getLog(); + private final static InternalLogger log = ClientLogger.getLog(); LocalMessageCache(final DefaultMQPullConsumer rocketmqPullConsumer, final ClientConfig clientConfig) { consumeRequestCache = new LinkedBlockingQueue<>(clientConfig.getRmqPullMessageCacheCapacity()); diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PullConsumerImpl.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PullConsumerImpl.java index 8d396d43c5f..da4afdb7a6b 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PullConsumerImpl.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PullConsumerImpl.java @@ -34,9 +34,9 @@ import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.impl.consumer.ProcessQueue; import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; -import org.slf4j.Logger; public class PullConsumerImpl implements PullConsumer { private final DefaultMQPullConsumer rocketmqPullConsumer; @@ -47,7 +47,7 @@ public class PullConsumerImpl implements PullConsumer { private final LocalMessageCache localMessageCache; private final ClientConfig clientConfig; - final static Logger log = ClientLogger.getLog(); + final static InternalLogger log = ClientLogger.getLog(); public PullConsumerImpl(final String queueName, final KeyValue properties) { this.properties = properties; diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/AbstractOMSProducer.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/AbstractOMSProducer.java index 8246bcd294a..db25fc6529b 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/AbstractOMSProducer.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/AbstractOMSProducer.java @@ -34,15 +34,15 @@ import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.remoting.exception.RemotingConnectException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; -import org.slf4j.Logger; import static io.openmessaging.rocketmq.utils.OMSUtil.buildInstanceName; abstract class AbstractOMSProducer implements ServiceLifecycle, MessageFactory { - final static Logger log = ClientLogger.getLog(); + final static InternalLogger log = ClientLogger.getLog(); final KeyValue properties; final DefaultMQProducer rocketmqProducer; private boolean started = false; diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/promise/DefaultPromise.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/promise/DefaultPromise.java index 3e4bd266ce1..453b665c855 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/promise/DefaultPromise.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/promise/DefaultPromise.java @@ -19,13 +19,14 @@ import io.openmessaging.Promise; import io.openmessaging.PromiseListener; import io.openmessaging.exception.OMSRuntimeException; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; + import java.util.ArrayList; import java.util.List; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class DefaultPromise implements Promise { - private static final Logger LOG = LoggerFactory.getLogger(DefaultPromise.class); + private static final InternalLogger LOG = InternalLoggerFactory.getLogger(DefaultPromise.class); private final Object lock = new Object(); private volatile FutureState state = FutureState.DOING; private V result = null; diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/utils/BeanUtils.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/utils/BeanUtils.java index 104d3d964fa..ba7cd5973aa 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/utils/BeanUtils.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/utils/BeanUtils.java @@ -25,10 +25,10 @@ import java.util.Set; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.client.log.ClientLogger; -import org.slf4j.Logger; +import org.apache.rocketmq.logging.InternalLogger; public final class BeanUtils { - final static Logger log = ClientLogger.getLog(); + final static InternalLogger log = ClientLogger.getLog(); /** * Maps primitive {@code Class}es to their corresponding wrapper {@code Class}. diff --git a/pom.xml b/pom.xml index 4be64a6f692..6737ae4116e 100644 --- a/pom.xml +++ b/pom.xml @@ -504,6 +504,11 @@ rocketmq-remoting ${project.version} + + ${project.groupId} + rocketmq-logging + ${project.version} + ${project.groupId} rocketmq-test diff --git a/remoting/pom.xml b/remoting/pom.xml index c788202cbb5..21d9c8d5f00 100644 --- a/remoting/pom.xml +++ b/remoting/pom.xml @@ -42,10 +42,9 @@ netty-all - org.slf4j - slf4j-api + ${project.groupId} + rocketmq-logging - io.netty netty-tcnative diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingHelper.java b/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingHelper.java index 79957305c0f..585b60b798c 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingHelper.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingHelper.java @@ -20,9 +20,9 @@ import org.apache.rocketmq.remoting.exception.RemotingConnectException; import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.net.InetSocketAddress; @@ -34,7 +34,7 @@ public class RemotingHelper { public static final String ROCKETMQ_REMOTING = "RocketmqRemoting"; public static final String DEFAULT_CHARSET = "UTF-8"; - private static final Logger log = LoggerFactory.getLogger(ROCKETMQ_REMOTING); + private static final InternalLogger log = InternalLoggerFactory.getLogger(ROCKETMQ_REMOTING); public static String exceptionSimpleDesc(final Throwable e) { StringBuffer sb = new StringBuffer(); diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingUtil.java b/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingUtil.java index 8d24e76b4f8..3da3a183960 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingUtil.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingUtil.java @@ -31,13 +31,14 @@ import java.nio.channels.spi.SelectorProvider; import java.util.ArrayList; import java.util.Enumeration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; public class RemotingUtil { public static final String OS_NAME = System.getProperty("os.name"); - private static final Logger log = LoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING); + private static final InternalLogger log = InternalLoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING); private static boolean isLinuxPlatform = false; private static boolean isWindowsPlatform = false; diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/common/ServiceThread.java b/remoting/src/main/java/org/apache/rocketmq/remoting/common/ServiceThread.java index 346e72c5a2a..7a383e44368 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/common/ServiceThread.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/common/ServiceThread.java @@ -16,14 +16,15 @@ */ package org.apache.rocketmq.remoting.common; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; /** * Base class for background thread */ public abstract class ServiceThread implements Runnable { - private static final Logger log = LoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING); + private static final InternalLogger log = InternalLoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING); private static final long JOIN_TIME = 90 * 1000; protected final Thread thread; diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyDecoder.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyDecoder.java index 6e99b32f4cb..f64ab2d109b 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyDecoder.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyDecoder.java @@ -22,12 +22,12 @@ import java.nio.ByteBuffer; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.common.RemotingUtil; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class NettyDecoder extends LengthFieldBasedFrameDecoder { - private static final Logger log = LoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING); + private static final InternalLogger log = InternalLoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING); private static final int FRAME_MAX_LENGTH = Integer.parseInt(System.getProperty("com.rocketmq.remoting.frameMaxLength", "16777216")); diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyEncoder.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyEncoder.java index e46730411a7..8c3c56a08cc 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyEncoder.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyEncoder.java @@ -22,12 +22,12 @@ import java.nio.ByteBuffer; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.common.RemotingUtil; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class NettyEncoder extends MessageToByteEncoder { - private static final Logger log = LoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING); + private static final InternalLogger log = InternalLoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING); @Override public void encode(ChannelHandlerContext ctx, RemotingCommand remotingCommand, ByteBuf out) diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyLogger.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyLogger.java new file mode 100644 index 00000000000..4b4e86e6871 --- /dev/null +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyLogger.java @@ -0,0 +1,315 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.remoting.netty; + + +import io.netty.util.internal.logging.InternalLogLevel; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; + +import java.util.concurrent.atomic.AtomicBoolean; + +public class NettyLogger { + + private static AtomicBoolean nettyLoggerSeted = new AtomicBoolean(false); + + private static InternalLogLevel nettyLogLevel = InternalLogLevel.ERROR; + + public static void initNettyLogger() { + if (!nettyLoggerSeted.get()) { + try { + io.netty.util.internal.logging.InternalLoggerFactory.setDefaultFactory(new NettyBridgeLoggerFactory()); + } catch (Throwable e) { + //ignore + } + nettyLoggerSeted.set(true); + } + } + + private static class NettyBridgeLoggerFactory extends io.netty.util.internal.logging.InternalLoggerFactory { + @Override + protected io.netty.util.internal.logging.InternalLogger newInstance(String s) { + return new NettyBridgeLogger(s); + } + } + + private static class NettyBridgeLogger implements io.netty.util.internal.logging.InternalLogger { + + private InternalLogger logger = null; + + public NettyBridgeLogger(String name) { + logger = InternalLoggerFactory.getLogger(name); + } + + @Override + public String name() { + return logger.getName(); + } + + @Override + public boolean isEnabled(InternalLogLevel internalLogLevel) { + return nettyLogLevel.ordinal() <= internalLogLevel.ordinal(); + } + + @Override + public void log(InternalLogLevel internalLogLevel, String s) { + if (internalLogLevel.equals(InternalLogLevel.DEBUG)) { + logger.debug(s); + } + if (internalLogLevel.equals(InternalLogLevel.TRACE)) { + logger.info(s); + } + if (internalLogLevel.equals(InternalLogLevel.INFO)) { + logger.info(s); + } + if (internalLogLevel.equals(InternalLogLevel.WARN)) { + logger.warn(s); + } + if (internalLogLevel.equals(InternalLogLevel.ERROR)) { + logger.error(s); + } + } + + @Override + public void log(InternalLogLevel internalLogLevel, String s, Object o) { + if (internalLogLevel.equals(InternalLogLevel.DEBUG)) { + logger.debug(s, o); + } + if (internalLogLevel.equals(InternalLogLevel.TRACE)) { + logger.info(s, o); + } + if (internalLogLevel.equals(InternalLogLevel.INFO)) { + logger.info(s, o); + } + if (internalLogLevel.equals(InternalLogLevel.WARN)) { + logger.warn(s, o); + } + if (internalLogLevel.equals(InternalLogLevel.ERROR)) { + logger.error(s, o); + } + } + + @Override + public void log(InternalLogLevel internalLogLevel, String s, Object o, Object o1) { + if (internalLogLevel.equals(InternalLogLevel.DEBUG)) { + logger.debug(s, o, o1); + } + if (internalLogLevel.equals(InternalLogLevel.TRACE)) { + logger.info(s, o, o1); + } + if (internalLogLevel.equals(InternalLogLevel.INFO)) { + logger.info(s, o, o1); + } + if (internalLogLevel.equals(InternalLogLevel.WARN)) { + logger.warn(s, o, o1); + } + if (internalLogLevel.equals(InternalLogLevel.ERROR)) { + logger.error(s, o, o1); + } + } + + @Override + public void log(InternalLogLevel internalLogLevel, String s, Object... objects) { + if (internalLogLevel.equals(InternalLogLevel.DEBUG)) { + logger.debug(s, objects); + } + if (internalLogLevel.equals(InternalLogLevel.TRACE)) { + logger.info(s, objects); + } + if (internalLogLevel.equals(InternalLogLevel.INFO)) { + logger.info(s, objects); + } + if (internalLogLevel.equals(InternalLogLevel.WARN)) { + logger.warn(s, objects); + } + if (internalLogLevel.equals(InternalLogLevel.ERROR)) { + logger.error(s, objects); + } + } + + @Override + public void log(InternalLogLevel internalLogLevel, String s, Throwable throwable) { + if (internalLogLevel.equals(InternalLogLevel.DEBUG)) { + logger.debug(s, throwable); + } + if (internalLogLevel.equals(InternalLogLevel.TRACE)) { + logger.info(s, throwable); + } + if (internalLogLevel.equals(InternalLogLevel.INFO)) { + logger.info(s, throwable); + } + if (internalLogLevel.equals(InternalLogLevel.WARN)) { + logger.warn(s, throwable); + } + if (internalLogLevel.equals(InternalLogLevel.ERROR)) { + logger.error(s, throwable); + } + } + + @Override + public boolean isTraceEnabled() { + return isEnabled(InternalLogLevel.TRACE); + } + + @Override + public void trace(String var1) { + logger.info(var1); + } + + @Override + public void trace(String var1, Object var2) { + logger.info(var1, var2); + } + + @Override + public void trace(String var1, Object var2, Object var3) { + logger.info(var1, var2, var3); + } + + @Override + public void trace(String var1, Object... var2) { + logger.info(var1, var2); + } + + @Override + public void trace(String var1, Throwable var2) { + logger.info(var1, var2); + } + + @Override + public boolean isDebugEnabled() { + return isEnabled(InternalLogLevel.DEBUG); + } + + @Override + public void debug(String var1) { + logger.debug(var1); + } + + @Override + public void debug(String var1, Object var2) { + logger.debug(var1, var2); + } + + @Override + public void debug(String var1, Object var2, Object var3) { + logger.debug(var1, var2, var3); + } + + @Override + public void debug(String var1, Object... var2) { + logger.debug(var1, var2); + } + + @Override + public void debug(String var1, Throwable var2) { + logger.debug(var1, var2); + } + + @Override + public boolean isInfoEnabled() { + return isEnabled(InternalLogLevel.INFO); + } + + @Override + public void info(String var1) { + logger.info(var1); + } + + @Override + public void info(String var1, Object var2) { + logger.info(var1, var2); + } + + @Override + public void info(String var1, Object var2, Object var3) { + logger.info(var1, var2, var3); + } + + @Override + public void info(String var1, Object... var2) { + logger.info(var1, var2); + } + + @Override + public void info(String var1, Throwable var2) { + logger.info(var1, var2); + } + + @Override + public boolean isWarnEnabled() { + return isEnabled(InternalLogLevel.WARN); + } + + @Override + public void warn(String var1) { + logger.warn(var1); + } + + @Override + public void warn(String var1, Object var2) { + logger.warn(var1, var2); + } + + @Override + public void warn(String var1, Object... var2) { + logger.warn(var1, var2); + } + + @Override + public void warn(String var1, Object var2, Object var3) { + logger.warn(var1, var2, var3); + } + + @Override + public void warn(String var1, Throwable var2) { + logger.warn(var1, var2); + } + + @Override + public boolean isErrorEnabled() { + return isEnabled(InternalLogLevel.ERROR); + } + + @Override + public void error(String var1) { + logger.error(var1); + } + + @Override + public void error(String var1, Object var2) { + logger.error(var1, var2); + } + + @Override + public void error(String var1, Object var2, Object var3) { + logger.error(var1, var2, var3); + } + + @Override + public void error(String var1, Object... var2) { + logger.error(var1, var2); + } + + @Override + public void error(String var1, Throwable var2) { + logger.error(var1, var2); + } + } + +} diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java index 557ad5602b0..7c414e9dd65 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java @@ -45,17 +45,17 @@ import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public abstract class NettyRemotingAbstract { /** * Remoting logger instance. */ - private static final Logger log = LoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING); + private static final InternalLogger log = InternalLoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING); /** * Semaphore to limit maximum number of on-going one-way requests, which protects system memory footprint. @@ -95,6 +95,10 @@ public abstract class NettyRemotingAbstract { */ protected volatile SslContext sslContext; + static { + NettyLogger.initNettyLogger(); + } + /** * Constructor, specifying capacity of one-way and asynchronous semaphores. * diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java index dcc80cba050..7cdfb80accb 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java @@ -64,12 +64,12 @@ import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class NettyRemotingClient extends NettyRemotingAbstract implements RemotingClient { - private static final Logger log = LoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING); + private static final InternalLogger log = InternalLoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING); private static final long LOCK_TIMEOUT_MILLIS = 3000; diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java index c8709a50149..198484251c0 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java @@ -58,12 +58,12 @@ import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class NettyRemotingServer extends NettyRemotingAbstract implements RemotingServer { - private static final Logger log = LoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING); + private static final InternalLogger log = InternalLoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING); private final ServerBootstrap serverBootstrap; private final EventLoopGroup eventLoopGroupSelector; private final EventLoopGroup eventLoopGroupBoss; diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/TlsHelper.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/TlsHelper.java index 3a74b4b67bc..efbdd52c073 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/TlsHelper.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/TlsHelper.java @@ -31,8 +31,8 @@ import java.security.cert.CertificateException; import java.util.Properties; import org.apache.rocketmq.remoting.common.RemotingHelper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_CLIENT_AUTHSERVER; import static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_CLIENT_CERTPATH; @@ -73,7 +73,7 @@ public interface DecryptionStrategy { InputStream decryptPrivateKey(String privateKeyEncryptPath, boolean forClient) throws IOException; } - private static final Logger LOGGER = LoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING); + private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING); private static DecryptionStrategy decryptionStrategy = new DecryptionStrategy() { @Override diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java index 2f8cb388440..cadcab1b11e 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java @@ -28,14 +28,14 @@ import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingCommandException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; public class RemotingCommand { public static final String SERIALIZE_TYPE_PROPERTY = "rocketmq.serialize.type"; public static final String SERIALIZE_TYPE_ENV = "ROCKETMQ_SERIALIZE_TYPE"; public static final String REMOTING_VERSION_KEY = "rocketmq.remoting.version"; - private static final Logger log = LoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING); + private static final InternalLogger log = InternalLoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING); private static final int RPC_TYPE = 0; // 0, REQUEST_COMMAND private static final int RPC_ONEWAY = 1; // 0, RPC private static final Map, Field[]> CLASS_HASH_MAP = diff --git a/srvutil/src/main/java/org/apache/rocketmq/srvutil/FileWatchService.java b/srvutil/src/main/java/org/apache/rocketmq/srvutil/FileWatchService.java index bc68d6a3ca6..adf61adbd83 100644 --- a/srvutil/src/main/java/org/apache/rocketmq/srvutil/FileWatchService.java +++ b/srvutil/src/main/java/org/apache/rocketmq/srvutil/FileWatchService.java @@ -30,11 +30,11 @@ import org.apache.rocketmq.common.ServiceThread; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; public class FileWatchService extends ServiceThread { - private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); private final List watchFiles; private final List fileCurrentHash; diff --git a/srvutil/src/main/java/org/apache/rocketmq/srvutil/ShutdownHookThread.java b/srvutil/src/main/java/org/apache/rocketmq/srvutil/ShutdownHookThread.java index 11f9b2c3365..ba01e1ebae2 100644 --- a/srvutil/src/main/java/org/apache/rocketmq/srvutil/ShutdownHookThread.java +++ b/srvutil/src/main/java/org/apache/rocketmq/srvutil/ShutdownHookThread.java @@ -17,9 +17,10 @@ package org.apache.rocketmq.srvutil; +import org.apache.rocketmq.logging.InternalLogger; + import java.util.concurrent.Callable; import java.util.concurrent.atomic.AtomicInteger; -import org.slf4j.Logger; /** * {@link ShutdownHookThread} is the standard hook for filtersrv and namesrv modules. @@ -28,7 +29,7 @@ public class ShutdownHookThread extends Thread { private volatile boolean hasShutdown = false; private AtomicInteger shutdownTimes = new AtomicInteger(0); - private final Logger log; + private final InternalLogger log; private final Callable callback; /** @@ -37,7 +38,7 @@ public class ShutdownHookThread extends Thread { * @param log The log instance is used in hook thread. * @param callback The call back function. */ - public ShutdownHookThread(Logger log, Callable callback) { + public ShutdownHookThread(InternalLogger log, Callable callback) { super("ShutdownHook"); this.log = log; this.callback = callback; diff --git a/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java b/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java index ad8e65dca40..dbb27cd744c 100644 --- a/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java +++ b/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java @@ -27,15 +27,15 @@ import org.apache.rocketmq.common.ServiceThread; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.store.config.BrokerRole; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Create MappedFile in advance */ public class AllocateMappedFileService extends ServiceThread { - private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); private static int waitTimeOut = 1000 * 5; private ConcurrentMap requestTable = new ConcurrentHashMap(); diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 03d98d31925..7f772b243b4 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -26,6 +26,8 @@ import org.apache.rocketmq.common.ServiceThread; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; @@ -36,8 +38,6 @@ import org.apache.rocketmq.store.config.FlushDiskType; import org.apache.rocketmq.store.ha.HAService; import org.apache.rocketmq.store.schedule.ScheduleMessageService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Store all metadata downtime for recovery, data protection reliability @@ -45,7 +45,7 @@ public class CommitLog { // Message's MAGIC CODE daa320a7 public final static int MESSAGE_MAGIC_CODE = 0xAABBCCDD ^ 1880681586 + 8; - private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); // End of file empty MAGIC CODE cbd43194 private final static int BLANK_MAGIC_CODE = 0xBBCCDDEE ^ 1880681586 + 8; private final MappedFileQueue mappedFileQueue; @@ -215,9 +215,7 @@ public DispatchRequest checkMessageAndReturnSize(java.nio.ByteBuffer byteBuffer, private void doNothingForDeadCode(final Object obj) { if (obj != null) { - if (log.isDebugEnabled()) { - log.debug(String.valueOf(obj.hashCode())); - } + log.debug(String.valueOf(obj.hashCode())); } } diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java index 09807c36e12..08c7f999069 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java +++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java @@ -20,15 +20,15 @@ import java.nio.ByteBuffer; import java.util.List; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.store.config.StorePathConfigHelper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class ConsumeQueue { - private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); public static final int CQ_STORE_UNIT_SIZE = 20; - private static final Logger LOG_ERROR = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); + private static final InternalLogger LOG_ERROR = InternalLoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); private final DefaultMessageStore defaultMessageStore; diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueueExt.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueueExt.java index aeb2803e23f..117a70b7138 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueueExt.java +++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueueExt.java @@ -18,8 +18,8 @@ package org.apache.rocketmq.store; import org.apache.rocketmq.common.constant.LoggerName; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import java.io.File; import java.nio.ByteBuffer; @@ -37,7 +37,7 @@ *
  • 4. Pls keep this file small.
    • */ public class ConsumeQueueExt { - private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); private final MappedFileQueue mappedFileQueue; private final String topic; diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index 7a5647c3e0a..4fc7412b1fd 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -42,6 +42,8 @@ import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageExtBatch; @@ -55,13 +57,11 @@ import org.apache.rocketmq.store.index.QueryOffsetResult; import org.apache.rocketmq.store.schedule.ScheduleMessageService; import org.apache.rocketmq.store.stats.BrokerStatsManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import static org.apache.rocketmq.store.config.BrokerRole.SLAVE; public class DefaultMessageStore implements MessageStore { - private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); private final MessageStoreConfig messageStoreConfig; // CommitLog diff --git a/store/src/main/java/org/apache/rocketmq/store/MappedFile.java b/store/src/main/java/org/apache/rocketmq/store/MappedFile.java index 0a43d47f8ff..93194656f41 100644 --- a/store/src/main/java/org/apache/rocketmq/store/MappedFile.java +++ b/store/src/main/java/org/apache/rocketmq/store/MappedFile.java @@ -33,17 +33,17 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageExtBatch; import org.apache.rocketmq.store.config.FlushDiskType; import org.apache.rocketmq.store.util.LibC; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import sun.nio.ch.DirectBuffer; public class MappedFile extends ReferenceResource { public static final int OS_PAGE_SIZE = 1024 * 4; - protected static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + protected static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); private static final AtomicLong TOTAL_MAPPED_VIRTUAL_MEMORY = new AtomicLong(0); diff --git a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java index c30316f1904..86de3d2c1de 100644 --- a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java +++ b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java @@ -26,12 +26,12 @@ import java.util.concurrent.CopyOnWriteArrayList; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; public class MappedFileQueue { - private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); - private static final Logger LOG_ERROR = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static final InternalLogger LOG_ERROR = InternalLoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); private static final int DELETE_FILES_BATCH_MAX = 10; diff --git a/store/src/main/java/org/apache/rocketmq/store/StoreCheckpoint.java b/store/src/main/java/org/apache/rocketmq/store/StoreCheckpoint.java index c5981c6d476..7e6c706942b 100644 --- a/store/src/main/java/org/apache/rocketmq/store/StoreCheckpoint.java +++ b/store/src/main/java/org/apache/rocketmq/store/StoreCheckpoint.java @@ -24,11 +24,11 @@ import java.nio.channels.FileChannel.MapMode; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; public class StoreCheckpoint { - private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); private final RandomAccessFile randomAccessFile; private final FileChannel fileChannel; private final MappedByteBuffer mappedByteBuffer; diff --git a/store/src/main/java/org/apache/rocketmq/store/StoreStatsService.java b/store/src/main/java/org/apache/rocketmq/store/StoreStatsService.java index 586947ce6fc..bc6493bfa61 100644 --- a/store/src/main/java/org/apache/rocketmq/store/StoreStatsService.java +++ b/store/src/main/java/org/apache/rocketmq/store/StoreStatsService.java @@ -25,11 +25,11 @@ import java.util.concurrent.locks.ReentrantLock; import org.apache.rocketmq.common.ServiceThread; import org.apache.rocketmq.common.constant.LoggerName; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; public class StoreStatsService extends ServiceThread { - private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); private static final int FREQUENCY_OF_SAMPLING = 1000; diff --git a/store/src/main/java/org/apache/rocketmq/store/TransientStorePool.java b/store/src/main/java/org/apache/rocketmq/store/TransientStorePool.java index 21da03eeb42..8684fe95d1a 100644 --- a/store/src/main/java/org/apache/rocketmq/store/TransientStorePool.java +++ b/store/src/main/java/org/apache/rocketmq/store/TransientStorePool.java @@ -22,14 +22,14 @@ import java.util.Deque; import java.util.concurrent.ConcurrentLinkedDeque; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.util.LibC; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import sun.nio.ch.DirectBuffer; public class TransientStorePool { - private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); private final int poolSize; private final int fileSize; diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/HAConnection.java b/store/src/main/java/org/apache/rocketmq/store/ha/HAConnection.java index 8b9750464f1..e94a3ee3ee9 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ha/HAConnection.java +++ b/store/src/main/java/org/apache/rocketmq/store/ha/HAConnection.java @@ -23,13 +23,13 @@ import java.nio.channels.SocketChannel; import org.apache.rocketmq.common.ServiceThread; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.common.RemotingUtil; import org.apache.rocketmq.store.SelectMappedBufferResult; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class HAConnection { - private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); private final HAService haService; private final SocketChannel socketChannel; private final String clientAddr; diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java index 51a8a27035a..55bea0b2d7d 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java +++ b/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java @@ -34,14 +34,14 @@ import java.util.concurrent.atomic.AtomicReference; import org.apache.rocketmq.common.ServiceThread; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.common.RemotingUtil; import org.apache.rocketmq.store.CommitLog; import org.apache.rocketmq.store.DefaultMessageStore; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class HAService { - private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); private final AtomicInteger connectionCount = new AtomicInteger(0); diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/WaitNotifyObject.java b/store/src/main/java/org/apache/rocketmq/store/ha/WaitNotifyObject.java index 6aba37529a4..d2cc04c0caf 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ha/WaitNotifyObject.java +++ b/store/src/main/java/org/apache/rocketmq/store/ha/WaitNotifyObject.java @@ -17,13 +17,13 @@ package org.apache.rocketmq.store.ha; import org.apache.rocketmq.common.constant.LoggerName; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import java.util.HashMap; public class WaitNotifyObject { - private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); protected final HashMap waitingThreadTable = new HashMap(16); diff --git a/store/src/main/java/org/apache/rocketmq/store/index/IndexFile.java b/store/src/main/java/org/apache/rocketmq/store/index/IndexFile.java index edc24764e75..28505984f49 100644 --- a/store/src/main/java/org/apache/rocketmq/store/index/IndexFile.java +++ b/store/src/main/java/org/apache/rocketmq/store/index/IndexFile.java @@ -23,12 +23,12 @@ import java.nio.channels.FileLock; import java.util.List; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.store.MappedFile; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class IndexFile { - private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); private static int hashSlotSize = 4; private static int indexSize = 20; private static int invalidIndex = 0; diff --git a/store/src/main/java/org/apache/rocketmq/store/index/IndexService.java b/store/src/main/java/org/apache/rocketmq/store/index/IndexService.java index b565349e12f..bf17ecffeaf 100644 --- a/store/src/main/java/org/apache/rocketmq/store/index/IndexService.java +++ b/store/src/main/java/org/apache/rocketmq/store/index/IndexService.java @@ -25,16 +25,16 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.DispatchRequest; import org.apache.rocketmq.store.config.StorePathConfigHelper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class IndexService { - private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); /** * Maximum times to attempt index file creation. */ diff --git a/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java b/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java index 35b8e8565ea..e707463c795 100644 --- a/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java +++ b/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java @@ -27,6 +27,8 @@ import org.apache.rocketmq.common.ConfigManager; import org.apache.rocketmq.common.TopicFilterType; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; @@ -40,11 +42,9 @@ import org.apache.rocketmq.store.PutMessageStatus; import org.apache.rocketmq.store.SelectMappedBufferResult; import org.apache.rocketmq.store.config.StorePathConfigHelper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class ScheduleMessageService extends ConfigManager { - private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); public static final String SCHEDULE_TOPIC = "SCHEDULE_TOPIC_XXXX"; private static final long FIRST_DELAY_TIME = 1000L; diff --git a/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStats.java b/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStats.java index a3240a4695a..38ace7d3814 100644 --- a/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStats.java +++ b/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStats.java @@ -17,12 +17,12 @@ package org.apache.rocketmq.store.stats; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.store.DefaultMessageStore; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class BrokerStats { - private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private final DefaultMessageStore defaultMessageStore; diff --git a/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java b/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java index 64f76cabacd..ac8ae3cb645 100644 --- a/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java +++ b/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java @@ -21,11 +21,11 @@ import java.util.concurrent.ScheduledExecutorService; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.stats.MomentStatsItemSet; import org.apache.rocketmq.common.stats.StatsItem; import org.apache.rocketmq.common.stats.StatsItemSet; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class BrokerStatsManager { @@ -60,8 +60,8 @@ public class BrokerStatsManager { /** * read disk follow stats */ - private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_STATS_LOGGER_NAME); - private static final Logger COMMERCIAL_LOG = LoggerFactory.getLogger(LoggerName.COMMERCIAL_LOGGER_NAME); + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.ROCKETMQ_STATS_LOGGER_NAME); + private static final InternalLogger COMMERCIAL_LOG = InternalLoggerFactory.getLogger(LoggerName.COMMERCIAL_LOGGER_NAME); private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl( "BrokerStatsThread")); private final ScheduledExecutorService commercialExecutor = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl( diff --git a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java index e1b8c9102d1..9dcb2d8b755 100644 --- a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java +++ b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java @@ -26,6 +26,8 @@ import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.namesrv.NamesrvConfig; import org.apache.rocketmq.namesrv.NamesrvController; import org.apache.rocketmq.remoting.netty.NettyClientConfig; @@ -34,11 +36,9 @@ import org.apache.rocketmq.test.util.MQAdmin; import org.apache.rocketmq.test.util.TestUtils; import org.junit.Assert; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class IntegrationTestBase { - public static Logger logger = LoggerFactory.getLogger(IntegrationTestBase.class); + public static InternalLogger logger = InternalLoggerFactory.getLogger(IntegrationTestBase.class); protected static final String SEP = File.separator; protected static final String BROKER_NAME_PREFIX = "TestBrokerName_"; diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java index c93c40067a1..bcd66669ce2 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java @@ -47,6 +47,7 @@ import org.apache.rocketmq.common.admin.TopicOffset; import org.apache.rocketmq.common.admin.TopicStatsTable; import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageClientExt; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; @@ -84,10 +85,9 @@ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.tools.admin.api.MessageTrack; import org.apache.rocketmq.tools.admin.api.TrackType; -import org.slf4j.Logger; public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner { - private final Logger log = ClientLogger.getLog(); + private final InternalLogger log = ClientLogger.getLog(); private final DefaultMQAdminExt defaultMQAdminExt; private ServiceState serviceState = ServiceState.CREATE_JUST; private MQClientInstance mqClientInstance; diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java index f341362c3f1..67a9197c1cd 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java @@ -25,6 +25,7 @@ import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.admin.ConsumeStats; import org.apache.rocketmq.common.admin.OffsetWrapper; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.body.Connection; import org.apache.rocketmq.common.protocol.body.ConsumerConnection; @@ -36,7 +37,6 @@ import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; import org.apache.rocketmq.tools.command.SubCommand; import org.apache.rocketmq.tools.command.SubCommandException; -import org.slf4j.Logger; import java.util.Collections; import java.util.Date; @@ -46,7 +46,7 @@ import java.util.Map; public class ConsumerProgressSubCommand implements SubCommand { - private final Logger log = ClientLogger.getLog(); + private final InternalLogger log = ClientLogger.getLog(); @Override public String commandName() { diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java index 14805752772..bb66e89f3a2 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java @@ -19,16 +19,16 @@ import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.tools.command.SubCommand; import org.apache.rocketmq.tools.command.SubCommandException; import org.apache.rocketmq.tools.monitor.DefaultMonitorListener; import org.apache.rocketmq.tools.monitor.MonitorConfig; import org.apache.rocketmq.tools.monitor.MonitorService; -import org.slf4j.Logger; public class StartMonitoringSubCommand implements SubCommand { - private final Logger log = ClientLogger.getLog(); + private final InternalLogger log = ClientLogger.getLog(); @Override public String commandName() { diff --git a/tools/src/main/java/org/apache/rocketmq/tools/monitor/DefaultMonitorListener.java b/tools/src/main/java/org/apache/rocketmq/tools/monitor/DefaultMonitorListener.java index 8648774e92d..2eddf2fbe9a 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/monitor/DefaultMonitorListener.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/monitor/DefaultMonitorListener.java @@ -21,13 +21,13 @@ import java.util.Map.Entry; import java.util.TreeMap; import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; -import org.slf4j.Logger; public class DefaultMonitorListener implements MonitorListener { private final static String LOG_PREFIX = "[MONITOR] "; private final static String LOG_NOTIFY = LOG_PREFIX + " [NOTIFY] "; - private final Logger log = ClientLogger.getLog(); + private final InternalLogger log = ClientLogger.getLog(); public DefaultMonitorListener() { } diff --git a/tools/src/main/java/org/apache/rocketmq/tools/monitor/MonitorService.java b/tools/src/main/java/org/apache/rocketmq/tools/monitor/MonitorService.java index 0c9ac36f108..9bf09ad4107 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/monitor/MonitorService.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/monitor/MonitorService.java @@ -40,6 +40,7 @@ import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.admin.ConsumeStats; import org.apache.rocketmq.common.admin.OffsetWrapper; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.body.Connection; @@ -50,10 +51,9 @@ import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; -import org.slf4j.Logger; public class MonitorService { - private final Logger log = ClientLogger.getLog(); + private final InternalLogger log = ClientLogger.getLog(); private final ScheduledExecutorService scheduledExecutorService = Executors .newSingleThreadScheduledExecutor(new ThreadFactoryImpl("MonitorService")); From e74422252719cfae89e13b0231f52f1a6538db48 Mon Sep 17 00:00:00 2001 From: fuyou001 Date: Wed, 7 Mar 2018 14:38:54 +0800 Subject: [PATCH 16/73] [ROCKETMQ-319] Improve broker register performance and reduce memory usage (#205) --- .../rocketmq/broker/BrokerController.java | 62 ++++-- .../rocketmq/broker/out/BrokerOuterAPI.java | 127 ++++++++++-- .../processor/AdminBrokerProcessor.java | 4 +- .../broker/topic/TopicConfigManager.java | 8 +- .../rocketmq/broker/BrokerOuterAPITest.java | 191 ++++++++++++++++++ .../apache/rocketmq/common/BrokerConfig.java | 25 ++- .../apache/rocketmq/common/DataVersion.java | 9 + .../rocketmq/common/ThreadFactoryImpl.java | 11 +- .../rocketmq/common/protocol/RequestCode.java | 2 + .../protocol/body/RegisterBrokerBody.java | 153 ++++++++++++++ .../QueryDataVersionRequestHeader.java | 70 +++++++ .../QueryDataVersionResponseHeader.java | 48 +++++ .../namesrv/RegisterBrokerRequestHeader.java | 11 +- .../common/RegisterBrokerBodyTest.java | 51 +++++ .../processor/DefaultRequestProcessor.java | 35 +++- .../namesrv/routeinfo/RouteInfoManager.java | 19 +- 16 files changed, 775 insertions(+), 51 deletions(-) create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/QueryDataVersionRequestHeader.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/QueryDataVersionResponseHeader.java create mode 100644 common/src/test/java/org/apache/rocketmq/common/RegisterBrokerBodyTest.java diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 409b1d06b64..60f287af584 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -729,14 +729,14 @@ public void start() throws Exception { this.filterServerManager.start(); } - this.registerBrokerAll(true, false); + this.registerBrokerAll(true, false, true); this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { @Override public void run() { try { - BrokerController.this.registerBrokerAll(true, false); + BrokerController.this.registerBrokerAll(true, false, brokerConfig.isForceRegister()); } catch (Throwable e) { log.error("registerBrokerAll Exception", e); } @@ -752,7 +752,7 @@ public void run() { } } - public synchronized void registerBrokerAll(final boolean checkOrderConfig, boolean oneway) { + public synchronized void registerBrokerAll(final boolean checkOrderConfig, boolean oneway, boolean forceRegister) { TopicConfigSerializeWrapper topicConfigWrapper = this.getTopicConfigManager().buildTopicConfigSerializeWrapper(); if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission()) @@ -767,28 +767,56 @@ public synchronized void registerBrokerAll(final boolean checkOrderConfig, boole topicConfigWrapper.setTopicConfigTable(topicConfigTable); } - RegisterBrokerResult registerBrokerResult = this.brokerOuterAPI.registerBrokerAll( - this.brokerConfig.getBrokerClusterName(), + if (forceRegister || needRegister(this.brokerConfig.getBrokerClusterName(), this.getBrokerAddr(), this.brokerConfig.getBrokerName(), this.brokerConfig.getBrokerId(), - this.getHAServerAddr(), - topicConfigWrapper, - this.filterServerManager.buildNewFilterServerList(), - oneway, - this.brokerConfig.getRegisterBrokerTimeoutMills()); - - if (registerBrokerResult != null) { - if (this.updateMasterHAServerAddrPeriodically && registerBrokerResult.getHaServerAddr() != null) { - this.messageStore.updateHaMasterAddress(registerBrokerResult.getHaServerAddr()); + this.brokerConfig.getRegisterBrokerTimeoutMills())) { + List registerBrokerResultList = this.brokerOuterAPI.registerBrokerAll( + this.brokerConfig.getBrokerClusterName(), + this.getBrokerAddr(), + this.brokerConfig.getBrokerName(), + this.brokerConfig.getBrokerId(), + this.getHAServerAddr(), + topicConfigWrapper, + this.filterServerManager.buildNewFilterServerList(), + oneway, + this.brokerConfig.getRegisterBrokerTimeoutMills(), + this.brokerConfig.isCompressedRegister()); + + if (registerBrokerResultList.size() > 0) { + RegisterBrokerResult registerBrokerResult = registerBrokerResultList.get(0); + if (registerBrokerResult != null) { + if (this.updateMasterHAServerAddrPeriodically && registerBrokerResult.getHaServerAddr() != null) { + this.messageStore.updateHaMasterAddress(registerBrokerResult.getHaServerAddr()); + } + + this.slaveSynchronize.setMasterAddr(registerBrokerResult.getMasterAddr()); + + if (checkOrderConfig) { + this.getTopicConfigManager().updateOrderTopicConfig(registerBrokerResult.getKvTable()); + } + } } + } + } - this.slaveSynchronize.setMasterAddr(registerBrokerResult.getMasterAddr()); + private boolean needRegister(final String clusterName, + final String brokerAddr, + final String brokerName, + final long brokerId, + final int timeoutMills) { - if (checkOrderConfig) { - this.getTopicConfigManager().updateOrderTopicConfig(registerBrokerResult.getKvTable()); + TopicConfigSerializeWrapper topicConfigWrapper = this.getTopicConfigManager().buildTopicConfigSerializeWrapper(); + List changeList = brokerOuterAPI.needRegister(clusterName, brokerAddr, brokerName, brokerId, topicConfigWrapper, timeoutMills); + boolean needRegister = false; + for (Boolean changed : changeList) { + if (changed) { + needRegister = true; + break; } } + return needRegister; } public TopicConfigManager getTopicConfigManager() { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java index 87c00a3f33d..262e2d2c911 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java @@ -16,11 +16,19 @@ */ package org.apache.rocketmq.broker.out; +import com.google.common.collect.Lists; import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.broker.latency.BrokerFixedThreadPoolExecutor; import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.common.DataVersion; import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; @@ -33,6 +41,8 @@ import org.apache.rocketmq.common.protocol.body.RegisterBrokerBody; import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; +import org.apache.rocketmq.common.protocol.header.namesrv.QueryDataVersionRequestHeader; +import org.apache.rocketmq.common.protocol.header.namesrv.QueryDataVersionResponseHeader; import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerResponseHeader; import org.apache.rocketmq.common.protocol.header.namesrv.UnRegisterBrokerRequestHeader; @@ -52,6 +62,8 @@ public class BrokerOuterAPI { private final RemotingClient remotingClient; private final TopAddressing topAddressing = new TopAddressing(MixAll.getWSAddr()); private String nameSrvAddr = null; + private BrokerFixedThreadPoolExecutor brokerOuterExecutor = new BrokerFixedThreadPoolExecutor(4, 10, 1, TimeUnit.MINUTES, + new ArrayBlockingQueue(32), new ThreadFactoryImpl("brokerOutApi_thread_", true)); public BrokerOuterAPI(final NettyClientConfig nettyClientConfig) { this(nettyClientConfig, null); @@ -97,7 +109,7 @@ public void updateNameServerAddressList(final String addrs) { this.remotingClient.updateNameServerAddressList(lst); } - public RegisterBrokerResult registerBrokerAll( + public List registerBrokerAll( final String clusterName, final String brokerAddr, final String brokerName, @@ -106,27 +118,41 @@ public RegisterBrokerResult registerBrokerAll( final TopicConfigSerializeWrapper topicConfigWrapper, final List filterServerList, final boolean oneway, - final int timeoutMills) { - RegisterBrokerResult registerBrokerResult = null; + final int timeoutMills, + final boolean compressed) { + final List registerBrokerResultList = Lists.newArrayList(); List nameServerAddressList = this.remotingClient.getNameServerAddressList(); - if (nameServerAddressList != null) { - for (String namesrvAddr : nameServerAddressList) { - try { - RegisterBrokerResult result = this.registerBroker(namesrvAddr, clusterName, brokerAddr, brokerName, brokerId, - haServerAddr, topicConfigWrapper, filterServerList, oneway, timeoutMills); - if (result != null) { - registerBrokerResult = result; + if (nameServerAddressList != null && nameServerAddressList.size() > 0) { + final CountDownLatch countDownLatch = new CountDownLatch(nameServerAddressList.size()); + for (final String namesrvAddr : nameServerAddressList) { + brokerOuterExecutor.execute(new Runnable() { + @Override + public void run() { + try { + RegisterBrokerResult result = registerBroker(namesrvAddr, clusterName, brokerAddr, brokerName, brokerId, + haServerAddr, topicConfigWrapper, filterServerList, oneway, timeoutMills, compressed); + if (result != null) { + registerBrokerResultList.add(result); + } + + log.info("register broker to name server {} OK", namesrvAddr); + } catch (Exception e) { + log.warn("registerBroker Exception, {}", namesrvAddr, e); + } finally { + countDownLatch.countDown(); + } } + }); + } - log.info("register broker to name server {} OK", namesrvAddr); - } catch (Exception e) { - log.warn("registerBroker Exception, {}", namesrvAddr, e); - } + try { + countDownLatch.await(timeoutMills, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { } } - return registerBrokerResult; + return registerBrokerResultList; } private RegisterBrokerResult registerBroker( @@ -139,7 +165,8 @@ private RegisterBrokerResult registerBroker( final TopicConfigSerializeWrapper topicConfigWrapper, final List filterServerList, final boolean oneway, - final int timeoutMills + final int timeoutMills, + final boolean compressed ) throws RemotingCommandException, MQBrokerException, RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException { RegisterBrokerRequestHeader requestHeader = new RegisterBrokerRequestHeader(); @@ -148,12 +175,13 @@ private RegisterBrokerResult registerBroker( requestHeader.setBrokerName(brokerName); requestHeader.setClusterName(clusterName); requestHeader.setHaServerAddr(haServerAddr); + requestHeader.setCompressed(compressed); RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.REGISTER_BROKER, requestHeader); RegisterBrokerBody requestBody = new RegisterBrokerBody(); requestBody.setTopicConfigSerializeWrapper(topicConfigWrapper); requestBody.setFilterServerList(filterServerList); - request.setBody(requestBody.encode()); + request.setBody(requestBody.encode(requestHeader.isCompressed())); if (oneway) { try { @@ -231,6 +259,71 @@ public void unregisterBroker( throw new MQBrokerException(response.getCode(), response.getRemark()); } + public List needRegister( + final String clusterName, + final String brokerAddr, + final String brokerName, + final long brokerId, + final TopicConfigSerializeWrapper topicConfigWrapper, + final int timeoutMills) { + final List changedList = new CopyOnWriteArrayList<>(); + List nameServerAddressList = this.remotingClient.getNameServerAddressList(); + if (nameServerAddressList != null && nameServerAddressList.size() > 0) { + final CountDownLatch countDownLatch = new CountDownLatch(nameServerAddressList.size()); + for (final String namesrvAddr : nameServerAddressList) { + brokerOuterExecutor.execute(new Runnable() { + @Override + public void run() { + try { + QueryDataVersionRequestHeader requestHeader = new QueryDataVersionRequestHeader(); + requestHeader.setBrokerAddr(brokerAddr); + requestHeader.setBrokerId(brokerId); + requestHeader.setBrokerName(brokerName); + requestHeader.setClusterName(clusterName); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_DATA_VERSION, requestHeader); + request.setBody(topicConfigWrapper.getDataVersion().encode()); + RemotingCommand response = remotingClient.invokeSync(namesrvAddr, request, timeoutMills); + DataVersion nameServerDataVersion = null; + Boolean changed = false; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + QueryDataVersionResponseHeader queryDataVersionResponseHeader = + (QueryDataVersionResponseHeader) response.decodeCommandCustomHeader(QueryDataVersionResponseHeader.class); + changed = queryDataVersionResponseHeader.getChanged(); + byte[] body = response.getBody(); + if (body != null) { + nameServerDataVersion = DataVersion.decode(body, DataVersion.class); + if (!topicConfigWrapper.getDataVersion().equals(nameServerDataVersion)) { + changed = true; + } + } + if (changed == null || changed) { + changedList.add(Boolean.TRUE); + } + } + default: + break; + } + log.warn("Query data version from name server {} OK,changed {}, broker {},name server {}", namesrvAddr, changed, topicConfigWrapper.getDataVersion(), nameServerDataVersion == null ? "" : nameServerDataVersion); + } catch (Exception e) { + changedList.add(Boolean.TRUE); + log.error("Query data version from name server {} Exception, {}", namesrvAddr, e); + } finally { + countDownLatch.countDown(); + } + } + }); + + } + try { + countDownLatch.await(timeoutMills, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + log.error("query dataversion from nameserver countDownLatch await Exception", e); + } + } + return changedList; + } + public TopicConfigSerializeWrapper getAllTopicConfig( final String addr) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index e8be2d4f4b5..a9e54aa3e65 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -245,7 +245,7 @@ private RemotingCommand updateAndCreateTopic(ChannelHandlerContext ctx, topicConfig.setTopicSysFlag(requestHeader.getTopicSysFlag() == null ? 0 : requestHeader.getTopicSysFlag()); this.brokerController.getTopicConfigManager().updateTopicConfig(topicConfig); - this.brokerController.registerBrokerAll(false, true); + this.brokerController.registerBrokerAll(false, true, true); return null; } @@ -310,8 +310,8 @@ private RemotingCommand updateBrokerConfig(ChannelHandlerContext ctx, RemotingCo log.info("updateBrokerConfig, new config: [{}] client: {} ", properties, ctx.channel().remoteAddress()); this.brokerController.getConfiguration().update(properties); if (properties.containsKey("brokerPermission")) { - this.brokerController.registerBrokerAll(false, false); this.brokerController.getTopicConfigManager().getDataVersion().nextVersion(); + this.brokerController.registerBrokerAll(false, false, true); } } else { log.error("string2Properties error"); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java index 29e22808c5a..cdae66f2aba 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java @@ -210,7 +210,7 @@ public TopicConfig createTopicInSendMessageMethod(final String topic, final Stri } if (createNew) { - this.brokerController.registerBrokerAll(false, true); + this.brokerController.registerBrokerAll(false, true,true); } return topicConfig; @@ -254,7 +254,7 @@ public TopicConfig createTopicInSendMessageBackMethod( } if (createNew) { - this.brokerController.registerBrokerAll(false, true); + this.brokerController.registerBrokerAll(false, true,true); } return topicConfig; @@ -279,7 +279,7 @@ public void updateTopicUnitFlag(final String topic, final boolean unit) { this.dataVersion.nextVersion(); this.persist(); - this.brokerController.registerBrokerAll(false, true); + this.brokerController.registerBrokerAll(false, true,true); } } @@ -299,7 +299,7 @@ public void updateTopicUnitSubFlag(final String topic, final boolean hasUnitSub) this.dataVersion.nextVersion(); this.persist(); - this.brokerController.registerBrokerAll(false, true); + this.brokerController.registerBrokerAll(false, true,true); } } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java b/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java new file mode 100644 index 00000000000..69e0dd382c5 --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.broker; + +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import io.netty.channel.ChannelHandlerContext; +import java.lang.reflect.Field; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.broker.out.BrokerOuterAPI; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.namesrv.RegisterBrokerResult; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; +import org.apache.rocketmq.common.protocol.header.namesrv.QueryDataVersionResponseHeader; +import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerResponseHeader; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.netty.NettyRemotingClient; +import org.apache.rocketmq.remoting.netty.NettyServerConfig; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import static org.junit.Assert.assertEquals; +import org.junit.Test; +import org.junit.runner.RunWith; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import org.mockito.Mock; +import static org.mockito.Mockito.when; +import org.mockito.Spy; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +@RunWith(MockitoJUnitRunner.class) +public class BrokerOuterAPITest { + @Mock + private ChannelHandlerContext handlerContext; + @Spy + private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig()); + @Mock + private MessageStore messageStore; + private String clusterName = "clusterName"; + private String brokerName = "brokerName"; + private String brokerAddr = "brokerAddr"; + private long brokerId = 0L; + private String nameserver1 = "127.0.0.1"; + private String nameserver2 = "127.0.0.2"; + private String nameserver3 = "127.0.0.3"; + private int timeOut = 3000; + + @Mock + private NettyRemotingClient nettyRemotingClient; + + private BrokerOuterAPI brokerOuterAPI; + + public void init() throws Exception { + brokerOuterAPI = new BrokerOuterAPI(new NettyClientConfig(), null); + Field field = BrokerOuterAPI.class.getDeclaredField("remotingClient"); + field.setAccessible(true); + field.set(brokerOuterAPI, nettyRemotingClient); + } + + @Test + public void test_needRegister_normal() throws Exception { + init(); + brokerOuterAPI.start(); + final RemotingCommand response = buildResponse(Boolean.TRUE); + + TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + + when(nettyRemotingClient.getNameServerAddressList()).thenReturn(Lists.asList(nameserver1, nameserver2, new String[] {nameserver3})); + when(nettyRemotingClient.invokeSync(anyString(), any(RemotingCommand.class), anyLong())).thenReturn(response); + List booleanList = brokerOuterAPI.needRegister(clusterName, brokerAddr, brokerName, brokerId, topicConfigSerializeWrapper, timeOut); + assertEquals(3, booleanList.size()); + assertEquals(false, booleanList.contains(Boolean.FALSE)); + } + + @Test + public void test_needRegister_timeout() throws Exception { + init(); + brokerOuterAPI.start(); + + TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + + when(nettyRemotingClient.getNameServerAddressList()).thenReturn(Lists.asList(nameserver1, nameserver2, new String[] {nameserver3})); + + when(nettyRemotingClient.invokeSync(anyString(), any(RemotingCommand.class), anyLong())).thenAnswer(new Answer() { + @Override + public RemotingCommand answer(InvocationOnMock invocation) throws Throwable { + if (invocation.getArgument(0) == nameserver1) { + return buildResponse(Boolean.TRUE); + } else if (invocation.getArgument(0) == nameserver2) { + return buildResponse(Boolean.FALSE); + } else if (invocation.getArgument(0) == nameserver3) { + TimeUnit.MILLISECONDS.sleep(timeOut + 20); + return buildResponse(Boolean.TRUE); + } + return buildResponse(Boolean.TRUE); + } + }); + List booleanList = brokerOuterAPI.needRegister(clusterName, brokerAddr, brokerName, brokerId, topicConfigSerializeWrapper, timeOut); + assertEquals(2, booleanList.size()); + boolean success = Iterables.any(booleanList, + new Predicate() { + public boolean apply(Boolean input) { + return input ? true : false; + } + }); + + assertEquals(true, success); + + } + + @Test + public void test_register_normal() throws Exception { + init(); + brokerOuterAPI.start(); + + final RemotingCommand response = RemotingCommand.createResponseCommand(RegisterBrokerResponseHeader.class); + final RegisterBrokerResponseHeader responseHeader = (RegisterBrokerResponseHeader) response.readCustomHeader(); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + + TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + + when(nettyRemotingClient.getNameServerAddressList()).thenReturn(Lists.asList(nameserver1, nameserver2, new String[] {nameserver3})); + when(nettyRemotingClient.invokeSync(anyString(), any(RemotingCommand.class), anyLong())).thenReturn(response); + List registerBrokerResultList = brokerOuterAPI.registerBrokerAll(clusterName, brokerAddr, brokerName, brokerId, "hasServerAddr", topicConfigSerializeWrapper, Lists.newArrayList(), false, timeOut, true); + + assertEquals(3, registerBrokerResultList.size()); + } + + @Test + public void test_register_timeout() throws Exception { + init(); + brokerOuterAPI.start(); + + final RemotingCommand response = RemotingCommand.createResponseCommand(RegisterBrokerResponseHeader.class); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + + TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + + when(nettyRemotingClient.getNameServerAddressList()).thenReturn(Lists.asList(nameserver1, nameserver2, new String[] {nameserver3})); + when(nettyRemotingClient.invokeSync(anyString(), any(RemotingCommand.class), anyLong())).thenAnswer(new Answer() { + @Override + public RemotingCommand answer(InvocationOnMock invocation) throws Throwable { + if (invocation.getArgument(0) == nameserver1) { + return response; + } else if (invocation.getArgument(0) == nameserver2) { + return response; + } else if (invocation.getArgument(0) == nameserver3) { + TimeUnit.MILLISECONDS.sleep(timeOut + 20); + return response; + } + return response; + } + }); + List registerBrokerResultList = brokerOuterAPI.registerBrokerAll(clusterName, brokerAddr, brokerName, brokerId, "hasServerAddr", topicConfigSerializeWrapper, Lists.newArrayList(), false, timeOut, true); + + assertEquals(2, registerBrokerResultList.size()); + } + + private RemotingCommand buildResponse(Boolean changed) { + final RemotingCommand response = RemotingCommand.createResponseCommand(QueryDataVersionResponseHeader.class); + final QueryDataVersionResponseHeader responseHeader = (QueryDataVersionResponseHeader) response.readCustomHeader(); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + responseHeader.setChanged(changed); + return response; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index 4d7eb469ead..4468b2d5573 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -16,6 +16,8 @@ */ package org.apache.rocketmq.common; +import java.net.InetAddress; +import java.net.UnknownHostException; import org.apache.rocketmq.common.annotation.ImportantField; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.PermName; @@ -23,9 +25,6 @@ import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.common.RemotingUtil; -import java.net.InetAddress; -import java.net.UnknownHostException; - public class BrokerConfig { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); @@ -133,6 +132,10 @@ public class BrokerConfig { private boolean filterSupportRetry = false; private boolean enablePropertyFilter = false; + private boolean compressedRegister = false; + + private boolean forceRegister = false; + public boolean isTraceOn() { return traceOn; } @@ -598,4 +601,20 @@ public boolean isEnablePropertyFilter() { public void setEnablePropertyFilter(boolean enablePropertyFilter) { this.enablePropertyFilter = enablePropertyFilter; } + + public boolean isCompressedRegister() { + return compressedRegister; + } + + public void setCompressedRegister(boolean compressedRegister) { + this.compressedRegister = compressedRegister; + } + + public boolean isForceRegister() { + return forceRegister; + } + + public void setForceRegister(boolean forceRegister) { + this.forceRegister = forceRegister; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/DataVersion.java b/common/src/main/java/org/apache/rocketmq/common/DataVersion.java index 71b00fdd72d..e54000deb65 100644 --- a/common/src/main/java/org/apache/rocketmq/common/DataVersion.java +++ b/common/src/main/java/org/apache/rocketmq/common/DataVersion.java @@ -78,4 +78,13 @@ public int hashCode() { } return result; } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("DataVersion["); + sb.append("timestamp=").append(timestamp); + sb.append(", counter=").append(counter); + sb.append(']'); + return sb.toString(); + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/ThreadFactoryImpl.java b/common/src/main/java/org/apache/rocketmq/common/ThreadFactoryImpl.java index 3860ec3cc72..564d60c54e4 100644 --- a/common/src/main/java/org/apache/rocketmq/common/ThreadFactoryImpl.java +++ b/common/src/main/java/org/apache/rocketmq/common/ThreadFactoryImpl.java @@ -23,14 +23,21 @@ public class ThreadFactoryImpl implements ThreadFactory { private final AtomicLong threadIndex = new AtomicLong(0); private final String threadNamePrefix; + private final boolean daemon; public ThreadFactoryImpl(final String threadNamePrefix) { + this(threadNamePrefix, false); + } + + public ThreadFactoryImpl(final String threadNamePrefix, boolean daemon) { this.threadNamePrefix = threadNamePrefix; + this.daemon = daemon; } @Override public Thread newThread(Runnable r) { - return new Thread(r, threadNamePrefix + this.threadIndex.incrementAndGet()); - + Thread thread = new Thread(r, threadNamePrefix + this.threadIndex.incrementAndGet()); + thread.setDaemon(daemon); + return thread; } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java b/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java index 5900c0b9d0c..8cf2d46adad 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java @@ -165,4 +165,6 @@ public class RequestCode { public static final int SEND_BATCH_MESSAGE = 320; public static final int QUERY_CONSUME_QUEUE = 321; + + public static final int QUERY_DATA_VERSION = 322; } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/RegisterBrokerBody.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/RegisterBrokerBody.java index c220927c23b..2b49b6d144b 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/body/RegisterBrokerBody.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/RegisterBrokerBody.java @@ -17,14 +17,155 @@ package org.apache.rocketmq.common.protocol.body; +import com.alibaba.fastjson.JSON; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.zip.Deflater; +import java.util.zip.DeflaterOutputStream; +import java.util.zip.InflaterInputStream; +import org.apache.rocketmq.common.DataVersion; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.remoting.protocol.RemotingSerializable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class RegisterBrokerBody extends RemotingSerializable { + + private static final Logger LOGGER = LoggerFactory.getLogger(RegisterBrokerBody.class); private TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); private List filterServerList = new ArrayList(); + public byte[] encode(boolean compress) { + + if (!compress) { + return super.encode(); + } + long start = System.currentTimeMillis(); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + DeflaterOutputStream outputStream = new DeflaterOutputStream(byteArrayOutputStream, new Deflater(Deflater.BEST_COMPRESSION)); + DataVersion dataVersion = topicConfigSerializeWrapper.getDataVersion(); + ConcurrentMap topicConfigTable = cloneTopicConfigTable(topicConfigSerializeWrapper.getTopicConfigTable()); + assert topicConfigTable != null; + try { + byte[] buffer = dataVersion.encode(); + + // write data version + outputStream.write(convertIntToByteArray(buffer.length)); + outputStream.write(buffer); + + int topicNumber = topicConfigTable.size(); + + // write number of topic configs + outputStream.write(convertIntToByteArray(topicNumber)); + + // write topic config entry one by one. + for (ConcurrentMap.Entry next : topicConfigTable.entrySet()) { + buffer = next.getValue().encode().getBytes(MixAll.DEFAULT_CHARSET); + outputStream.write(convertIntToByteArray(buffer.length)); + outputStream.write(buffer); + } + + buffer = JSON.toJSONString(filterServerList).getBytes(MixAll.DEFAULT_CHARSET); + + // write filter server list json length + outputStream.write(convertIntToByteArray(buffer.length)); + + // write filter server list json + outputStream.write(buffer); + + outputStream.finish(); + long interval = System.currentTimeMillis() - start; + if (interval > 50) { + LOGGER.info("Compressing takes {}ms", interval); + } + return byteArrayOutputStream.toByteArray(); + } catch (IOException e) { + LOGGER.error("Failed to compress RegisterBrokerBody object", e); + } + + return null; + } + + public static RegisterBrokerBody decode(byte[] data, boolean compressed) throws IOException { + if (!compressed) { + return RegisterBrokerBody.decode(data, RegisterBrokerBody.class); + } + long start = System.currentTimeMillis(); + InflaterInputStream inflaterInputStream = new InflaterInputStream(new ByteArrayInputStream(data)); + int dataVersionLength = readInt(inflaterInputStream); + byte[] dataVersionBytes = readBytes(inflaterInputStream, dataVersionLength); + DataVersion dataVersion = DataVersion.decode(dataVersionBytes, DataVersion.class); + + RegisterBrokerBody registerBrokerBody = new RegisterBrokerBody(); + registerBrokerBody.getTopicConfigSerializeWrapper().setDataVersion(dataVersion); + ConcurrentMap topicConfigTable = registerBrokerBody.getTopicConfigSerializeWrapper().getTopicConfigTable(); + + int topicConfigNumber = readInt(inflaterInputStream); + LOGGER.debug("{} topic configs to extract", topicConfigNumber); + + for (int i = 0; i < topicConfigNumber; i++) { + int topicConfigJsonLength = readInt(inflaterInputStream); + + byte[] buffer = readBytes(inflaterInputStream, topicConfigJsonLength); + TopicConfig topicConfig = new TopicConfig(); + String topicConfigJson = new String(buffer, MixAll.DEFAULT_CHARSET); + topicConfig.decode(topicConfigJson); + topicConfigTable.put(topicConfig.getTopicName(), topicConfig); + } + + int filterServerListJsonLength = readInt(inflaterInputStream); + + byte[] filterServerListBuffer = readBytes(inflaterInputStream, filterServerListJsonLength); + String filterServerListJson = new String(filterServerListBuffer, MixAll.DEFAULT_CHARSET); + List filterServerList = new ArrayList(); + try { + filterServerList = JSON.parseArray(filterServerListJson, String.class); + } catch (Exception e) { + LOGGER.error("Decompressing occur Exception {}", filterServerListJson); + } + + registerBrokerBody.setFilterServerList(filterServerList); + long interval = System.currentTimeMillis() - start; + if (interval > 50) { + LOGGER.info("Decompressing takes {}ms", interval); + } + return registerBrokerBody; + } + + private static byte[] convertIntToByteArray(int n) { + ByteBuffer byteBuffer = ByteBuffer.allocate(4); + byteBuffer.putInt(n); + return byteBuffer.array(); + } + + private static byte[] readBytes(InflaterInputStream inflaterInputStream, int length) throws IOException { + byte[] buffer = new byte[length]; + int bytesRead = 0; + while (bytesRead < length) { + int len = inflaterInputStream.read(buffer, bytesRead, length - bytesRead); + if (len == -1) { + throw new IOException("End of compressed data has reached"); + } else { + bytesRead += len; + } + } + return buffer; + } + + private static int readInt(InflaterInputStream inflaterInputStream) throws IOException { + byte[] buffer = readBytes(inflaterInputStream, 4); + ByteBuffer byteBuffer = ByteBuffer.wrap(buffer); + return byteBuffer.getInt(); + } + public TopicConfigSerializeWrapper getTopicConfigSerializeWrapper() { return topicConfigSerializeWrapper; } @@ -40,4 +181,16 @@ public List getFilterServerList() { public void setFilterServerList(List filterServerList) { this.filterServerList = filterServerList; } + + public static ConcurrentMap cloneTopicConfigTable( + ConcurrentMap topicConfigConcurrentMap) { + ConcurrentHashMap result = new ConcurrentHashMap(); + if (topicConfigConcurrentMap != null) { + for (Map.Entry entry : topicConfigConcurrentMap.entrySet()) { + result.put(entry.getKey(), entry.getValue()); + } + } + return result; + + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/QueryDataVersionRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/QueryDataVersionRequestHeader.java new file mode 100644 index 00000000000..ac6a617db2f --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/QueryDataVersionRequestHeader.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.protocol.header.namesrv; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +public class QueryDataVersionRequestHeader implements CommandCustomHeader { + @CFNotNull + private String brokerName; + @CFNotNull + private String brokerAddr; + @CFNotNull + private String clusterName; + @CFNotNull + private Long brokerId; + + @Override + public void checkFields() throws RemotingCommandException { + + } + + public String getBrokerName() { + return brokerName; + } + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } + + public String getBrokerAddr() { + return brokerAddr; + } + + public void setBrokerAddr(String brokerAddr) { + this.brokerAddr = brokerAddr; + } + + public String getClusterName() { + return clusterName; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + public Long getBrokerId() { + return brokerId; + } + + public void setBrokerId(Long brokerId) { + this.brokerId = brokerId; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/QueryDataVersionResponseHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/QueryDataVersionResponseHeader.java new file mode 100644 index 00000000000..90741e5f5b3 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/QueryDataVersionResponseHeader.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.protocol.header.namesrv; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +public class QueryDataVersionResponseHeader implements CommandCustomHeader { + @CFNotNull + private Boolean changed; + + @Override + public void checkFields() throws RemotingCommandException { + + } + + public Boolean getChanged() { + return changed; + } + + public void setChanged(Boolean changed) { + this.changed = changed; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("QueryDataVersionResponseHeader{"); + sb.append("changed=").append(changed); + sb.append('}'); + return sb.toString(); + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/RegisterBrokerRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/RegisterBrokerRequestHeader.java index 45d5b6e9ebd..7ed7a403d6b 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/RegisterBrokerRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/RegisterBrokerRequestHeader.java @@ -36,7 +36,8 @@ public class RegisterBrokerRequestHeader implements CommandCustomHeader { @CFNotNull private Long brokerId; - @Override + private boolean compressed; + public void checkFields() throws RemotingCommandException { } @@ -79,4 +80,12 @@ public Long getBrokerId() { public void setBrokerId(Long brokerId) { this.brokerId = brokerId; } + + public boolean isCompressed() { + return compressed; + } + + public void setCompressed(boolean compressed) { + this.compressed = compressed; + } } diff --git a/common/src/test/java/org/apache/rocketmq/common/RegisterBrokerBodyTest.java b/common/src/test/java/org/apache/rocketmq/common/RegisterBrokerBodyTest.java new file mode 100644 index 00000000000..87a0fc0085d --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/RegisterBrokerBodyTest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common; + +import java.io.IOException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import org.apache.rocketmq.common.protocol.body.RegisterBrokerBody; +import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; +import static org.junit.Assert.assertEquals; +import org.junit.Test; + +public class RegisterBrokerBodyTest { + @Test + public void test_encode_decode() throws IOException { + RegisterBrokerBody registerBrokerBody = new RegisterBrokerBody(); + TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + registerBrokerBody.setTopicConfigSerializeWrapper(topicConfigSerializeWrapper); + + ConcurrentMap topicConfigTable = new ConcurrentHashMap(); + for (int i = 0; i < 10000; i++) { + topicConfigTable.put(String.valueOf(i), new TopicConfig(String.valueOf(i))); + } + + topicConfigSerializeWrapper.setTopicConfigTable(topicConfigTable); + + byte[] compareEncode = registerBrokerBody.encode(true); + byte[] encode2 = registerBrokerBody.encode(false); + System.out.println(compareEncode.length); + System.out.println(encode2.length); + RegisterBrokerBody decodeRegisterBrokerBody = RegisterBrokerBody.decode(compareEncode, true); + + assertEquals(registerBrokerBody.getTopicConfigSerializeWrapper().getTopicConfigTable().size(), decodeRegisterBrokerBody.getTopicConfigSerializeWrapper().getTopicConfigTable().size()); + + } +} diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java index 490688601b9..236e6a12ca0 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java @@ -20,6 +20,7 @@ import java.io.UnsupportedEncodingException; import java.util.Properties; import java.util.concurrent.atomic.AtomicLong; +import org.apache.rocketmq.common.DataVersion; import org.apache.rocketmq.common.MQVersion; import org.apache.rocketmq.common.MQVersion.Version; import org.apache.rocketmq.common.MixAll; @@ -41,6 +42,8 @@ import org.apache.rocketmq.common.protocol.header.namesrv.GetKVListByNamespaceRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.GetRouteInfoRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.PutKVConfigRequestHeader; +import org.apache.rocketmq.common.protocol.header.namesrv.QueryDataVersionRequestHeader; +import org.apache.rocketmq.common.protocol.header.namesrv.QueryDataVersionResponseHeader; import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerResponseHeader; import org.apache.rocketmq.common.protocol.header.namesrv.UnRegisterBrokerRequestHeader; @@ -81,6 +84,8 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx, return this.getKVConfig(ctx, request); case RequestCode.DELETE_KV_CONFIG: return this.deleteKVConfig(ctx, request); + case RequestCode.QUERY_DATA_VERSION: + return queryBrokerTopicConfig(ctx, request); case RequestCode.REGISTER_BROKER: Version brokerVersion = MQVersion.value2Version(request.getVersion()); if (brokerVersion.ordinal() >= MQVersion.Version.V3_0_11.ordinal()) { @@ -194,7 +199,11 @@ public RemotingCommand registerBrokerWithFilterServer(ChannelHandlerContext ctx, RegisterBrokerBody registerBrokerBody = new RegisterBrokerBody(); if (request.getBody() != null) { - registerBrokerBody = RegisterBrokerBody.decode(request.getBody(), RegisterBrokerBody.class); + try { + registerBrokerBody = RegisterBrokerBody.decode(request.getBody(), requestHeader.isCompressed()); + } catch (Exception e) { + throw new RemotingCommandException("Failed to decode RegisterBrokerBody", e); + } } else { registerBrokerBody.getTopicConfigSerializeWrapper().getDataVersion().setCounter(new AtomicLong(0)); registerBrokerBody.getTopicConfigSerializeWrapper().getDataVersion().setTimestamp(0); @@ -221,6 +230,30 @@ public RemotingCommand registerBrokerWithFilterServer(ChannelHandlerContext ctx, return response; } + public RemotingCommand queryBrokerTopicConfig(ChannelHandlerContext ctx, + RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(QueryDataVersionResponseHeader.class); + final QueryDataVersionResponseHeader responseHeader = (QueryDataVersionResponseHeader) response.readCustomHeader(); + final QueryDataVersionRequestHeader requestHeader = + (QueryDataVersionRequestHeader) request.decodeCommandCustomHeader(QueryDataVersionRequestHeader.class); + DataVersion dataVersion = DataVersion.decode(request.getBody(), DataVersion.class); + + Boolean changed = this.namesrvController.getRouteInfoManager().isBrokerTopicConfigChanged(requestHeader.getBrokerAddr(), dataVersion); + if (!changed) { + this.namesrvController.getRouteInfoManager().updateBrokerInfoUpdateTimestamp(requestHeader.getBrokerAddr()); + } + + DataVersion nameSeverDataVersion = this.namesrvController.getRouteInfoManager().queryBrokerTopicConfig(requestHeader.getBrokerAddr()); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + + if (nameSeverDataVersion != null) { + response.setBody(nameSeverDataVersion.encode()); + } + responseHeader.setChanged(changed); + return response; + } + public RemotingCommand registerBroker(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(RegisterBrokerResponseHeader.class); diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java index ef02dd0087f..00962ef246e 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java @@ -183,13 +183,24 @@ public RegisterBrokerResult registerBroker( return result; } - private boolean isBrokerTopicConfigChanged(final String brokerAddr, final DataVersion dataVersion) { + public boolean isBrokerTopicConfigChanged(final String brokerAddr, final DataVersion dataVersion) { + DataVersion prev = queryBrokerTopicConfig(brokerAddr); + return null == prev || !prev.equals(dataVersion); + } + + public DataVersion queryBrokerTopicConfig(final String brokerAddr) { BrokerLiveInfo prev = this.brokerLiveTable.get(brokerAddr); - if (null == prev || !prev.getDataVersion().equals(dataVersion)) { - return true; + if (prev != null) { + return prev.getDataVersion(); } + return null; + } - return false; + public void updateBrokerInfoUpdateTimestamp(final String brokerAddr) { + BrokerLiveInfo prev = this.brokerLiveTable.get(brokerAddr); + if (prev != null) { + prev.setLastUpdateTimestamp(System.currentTimeMillis()); + } } private void createAndUpdateQueueData(final String brokerName, final TopicConfig topicConfig) { From 06d63076e0f251a5e72756617c93324b918ddf37 Mon Sep 17 00:00:00 2001 From: yukon Date: Wed, 7 Mar 2018 14:58:22 +0800 Subject: [PATCH 17/73] [ROCKETMQ-319] Replace additional logger with InternalLogger --- .../rocketmq/common/protocol/body/RegisterBrokerBody.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/RegisterBrokerBody.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/RegisterBrokerBody.java index 2b49b6d144b..4065c087922 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/body/RegisterBrokerBody.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/RegisterBrokerBody.java @@ -33,13 +33,14 @@ import org.apache.rocketmq.common.DataVersion; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.protocol.RemotingSerializable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class RegisterBrokerBody extends RemotingSerializable { - private static final Logger LOGGER = LoggerFactory.getLogger(RegisterBrokerBody.class); + private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); private TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); private List filterServerList = new ArrayList(); From 5db240f39208374117342c5852b3a32061c8e6ef Mon Sep 17 00:00:00 2001 From: lindzh Date: Tue, 27 Mar 2018 19:46:19 +0800 Subject: [PATCH 18/73] Add slf4j-api pom for nameserver and broker (#257) --- broker/pom.xml | 4 ++++ namesrv/pom.xml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/broker/pom.xml b/broker/pom.xml index 3da70fa3368..e3787c36503 100644 --- a/broker/pom.xml +++ b/broker/pom.xml @@ -68,6 +68,10 @@ org.javassist javassist     + + org.slf4j + slf4j-api + diff --git a/namesrv/pom.xml b/namesrv/pom.xml index 02ed402bf2d..959814cc28a 100644 --- a/namesrv/pom.xml +++ b/namesrv/pom.xml @@ -48,5 +48,9 @@ ch.qos.logback logback-core + + org.slf4j + slf4j-api + From d035ccb08a3f75f6211a374e02e4a8f6ea520e67 Mon Sep 17 00:00:00 2001 From: fuyou001 Date: Mon, 2 Apr 2018 16:53:06 +0800 Subject: [PATCH 19/73] [ISSUE #270] broker force register nameserver . (#271) --- .../src/main/java/org/apache/rocketmq/common/BrokerConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index 4468b2d5573..fa9a9ef5615 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -134,7 +134,7 @@ public class BrokerConfig { private boolean compressedRegister = false; - private boolean forceRegister = false; + private boolean forceRegister = true; public boolean isTraceOn() { return traceOn; From bf848c14e1a9540f956aff6856bf4ee62b865816 Mon Sep 17 00:00:00 2001 From: Zhanhui Li Date: Sun, 8 Apr 2018 16:32:31 +0800 Subject: [PATCH 20/73] Check if pull message service has shutdown before scheduling pull requests (#277) --- .../impl/consumer/PullMessageService.java | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullMessageService.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullMessageService.java index 5f22f041eb2..bd46a58859a 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullMessageService.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullMessageService.java @@ -44,13 +44,16 @@ public PullMessageService(MQClientInstance mQClientFactory) { } public void executePullRequestLater(final PullRequest pullRequest, final long timeDelay) { - this.scheduledExecutorService.schedule(new Runnable() { - - @Override - public void run() { - PullMessageService.this.executePullRequestImmediately(pullRequest); - } - }, timeDelay, TimeUnit.MILLISECONDS); + if (!isStopped()) { + this.scheduledExecutorService.schedule(new Runnable() { + @Override + public void run() { + PullMessageService.this.executePullRequestImmediately(pullRequest); + } + }, timeDelay, TimeUnit.MILLISECONDS); + } else { + log.warn("PullMessageServiceScheduledThread has shutdown"); + } } public void executePullRequestImmediately(final PullRequest pullRequest) { @@ -62,7 +65,11 @@ public void executePullRequestImmediately(final PullRequest pullRequest) { } public void executeTaskLater(final Runnable r, final long timeDelay) { - this.scheduledExecutorService.schedule(r, timeDelay, TimeUnit.MILLISECONDS); + if (!isStopped()) { + this.scheduledExecutorService.schedule(r, timeDelay, TimeUnit.MILLISECONDS); + } else { + log.warn("PullMessageServiceScheduledThread has shutdown"); + } } public ScheduledExecutorService getScheduledExecutorService() { @@ -86,10 +93,8 @@ public void run() { while (!this.isStopped()) { try { PullRequest pullRequest = this.pullRequestQueue.take(); - if (pullRequest != null) { - this.pullMessage(pullRequest); - } - } catch (InterruptedException e) { + this.pullMessage(pullRequest); + } catch (InterruptedException ignored) { } catch (Exception e) { log.error("Pull Message Service Run Method exception", e); } From 48476ae587fa91f04eacf7486b6c7cf4bb988977 Mon Sep 17 00:00:00 2001 From: Zhanhui Li Date: Sun, 8 Apr 2018 16:33:45 +0800 Subject: [PATCH 21/73] Standardize the startup class structure (#243) --- .../apache/rocketmq/broker/BrokerStartup.java | 6 + .../rocketmq/namesrv/NamesrvStartup.java | 162 ++++++++++-------- 2 files changed, 99 insertions(+), 69 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java index 7627bb967aa..f0a1150122b 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java @@ -79,6 +79,12 @@ public static BrokerController start(BrokerController controller) { return null; } + public static void shutdown(final BrokerController controller) { + if (null != controller) { + controller.shutdown(); + } + } + public static BrokerController createBrokerController(String[] args) { System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, Integer.toString(MQVersion.CURRENT_VERSION)); diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java index a55735a9b88..a073dafb21d 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java @@ -18,8 +18,10 @@ import ch.qos.logback.classic.LoggerContext; import ch.qos.logback.classic.joran.JoranConfigurator; +import ch.qos.logback.core.joran.spi.JoranException; import java.io.BufferedInputStream; import java.io.FileInputStream; +import java.io.IOException; import java.io.InputStream; import java.util.Properties; import java.util.concurrent.Callable; @@ -40,99 +42,121 @@ import org.slf4j.LoggerFactory; public class NamesrvStartup { - public static Properties properties = null; - public static CommandLine commandLine = null; + + private static InternalLogger log; + private static Properties properties = null; + private static CommandLine commandLine = null; public static void main(String[] args) { main0(args); } public static NamesrvController main0(String[] args) { - System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, Integer.toString(MQVersion.CURRENT_VERSION)); + try { - //PackageConflictDetect.detectFastjson(); + NamesrvController controller = createNamesrvController(args); + start(controller); + String tip = "The Name Server boot success. serializeType=" + RemotingCommand.getSerializeTypeConfigInThisServer(); + log.info(tip); + System.out.printf("%s%n", tip); + return controller; + } catch (Throwable e) { + e.printStackTrace(); + System.exit(-1); + } - Options options = ServerUtil.buildCommandlineOptions(new Options()); - commandLine = ServerUtil.parseCmdLine("mqnamesrv", args, buildCommandlineOptions(options), new PosixParser()); - if (null == commandLine) { - System.exit(-1); - return null; - } + return null; + } - final NamesrvConfig namesrvConfig = new NamesrvConfig(); - final NettyServerConfig nettyServerConfig = new NettyServerConfig(); - nettyServerConfig.setListenPort(9876); - if (commandLine.hasOption('c')) { - String file = commandLine.getOptionValue('c'); - if (file != null) { - InputStream in = new BufferedInputStream(new FileInputStream(file)); - properties = new Properties(); - properties.load(in); - MixAll.properties2Object(properties, namesrvConfig); - MixAll.properties2Object(properties, nettyServerConfig); - - namesrvConfig.setConfigStorePath(file); - - System.out.printf("load config properties file OK, %s%n", file); - in.close(); - } - } + public static NamesrvController createNamesrvController(String[] args) throws IOException, JoranException { + System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, Integer.toString(MQVersion.CURRENT_VERSION)); + //PackageConflictDetect.detectFastjson(); - if (commandLine.hasOption('p')) { - MixAll.printObjectProperties(null, namesrvConfig); - MixAll.printObjectProperties(null, nettyServerConfig); - System.exit(0); + Options options = ServerUtil.buildCommandlineOptions(new Options()); + commandLine = ServerUtil.parseCmdLine("mqnamesrv", args, buildCommandlineOptions(options), new PosixParser()); + if (null == commandLine) { + System.exit(-1); + return null; + } + + final NamesrvConfig namesrvConfig = new NamesrvConfig(); + final NettyServerConfig nettyServerConfig = new NettyServerConfig(); + nettyServerConfig.setListenPort(9876); + if (commandLine.hasOption('c')) { + String file = commandLine.getOptionValue('c'); + if (file != null) { + InputStream in = new BufferedInputStream(new FileInputStream(file)); + properties = new Properties(); + properties.load(in); + MixAll.properties2Object(properties, namesrvConfig); + MixAll.properties2Object(properties, nettyServerConfig); + + namesrvConfig.setConfigStorePath(file); + + System.out.printf("load config properties file OK, %s%n", file); + in.close(); } + } + + if (commandLine.hasOption('p')) { + MixAll.printObjectProperties(null, namesrvConfig); + MixAll.printObjectProperties(null, nettyServerConfig); + System.exit(0); + } - MixAll.properties2Object(ServerUtil.commandLine2Properties(commandLine), namesrvConfig); + MixAll.properties2Object(ServerUtil.commandLine2Properties(commandLine), namesrvConfig); - if (null == namesrvConfig.getRocketmqHome()) { - System.out.printf("Please set the %s variable in your environment to match the location of the RocketMQ installation%n", MixAll.ROCKETMQ_HOME_ENV); - System.exit(-2); - } + if (null == namesrvConfig.getRocketmqHome()) { + System.out.printf("Please set the %s variable in your environment to match the location of the RocketMQ installation%n", MixAll.ROCKETMQ_HOME_ENV); + System.exit(-2); + } - LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory(); - JoranConfigurator configurator = new JoranConfigurator(); - configurator.setContext(lc); - lc.reset(); - configurator.doConfigure(namesrvConfig.getRocketmqHome() + "/conf/logback_namesrv.xml"); - final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); + LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory(); + JoranConfigurator configurator = new JoranConfigurator(); + configurator.setContext(lc); + lc.reset(); + configurator.doConfigure(namesrvConfig.getRocketmqHome() + "/conf/logback_namesrv.xml"); - MixAll.printObjectProperties(log, namesrvConfig); - MixAll.printObjectProperties(log, nettyServerConfig); + log = InternalLoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); - final NamesrvController controller = new NamesrvController(namesrvConfig, nettyServerConfig); + MixAll.printObjectProperties(log, namesrvConfig); + MixAll.printObjectProperties(log, nettyServerConfig); - // remember all configs to prevent discard - controller.getConfiguration().registerConfig(properties); + final NamesrvController controller = new NamesrvController(namesrvConfig, nettyServerConfig); - boolean initResult = controller.initialize(); - if (!initResult) { - controller.shutdown(); - System.exit(-3); - } + // remember all configs to prevent discard + controller.getConfiguration().registerConfig(properties); - Runtime.getRuntime().addShutdownHook(new ShutdownHookThread(log, new Callable() { - @Override - public Void call() throws Exception { - controller.shutdown(); - return null; - } - })); + return controller; + } - controller.start(); + public static NamesrvController start(final NamesrvController controller) throws Exception { - String tip = "The Name Server boot success. serializeType=" + RemotingCommand.getSerializeTypeConfigInThisServer(); - log.info(tip); - System.out.printf("%s%n", tip); + if (null == controller) { + throw new IllegalArgumentException("NamesrvController is null"); + } - return controller; - } catch (Throwable e) { - e.printStackTrace(); - System.exit(-1); + boolean initResult = controller.initialize(); + if (!initResult) { + controller.shutdown(); + System.exit(-3); } - return null; + Runtime.getRuntime().addShutdownHook(new ShutdownHookThread(log, new Callable() { + @Override + public Void call() throws Exception { + controller.shutdown(); + return null; + } + })); + + controller.start(); + + return controller; + } + + public static void shutdown(final NamesrvController controller) { + controller.shutdown(); } public static Options buildCommandlineOptions(final Options options) { From 2169e3c075e48f8f016337c9b81eafc258b9b9b1 Mon Sep 17 00:00:00 2001 From: "shutian.lzh" Date: Sun, 15 Apr 2018 16:13:35 +0800 Subject: [PATCH 22/73] Make code compatible to OMS 0.3.0 --- .../apache/rocketmq/broker/BrokerStartup.java | 1 + .../rocketmq/client/log/ClientLoggerTest.java | 5 +- .../example/openmessaging/SimpleProducer.java | 54 ++++---- .../openmessaging/SimplePullConsumer.java | 54 +++++--- .../openmessaging/SimplePushConsumer.java | 19 ++- .../rocketmq/example/simple/Producer.java | 2 +- .../example/simple/PullScheduleService.java | 2 +- .../rocketmq/MessagingAccessPointImpl.java | 68 +++------- .../rocketmq/config/ClientConfig.java | 128 +++++++++--------- .../rocketmq/consumer/LocalMessageCache.java | 12 +- .../rocketmq/consumer/PullConsumerImpl.java | 44 ++++-- .../rocketmq/consumer/PushConsumerImpl.java | 60 +++++--- .../rocketmq/domain/BytesMessageImpl.java | 48 +++---- .../rocketmq/domain/RocketMQConstants.java | 7 + .../rocketmq/domain/SendResultImpl.java | 3 +- .../producer/AbstractOMSProducer.java | 27 ++-- .../rocketmq/producer/ProducerImpl.java | 45 ++++-- .../producer/SequenceProducerImpl.java | 95 ------------- .../rocketmq/promise/DefaultPromise.java | 15 +- .../rocketmq/utils/BeanUtils.java | 2 +- .../openmessaging/rocketmq/utils/OMSUtil.java | 62 ++++----- .../consumer/PullConsumerImplTest.java | 24 ++-- .../consumer/PushConsumerImplTest.java | 18 ++- .../rocketmq/producer/ProducerImplTest.java | 16 +-- .../producer/SequenceProducerImplTest.java | 86 ------------ .../rocketmq/promise/DefaultPromiseTest.java | 38 ++---- .../rocketmq/utils/BeanUtilsTest.java | 4 +- pom.xml | 2 +- 28 files changed, 392 insertions(+), 549 deletions(-) create mode 100644 openmessaging/src/main/java/io/openmessaging/rocketmq/domain/RocketMQConstants.java delete mode 100644 openmessaging/src/main/java/io/openmessaging/rocketmq/producer/SequenceProducerImpl.java delete mode 100644 openmessaging/src/test/java/io/openmessaging/rocketmq/producer/SequenceProducerImplTest.java diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java index f0a1150122b..1fc1b3b8474 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java @@ -70,6 +70,7 @@ public static BrokerController start(BrokerController controller) { } log.info(tip); + System.out.printf("%s%n", tip); return controller; } catch (Throwable e) { e.printStackTrace(); diff --git a/client/src/test/java/org/apache/rocketmq/client/log/ClientLoggerTest.java b/client/src/test/java/org/apache/rocketmq/client/log/ClientLoggerTest.java index 9fe0d8b9c45..4888186b1e7 100644 --- a/client/src/test/java/org/apache/rocketmq/client/log/ClientLoggerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/log/ClientLoggerTest.java @@ -49,7 +49,10 @@ public void testClientlog() throws IOException { rocketmqCommon.info("common message {}", i, new RuntimeException()); rocketmqRemoting.info("remoting message {}", i, new RuntimeException()); } - + try { + Thread.sleep(10); + } catch (InterruptedException ignore) { + } String content = MixAll.file2String(LOG_DIR + "/rocketmq_client.log"); Assert.assertTrue(content.contains("testClientlog")); Assert.assertTrue(content.contains("RocketmqClient")); diff --git a/example/src/main/java/org/apache/rocketmq/example/openmessaging/SimpleProducer.java b/example/src/main/java/org/apache/rocketmq/example/openmessaging/SimpleProducer.java index 9d162ac1463..dbe1d105688 100644 --- a/example/src/main/java/org/apache/rocketmq/example/openmessaging/SimpleProducer.java +++ b/example/src/main/java/org/apache/rocketmq/example/openmessaging/SimpleProducer.java @@ -16,19 +16,20 @@ */ package org.apache.rocketmq.example.openmessaging; +import io.openmessaging.Future; +import io.openmessaging.FutureListener; import io.openmessaging.Message; import io.openmessaging.MessagingAccessPoint; -import io.openmessaging.MessagingAccessPointFactory; -import io.openmessaging.Producer; -import io.openmessaging.Promise; -import io.openmessaging.PromiseListener; -import io.openmessaging.SendResult; +import io.openmessaging.OMS; +import io.openmessaging.producer.Producer; +import io.openmessaging.producer.SendResult; import java.nio.charset.Charset; +import java.util.concurrent.CountDownLatch; public class SimpleProducer { public static void main(String[] args) { - final MessagingAccessPoint messagingAccessPoint = MessagingAccessPointFactory - .getMessagingAccessPoint("openmessaging:rocketmq://IP1:9876,IP2:9876/namespace"); + final MessagingAccessPoint messagingAccessPoint = + OMS.getMessagingAccessPoint("oms:rocketmq://localhost:9876/default:default"); final Producer producer = messagingAccessPoint.createProducer(); @@ -38,39 +39,40 @@ public static void main(String[] args) { producer.startup(); System.out.printf("Producer startup OK%n"); - Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { - @Override - public void run() { - producer.shutdown(); - messagingAccessPoint.shutdown(); - } - })); - { - Message message = producer.createBytesMessageToTopic("OMS_HELLO_TOPIC", "OMS_HELLO_BODY".getBytes(Charset.forName("UTF-8"))); + Message message = producer.createBytesMessage("OMS_HELLO_TOPIC", "OMS_HELLO_BODY".getBytes(Charset.forName("UTF-8"))); SendResult sendResult = producer.send(message); //final Void aVoid = result.get(3000L); System.out.printf("Send async message OK, msgId: %s%n", sendResult.messageId()); } + final CountDownLatch countDownLatch = new CountDownLatch(1); { - final Promise result = producer.sendAsync(producer.createBytesMessageToTopic("OMS_HELLO_TOPIC", "OMS_HELLO_BODY".getBytes(Charset.forName("UTF-8")))); - result.addListener(new PromiseListener() { - @Override - public void operationCompleted(Promise promise) { - System.out.printf("Send async message OK, msgId: %s%n", promise.get().messageId()); - } - + final Future result = producer.sendAsync(producer.createBytesMessage("OMS_HELLO_TOPIC", "OMS_HELLO_BODY".getBytes(Charset.forName("UTF-8")))); + result.addListener(new FutureListener() { @Override - public void operationFailed(Promise promise) { - System.out.printf("Send async message Failed, error: %s%n", promise.getThrowable().getMessage()); + public void operationComplete(Future future) { + if (future.getThrowable() != null) { + System.out.printf("Send async message Failed, error: %s%n", future.getThrowable().getMessage()); + } else { + System.out.printf("Send async message OK, msgId: %s%n", future.get().messageId()); + } + countDownLatch.countDown(); } }); } { - producer.sendOneway(producer.createBytesMessageToTopic("OMS_HELLO_TOPIC", "OMS_HELLO_BODY".getBytes(Charset.forName("UTF-8")))); + producer.sendOneway(producer.createBytesMessage("OMS_HELLO_TOPIC", "OMS_HELLO_BODY".getBytes(Charset.forName("UTF-8")))); System.out.printf("Send oneway message OK%n"); } + + try { + countDownLatch.await(); + Thread.sleep(500); // Wait some time for one-way delivery. + } catch (InterruptedException ignore) { + } + + producer.shutdown(); } } diff --git a/example/src/main/java/org/apache/rocketmq/example/openmessaging/SimplePullConsumer.java b/example/src/main/java/org/apache/rocketmq/example/openmessaging/SimplePullConsumer.java index 8e067724dee..86aba410aee 100644 --- a/example/src/main/java/org/apache/rocketmq/example/openmessaging/SimplePullConsumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/openmessaging/SimplePullConsumer.java @@ -17,42 +17,60 @@ package org.apache.rocketmq.example.openmessaging; import io.openmessaging.Message; -import io.openmessaging.MessageHeader; import io.openmessaging.MessagingAccessPoint; -import io.openmessaging.MessagingAccessPointFactory; import io.openmessaging.OMS; -import io.openmessaging.PullConsumer; -import io.openmessaging.rocketmq.domain.NonStandardKeys; +import io.openmessaging.OMSBuiltinKeys; +import io.openmessaging.consumer.PullConsumer; +import io.openmessaging.producer.Producer; +import io.openmessaging.producer.SendResult; public class SimplePullConsumer { public static void main(String[] args) { - final MessagingAccessPoint messagingAccessPoint = MessagingAccessPointFactory - .getMessagingAccessPoint("openmessaging:rocketmq://IP1:9876,IP2:9876/namespace"); + final MessagingAccessPoint messagingAccessPoint = + OMS.getMessagingAccessPoint("oms:rocketmq://localhost:9876/default:default"); - final PullConsumer consumer = messagingAccessPoint.createPullConsumer("OMS_HELLO_TOPIC", - OMS.newKeyValue().put(NonStandardKeys.CONSUMER_GROUP, "OMS_CONSUMER")); + messagingAccessPoint.startup(); + + final Producer producer = messagingAccessPoint.createProducer(); + + final PullConsumer consumer = messagingAccessPoint.createPullConsumer( + OMS.newKeyValue().put(OMSBuiltinKeys.CONSUMER_ID, "OMS_CONSUMER")); messagingAccessPoint.startup(); System.out.printf("MessagingAccessPoint startup OK%n"); - Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { - @Override - public void run() { - consumer.shutdown(); - messagingAccessPoint.shutdown(); - } - })); + final String queueName = "TopicTest"; + + producer.startup(); + Message msg = producer.createBytesMessage(queueName, "Hello Open Messaging".getBytes()); + SendResult sendResult = producer.send(msg); + System.out.printf("Send Message OK. MsgId: %s%n", sendResult.messageId()); + producer.shutdown(); + + consumer.attachQueue(queueName); consumer.startup(); System.out.printf("Consumer startup OK%n"); - while (true) { - Message message = consumer.poll(); + // Keep running until we find the one that has just been sent + boolean stop = false; + while (!stop) { + Message message = consumer.receive(); if (message != null) { - String msgId = message.headers().getString(MessageHeader.MESSAGE_ID); + String msgId = message.sysHeaders().getString(Message.BuiltinKeys.MESSAGE_ID); System.out.printf("Received one message: %s%n", msgId); consumer.ack(msgId); + + if (!stop) { + stop = msgId.equalsIgnoreCase(sendResult.messageId()); + } + + } else { + System.out.printf("Return without any message%n"); } } + + consumer.shutdown(); + messagingAccessPoint.shutdown(); } } diff --git a/example/src/main/java/org/apache/rocketmq/example/openmessaging/SimplePushConsumer.java b/example/src/main/java/org/apache/rocketmq/example/openmessaging/SimplePushConsumer.java index b0935d4c743..220c1323061 100644 --- a/example/src/main/java/org/apache/rocketmq/example/openmessaging/SimplePushConsumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/openmessaging/SimplePushConsumer.java @@ -17,22 +17,19 @@ package org.apache.rocketmq.example.openmessaging; import io.openmessaging.Message; -import io.openmessaging.MessageHeader; -import io.openmessaging.MessageListener; import io.openmessaging.MessagingAccessPoint; -import io.openmessaging.MessagingAccessPointFactory; import io.openmessaging.OMS; -import io.openmessaging.PushConsumer; -import io.openmessaging.ReceivedMessageContext; -import io.openmessaging.rocketmq.domain.NonStandardKeys; +import io.openmessaging.OMSBuiltinKeys; +import io.openmessaging.consumer.MessageListener; +import io.openmessaging.consumer.PushConsumer; public class SimplePushConsumer { public static void main(String[] args) { - final MessagingAccessPoint messagingAccessPoint = MessagingAccessPointFactory - .getMessagingAccessPoint("openmessaging:rocketmq://IP1:9876,IP2:9876/namespace"); + final MessagingAccessPoint messagingAccessPoint = OMS + .getMessagingAccessPoint("oms:rocketmq://localhost:9876/default:default"); final PushConsumer consumer = messagingAccessPoint. - createPushConsumer(OMS.newKeyValue().put(NonStandardKeys.CONSUMER_GROUP, "OMS_CONSUMER")); + createPushConsumer(OMS.newKeyValue().put(OMSBuiltinKeys.CONSUMER_ID, "OMS_CONSUMER")); messagingAccessPoint.startup(); System.out.printf("MessagingAccessPoint startup OK%n"); @@ -47,8 +44,8 @@ public void run() { consumer.attachQueue("OMS_HELLO_TOPIC", new MessageListener() { @Override - public void onMessage(final Message message, final ReceivedMessageContext context) { - System.out.printf("Received one message: %s%n", message.headers().getString(MessageHeader.MESSAGE_ID)); + public void onReceived(Message message, Context context) { + System.out.printf("Received one message: %s%n", message.sysHeaders().getString(Message.BuiltinKeys.MESSAGE_ID)); context.ack(); } }); diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/Producer.java b/example/src/main/java/org/apache/rocketmq/example/simple/Producer.java index 5751d22c7f1..7b504dd2a3e 100644 --- a/example/src/main/java/org/apache/rocketmq/example/simple/Producer.java +++ b/example/src/main/java/org/apache/rocketmq/example/simple/Producer.java @@ -29,7 +29,7 @@ public static void main(String[] args) throws MQClientException, InterruptedExce producer.start(); - for (int i = 0; i < 10000000; i++) + for (int i = 0; i < 128; i++) try { { Message msg = new Message("TopicTest", diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/PullScheduleService.java b/example/src/main/java/org/apache/rocketmq/example/simple/PullScheduleService.java index 151628f9a80..8cfdd9bbd0c 100644 --- a/example/src/main/java/org/apache/rocketmq/example/simple/PullScheduleService.java +++ b/example/src/main/java/org/apache/rocketmq/example/simple/PullScheduleService.java @@ -32,7 +32,7 @@ public static void main(String[] args) throws MQClientException { final MQPullConsumerScheduleService scheduleService = new MQPullConsumerScheduleService("GroupName1"); scheduleService.setMessageModel(MessageModel.CLUSTERING); - scheduleService.registerPullTaskCallback("TopicTest1", new PullTaskCallback() { + scheduleService.registerPullTaskCallback("TopicTest", new PullTaskCallback() { @Override public void doPullTask(MessageQueue mq, PullTaskContext context) { diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/MessagingAccessPointImpl.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/MessagingAccessPointImpl.java index 65caf84084c..51388f9c610 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/MessagingAccessPointImpl.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/MessagingAccessPointImpl.java @@ -16,24 +16,21 @@ */ package io.openmessaging.rocketmq; -import io.openmessaging.IterableConsumer; import io.openmessaging.KeyValue; import io.openmessaging.MessagingAccessPoint; -import io.openmessaging.Producer; -import io.openmessaging.PullConsumer; -import io.openmessaging.PushConsumer; import io.openmessaging.ResourceManager; -import io.openmessaging.SequenceProducer; -import io.openmessaging.ServiceEndPoint; +import io.openmessaging.consumer.PullConsumer; +import io.openmessaging.consumer.PushConsumer; +import io.openmessaging.consumer.StreamingConsumer; import io.openmessaging.exception.OMSNotSupportedException; -import io.openmessaging.observer.Observer; +import io.openmessaging.producer.Producer; import io.openmessaging.rocketmq.consumer.PullConsumerImpl; import io.openmessaging.rocketmq.consumer.PushConsumerImpl; import io.openmessaging.rocketmq.producer.ProducerImpl; -import io.openmessaging.rocketmq.producer.SequenceProducerImpl; import io.openmessaging.rocketmq.utils.OMSUtil; public class MessagingAccessPointImpl implements MessagingAccessPoint { + private final KeyValue accessPointProperties; public MessagingAccessPointImpl(final KeyValue accessPointProperties) { @@ -41,10 +38,15 @@ public MessagingAccessPointImpl(final KeyValue accessPointProperties) { } @Override - public KeyValue properties() { + public KeyValue attributes() { return accessPointProperties; } + @Override + public String implVersion() { + return "0.3.0"; + } + @Override public Producer createProducer() { return new ProducerImpl(this.accessPointProperties); @@ -55,16 +57,6 @@ public Producer createProducer(KeyValue properties) { return new ProducerImpl(OMSUtil.buildKeyValue(this.accessPointProperties, properties)); } - @Override - public SequenceProducer createSequenceProducer() { - return new SequenceProducerImpl(this.accessPointProperties); - } - - @Override - public SequenceProducer createSequenceProducer(KeyValue properties) { - return new SequenceProducerImpl(OMSUtil.buildKeyValue(this.accessPointProperties, properties)); - } - @Override public PushConsumer createPushConsumer() { return new PushConsumerImpl(accessPointProperties); @@ -76,50 +68,30 @@ public PushConsumer createPushConsumer(KeyValue properties) { } @Override - public PullConsumer createPullConsumer(String queueName) { - return new PullConsumerImpl(queueName, accessPointProperties); + public PullConsumer createPullConsumer() { + return new PullConsumerImpl(accessPointProperties); } @Override - public PullConsumer createPullConsumer(String queueName, KeyValue properties) { - return new PullConsumerImpl(queueName, OMSUtil.buildKeyValue(this.accessPointProperties, properties)); + public PullConsumer createPullConsumer(KeyValue attributes) { + return new PullConsumerImpl(OMSUtil.buildKeyValue(this.accessPointProperties, attributes)); } @Override - public IterableConsumer createIterableConsumer(String queueName) { - throw new OMSNotSupportedException("-1", "IterableConsumer is not supported in current version"); + public StreamingConsumer createStreamingConsumer() { + return null; } @Override - public IterableConsumer createIterableConsumer(String queueName, KeyValue properties) { - throw new OMSNotSupportedException("-1", "IterableConsumer is not supported in current version"); + public StreamingConsumer createStreamingConsumer(KeyValue attributes) { + return null; } @Override - public ResourceManager getResourceManager() { + public ResourceManager resourceManager() { throw new OMSNotSupportedException("-1", "ResourceManager is not supported in current version."); } - @Override - public ServiceEndPoint createServiceEndPoint() { - throw new OMSNotSupportedException("-1", "ServiceEndPoint is not supported in current version."); - } - - @Override - public ServiceEndPoint createServiceEndPoint(KeyValue properties) { - throw new OMSNotSupportedException("-1", "ServiceEndPoint is not supported in current version."); - } - - @Override - public void addObserver(Observer observer) { - //Ignore - } - - @Override - public void deleteObserver(Observer observer) { - //Ignore - } - @Override public void startup() { //Ignore diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/config/ClientConfig.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/config/ClientConfig.java index 7077c6dc997..a5dfe49484a 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/config/ClientConfig.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/config/ClientConfig.java @@ -16,20 +16,20 @@ */ package io.openmessaging.rocketmq.config; -import io.openmessaging.PropertyKeys; +import io.openmessaging.OMSBuiltinKeys; import io.openmessaging.rocketmq.domain.NonStandardKeys; -public class ClientConfig implements PropertyKeys, NonStandardKeys { - private String omsDriverImpl; - private String omsAccessPoints; - private String omsNamespace; - private String omsProducerId; - private String omsConsumerId; - private int omsOperationTimeout = 5000; - private String omsRoutingName; - private String omsOperatorName; - private String omsDstQueue; - private String omsSrcTopic; +public class ClientConfig implements OMSBuiltinKeys, NonStandardKeys { + private String driverImpl; + private String accessPoints; + private String namespace; + private String producerId; + private String consumerId; + private int operationTimeout = 5000; + private String region; + private String routingSource; + private String routingDestination; + private String routingExpression; private String rmqConsumerGroup; private String rmqProducerGroup = "__OMS_PRODUCER_DEFAULT_GROUP"; private int rmqMaxRedeliveryTimes = 16; @@ -40,84 +40,60 @@ public class ClientConfig implements PropertyKeys, NonStandardKeys { private int rmqPullMessageBatchNums = 32; private int rmqPullMessageCacheCapacity = 1000; - public String getOmsDriverImpl() { - return omsDriverImpl; + public String getDriverImpl() { + return driverImpl; } - public void setOmsDriverImpl(final String omsDriverImpl) { - this.omsDriverImpl = omsDriverImpl; + public void setDriverImpl(final String driverImpl) { + this.driverImpl = driverImpl; } - public String getOmsAccessPoints() { - return omsAccessPoints; + public String getAccessPoints() { + return accessPoints; } - public void setOmsAccessPoints(final String omsAccessPoints) { - this.omsAccessPoints = omsAccessPoints; + public void setAccessPoints(final String accessPoints) { + this.accessPoints = accessPoints; } - public String getOmsNamespace() { - return omsNamespace; + public String getNamespace() { + return namespace; } - public void setOmsNamespace(final String omsNamespace) { - this.omsNamespace = omsNamespace; + public void setNamespace(final String namespace) { + this.namespace = namespace; } - public String getOmsProducerId() { - return omsProducerId; + public String getProducerId() { + return producerId; } - public void setOmsProducerId(final String omsProducerId) { - this.omsProducerId = omsProducerId; + public void setProducerId(final String producerId) { + this.producerId = producerId; } - public String getOmsConsumerId() { - return omsConsumerId; + public String getConsumerId() { + return consumerId; } - public void setOmsConsumerId(final String omsConsumerId) { - this.omsConsumerId = omsConsumerId; + public void setConsumerId(final String consumerId) { + this.consumerId = consumerId; } - public int getOmsOperationTimeout() { - return omsOperationTimeout; + public int getOperationTimeout() { + return operationTimeout; } - public void setOmsOperationTimeout(final int omsOperationTimeout) { - this.omsOperationTimeout = omsOperationTimeout; + public void setOperationTimeout(final int operationTimeout) { + this.operationTimeout = operationTimeout; } - public String getOmsRoutingName() { - return omsRoutingName; + public String getRoutingSource() { + return routingSource; } - public void setOmsRoutingName(final String omsRoutingName) { - this.omsRoutingName = omsRoutingName; - } - - public String getOmsOperatorName() { - return omsOperatorName; - } - - public void setOmsOperatorName(final String omsOperatorName) { - this.omsOperatorName = omsOperatorName; - } - - public String getOmsDstQueue() { - return omsDstQueue; - } - - public void setOmsDstQueue(final String omsDstQueue) { - this.omsDstQueue = omsDstQueue; - } - - public String getOmsSrcTopic() { - return omsSrcTopic; - } - - public void setOmsSrcTopic(final String omsSrcTopic) { - this.omsSrcTopic = omsSrcTopic; + public void setRoutingSource(final String routingSource) { + this.routingSource = routingSource; } public String getRmqConsumerGroup() { @@ -191,4 +167,28 @@ public int getRmqPullMessageCacheCapacity() { public void setRmqPullMessageCacheCapacity(final int rmqPullMessageCacheCapacity) { this.rmqPullMessageCacheCapacity = rmqPullMessageCacheCapacity; } + + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + public String getRoutingDestination() { + return routingDestination; + } + + public void setRoutingDestination(String routingDestination) { + this.routingDestination = routingDestination; + } + + public String getRoutingExpression() { + return routingExpression; + } + + public void setRoutingExpression(String routingExpression) { + this.routingExpression = routingExpression; + } } diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/LocalMessageCache.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/LocalMessageCache.java index cc1a5157152..93e60a738e9 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/LocalMessageCache.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/LocalMessageCache.java @@ -17,7 +17,7 @@ package io.openmessaging.rocketmq.consumer; import io.openmessaging.KeyValue; -import io.openmessaging.PropertyKeys; +import io.openmessaging.Message; import io.openmessaging.ServiceLifecycle; import io.openmessaging.rocketmq.config.ClientConfig; import io.openmessaging.rocketmq.domain.ConsumeRequest; @@ -37,11 +37,11 @@ import org.apache.rocketmq.client.impl.consumer.ProcessQueue; import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.common.ThreadFactoryImpl; -import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.utils.ThreadUtils; +import org.apache.rocketmq.logging.InternalLogger; class LocalMessageCache implements ServiceLifecycle { private final BlockingQueue consumeRequestCache; @@ -91,13 +91,13 @@ void submitConsumeRequest(ConsumeRequest consumeRequest) { } MessageExt poll() { - return poll(clientConfig.getOmsOperationTimeout()); + return poll(clientConfig.getOperationTimeout()); } MessageExt poll(final KeyValue properties) { - int currentPollTimeout = clientConfig.getOmsOperationTimeout(); - if (properties.containsKey(PropertyKeys.OPERATION_TIMEOUT)) { - currentPollTimeout = properties.getInt(PropertyKeys.OPERATION_TIMEOUT); + int currentPollTimeout = clientConfig.getOperationTimeout(); + if (properties.containsKey(Message.BuiltinKeys.TIMEOUT)) { + currentPollTimeout = properties.getInt(Message.BuiltinKeys.TIMEOUT); } return poll(currentPollTimeout); } diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PullConsumerImpl.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PullConsumerImpl.java index da4afdb7a6b..2e22509a2a1 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PullConsumerImpl.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PullConsumerImpl.java @@ -18,8 +18,8 @@ import io.openmessaging.KeyValue; import io.openmessaging.Message; -import io.openmessaging.PropertyKeys; -import io.openmessaging.PullConsumer; +import io.openmessaging.OMSBuiltinKeys; +import io.openmessaging.consumer.PullConsumer; import io.openmessaging.exception.OMSRuntimeException; import io.openmessaging.rocketmq.config.ClientConfig; import io.openmessaging.rocketmq.domain.ConsumeRequest; @@ -34,28 +34,25 @@ import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.impl.consumer.ProcessQueue; import org.apache.rocketmq.client.log.ClientLogger; -import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.logging.InternalLogger; public class PullConsumerImpl implements PullConsumer { private final DefaultMQPullConsumer rocketmqPullConsumer; private final KeyValue properties; private boolean started = false; - private String targetQueueName; private final MQPullConsumerScheduleService pullConsumerScheduleService; private final LocalMessageCache localMessageCache; private final ClientConfig clientConfig; final static InternalLogger log = ClientLogger.getLog(); - public PullConsumerImpl(final String queueName, final KeyValue properties) { + public PullConsumerImpl(final KeyValue properties) { this.properties = properties; - this.targetQueueName = queueName; - this.clientConfig = BeanUtils.populate(properties, ClientConfig.class); - String consumerGroup = clientConfig.getRmqConsumerGroup(); + String consumerGroup = clientConfig.getConsumerId(); if (null == consumerGroup || consumerGroup.isEmpty()) { throw new OMSRuntimeException("-1", "Consumer Group is necessary for RocketMQ, please set it."); } @@ -63,7 +60,7 @@ public PullConsumerImpl(final String queueName, final KeyValue properties) { this.rocketmqPullConsumer = pullConsumerScheduleService.getDefaultMQPullConsumer(); - String accessPoints = clientConfig.getOmsAccessPoints(); + String accessPoints = clientConfig.getAccessPoints(); if (accessPoints == null || accessPoints.isEmpty()) { throw new OMSRuntimeException("-1", "OMS AccessPoints is null or empty."); } @@ -76,24 +73,42 @@ public PullConsumerImpl(final String queueName, final KeyValue properties) { String consumerId = OMSUtil.buildInstanceName(); this.rocketmqPullConsumer.setInstanceName(consumerId); - properties.put(PropertyKeys.CONSUMER_ID, consumerId); + properties.put(OMSBuiltinKeys.CONSUMER_ID, consumerId); this.localMessageCache = new LocalMessageCache(this.rocketmqPullConsumer, clientConfig); } @Override - public KeyValue properties() { + public KeyValue attributes() { return properties; } @Override - public Message poll() { + public PullConsumer attachQueue(String queueName) { + registerPullTaskCallback(queueName); + return this; + } + + @Override + public PullConsumer attachQueue(String queueName, KeyValue attributes) { + registerPullTaskCallback(queueName); + return this; + } + + @Override + public PullConsumer detachQueue(String queueName) { + this.rocketmqPullConsumer.getRegisterTopics().remove(queueName); + return this; + } + + @Override + public Message receive() { MessageExt rmqMsg = localMessageCache.poll(); return rmqMsg == null ? null : OMSUtil.msgConvert(rmqMsg); } @Override - public Message poll(final KeyValue properties) { + public Message receive(final KeyValue properties) { MessageExt rmqMsg = localMessageCache.poll(properties); return rmqMsg == null ? null : OMSUtil.msgConvert(rmqMsg); } @@ -112,7 +127,6 @@ public void ack(final String messageId, final KeyValue properties) { public synchronized void startup() { if (!started) { try { - registerPullTaskCallback(); this.pullConsumerScheduleService.start(); this.localMessageCache.startup(); } catch (MQClientException e) { @@ -122,7 +136,7 @@ public synchronized void startup() { this.started = true; } - private void registerPullTaskCallback() { + private void registerPullTaskCallback(final String targetQueueName) { this.pullConsumerScheduleService.registerPullTaskCallback(targetQueueName, new PullTaskCallback() { @Override public void doPullTask(final MessageQueue mq, final PullTaskContext context) { diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PushConsumerImpl.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PushConsumerImpl.java index f9b8058e04a..9bfd7c8b622 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PushConsumerImpl.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PushConsumerImpl.java @@ -18,12 +18,12 @@ import io.openmessaging.BytesMessage; import io.openmessaging.KeyValue; -import io.openmessaging.MessageListener; import io.openmessaging.OMS; -import io.openmessaging.PropertyKeys; -import io.openmessaging.PushConsumer; -import io.openmessaging.ReceivedMessageContext; +import io.openmessaging.OMSBuiltinKeys; +import io.openmessaging.consumer.MessageListener; +import io.openmessaging.consumer.PushConsumer; import io.openmessaging.exception.OMSRuntimeException; +import io.openmessaging.interceptor.ConsumerInterceptor; import io.openmessaging.rocketmq.config.ClientConfig; import io.openmessaging.rocketmq.domain.NonStandardKeys; import io.openmessaging.rocketmq.utils.BeanUtils; @@ -52,13 +52,13 @@ public PushConsumerImpl(final KeyValue properties) { this.properties = properties; this.clientConfig = BeanUtils.populate(properties, ClientConfig.class); - String accessPoints = clientConfig.getOmsAccessPoints(); + String accessPoints = clientConfig.getAccessPoints(); if (accessPoints == null || accessPoints.isEmpty()) { throw new OMSRuntimeException("-1", "OMS AccessPoints is null or empty."); } this.rocketmqPushConsumer.setNamesrvAddr(accessPoints.replace(',', ';')); - String consumerGroup = clientConfig.getRmqConsumerGroup(); + String consumerGroup = clientConfig.getConsumerId(); if (null == consumerGroup || consumerGroup.isEmpty()) { throw new OMSRuntimeException("-1", "Consumer Group is necessary for RocketMQ, please set it."); } @@ -70,13 +70,13 @@ public PushConsumerImpl(final KeyValue properties) { String consumerId = OMSUtil.buildInstanceName(); this.rocketmqPushConsumer.setInstanceName(consumerId); - properties.put(PropertyKeys.CONSUMER_ID, consumerId); + properties.put(OMSBuiltinKeys.CONSUMER_ID, consumerId); this.rocketmqPushConsumer.registerMessageListener(new MessageListenerImpl()); } @Override - public KeyValue properties() { + public KeyValue attributes() { return properties; } @@ -90,6 +90,11 @@ public void suspend() { this.rocketmqPushConsumer.suspend(); } + @Override + public void suspend(long timeout) { + + } + @Override public boolean isSuspended() { return this.rocketmqPushConsumer.getDefaultMQPushConsumerImpl().isPause(); @@ -106,6 +111,32 @@ public PushConsumer attachQueue(final String queueName, final MessageListener li return this; } + @Override + public PushConsumer attachQueue(String queueName, MessageListener listener, KeyValue attributes) { + return this.attachQueue(queueName, listener); + } + + @Override + public PushConsumer detachQueue(String queueName) { + this.subscribeTable.remove(queueName); + try { + this.rocketmqPushConsumer.unsubscribe(queueName); + } catch (Exception e) { + throw new OMSRuntimeException("-1", String.format("RocketMQ push consumer fails to unsubscribe topic: %s", queueName)); + } + return null; + } + + @Override + public void addInterceptor(ConsumerInterceptor interceptor) { + + } + + @Override + public void removeInterceptor(ConsumerInterceptor interceptor) { + + } + @Override public synchronized void startup() { if (!started) { @@ -146,9 +177,9 @@ public ConsumeConcurrentlyStatus consumeMessage(List rmqMsgList, contextProperties.put(NonStandardKeys.MESSAGE_CONSUME_STATUS, ConsumeConcurrentlyStatus.RECONSUME_LATER.name()); - ReceivedMessageContext context = new ReceivedMessageContext() { + MessageListener.Context context = new MessageListener.Context() { @Override - public KeyValue properties() { + public KeyValue attributes() { return contextProperties; } @@ -158,16 +189,9 @@ public void ack() { contextProperties.put(NonStandardKeys.MESSAGE_CONSUME_STATUS, ConsumeConcurrentlyStatus.CONSUME_SUCCESS.name()); } - - @Override - public void ack(final KeyValue properties) { - sync.countDown(); - contextProperties.put(NonStandardKeys.MESSAGE_CONSUME_STATUS, - properties.getString(NonStandardKeys.MESSAGE_CONSUME_STATUS)); - } }; long begin = System.currentTimeMillis(); - listener.onMessage(omsMsg, context); + listener.onReceived(omsMsg, context); long costs = System.currentTimeMillis() - begin; long timeoutMills = clientConfig.getRmqMessageConsumeTimeout() * 60 * 1000; try { diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/BytesMessageImpl.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/BytesMessageImpl.java index 43f80ae5be4..702d561fafa 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/BytesMessageImpl.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/BytesMessageImpl.java @@ -23,13 +23,13 @@ import org.apache.commons.lang3.builder.ToStringBuilder; public class BytesMessageImpl implements BytesMessage { - private KeyValue headers; - private KeyValue properties; + private KeyValue sysHeaders; + private KeyValue userHeaders; private byte[] body; public BytesMessageImpl() { - this.headers = OMS.newKeyValue(); - this.properties = OMS.newKeyValue(); + this.sysHeaders = OMS.newKeyValue(); + this.userHeaders = OMS.newKeyValue(); } @Override @@ -44,60 +44,60 @@ public BytesMessage setBody(final byte[] body) { } @Override - public KeyValue headers() { - return headers; + public KeyValue sysHeaders() { + return sysHeaders; } @Override - public KeyValue properties() { - return properties; + public KeyValue userHeaders() { + return userHeaders; } @Override - public Message putHeaders(final String key, final int value) { - headers.put(key, value); + public Message putSysHeaders(String key, int value) { + sysHeaders.put(key, value); return this; } @Override - public Message putHeaders(final String key, final long value) { - headers.put(key, value); + public Message putSysHeaders(String key, long value) { + sysHeaders.put(key, value); return this; } @Override - public Message putHeaders(final String key, final double value) { - headers.put(key, value); + public Message putSysHeaders(String key, double value) { + sysHeaders.put(key, value); return this; } @Override - public Message putHeaders(final String key, final String value) { - headers.put(key, value); + public Message putSysHeaders(String key, String value) { + sysHeaders.put(key, value); return this; } @Override - public Message putProperties(final String key, final int value) { - properties.put(key, value); + public Message putUserHeaders(String key, int value) { + userHeaders.put(key, value); return this; } @Override - public Message putProperties(final String key, final long value) { - properties.put(key, value); + public Message putUserHeaders(String key, long value) { + userHeaders.put(key, value); return this; } @Override - public Message putProperties(final String key, final double value) { - properties.put(key, value); + public Message putUserHeaders(String key, double value) { + userHeaders.put(key, value); return this; } @Override - public Message putProperties(final String key, final String value) { - properties.put(key, value); + public Message putUserHeaders(String key, String value) { + userHeaders.put(key, value); return this; } diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/RocketMQConstants.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/RocketMQConstants.java new file mode 100644 index 00000000000..4c6568ae0f9 --- /dev/null +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/RocketMQConstants.java @@ -0,0 +1,7 @@ +package io.openmessaging.rocketmq.domain; + +public interface RocketMQConstants { + + String START_DELIVER_TIME = "__STARTDELIVERTIME"; + +} diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/SendResultImpl.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/SendResultImpl.java index 228a9f0b595..85bcd685493 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/SendResultImpl.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/SendResultImpl.java @@ -17,7 +17,7 @@ package io.openmessaging.rocketmq.domain; import io.openmessaging.KeyValue; -import io.openmessaging.SendResult; +import io.openmessaging.producer.SendResult; public class SendResultImpl implements SendResult { private String messageId; @@ -33,7 +33,6 @@ public String messageId() { return messageId; } - @Override public KeyValue properties() { return properties; } diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/AbstractOMSProducer.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/AbstractOMSProducer.java index db25fc6529b..f7337566098 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/AbstractOMSProducer.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/AbstractOMSProducer.java @@ -20,8 +20,7 @@ import io.openmessaging.KeyValue; import io.openmessaging.Message; import io.openmessaging.MessageFactory; -import io.openmessaging.MessageHeader; -import io.openmessaging.PropertyKeys; +import io.openmessaging.OMSBuiltinKeys; import io.openmessaging.ServiceLifecycle; import io.openmessaging.exception.OMSMessageFormatException; import io.openmessaging.exception.OMSNotSupportedException; @@ -53,7 +52,7 @@ abstract class AbstractOMSProducer implements ServiceLifecycle, MessageFactory { this.rocketmqProducer = new DefaultMQProducer(); this.clientConfig = BeanUtils.populate(properties, ClientConfig.class); - String accessPoints = clientConfig.getOmsAccessPoints(); + String accessPoints = clientConfig.getAccessPoints(); if (accessPoints == null || accessPoints.isEmpty()) { throw new OMSRuntimeException("-1", "OMS AccessPoints is null or empty."); } @@ -61,10 +60,10 @@ abstract class AbstractOMSProducer implements ServiceLifecycle, MessageFactory { this.rocketmqProducer.setProducerGroup(clientConfig.getRmqProducerGroup()); String producerId = buildInstanceName(); - this.rocketmqProducer.setSendMsgTimeout(clientConfig.getOmsOperationTimeout()); + this.rocketmqProducer.setSendMsgTimeout(clientConfig.getOperationTimeout()); this.rocketmqProducer.setInstanceName(producerId); this.rocketmqProducer.setMaxMessageSize(1024 * 1024 * 4); - properties.put(PropertyKeys.PRODUCER_ID, producerId); + properties.put(OMSBuiltinKeys.PRODUCER_ID, producerId); } @Override @@ -121,18 +120,10 @@ protected void checkMessageType(Message message) { } @Override - public BytesMessage createBytesMessageToTopic(final String topic, final byte[] body) { - BytesMessage bytesMessage = new BytesMessageImpl(); - bytesMessage.setBody(body); - bytesMessage.headers().put(MessageHeader.TOPIC, topic); - return bytesMessage; - } - - @Override - public BytesMessage createBytesMessageToQueue(final String queue, final byte[] body) { - BytesMessage bytesMessage = new BytesMessageImpl(); - bytesMessage.setBody(body); - bytesMessage.headers().put(MessageHeader.QUEUE, queue); - return bytesMessage; + public BytesMessage createBytesMessage(String queue, byte[] body) { + BytesMessage message = new BytesMessageImpl(); + message.setBody(body); + message.sysHeaders().put(Message.BuiltinKeys.DESTINATION, queue); + return message; } } diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/ProducerImpl.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/ProducerImpl.java index 2c00c60ebc4..c2b6d3e3c77 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/ProducerImpl.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/ProducerImpl.java @@ -19,12 +19,13 @@ import io.openmessaging.BytesMessage; import io.openmessaging.KeyValue; import io.openmessaging.Message; -import io.openmessaging.MessageHeader; -import io.openmessaging.Producer; import io.openmessaging.Promise; -import io.openmessaging.PropertyKeys; -import io.openmessaging.SendResult; import io.openmessaging.exception.OMSRuntimeException; +import io.openmessaging.interceptor.ProducerInterceptor; +import io.openmessaging.producer.BatchMessageSender; +import io.openmessaging.producer.LocalTransactionExecutor; +import io.openmessaging.producer.Producer; +import io.openmessaging.producer.SendResult; import io.openmessaging.rocketmq.promise.DefaultPromise; import io.openmessaging.rocketmq.utils.OMSUtil; import org.apache.rocketmq.client.producer.SendCallback; @@ -39,7 +40,7 @@ public ProducerImpl(final KeyValue properties) { } @Override - public KeyValue properties() { + public KeyValue attributes() { return properties; } @@ -50,11 +51,16 @@ public SendResult send(final Message message) { @Override public SendResult send(final Message message, final KeyValue properties) { - long timeout = properties.containsKey(PropertyKeys.OPERATION_TIMEOUT) - ? properties.getInt(PropertyKeys.OPERATION_TIMEOUT) : this.rocketmqProducer.getSendMsgTimeout(); + long timeout = properties.containsKey(Message.BuiltinKeys.TIMEOUT) + ? properties.getInt(Message.BuiltinKeys.TIMEOUT) : this.rocketmqProducer.getSendMsgTimeout(); return send(message, timeout); } + @Override + public SendResult send(Message message, LocalTransactionExecutor branchExecutor, KeyValue attributes) { + return null; + } + private SendResult send(final Message message, long timeout) { checkMessageType(message); org.apache.rocketmq.common.message.Message rmqMessage = msgConvert((BytesMessage) message); @@ -64,11 +70,11 @@ private SendResult send(final Message message, long timeout) { log.error(String.format("Send message to RocketMQ failed, %s", message)); throw new OMSRuntimeException("-1", "Send message to RocketMQ broker failed."); } - message.headers().put(MessageHeader.MESSAGE_ID, rmqResult.getMsgId()); + message.sysHeaders().put(Message.BuiltinKeys.MESSAGE_ID, rmqResult.getMsgId()); return OMSUtil.sendResultConvert(rmqResult); } catch (Exception e) { log.error(String.format("Send message to RocketMQ failed, %s", message), e); - throw checkProducerException(rmqMessage.getTopic(), message.headers().getString(MessageHeader.MESSAGE_ID), e); + throw checkProducerException(rmqMessage.getTopic(), message.sysHeaders().getString(Message.BuiltinKeys.MESSAGE_ID), e); } } @@ -79,8 +85,8 @@ public Promise sendAsync(final Message message) { @Override public Promise sendAsync(final Message message, final KeyValue properties) { - long timeout = properties.containsKey(PropertyKeys.OPERATION_TIMEOUT) - ? properties.getInt(PropertyKeys.OPERATION_TIMEOUT) : this.rocketmqProducer.getSendMsgTimeout(); + long timeout = properties.containsKey(Message.BuiltinKeys.TIMEOUT) + ? properties.getInt(Message.BuiltinKeys.TIMEOUT) : this.rocketmqProducer.getSendMsgTimeout(); return sendAsync(message, timeout); } @@ -92,7 +98,7 @@ private Promise sendAsync(final Message message, long timeout) { this.rocketmqProducer.send(rmqMessage, new SendCallback() { @Override public void onSuccess(final org.apache.rocketmq.client.producer.SendResult rmqResult) { - message.headers().put(MessageHeader.MESSAGE_ID, rmqResult.getMsgId()); + message.sysHeaders().put(Message.BuiltinKeys.MESSAGE_ID, rmqResult.getMsgId()); promise.set(OMSUtil.sendResultConvert(rmqResult)); } @@ -121,4 +127,19 @@ public void sendOneway(final Message message) { public void sendOneway(final Message message, final KeyValue properties) { sendOneway(message); } + + @Override + public BatchMessageSender createBatchMessageSender() { + return null; + } + + @Override + public void addInterceptor(ProducerInterceptor interceptor) { + + } + + @Override + public void removeInterceptor(ProducerInterceptor interceptor) { + + } } diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/SequenceProducerImpl.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/SequenceProducerImpl.java deleted file mode 100644 index 05225cc5058..00000000000 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/SequenceProducerImpl.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.openmessaging.rocketmq.producer; - -import io.openmessaging.BytesMessage; -import io.openmessaging.KeyValue; -import io.openmessaging.Message; -import io.openmessaging.MessageHeader; -import io.openmessaging.SequenceProducer; -import io.openmessaging.rocketmq.utils.OMSUtil; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import org.apache.rocketmq.client.Validators; -import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.producer.SendResult; - -public class SequenceProducerImpl extends AbstractOMSProducer implements SequenceProducer { - - private BlockingQueue msgCacheQueue; - - public SequenceProducerImpl(final KeyValue properties) { - super(properties); - this.msgCacheQueue = new LinkedBlockingQueue<>(); - } - - @Override - public KeyValue properties() { - return properties; - } - - @Override - public void send(final Message message) { - checkMessageType(message); - org.apache.rocketmq.common.message.Message rmqMessage = OMSUtil.msgConvert((BytesMessage) message); - try { - Validators.checkMessage(rmqMessage, this.rocketmqProducer); - } catch (MQClientException e) { - throw checkProducerException(rmqMessage.getTopic(), message.headers().getString(MessageHeader.MESSAGE_ID), e); - } - msgCacheQueue.add(message); - } - - @Override - public void send(final Message message, final KeyValue properties) { - send(message); - } - - @Override - public synchronized void commit() { - List messages = new ArrayList<>(); - msgCacheQueue.drainTo(messages); - - List rmqMessages = new ArrayList<>(); - - for (Message message : messages) { - rmqMessages.add(OMSUtil.msgConvert((BytesMessage) message)); - } - - if (rmqMessages.size() == 0) { - return; - } - - try { - SendResult sendResult = this.rocketmqProducer.send(rmqMessages); - String[] msgIdArray = sendResult.getMsgId().split(","); - for (int i = 0; i < messages.size(); i++) { - Message message = messages.get(i); - message.headers().put(MessageHeader.MESSAGE_ID, msgIdArray[i]); - } - } catch (Exception e) { - throw checkProducerException("", "", e); - } - } - - @Override - public synchronized void rollback() { - msgCacheQueue.clear(); - } -} diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/promise/DefaultPromise.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/promise/DefaultPromise.java index 453b665c855..c1b59993f6f 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/promise/DefaultPromise.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/promise/DefaultPromise.java @@ -17,7 +17,7 @@ package io.openmessaging.rocketmq.promise; import io.openmessaging.Promise; -import io.openmessaging.PromiseListener; +import io.openmessaging.FutureListener; import io.openmessaging.exception.OMSRuntimeException; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; @@ -33,7 +33,7 @@ public class DefaultPromise implements Promise { private long timeout; private long createTime; private Throwable exception = null; - private List> promiseListenerList; + private List> promiseListenerList; public DefaultPromise() { createTime = System.currentTimeMillis(); @@ -121,7 +121,7 @@ public boolean setFailure(final Throwable cause) { } @Override - public void addListener(final PromiseListener listener) { + public void addListener(final FutureListener listener) { if (listener == null) { throw new NullPointerException("FutureListener is null"); } @@ -150,7 +150,7 @@ public Throwable getThrowable() { private void notifyListeners() { if (promiseListenerList != null) { - for (PromiseListener listener : promiseListenerList) { + for (FutureListener listener : promiseListenerList) { notifyListener(listener); } } @@ -199,12 +199,9 @@ private boolean done() { return true; } - private void notifyListener(final PromiseListener listener) { + private void notifyListener(final FutureListener listener) { try { - if (exception != null) - listener.operationFailed(this); - else - listener.operationCompleted(this); + listener.operationComplete(this); } catch (Throwable t) { LOG.error("notifyListener {} Error:{}", listener.getClass().getSimpleName(), t); } diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/utils/BeanUtils.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/utils/BeanUtils.java index ba7cd5973aa..ef9236f09c1 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/utils/BeanUtils.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/utils/BeanUtils.java @@ -164,7 +164,7 @@ public static T populate(final KeyValue properties, final T obj) { final Set keySet = properties.keySet(); for (String key : keySet) { - String[] keyGroup = key.split("\\."); + String[] keyGroup = key.split("[\\._]"); for (int i = 0; i < keyGroup.length; i++) { keyGroup[i] = keyGroup[i].toLowerCase(); keyGroup[i] = StringUtils.capitalize(keyGroup[i]); diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/utils/OMSUtil.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/utils/OMSUtil.java index 60c840813cf..23021413c80 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/utils/OMSUtil.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/utils/OMSUtil.java @@ -18,11 +18,11 @@ import io.openmessaging.BytesMessage; import io.openmessaging.KeyValue; -import io.openmessaging.MessageHeader; +import io.openmessaging.Message.BuiltinKeys; import io.openmessaging.OMS; -import io.openmessaging.SendResult; +import io.openmessaging.producer.SendResult; import io.openmessaging.rocketmq.domain.BytesMessageImpl; -import io.openmessaging.rocketmq.domain.NonStandardKeys; +import io.openmessaging.rocketmq.domain.RocketMQConstants; import io.openmessaging.rocketmq.domain.SendResultImpl; import java.lang.reflect.Field; import java.util.Iterator; @@ -48,25 +48,26 @@ public static org.apache.rocketmq.common.message.Message msgConvert(BytesMessage org.apache.rocketmq.common.message.Message rmqMessage = new org.apache.rocketmq.common.message.Message(); rmqMessage.setBody(omsMessage.getBody()); - KeyValue headers = omsMessage.headers(); - KeyValue properties = omsMessage.properties(); + KeyValue sysHeaders = omsMessage.sysHeaders(); + KeyValue userHeaders = omsMessage.userHeaders(); //All destinations in RocketMQ use Topic - if (headers.containsKey(MessageHeader.TOPIC)) { - rmqMessage.setTopic(headers.getString(MessageHeader.TOPIC)); - rmqMessage.putUserProperty(NonStandardKeys.MESSAGE_DESTINATION, "TOPIC"); - } else { - rmqMessage.setTopic(headers.getString(MessageHeader.QUEUE)); - rmqMessage.putUserProperty(NonStandardKeys.MESSAGE_DESTINATION, "QUEUE"); + rmqMessage.setTopic(sysHeaders.getString(BuiltinKeys.DESTINATION)); + + if (sysHeaders.containsKey(BuiltinKeys.START_TIME)) { + long deliverTime = sysHeaders.getLong(BuiltinKeys.START_TIME, 0); + if (deliverTime > 0) { + rmqMessage.putUserProperty(RocketMQConstants.START_DELIVER_TIME, String.valueOf(deliverTime)); + } } - for (String key : properties.keySet()) { - MessageAccessor.putProperty(rmqMessage, key, properties.getString(key)); + for (String key : userHeaders.keySet()) { + MessageAccessor.putProperty(rmqMessage, key, userHeaders.getString(key)); } - //Headers has a high priority - for (String key : headers.keySet()) { - MessageAccessor.putProperty(rmqMessage, key, headers.getString(key)); + //System headers has a high priority + for (String key : sysHeaders.keySet()) { + MessageAccessor.putProperty(rmqMessage, key, sysHeaders.getString(key)); } return rmqMessage; @@ -76,8 +77,8 @@ public static BytesMessage msgConvert(org.apache.rocketmq.common.message.Message BytesMessage omsMsg = new BytesMessageImpl(); omsMsg.setBody(rmqMsg.getBody()); - KeyValue headers = omsMsg.headers(); - KeyValue properties = omsMsg.properties(); + KeyValue headers = omsMsg.sysHeaders(); + KeyValue properties = omsMsg.userHeaders(); final Set> entries = rmqMsg.getProperties().entrySet(); @@ -89,25 +90,22 @@ public static BytesMessage msgConvert(org.apache.rocketmq.common.message.Message } } - omsMsg.putHeaders(MessageHeader.MESSAGE_ID, rmqMsg.getMsgId()); - if (!rmqMsg.getProperties().containsKey(NonStandardKeys.MESSAGE_DESTINATION) || - rmqMsg.getProperties().get(NonStandardKeys.MESSAGE_DESTINATION).equals("TOPIC")) { - omsMsg.putHeaders(MessageHeader.TOPIC, rmqMsg.getTopic()); - } else { - omsMsg.putHeaders(MessageHeader.QUEUE, rmqMsg.getTopic()); - } - omsMsg.putHeaders(MessageHeader.SEARCH_KEY, rmqMsg.getKeys()); - omsMsg.putHeaders(MessageHeader.BORN_HOST, String.valueOf(rmqMsg.getBornHost())); - omsMsg.putHeaders(MessageHeader.BORN_TIMESTAMP, rmqMsg.getBornTimestamp()); - omsMsg.putHeaders(MessageHeader.STORE_HOST, String.valueOf(rmqMsg.getStoreHost())); - omsMsg.putHeaders(MessageHeader.STORE_TIMESTAMP, rmqMsg.getStoreTimestamp()); + omsMsg.putSysHeaders(BuiltinKeys.MESSAGE_ID, rmqMsg.getMsgId()); + + omsMsg.putSysHeaders(BuiltinKeys.DESTINATION, rmqMsg.getTopic()); + + omsMsg.putSysHeaders(BuiltinKeys.SEARCH_KEYS, rmqMsg.getKeys()); + omsMsg.putSysHeaders(BuiltinKeys.BORN_HOST, String.valueOf(rmqMsg.getBornHost())); + omsMsg.putSysHeaders(BuiltinKeys.BORN_TIMESTAMP, rmqMsg.getBornTimestamp()); + omsMsg.putSysHeaders(BuiltinKeys.STORE_HOST, String.valueOf(rmqMsg.getStoreHost())); + omsMsg.putSysHeaders(BuiltinKeys.STORE_TIMESTAMP, rmqMsg.getStoreTimestamp()); return omsMsg; } public static boolean isOMSHeader(String value) { - for (Field field : MessageHeader.class.getDeclaredFields()) { + for (Field field : BuiltinKeys.class.getDeclaredFields()) { try { - if (field.get(MessageHeader.class).equals(value)) { + if (field.get(BuiltinKeys.class).equals(value)) { return true; } } catch (IllegalAccessException e) { diff --git a/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/PullConsumerImplTest.java b/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/PullConsumerImplTest.java index 843ddb78805..da2e8a084fe 100644 --- a/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/PullConsumerImplTest.java +++ b/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/PullConsumerImplTest.java @@ -18,12 +18,10 @@ import io.openmessaging.BytesMessage; import io.openmessaging.Message; -import io.openmessaging.MessageHeader; import io.openmessaging.MessagingAccessPoint; -import io.openmessaging.MessagingAccessPointFactory; import io.openmessaging.OMS; -import io.openmessaging.PropertyKeys; -import io.openmessaging.PullConsumer; +import io.openmessaging.OMSBuiltinKeys; +import io.openmessaging.consumer.PullConsumer; import io.openmessaging.rocketmq.config.ClientConfig; import io.openmessaging.rocketmq.domain.NonStandardKeys; import java.lang.reflect.Field; @@ -50,18 +48,18 @@ public class PullConsumerImplTest { @Before public void init() throws NoSuchFieldException, IllegalAccessException { - final MessagingAccessPoint messagingAccessPoint = MessagingAccessPointFactory - .getMessagingAccessPoint("openmessaging:rocketmq://IP1:9876,IP2:9876/namespace"); + final MessagingAccessPoint messagingAccessPoint = OMS + .getMessagingAccessPoint("oms:rocketmq://IP1:9876,IP2:9876/namespace"); - consumer = messagingAccessPoint.createPullConsumer(queueName, - OMS.newKeyValue().put(NonStandardKeys.CONSUMER_GROUP, "TestGroup")); + consumer = messagingAccessPoint.createPullConsumer(OMS.newKeyValue().put(OMSBuiltinKeys.CONSUMER_ID, "TestGroup")); + consumer.attachQueue(queueName); Field field = PullConsumerImpl.class.getDeclaredField("rocketmqPullConsumer"); field.setAccessible(true); field.set(consumer, rocketmqPullConsumer); //Replace ClientConfig clientConfig = new ClientConfig(); - clientConfig.setOmsOperationTimeout(200); + clientConfig.setOperationTimeout(200); localMessageCache = spy(new LocalMessageCache(rocketmqPullConsumer, clientConfig)); field = PullConsumerImpl.class.getDeclaredField("localMessageCache"); @@ -83,18 +81,18 @@ public void testPoll() { when(localMessageCache.poll()).thenReturn(consumedMsg); - Message message = consumer.poll(); - assertThat(message.headers().getString(MessageHeader.MESSAGE_ID)).isEqualTo("NewMsgId"); + Message message = consumer.receive(); + assertThat(message.sysHeaders().getString(Message.BuiltinKeys.MESSAGE_ID)).isEqualTo("NewMsgId"); assertThat(((BytesMessage) message).getBody()).isEqualTo(testBody); } @Test public void testPoll_WithTimeout() { //There is a default timeout value, @see ClientConfig#omsOperationTimeout. - Message message = consumer.poll(); + Message message = consumer.receive(); assertThat(message).isNull(); - message = consumer.poll(OMS.newKeyValue().put(PropertyKeys.OPERATION_TIMEOUT, 100)); + message = consumer.receive(OMS.newKeyValue().put(Message.BuiltinKeys.TIMEOUT, 100)); assertThat(message).isNull(); } } \ No newline at end of file diff --git a/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/PushConsumerImplTest.java b/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/PushConsumerImplTest.java index 882e57ea5ee..b55816b8989 100644 --- a/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/PushConsumerImplTest.java +++ b/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/PushConsumerImplTest.java @@ -18,13 +18,11 @@ import io.openmessaging.BytesMessage; import io.openmessaging.Message; -import io.openmessaging.MessageHeader; -import io.openmessaging.MessageListener; +import io.openmessaging.OMSBuiltinKeys; +import io.openmessaging.consumer.MessageListener; import io.openmessaging.MessagingAccessPoint; -import io.openmessaging.MessagingAccessPointFactory; import io.openmessaging.OMS; -import io.openmessaging.PushConsumer; -import io.openmessaging.ReceivedMessageContext; +import io.openmessaging.consumer.PushConsumer; import io.openmessaging.rocketmq.domain.NonStandardKeys; import java.lang.reflect.Field; import java.util.Collections; @@ -49,10 +47,10 @@ public class PushConsumerImplTest { @Before public void init() throws NoSuchFieldException, IllegalAccessException { - final MessagingAccessPoint messagingAccessPoint = MessagingAccessPointFactory - .getMessagingAccessPoint("openmessaging:rocketmq://IP1:9876,IP2:9876/namespace"); + final MessagingAccessPoint messagingAccessPoint = OMS + .getMessagingAccessPoint("oms:rocketmq://IP1:9876,IP2:9876/namespace"); consumer = messagingAccessPoint.createPushConsumer( - OMS.newKeyValue().put(NonStandardKeys.CONSUMER_GROUP, "TestGroup")); + OMS.newKeyValue().put(OMSBuiltinKeys.CONSUMER_ID, "TestGroup")); Field field = PushConsumerImpl.class.getDeclaredField("rocketmqPushConsumer"); field.setAccessible(true); @@ -75,8 +73,8 @@ public void testConsumeMessage() { consumedMsg.setTopic("HELLO_QUEUE"); consumer.attachQueue("HELLO_QUEUE", new MessageListener() { @Override - public void onMessage(final Message message, final ReceivedMessageContext context) { - assertThat(message.headers().getString(MessageHeader.MESSAGE_ID)).isEqualTo("NewMsgId"); + public void onReceived(Message message, Context context) { + assertThat(message.sysHeaders().getString(Message.BuiltinKeys.MESSAGE_ID)).isEqualTo("NewMsgId"); assertThat(((BytesMessage) message).getBody()).isEqualTo(testBody); context.ack(); } diff --git a/openmessaging/src/test/java/io/openmessaging/rocketmq/producer/ProducerImplTest.java b/openmessaging/src/test/java/io/openmessaging/rocketmq/producer/ProducerImplTest.java index 1db80c3efac..fc6515ea8d1 100644 --- a/openmessaging/src/test/java/io/openmessaging/rocketmq/producer/ProducerImplTest.java +++ b/openmessaging/src/test/java/io/openmessaging/rocketmq/producer/ProducerImplTest.java @@ -17,9 +17,9 @@ package io.openmessaging.rocketmq.producer; import io.openmessaging.MessagingAccessPoint; -import io.openmessaging.MessagingAccessPointFactory; -import io.openmessaging.Producer; +import io.openmessaging.OMS; import io.openmessaging.exception.OMSRuntimeException; +import io.openmessaging.producer.Producer; import java.lang.reflect.Field; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; @@ -49,8 +49,8 @@ public class ProducerImplTest { @Before public void init() throws NoSuchFieldException, IllegalAccessException { - final MessagingAccessPoint messagingAccessPoint = MessagingAccessPointFactory - .getMessagingAccessPoint("openmessaging:rocketmq://IP1:9876,IP2:9876/namespace"); + final MessagingAccessPoint messagingAccessPoint = OMS + .getMessagingAccessPoint("oms:rocketmq://IP1:9876,IP2:9876/namespace"); producer = messagingAccessPoint.createProducer(); Field field = AbstractOMSProducer.class.getDeclaredField("rocketmqProducer"); @@ -67,8 +67,8 @@ public void testSend_OK() throws InterruptedException, RemotingException, MQClie sendResult.setMsgId("TestMsgID"); sendResult.setSendStatus(SendStatus.SEND_OK); when(rocketmqProducer.send(any(Message.class), anyLong())).thenReturn(sendResult); - io.openmessaging.SendResult omsResult = - producer.send(producer.createBytesMessageToTopic("HELLO_TOPIC", new byte[] {'a'})); + io.openmessaging.producer.SendResult omsResult = + producer.send(producer.createBytesMessage("HELLO_TOPIC", new byte[] {'a'})); assertThat(omsResult.messageId()).isEqualTo("TestMsgID"); } @@ -80,7 +80,7 @@ public void testSend_Not_OK() throws InterruptedException, RemotingException, MQ when(rocketmqProducer.send(any(Message.class), anyLong())).thenReturn(sendResult); try { - producer.send(producer.createBytesMessageToTopic("HELLO_TOPIC", new byte[] {'a'})); + producer.send(producer.createBytesMessage("HELLO_TOPIC", new byte[] {'a'})); failBecauseExceptionWasNotThrown(OMSRuntimeException.class); } catch (Exception e) { assertThat(e).hasMessageContaining("Send message to RocketMQ broker failed."); @@ -91,7 +91,7 @@ public void testSend_Not_OK() throws InterruptedException, RemotingException, MQ public void testSend_WithException() throws InterruptedException, RemotingException, MQClientException, MQBrokerException { when(rocketmqProducer.send(any(Message.class), anyLong())).thenThrow(MQClientException.class); try { - producer.send(producer.createBytesMessageToTopic("HELLO_TOPIC", new byte[] {'a'})); + producer.send(producer.createBytesMessage("HELLO_TOPIC", new byte[] {'a'})); failBecauseExceptionWasNotThrown(OMSRuntimeException.class); } catch (Exception e) { assertThat(e).hasMessageContaining("Send message to RocketMQ broker failed."); diff --git a/openmessaging/src/test/java/io/openmessaging/rocketmq/producer/SequenceProducerImplTest.java b/openmessaging/src/test/java/io/openmessaging/rocketmq/producer/SequenceProducerImplTest.java deleted file mode 100644 index 823fe015cca..00000000000 --- a/openmessaging/src/test/java/io/openmessaging/rocketmq/producer/SequenceProducerImplTest.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.openmessaging.rocketmq.producer; - -import io.openmessaging.BytesMessage; -import io.openmessaging.MessageHeader; -import io.openmessaging.MessagingAccessPoint; -import io.openmessaging.MessagingAccessPointFactory; -import io.openmessaging.SequenceProducer; -import java.lang.reflect.Field; -import org.apache.rocketmq.client.exception.MQBrokerException; -import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.producer.DefaultMQProducer; -import org.apache.rocketmq.client.producer.SendResult; -import org.apache.rocketmq.client.producer.SendStatus; -import org.apache.rocketmq.common.message.Message; -import org.apache.rocketmq.remoting.exception.RemotingException; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.ArgumentMatchers; -import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; - -@RunWith(MockitoJUnitRunner.class) -public class SequenceProducerImplTest { - - private SequenceProducer producer; - - @Mock - private DefaultMQProducer rocketmqProducer; - - @Before - public void init() throws NoSuchFieldException, IllegalAccessException { - final MessagingAccessPoint messagingAccessPoint = MessagingAccessPointFactory - .getMessagingAccessPoint("openmessaging:rocketmq://IP1:9876,IP2:9876/namespace"); - producer = messagingAccessPoint.createSequenceProducer(); - - Field field = AbstractOMSProducer.class.getDeclaredField("rocketmqProducer"); - field.setAccessible(true); - field.set(producer, rocketmqProducer); - - messagingAccessPoint.startup(); - producer.startup(); - } - - @Test - public void testSend_WithCommit() throws InterruptedException, RemotingException, MQClientException, MQBrokerException { - SendResult sendResult = new SendResult(); - sendResult.setMsgId("TestMsgID"); - sendResult.setSendStatus(SendStatus.SEND_OK); - when(rocketmqProducer.send(ArgumentMatchers.anyList())).thenReturn(sendResult); - when(rocketmqProducer.getMaxMessageSize()).thenReturn(1024); - final BytesMessage message = producer.createBytesMessageToTopic("HELLO_TOPIC", new byte[] {'a'}); - producer.send(message); - producer.commit(); - assertThat(message.headers().getString(MessageHeader.MESSAGE_ID)).isEqualTo("TestMsgID"); - } - - @Test - public void testRollback() { - when(rocketmqProducer.getMaxMessageSize()).thenReturn(1024); - final BytesMessage message = producer.createBytesMessageToTopic("HELLO_TOPIC", new byte[] {'a'}); - producer.send(message); - producer.rollback(); - producer.commit(); //Commit nothing. - assertThat(message.headers().getString(MessageHeader.MESSAGE_ID)).isEqualTo(null); - } -} \ No newline at end of file diff --git a/openmessaging/src/test/java/io/openmessaging/rocketmq/promise/DefaultPromiseTest.java b/openmessaging/src/test/java/io/openmessaging/rocketmq/promise/DefaultPromiseTest.java index 2240ff2dd11..f226edef0a2 100644 --- a/openmessaging/src/test/java/io/openmessaging/rocketmq/promise/DefaultPromiseTest.java +++ b/openmessaging/src/test/java/io/openmessaging/rocketmq/promise/DefaultPromiseTest.java @@ -16,8 +16,9 @@ */ package io.openmessaging.rocketmq.promise; +import io.openmessaging.Future; +import io.openmessaging.FutureListener; import io.openmessaging.Promise; -import io.openmessaging.PromiseListener; import io.openmessaging.exception.OMSRuntimeException; import org.junit.Before; import org.junit.Test; @@ -63,14 +64,10 @@ public void testGet_WithTimeout() throws Exception { @Test public void testAddListener() throws Exception { - promise.addListener(new PromiseListener() { + promise.addListener(new FutureListener() { @Override - public void operationCompleted(final Promise promise) { + public void operationComplete(Future future) { assertThat(promise.get()).isEqualTo("Done"); - } - - @Override - public void operationFailed(final Promise promise) { } }); @@ -80,15 +77,10 @@ public void operationFailed(final Promise promise) { @Test public void testAddListener_ListenerAfterSet() throws Exception { promise.set("Done"); - promise.addListener(new PromiseListener() { - @Override - public void operationCompleted(final Promise promise) { - assertThat(promise.get()).isEqualTo("Done"); - } - + promise.addListener(new FutureListener() { @Override - public void operationFailed(final Promise promise) { - + public void operationComplete(Future future) { + assertThat(future.get()).isEqualTo("Done"); } }); } @@ -97,13 +89,9 @@ public void operationFailed(final Promise promise) { public void testAddListener_WithException_ListenerAfterSet() throws Exception { final Throwable exception = new OMSRuntimeException("-1", "Test Error"); promise.setFailure(exception); - promise.addListener(new PromiseListener() { - @Override - public void operationCompleted(final Promise promise) { - } - + promise.addListener(new FutureListener() { @Override - public void operationFailed(final Promise promise) { + public void operationComplete(Future future) { assertThat(promise.getThrowable()).isEqualTo(exception); } }); @@ -112,13 +100,9 @@ public void operationFailed(final Promise promise) { @Test public void testAddListener_WithException() throws Exception { final Throwable exception = new OMSRuntimeException("-1", "Test Error"); - promise.addListener(new PromiseListener() { - @Override - public void operationCompleted(final Promise promise) { - } - + promise.addListener(new FutureListener() { @Override - public void operationFailed(final Promise promise) { + public void operationComplete(Future future) { assertThat(promise.getThrowable()).isEqualTo(exception); } }); diff --git a/openmessaging/src/test/java/io/openmessaging/rocketmq/utils/BeanUtilsTest.java b/openmessaging/src/test/java/io/openmessaging/rocketmq/utils/BeanUtilsTest.java index 71ca11ccfdd..1a431d988fd 100644 --- a/openmessaging/src/test/java/io/openmessaging/rocketmq/utils/BeanUtilsTest.java +++ b/openmessaging/src/test/java/io/openmessaging/rocketmq/utils/BeanUtilsTest.java @@ -92,9 +92,9 @@ public void testPopulate() { @Test public void testPopulate_ExistObj() { CustomizedConfig config = new CustomizedConfig(); - config.setOmsConsumerId("NewConsumerId"); + config.setConsumerId("NewConsumerId"); - Assert.assertEquals(config.getOmsConsumerId(), "NewConsumerId"); + Assert.assertEquals(config.getConsumerId(), "NewConsumerId"); config = BeanUtils.populate(properties, config); diff --git a/pom.xml b/pom.xml index 6737ae4116e..f4184a5f32f 100644 --- a/pom.xml +++ b/pom.xml @@ -592,7 +592,7 @@ io.openmessaging openmessaging-api - 0.1.0-alpha + 0.3.0-alpha-SNAPSHOT log4j From e428de25db26658157d7d335400cf6e9f9d0a649 Mon Sep 17 00:00:00 2001 From: "shutian.lzh" Date: Tue, 24 Apr 2018 12:39:21 +0800 Subject: [PATCH 23/73] Fix Open Messaging Spec version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f4184a5f32f..5ec47f31554 100644 --- a/pom.xml +++ b/pom.xml @@ -592,7 +592,7 @@ io.openmessaging openmessaging-api - 0.3.0-alpha-SNAPSHOT + 0.3.0-alpha log4j From d6807c87f210d98324216d285f4f69a6814c6a02 Mon Sep 17 00:00:00 2001 From: "shutian.lzh" Date: Tue, 24 Apr 2018 12:44:23 +0800 Subject: [PATCH 24/73] Fix code style issue --- .../rocketmq/domain/RocketMQConstants.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/RocketMQConstants.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/RocketMQConstants.java index 4c6568ae0f9..2bebc8abfab 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/RocketMQConstants.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/RocketMQConstants.java @@ -1,7 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.openmessaging.rocketmq.domain; public interface RocketMQConstants { + /** + * Key of scheduled message delivery time + */ String START_DELIVER_TIME = "__STARTDELIVERTIME"; } From 61246957a7d453c4401abd64bf75bbc0034d044d Mon Sep 17 00:00:00 2001 From: "shutian.lzh" Date: Thu, 26 Apr 2018 13:48:30 +0800 Subject: [PATCH 25/73] Use access point url as name server conditionally --- .../rocketmq/producer/AbstractOMSProducer.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/AbstractOMSProducer.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/AbstractOMSProducer.java index f7337566098..e40e2d452e0 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/AbstractOMSProducer.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/AbstractOMSProducer.java @@ -52,11 +52,15 @@ abstract class AbstractOMSProducer implements ServiceLifecycle, MessageFactory { this.rocketmqProducer = new DefaultMQProducer(); this.clientConfig = BeanUtils.populate(properties, ClientConfig.class); - String accessPoints = clientConfig.getAccessPoints(); - if (accessPoints == null || accessPoints.isEmpty()) { - throw new OMSRuntimeException("-1", "OMS AccessPoints is null or empty."); + if ("true".equalsIgnoreCase(System.getenv("OMS_RMQ_DIRECT_NAME_SRV"))) { + String accessPoints = clientConfig.getAccessPoints(); + if (accessPoints == null || accessPoints.isEmpty()) { + throw new OMSRuntimeException("-1", "OMS AccessPoints is null or empty."); + } + + this.rocketmqProducer.setNamesrvAddr(accessPoints.replace(',', ';')); } - this.rocketmqProducer.setNamesrvAddr(accessPoints.replace(',', ';')); + this.rocketmqProducer.setProducerGroup(clientConfig.getRmqProducerGroup()); String producerId = buildInstanceName(); From 173f77d2c32019c8f5884a3adeb78d6f83aaa55c Mon Sep 17 00:00:00 2001 From: Zhanhui Li Date: Sat, 28 Apr 2018 05:01:29 +0800 Subject: [PATCH 26/73] BugFix: access point parsed as name server address when enabled for push / pull consumers --- .../rocketmq/consumer/PullConsumerImpl.java | 10 ++++++---- .../rocketmq/consumer/PushConsumerImpl.java | 10 ++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PullConsumerImpl.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PullConsumerImpl.java index 2e22509a2a1..c11da5857bf 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PullConsumerImpl.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PullConsumerImpl.java @@ -60,11 +60,13 @@ public PullConsumerImpl(final KeyValue properties) { this.rocketmqPullConsumer = pullConsumerScheduleService.getDefaultMQPullConsumer(); - String accessPoints = clientConfig.getAccessPoints(); - if (accessPoints == null || accessPoints.isEmpty()) { - throw new OMSRuntimeException("-1", "OMS AccessPoints is null or empty."); + if ("true".equalsIgnoreCase(System.getenv("OMS_RMQ_DIRECT_NAME_SRV"))) { + String accessPoints = clientConfig.getAccessPoints(); + if (accessPoints == null || accessPoints.isEmpty()) { + throw new OMSRuntimeException("-1", "OMS AccessPoints is null or empty."); + } + this.rocketmqPullConsumer.setNamesrvAddr(accessPoints.replace(',', ';')); } - this.rocketmqPullConsumer.setNamesrvAddr(accessPoints.replace(',', ';')); this.rocketmqPullConsumer.setConsumerGroup(consumerGroup); diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PushConsumerImpl.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PushConsumerImpl.java index 9bfd7c8b622..46f6775e42e 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PushConsumerImpl.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PushConsumerImpl.java @@ -52,11 +52,13 @@ public PushConsumerImpl(final KeyValue properties) { this.properties = properties; this.clientConfig = BeanUtils.populate(properties, ClientConfig.class); - String accessPoints = clientConfig.getAccessPoints(); - if (accessPoints == null || accessPoints.isEmpty()) { - throw new OMSRuntimeException("-1", "OMS AccessPoints is null or empty."); + if ("true".equalsIgnoreCase(System.getenv("OMS_RMQ_DIRECT_NAME_SRV"))) { + String accessPoints = clientConfig.getAccessPoints(); + if (accessPoints == null || accessPoints.isEmpty()) { + throw new OMSRuntimeException("-1", "OMS AccessPoints is null or empty."); + } + this.rocketmqPushConsumer.setNamesrvAddr(accessPoints.replace(',', ';')); } - this.rocketmqPushConsumer.setNamesrvAddr(accessPoints.replace(',', ';')); String consumerGroup = clientConfig.getConsumerId(); if (null == consumerGroup || consumerGroup.isEmpty()) { From c508cb33f644355bf2cf52b056a067b0ed0762b9 Mon Sep 17 00:00:00 2001 From: "shutian.lzh" Date: Tue, 8 May 2018 13:22:12 +0800 Subject: [PATCH 27/73] Accomodate updated openmessaging api --- .../openmessaging/rocketmq/domain/BytesMessageImpl.java | 9 +++++++-- .../java/io/openmessaging/rocketmq/utils/OMSUtil.java | 2 +- .../rocketmq/consumer/PullConsumerImplTest.java | 2 +- .../rocketmq/consumer/PushConsumerImplTest.java | 2 +- pom.xml | 2 +- 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/BytesMessageImpl.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/BytesMessageImpl.java index 702d561fafa..6d8995a1564 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/BytesMessageImpl.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/BytesMessageImpl.java @@ -20,6 +20,7 @@ import io.openmessaging.KeyValue; import io.openmessaging.Message; import io.openmessaging.OMS; +import io.openmessaging.exception.OMSMessageFormatException; import org.apache.commons.lang3.builder.ToStringBuilder; public class BytesMessageImpl implements BytesMessage { @@ -33,8 +34,12 @@ public BytesMessageImpl() { } @Override - public byte[] getBody() { - return body; + public T getBody(Class type) throws OMSMessageFormatException { + if (type == byte[].class) { + return (T)body; + } + + throw new OMSMessageFormatException("", "Cannot assign byte[] to " + type.getName()); } @Override diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/utils/OMSUtil.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/utils/OMSUtil.java index 23021413c80..66af8cebb91 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/utils/OMSUtil.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/utils/OMSUtil.java @@ -46,7 +46,7 @@ public static String buildInstanceName() { public static org.apache.rocketmq.common.message.Message msgConvert(BytesMessage omsMessage) { org.apache.rocketmq.common.message.Message rmqMessage = new org.apache.rocketmq.common.message.Message(); - rmqMessage.setBody(omsMessage.getBody()); + rmqMessage.setBody(omsMessage.getBody(byte[].class)); KeyValue sysHeaders = omsMessage.sysHeaders(); KeyValue userHeaders = omsMessage.userHeaders(); diff --git a/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/PullConsumerImplTest.java b/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/PullConsumerImplTest.java index da2e8a084fe..5a0fd9c8c38 100644 --- a/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/PullConsumerImplTest.java +++ b/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/PullConsumerImplTest.java @@ -83,7 +83,7 @@ public void testPoll() { Message message = consumer.receive(); assertThat(message.sysHeaders().getString(Message.BuiltinKeys.MESSAGE_ID)).isEqualTo("NewMsgId"); - assertThat(((BytesMessage) message).getBody()).isEqualTo(testBody); + assertThat(((BytesMessage) message).getBody(byte[].class)).isEqualTo(testBody); } @Test diff --git a/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/PushConsumerImplTest.java b/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/PushConsumerImplTest.java index b55816b8989..d80e02622db 100644 --- a/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/PushConsumerImplTest.java +++ b/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/PushConsumerImplTest.java @@ -75,7 +75,7 @@ public void testConsumeMessage() { @Override public void onReceived(Message message, Context context) { assertThat(message.sysHeaders().getString(Message.BuiltinKeys.MESSAGE_ID)).isEqualTo("NewMsgId"); - assertThat(((BytesMessage) message).getBody()).isEqualTo(testBody); + assertThat(((BytesMessage) message).getBody(byte[].class)).isEqualTo(testBody); context.ack(); } }); diff --git a/pom.xml b/pom.xml index 5ec47f31554..d26e78f7634 100644 --- a/pom.xml +++ b/pom.xml @@ -592,7 +592,7 @@ io.openmessaging openmessaging-api - 0.3.0-alpha + 0.3.1-alpha-SNAPSHOT log4j From 9f4934fc9846ed151aa6bb20b7e971ee9a043a32 Mon Sep 17 00:00:00 2001 From: Shannon Date: Tue, 15 May 2018 16:59:27 +0800 Subject: [PATCH 28/73] =?UTF-8?q?[ISSUE=20#66]=20duplicate=20compress=20me?= =?UTF-8?q?ssage=20body=20if=20retry=20to=20send=20msg=20when=E2=80=A6=20(?= =?UTF-8?q?#294)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix issue : duplicate compress message body if retry to send msg when exception occurs in async sending. --- .../rocketmq/client/impl/MQClientAPIImpl.java | 1 + .../impl/producer/DefaultMQProducerImpl.java | 13 ++++- .../producer/DefaultMQProducerTest.java | 49 +++++++++++++++++++ .../common/message/MessageAccessor.java | 7 +++ 4 files changed, 69 insertions(+), 1 deletion(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index b07778499b7..d4ed1ec4df1 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -27,6 +27,7 @@ import java.util.Properties; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; + import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.consumer.PullCallback; import org.apache.rocketmq.client.consumer.PullResult; diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index 9dd8ee3ce0a..81461f5e9b2 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -607,8 +607,10 @@ private SendResult sendKernelImpl(final Message msg, } int sysFlag = 0; + boolean msgBodyCompressed = false; if (this.tryToCompressMessage(msg)) { sysFlag |= MessageSysFlag.COMPRESSED_FLAG; + msgBodyCompressed = true; } final String tranMsg = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED); @@ -678,10 +680,19 @@ private SendResult sendKernelImpl(final Message msg, SendResult sendResult = null; switch (communicationMode) { case ASYNC: + Message tmpMessage = msg; + if (msgBodyCompressed) { + //If msg body was compressed, msgbody should be reset using prevBody. + //Clone new message using commpressed message body and recover origin massage. + //Fix bug:https://github.com/apache/rocketmq-externals/issues/66 + tmpMessage = MessageAccessor.cloneMessage(msg); + msg.setBody(prevBody); + } + sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage( brokerAddr, mq.getBrokerName(), - msg, + tmpMessage, requestHeader, timeout, communicationMode, diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java index ded22ada914..d3c6cc8b117 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java @@ -69,6 +69,7 @@ public class DefaultMQProducerTest { private DefaultMQProducer producer; private Message message; private Message zeroMsg; + private Message bigMessage; private String topic = "FooBar"; private String producerGroupPrefix = "FooBar_PID"; @@ -77,8 +78,10 @@ public void init() throws Exception { String producerGroupTemp = producerGroupPrefix + System.currentTimeMillis(); producer = new DefaultMQProducer(producerGroupTemp); producer.setNamesrvAddr("127.0.0.1:9876"); + producer.setCompressMsgBodyOverHowmuch(16); message = new Message(topic, new byte[] {'a'}); zeroMsg = new Message(topic, new byte[] {}); + bigMessage = new Message(topic, "This is a very huge message!".getBytes()); producer.start(); @@ -146,6 +149,52 @@ public void testSendMessageSync_Success() throws RemotingException, InterruptedE assertThat(sendResult.getQueueOffset()).isEqualTo(456L); } + @Test + public void testSendMessageSync_WithBodyCompressed() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + SendResult sendResult = producer.send(bigMessage); + + assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); + assertThat(sendResult.getOffsetMsgId()).isEqualTo("123"); + assertThat(sendResult.getQueueOffset()).isEqualTo(456L); + } + + @Test + public void testSendMessageAsync_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + producer.send(message, new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); + assertThat(sendResult.getOffsetMsgId()).isEqualTo("123"); + assertThat(sendResult.getQueueOffset()).isEqualTo(456L); + } + + @Override + public void onException(Throwable e) { + } + }); + + } + + @Test + public void testSendMessageAsync_BodyCompressed() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + producer.send(bigMessage, new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); + assertThat(sendResult.getOffsetMsgId()).isEqualTo("123"); + assertThat(sendResult.getQueueOffset()).isEqualTo(456L); + } + + @Override + public void onException(Throwable e) { + } + }); + + } + @Test public void testSendMessageSync_SuccessWithHook() throws Throwable { when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageAccessor.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageAccessor.java index 4cac404b9bd..1b7e2bba320 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageAccessor.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageAccessor.java @@ -89,4 +89,11 @@ public static String getConsumeStartTimeStamp(final Message msg) { return msg.getProperty(MessageConst.PROPERTY_CONSUME_START_TIMESTAMP); } + public static Message cloneMessage(final Message msg) { + Message newMsg = new Message(msg.getTopic(), msg.getBody()); + newMsg.setFlag(msg.getFlag()); + newMsg.setProperties(msg.getProperties()); + return newMsg; + } + } From 12a51c41942ec64046b7efe982e90d6fbadfaaaf Mon Sep 17 00:00:00 2001 From: Li Zhanhui Date: Thu, 17 May 2018 13:23:42 +0800 Subject: [PATCH 29/73] Use uber jar for netty-tcnative --- pom.xml | 5 ----- remoting/pom.xml | 15 ++------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/pom.xml b/pom.xml index d26e78f7634..1f9ce567667 100644 --- a/pom.xml +++ b/pom.xml @@ -534,11 +534,6 @@ rocketmq-example ${project.version} - - ${project.groupId} - rocketmq-logging - ${project.version} - org.slf4j slf4j-api diff --git a/remoting/pom.xml b/remoting/pom.xml index 21d9c8d5f00..55589137943 100644 --- a/remoting/pom.xml +++ b/remoting/pom.xml @@ -47,19 +47,8 @@ io.netty - netty-tcnative - 1.1.33.Fork22 - ${os.detected.classifier} + netty-tcnative-boringssl-static + 1.1.33.Fork26 - - - - - kr.motd.maven - os-maven-plugin - 1.4.0.Final - - - From 9c0e5360e109b2a5c4c86ed7053a59f868b078ee Mon Sep 17 00:00:00 2001 From: Li Zhanhui Date: Mon, 21 May 2018 11:30:12 +0800 Subject: [PATCH 30/73] Tag language of clients initialized through OMS as 'OMS' --- .../apache/rocketmq/client/ClientConfig.java | 15 ++++++++++++++- .../rocketmq/client/impl/MQClientAPIImpl.java | 17 ++++++----------- .../rocketmq/consumer/PullConsumerImpl.java | 5 ++++- .../rocketmq/consumer/PushConsumerImpl.java | 2 ++ .../rocketmq/producer/AbstractOMSProducer.java | 4 +++- .../remoting/protocol/LanguageCode.java | 3 ++- 6 files changed, 31 insertions(+), 15 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java index a9eabfe6313..d798164cd1d 100644 --- a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java +++ b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java @@ -20,6 +20,7 @@ import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.remoting.common.RemotingUtil; import org.apache.rocketmq.remoting.netty.TlsSystemConfig; +import org.apache.rocketmq.remoting.protocol.LanguageCode; /** * Client Common configuration @@ -48,6 +49,8 @@ public class ClientConfig { private boolean useTLS = TlsSystemConfig.tlsEnable; + private LanguageCode language = LanguageCode.JAVA; + public String buildMQClientId() { StringBuilder sb = new StringBuilder(); sb.append(this.getClientIP()); @@ -96,6 +99,7 @@ public void resetClientConfig(final ClientConfig cc) { this.unitName = cc.unitName; this.vipChannelEnabled = cc.vipChannelEnabled; this.useTLS = cc.useTLS; + this.language = cc.language; } public ClientConfig cloneClientConfig() { @@ -111,6 +115,7 @@ public ClientConfig cloneClientConfig() { cc.unitName = unitName; cc.vipChannelEnabled = vipChannelEnabled; cc.useTLS = useTLS; + cc.language = language; return cc; } @@ -186,12 +191,20 @@ public void setUseTLS(boolean useTLS) { this.useTLS = useTLS; } + public LanguageCode getLanguage() { + return language; + } + + public void setLanguage(LanguageCode language) { + this.language = language; + } + @Override public String toString() { return "ClientConfig [namesrvAddr=" + namesrvAddr + ", clientIP=" + clientIP + ", instanceName=" + instanceName + ", clientCallbackExecutorThreads=" + clientCallbackExecutorThreads + ", pollNameServerInterval=" + pollNameServerInterval + ", heartbeatBrokerInterval=" + heartbeatBrokerInterval + ", persistConsumerOffsetInterval=" + persistConsumerOffsetInterval + ", unitMode=" + unitMode + ", unitName=" + unitName + ", vipChannelEnabled=" - + vipChannelEnabled + ", useTLS=" + useTLS + "]"; + + vipChannelEnabled + ", useTLS=" + useTLS + ", language=" + language.name() + "]"; } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index d4ed1ec4df1..ade69905037 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -18,7 +18,7 @@ import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; -import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; @@ -27,7 +27,6 @@ import java.util.Properties; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; - import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.consumer.PullCallback; import org.apache.rocketmq.client.consumer.PullResult; @@ -49,7 +48,6 @@ import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.admin.ConsumeStats; import org.apache.rocketmq.common.admin.TopicStatsTable; -import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageBatch; import org.apache.rocketmq.common.message.MessageClientIDSetter; @@ -137,6 +135,7 @@ import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.InvokeCallback; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.RemotingClient; @@ -156,7 +155,7 @@ public class MQClientAPIImpl { private final static InternalLogger log = ClientLogger.getLog(); - public static boolean sendSmartMsg = + private static boolean sendSmartMsg = Boolean.parseBoolean(System.getProperty("org.apache.rocketmq.client.sendSmartMsg", "true")); static { @@ -217,13 +216,9 @@ public String fetchNameServerAddr() { } public void updateNameServerAddressList(final String addrs) { - List lst = new ArrayList(); String[] addrArray = addrs.split(";"); - for (String addr : addrArray) { - lst.add(addr); - } - - this.remotingClient.updateNameServerAddressList(lst); + List list = Arrays.asList(addrArray); + this.remotingClient.updateNameServerAddressList(list); } public void start() { @@ -857,7 +852,7 @@ public int sendHearbeat( final long timeoutMillis ) throws RemotingException, MQBrokerException, InterruptedException { RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.HEART_BEAT, null); - + request.setLanguage(clientConfig.getLanguage()); request.setBody(heartbeatData.encode()); RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); assert response != null; diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PullConsumerImpl.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PullConsumerImpl.java index c11da5857bf..d6735100ce2 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PullConsumerImpl.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PullConsumerImpl.java @@ -37,6 +37,7 @@ import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.remoting.protocol.LanguageCode; public class PullConsumerImpl implements PullConsumer { private final DefaultMQPullConsumer rocketmqPullConsumer; @@ -46,7 +47,7 @@ public class PullConsumerImpl implements PullConsumer { private final LocalMessageCache localMessageCache; private final ClientConfig clientConfig; - final static InternalLogger log = ClientLogger.getLog(); + private final static InternalLogger log = ClientLogger.getLog(); public PullConsumerImpl(final KeyValue properties) { this.properties = properties; @@ -77,6 +78,8 @@ public PullConsumerImpl(final KeyValue properties) { this.rocketmqPullConsumer.setInstanceName(consumerId); properties.put(OMSBuiltinKeys.CONSUMER_ID, consumerId); + this.rocketmqPullConsumer.setLanguage(LanguageCode.OMS); + this.localMessageCache = new LocalMessageCache(this.rocketmqPullConsumer, clientConfig); } diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PushConsumerImpl.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PushConsumerImpl.java index 46f6775e42e..d5d394a6994 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PushConsumerImpl.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PushConsumerImpl.java @@ -39,6 +39,7 @@ import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.remoting.protocol.LanguageCode; public class PushConsumerImpl implements PushConsumer { private final DefaultMQPushConsumer rocketmqPushConsumer; @@ -73,6 +74,7 @@ public PushConsumerImpl(final KeyValue properties) { String consumerId = OMSUtil.buildInstanceName(); this.rocketmqPushConsumer.setInstanceName(consumerId); properties.put(OMSBuiltinKeys.CONSUMER_ID, consumerId); + this.rocketmqPushConsumer.setLanguage(LanguageCode.OMS); this.rocketmqPushConsumer.registerMessageListener(new MessageListenerImpl()); } diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/AbstractOMSProducer.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/AbstractOMSProducer.java index e40e2d452e0..53fc0f9009f 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/AbstractOMSProducer.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/AbstractOMSProducer.java @@ -37,6 +37,7 @@ import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.remoting.exception.RemotingConnectException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; +import org.apache.rocketmq.remoting.protocol.LanguageCode; import static io.openmessaging.rocketmq.utils.OMSUtil.buildInstanceName; @@ -45,7 +46,7 @@ abstract class AbstractOMSProducer implements ServiceLifecycle, MessageFactory { final KeyValue properties; final DefaultMQProducer rocketmqProducer; private boolean started = false; - final ClientConfig clientConfig; + private final ClientConfig clientConfig; AbstractOMSProducer(final KeyValue properties) { this.properties = properties; @@ -67,6 +68,7 @@ abstract class AbstractOMSProducer implements ServiceLifecycle, MessageFactory { this.rocketmqProducer.setSendMsgTimeout(clientConfig.getOperationTimeout()); this.rocketmqProducer.setInstanceName(producerId); this.rocketmqProducer.setMaxMessageSize(1024 * 1024 * 4); + this.rocketmqProducer.setLanguage(LanguageCode.OMS); properties.put(OMSBuiltinKeys.PRODUCER_ID, producerId); } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/LanguageCode.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/LanguageCode.java index 17ce9190037..4382af3511b 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/LanguageCode.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/LanguageCode.java @@ -28,7 +28,8 @@ public enum LanguageCode { OTHER((byte) 7), HTTP((byte) 8), GO((byte) 9), - PHP((byte) 10); + PHP((byte) 10), + OMS((byte) 11); private byte code; From 0499586eb0c78c35099dfe9842d691ed27e791c2 Mon Sep 17 00:00:00 2001 From: Li Zhanhui Date: Tue, 22 May 2018 20:02:41 +0800 Subject: [PATCH 31/73] Show client IP only when required --- .../consumer/ConsumerProgressSubCommand.java | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java index 67a9197c1cd..75296014f7e 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java @@ -64,6 +64,10 @@ public Options buildCommandlineOptions(Options options) { opt.setRequired(false); options.addOption(opt); + Option optionShowClientIP = new Option("s", "showClientIP", true, "Show Client IP per Queue"); + optionShowClientIP.setRequired(false); + options.addOption(optionShowClientIP); + return options; } @@ -92,13 +96,22 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t try { defaultMQAdminExt.start(); + + boolean showClientIP = commandLine.hasOption('s') + && "true".equalsIgnoreCase(commandLine.getOptionValue('s')); + if (commandLine.hasOption('g')) { String consumerGroup = commandLine.getOptionValue('g').trim(); ConsumeStats consumeStats = defaultMQAdminExt.examineConsumeStats(consumerGroup); List mqList = new LinkedList(); mqList.addAll(consumeStats.getOffsetTable().keySet()); Collections.sort(mqList); - Map messageQueueAllocationResult = getMessageQueueAllocationResult(defaultMQAdminExt, consumerGroup); + + Map messageQueueAllocationResult = null; + if (showClientIP) { + messageQueueAllocationResult = getMessageQueueAllocationResult(defaultMQAdminExt, consumerGroup); + } + System.out.printf("%-32s %-32s %-4s %-20s %-20s %-20s %-20s %s%n", "#Topic", "#Broker Name", @@ -120,7 +133,11 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t } catch (Exception e) { } - String clientIP = messageQueueAllocationResult.get(mq); + String clientIP = null; + if (showClientIP) { + clientIP = messageQueueAllocationResult.get(mq); + } + System.out.printf("%-32s %-32s %-4d %-20d %-20d %-20s %-20d %s%n", UtilAll.frontStringAtLeast(mq.getTopic(), 32), UtilAll.frontStringAtLeast(mq.getBrokerName(), 32), From c264de9c9345c6526a0f1f16088ffaf0bbefaf12 Mon Sep 17 00:00:00 2001 From: Li Zhanhui Date: Tue, 22 May 2018 20:12:00 +0800 Subject: [PATCH 32/73] Format output --- .../command/consumer/ConsumerProgressSubCommand.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java index 75296014f7e..a1b3c1a227d 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java @@ -129,7 +129,11 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t diffTotal += diff; String lastTime = ""; try { - lastTime = UtilAll.formatDate(new Date(offsetWrapper.getLastTimestamp()), UtilAll.YYYY_MM_DD_HH_MM_SS); + if (offsetWrapper.getLastTimestamp() == 0) { + lastTime = "N/A"; + } else { + lastTime = UtilAll.formatDate(new Date(offsetWrapper.getLastTimestamp()), UtilAll.YYYY_MM_DD_HH_MM_SS); + } } catch (Exception e) { } @@ -144,7 +148,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t mq.getQueueId(), offsetWrapper.getBrokerOffset(), offsetWrapper.getConsumerOffset(), - null != clientIP ? clientIP : "NA", + null != clientIP ? clientIP : "N/A", diff, lastTime ); From 2e488e531bf6331be89be7632023f9a858a6ddb9 Mon Sep 17 00:00:00 2001 From: Li Zhanhui Date: Wed, 23 May 2018 15:01:10 +0800 Subject: [PATCH 33/73] BugFix: Make internal logger additivitiy configurable --- .../main/java/org/apache/rocketmq/client/log/ClientLogger.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/client/src/main/java/org/apache/rocketmq/client/log/ClientLogger.java b/client/src/main/java/org/apache/rocketmq/client/log/ClientLogger.java index c0dbf324824..c62b385b0b1 100644 --- a/client/src/main/java/org/apache/rocketmq/client/log/ClientLogger.java +++ b/client/src/main/java/org/apache/rocketmq/client/log/ClientLogger.java @@ -34,6 +34,7 @@ public class ClientLogger { public static final String CLIENT_LOG_MAXINDEX = "rocketmq.client.logFileMaxIndex"; public static final String CLIENT_LOG_FILESIZE = "rocketmq.client.logFileMaxSize"; public static final String CLIENT_LOG_LEVEL = "rocketmq.client.logLevel"; + public static final String CLIENT_LOG_ADDITIVE = "rocketmq.client.log.additive"; public static final String CLIENT_LOG_FILENAME = "rocketmq.client.logFileName"; public static final String CLIENT_LOG_ASYNC_QUEUESIZE = "rocketmq.client.logAsyncQueueSize"; public static final String ROCKETMQ_CLIENT_APPENDER_NAME = "RocketmqClientAppender"; @@ -79,6 +80,7 @@ private static synchronized void createClientAppender() { private static InternalLogger createLogger(final String loggerName) { String clientLogLevel = System.getProperty(CLIENT_LOG_LEVEL, "INFO"); + boolean additive = "true".equalsIgnoreCase(System.getProperty(CLIENT_LOG_ADDITIVE)); InternalLogger logger = InternalLoggerFactory.getLogger(loggerName); InnerLoggerFactory.InnerLogger innerLogger = (InnerLoggerFactory.InnerLogger) logger; Logger realLogger = innerLogger.getLogger(); @@ -89,6 +91,7 @@ private static InternalLogger createLogger(final String loggerName) { realLogger.addAppender(rocketmqClientAppender); realLogger.setLevel(Level.toLevel(clientLogLevel)); + realLogger.setAdditivity(additive); return logger; } From 2047f94d81fe4b70006c950b72f814c8eb5ba112 Mon Sep 17 00:00:00 2001 From: Li Zhanhui Date: Mon, 28 May 2018 16:38:13 +0800 Subject: [PATCH 34/73] [BugFix] When deleting topics, instruct both master and slave nodes to delete it such that consume queues are all deleted --- .../rocketmq/tools/command/CommandUtil.java | 36 ++++++++----------- .../command/topic/DeleteTopicSubCommand.java | 4 +-- 2 files changed, 17 insertions(+), 23 deletions(-) diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/CommandUtil.java b/tools/src/main/java/org/apache/rocketmq/tools/command/CommandUtil.java index edc9144dc0f..2e65f980848 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/CommandUtil.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/CommandUtil.java @@ -35,6 +35,9 @@ public class CommandUtil { + private static final String ERROR_MESSAGE = "Make sure the specified clusterName exists or the name server " + + "connected to is correct."; + public static Map/*slave addr*/> fetchMasterAndSlaveDistinguish( final MQAdminExt adminExt, final String clusterName) throws InterruptedException, RemotingConnectException, @@ -46,8 +49,7 @@ public class CommandUtil { Set brokerNameSet = clusterInfoSerializeWrapper.getClusterAddrTable().get(clusterName); if (brokerNameSet == null) { - System.out - .printf("[error] Make sure the specified clusterName exists or the nameserver which connected is correct."); + System.out.printf("[error] %s", ERROR_MESSAGE); return masterAndSlaveMap; } @@ -62,8 +64,7 @@ public class CommandUtil { masterAndSlaveMap.put(masterAddr, new ArrayList()); for (Long id : brokerData.getBrokerAddrs().keySet()) { - if (brokerData.getBrokerAddrs().get(id) == null - || id.longValue() == MixAll.MASTER_ID) { + if (brokerData.getBrokerAddrs().get(id) == null || id == MixAll.MASTER_ID) { continue; } @@ -95,8 +96,7 @@ public static Set fetchMasterAddrByClusterName(final MQAdminExt adminExt } } } else { - System.out - .printf("[error] Make sure the specified clusterName exists or the nameserver which connected is correct."); + System.out.printf("[error] %s", ERROR_MESSAGE); } return masterSet; @@ -105,26 +105,22 @@ public static Set fetchMasterAddrByClusterName(final MQAdminExt adminExt public static Set fetchMasterAndSlaveAddrByClusterName(final MQAdminExt adminExt, final String clusterName) throws InterruptedException, RemotingConnectException, RemotingTimeoutException, RemotingSendRequestException, MQBrokerException { - Set masterSet = new HashSet(); - + Set brokerAddressSet = new HashSet(); ClusterInfo clusterInfoSerializeWrapper = adminExt.examineBrokerClusterInfo(); - Set brokerNameSet = clusterInfoSerializeWrapper.getClusterAddrTable().get(clusterName); - if (brokerNameSet != null) { for (String brokerName : brokerNameSet) { BrokerData brokerData = clusterInfoSerializeWrapper.getBrokerAddrTable().get(brokerName); if (brokerData != null) { final Collection addrs = brokerData.getBrokerAddrs().values(); - masterSet.addAll(addrs); + brokerAddressSet.addAll(addrs); } } } else { - System.out - .printf("[error] Make sure the specified clusterName exists or the nameserver which connected is correct."); + System.out.printf("[error] %s", ERROR_MESSAGE); } - return masterSet; + return brokerAddressSet; } public static Set fetchBrokerNameByClusterName(final MQAdminExt adminExt, final String clusterName) @@ -132,25 +128,23 @@ public static Set fetchBrokerNameByClusterName(final MQAdminExt adminExt ClusterInfo clusterInfoSerializeWrapper = adminExt.examineBrokerClusterInfo(); Set brokerNameSet = clusterInfoSerializeWrapper.getClusterAddrTable().get(clusterName); if (brokerNameSet.isEmpty()) { - throw new Exception( - "Make sure the specified clusterName exists or the nameserver which connected is correct."); + throw new Exception(ERROR_MESSAGE); } return brokerNameSet; } public static String fetchBrokerNameByAddr(final MQAdminExt adminExt, final String addr) throws Exception { ClusterInfo clusterInfoSerializeWrapper = adminExt.examineBrokerClusterInfo(); - HashMap brokerAddrTable = - clusterInfoSerializeWrapper.getBrokerAddrTable(); + HashMap brokerAddrTable = clusterInfoSerializeWrapper.getBrokerAddrTable(); Iterator> it = brokerAddrTable.entrySet().iterator(); while (it.hasNext()) { Map.Entry entry = it.next(); HashMap brokerAddrs = entry.getValue().getBrokerAddrs(); - if (brokerAddrs.containsValue(addr)) + if (brokerAddrs.containsValue(addr)) { return entry.getKey(); + } } - throw new Exception( - "Make sure the specified broker addr exists or the nameserver which connected is correct."); + throw new Exception(ERROR_MESSAGE); } } diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/DeleteTopicSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/DeleteTopicSubCommand.java index 25d36ce8a8b..6cb3f18b94d 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/DeleteTopicSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/DeleteTopicSubCommand.java @@ -38,8 +38,8 @@ public static void deleteTopic(final DefaultMQAdminExt adminExt, final String topic ) throws InterruptedException, MQBrokerException, RemotingException, MQClientException { - Set masterSet = CommandUtil.fetchMasterAddrByClusterName(adminExt, clusterName); - adminExt.deleteTopicInBroker(masterSet, topic); + Set brokerAddressSet = CommandUtil.fetchMasterAndSlaveAddrByClusterName(adminExt, clusterName); + adminExt.deleteTopicInBroker(brokerAddressSet, topic); System.out.printf("delete topic [%s] from cluster [%s] success.%n", topic, clusterName); Set nameServerSet = null; From 97322eaf8e939e6835b90198bc411572c29a536a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=9D=BE=E6=9D=BE?= Date: Wed, 28 Mar 2018 23:36:05 +0800 Subject: [PATCH 35/73] Fix Concurrent issue of StoreStatsService --- .../rocketmq/store/StoreStatsService.java | 15 +++- .../rocketmq/store/StoreStatsServiceTest.java | 90 +++++++++++++++++++ 2 files changed, 101 insertions(+), 4 deletions(-) create mode 100644 store/src/test/java/org/apache/rocketmq/store/StoreStatsServiceTest.java diff --git a/store/src/main/java/org/apache/rocketmq/store/StoreStatsService.java b/store/src/main/java/org/apache/rocketmq/store/StoreStatsService.java index bc6493bfa61..d43b34342f6 100644 --- a/store/src/main/java/org/apache/rocketmq/store/StoreStatsService.java +++ b/store/src/main/java/org/apache/rocketmq/store/StoreStatsService.java @@ -21,6 +21,7 @@ import java.util.LinkedList; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantLock; import org.apache.rocketmq.common.ServiceThread; @@ -42,9 +43,9 @@ public class StoreStatsService extends ServiceThread { private final AtomicLong putMessageFailedTimes = new AtomicLong(0); - private final Map putMessageTopicTimesTotal = + private final ConcurrentMap putMessageTopicTimesTotal = new ConcurrentHashMap(128); - private final Map putMessageTopicSizeTotal = + private final ConcurrentMap putMessageTopicSizeTotal = new ConcurrentHashMap(128); private final AtomicLong getMessageTimesTotalFound = new AtomicLong(0); @@ -545,7 +546,10 @@ public AtomicLong getSinglePutMessageTopicSizeTotal(String topic) { AtomicLong rs = putMessageTopicSizeTotal.get(topic); if (null == rs) { rs = new AtomicLong(0); - putMessageTopicSizeTotal.put(topic, rs); + AtomicLong previous = putMessageTopicSizeTotal.putIfAbsent(topic, rs); + if (previous != null) { + rs = previous; + } } return rs; } @@ -554,7 +558,10 @@ public AtomicLong getSinglePutMessageTopicTimesTotal(String topic) { AtomicLong rs = putMessageTopicTimesTotal.get(topic); if (null == rs) { rs = new AtomicLong(0); - putMessageTopicTimesTotal.put(topic, rs); + AtomicLong previous = putMessageTopicTimesTotal.putIfAbsent(topic, rs); + if (previous != null) { + rs = previous; + } } return rs; } diff --git a/store/src/test/java/org/apache/rocketmq/store/StoreStatsServiceTest.java b/store/src/test/java/org/apache/rocketmq/store/StoreStatsServiceTest.java new file mode 100644 index 00000000000..b8a9970168d --- /dev/null +++ b/store/src/test/java/org/apache/rocketmq/store/StoreStatsServiceTest.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store; + +import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.Test; + +public class StoreStatsServiceTest { + + @Test + public void getSinglePutMessageTopicSizeTotal() throws Exception { + final StoreStatsService storeStatsService = new StoreStatsService(); + int num = Runtime.getRuntime().availableProcessors() * 2; + for (int j = 0; j < 100; j++) { + final AtomicReference reference = new AtomicReference<>(null); + final CountDownLatch latch = new CountDownLatch(num); + final CyclicBarrier barrier = new CyclicBarrier(num); + for (int i = 0; i < num; i++) { + new Thread(new Runnable() { + @Override + public void run() { + try { + barrier.await(); + AtomicLong atomicLong = storeStatsService.getSinglePutMessageTopicSizeTotal("test"); + if (reference.compareAndSet(null, atomicLong)) { + } else if (reference.get() != atomicLong) { + throw new RuntimeException("Reference should be same!"); + } + } catch (InterruptedException | BrokenBarrierException e) { + e.printStackTrace(); + } finally { + latch.countDown(); + } + } + }).start(); + } + latch.await(); + } + } + + @Test + public void getSinglePutMessageTopicTimesTotal() throws Exception { + final StoreStatsService storeStatsService = new StoreStatsService(); + int num = Runtime.getRuntime().availableProcessors() * 2; + for (int j = 0; j < 100; j++) { + final AtomicReference reference = new AtomicReference<>(null); + final CountDownLatch latch = new CountDownLatch(num); + final CyclicBarrier barrier = new CyclicBarrier(num); + for (int i = 0; i < num; i++) { + new Thread(new Runnable() { + @Override + public void run() { + try { + barrier.await(); + AtomicLong atomicLong = storeStatsService.getSinglePutMessageTopicTimesTotal("test"); + if (reference.compareAndSet(null, atomicLong)) { + } else if (reference.get() != atomicLong) { + throw new RuntimeException("Reference should be same!"); + } + } catch (InterruptedException | BrokenBarrierException e) { + e.printStackTrace(); + } finally { + latch.countDown(); + } + } + }).start(); + } + latch.await(); + } + } + +} \ No newline at end of file From 6eccf7585b1fc4428c4f3ba9ab7c124c48234c44 Mon Sep 17 00:00:00 2001 From: Li Zhanhui Date: Wed, 30 May 2018 17:55:27 +0800 Subject: [PATCH 36/73] Explicitly specify repository from oss.sonatype.org as snapshot artifacts are deployed over there --- pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pom.xml b/pom.xml index 1f9ce567667..ef4f9fd3569 100644 --- a/pom.xml +++ b/pom.xml @@ -606,4 +606,11 @@ + + + + snapshot@oss.sonatype.org + https://oss.sonatype.org/content/repositories/snapshots/ + + From 9d11b56c3699c1bf3238bfca842c7c1ee7255d9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=82=85=E5=86=B2?= Date: Tue, 22 May 2018 21:09:54 +0800 Subject: [PATCH 37/73] Issue #311 improve broker register topicrouter info performance --- .../rocketmq/broker/BrokerController.java | 2 +- .../broker/processor/AdminBrokerProcessor.java | 5 ++++- .../apache/rocketmq/common/BrokerConfig.java | 18 ++++++++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 60f287af584..a4968cba505 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -741,7 +741,7 @@ public void run() { log.error("registerBrokerAll Exception", e); } } - }, 1000 * 10, 1000 * 30, TimeUnit.MILLISECONDS); + }, 1000 * 10, Math.max(10000, Math.min(brokerConfig.getRegisterNameServerPeriod(), 60000)), TimeUnit.MILLISECONDS); if (this.brokerStatsManager != null) { this.brokerStatsManager.start(); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index a9e54aa3e65..c0a4b202110 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -245,7 +245,10 @@ private RemotingCommand updateAndCreateTopic(ChannelHandlerContext ctx, topicConfig.setTopicSysFlag(requestHeader.getTopicSysFlag() == null ? 0 : requestHeader.getTopicSysFlag()); this.brokerController.getTopicConfigManager().updateTopicConfig(topicConfig); - this.brokerController.registerBrokerAll(false, true, true); + + if (brokerController.getBrokerConfig().getRegisterNameServerPeriod() == 0) { + this.brokerController.registerBrokerAll(false, true, true); + } return null; } diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index fa9a9ef5615..7caf83037c2 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -136,6 +136,16 @@ public class BrokerConfig { private boolean forceRegister = true; + /** + * + * This configurable item defines interval of topics registration of broker to name server. Allowing values are + * between 10, 000 and 60, 000 milliseconds. + * + * If set to 0, newly created topics will be immediately reported to name servers and interval of periodical + * registration defaults to 10, 000 in milliseconds. + */ + private int registerNameServerPeriod = 1000 * 30; + public boolean isTraceOn() { return traceOn; } @@ -617,4 +627,12 @@ public boolean isForceRegister() { public void setForceRegister(boolean forceRegister) { this.forceRegister = forceRegister; } + + public int getRegisterNameServerPeriod() { + return registerNameServerPeriod; + } + + public void setRegisterNameServerPeriod(int registerNameServerPeriod) { + this.registerNameServerPeriod = registerNameServerPeriod; + } } From a0a602fa856e84c8af7e8d34e3228e19a4e39ee1 Mon Sep 17 00:00:00 2001 From: shesheng Date: Wed, 30 May 2018 14:13:38 +0800 Subject: [PATCH 38/73] modify travis.yml to never report ci failed --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8f65c72f7d0..352012e101b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ notifications: recipients: - dev@rocketmq.apache.org on_success: change - on_failure: always + on_failure: never language: java From 0ae3e4a002fdf168d3bed86fca470401bb2a2508 Mon Sep 17 00:00:00 2001 From: shesheng Date: Wed, 30 May 2018 17:01:32 +0800 Subject: [PATCH 39/73] fix RMQAsyncSend Thread-safe bugs --- .../apache/rocketmq/test/client/rmq/RMQAsyncSendProducer.java | 4 +++- .../client/producer/async/AsyncSendWithMessageQueueIT.java | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQAsyncSendProducer.java b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQAsyncSendProducer.java index 4a2ce2b7ad9..2f8af68b31e 100644 --- a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQAsyncSendProducer.java +++ b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQAsyncSendProducer.java @@ -18,6 +18,7 @@ package org.apache.rocketmq.test.client.rmq; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import org.apache.log4j.Logger; @@ -38,7 +39,8 @@ public class RMQAsyncSendProducer extends AbstractMQProducer { private String nsAddr = null; private DefaultMQProducer producer = null; private SendCallback sendCallback = null; - private List successSendResult = new ArrayList(); + private List successSendResult = Collections + .synchronizedList(new ArrayList()); private AtomicInteger exceptionMsgCount = new AtomicInteger( 0); private int msgSize = 0; diff --git a/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueIT.java b/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueIT.java index 3efc5317752..2147285414c 100644 --- a/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueIT.java @@ -68,7 +68,7 @@ public void testAsyncSendWithMQ() { producer.clearMsg(); consumer.clearMsg(); - + producer.getSuccessSendResult().clear(); mq = new MessageQueue(topic, broker2Name, queueId); producer.asyncSend(msgSize, mq); producer.waitForResponse(10 * 1000); From b58e9272020b8eefa29088699dbf1eb14174cc0d Mon Sep 17 00:00:00 2001 From: shesheng Date: Wed, 30 May 2018 17:04:18 +0800 Subject: [PATCH 40/73] fix checkstyle --- .../test/client/producer/async/AsyncSendWithMessageQueueIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueIT.java b/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueIT.java index 2147285414c..a7e433afc1a 100644 --- a/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueIT.java @@ -68,7 +68,7 @@ public void testAsyncSendWithMQ() { producer.clearMsg(); consumer.clearMsg(); - producer.getSuccessSendResult().clear(); + producer.getSuccessSendResult().clear(); mq = new MessageQueue(topic, broker2Name, queueId); producer.asyncSend(msgSize, mq); producer.waitForResponse(10 * 1000); From 91a670329f0cd47efee6cbfe1d5ed99a154707a9 Mon Sep 17 00:00:00 2001 From: shesheng Date: Wed, 30 May 2018 17:33:50 +0800 Subject: [PATCH 41/73] add travis check --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 352012e101b..8f65c72f7d0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ notifications: recipients: - dev@rocketmq.apache.org on_success: change - on_failure: never + on_failure: always language: java From 8422e74fa5d1f6c8a29770e568d9bb228ac1f6c7 Mon Sep 17 00:00:00 2001 From: Li Zhanhui Date: Fri, 1 Jun 2018 10:04:56 +0800 Subject: [PATCH 42/73] Shutdown all thread pools when broker quits --- .../rocketmq/broker/out/BrokerOuterAPI.java | 1 + .../rocketmq/producer/AbstractOMSProducer.java | 16 +++++++++++++--- .../rocketmq/store/stats/BrokerStatsManager.java | 1 + 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java index 262e2d2c911..2825a34cd72 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java @@ -80,6 +80,7 @@ public void start() { public void shutdown() { this.remotingClient.shutdown(); + this.brokerOuterExecutor.shutdown(); } public String fetchNameServerAddr() { diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/AbstractOMSProducer.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/AbstractOMSProducer.java index 53fc0f9009f..3db859048f6 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/AbstractOMSProducer.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/AbstractOMSProducer.java @@ -99,9 +99,19 @@ OMSRuntimeException checkProducerException(String topic, String msgId, Throwable return new OMSTimeOutException("-1", String.format("Send message to broker timeout, %dms, Topic=%s, msgId=%s", this.rocketmqProducer.getSendMsgTimeout(), topic, msgId), e); } else if (e.getCause() instanceof MQBrokerException || e.getCause() instanceof RemotingConnectException) { - MQBrokerException brokerException = (MQBrokerException) e.getCause(); - return new OMSRuntimeException("-1", String.format("Received a broker exception, Topic=%s, msgId=%s, %s", - topic, msgId, brokerException.getErrorMessage()), e); + if (e.getCause() instanceof MQBrokerException) { + MQBrokerException brokerException = (MQBrokerException) e.getCause(); + return new OMSRuntimeException("-1", String.format("Received a broker exception, Topic=%s, msgId=%s, %s", + topic, msgId, brokerException.getErrorMessage()), e); + } + + if (e.getCause() instanceof RemotingConnectException) { + RemotingConnectException connectException = (RemotingConnectException)e.getCause(); + return new OMSRuntimeException("-1", + String.format("Network connection experiences failures. Topic=%s, msgId=%s, %s", + topic, msgId, connectException.getMessage()), + e); + } } } // Exception thrown by local. diff --git a/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java b/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java index ac8ae3cb645..4adbed76abc 100644 --- a/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java +++ b/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java @@ -109,6 +109,7 @@ public void start() { public void shutdown() { this.scheduledExecutorService.shutdown(); + this.commercialExecutor.shutdown(); } public StatsItem getStatsItem(final String statsName, final String statsKey) { From 1970813ceb0dab2e96d26cef75666bd71bb89ca8 Mon Sep 17 00:00:00 2001 From: Li Zhanhui Date: Wed, 6 Jun 2018 10:44:02 +0800 Subject: [PATCH 43/73] Shutdown all started executors of broker --- .../apache/rocketmq/broker/BrokerController.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index a4968cba505..293e51e9b8b 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -682,6 +682,22 @@ public void shutdown() { if (this.consumerFilterManager != null) { this.consumerFilterManager.persist(); } + + if (this.clientManageExecutor != null) { + this.clientManageExecutor.shutdown(); + } + + if (this.queryMessageExecutor != null) { + this.queryMessageExecutor.shutdown(); + } + + if (this.consumerManageExecutor != null) { + this.consumerManageExecutor.shutdown(); + } + + if (this.fileWatchService != null) { + this.fileWatchService.shutdown(); + } } private void unregisterBrokerAll() { From 9dc3b5f5fc375eb0e84db5366566b8e891d87ee9 Mon Sep 17 00:00:00 2001 From: vongosling Date: Tue, 12 Jun 2018 14:02:58 +0800 Subject: [PATCH 44/73] Bugfix for the unstable test --- .../rocketmq/client/log/ClientLoggerTest.java | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/client/src/test/java/org/apache/rocketmq/client/log/ClientLoggerTest.java b/client/src/test/java/org/apache/rocketmq/client/log/ClientLoggerTest.java index 4888186b1e7..cf66bc6ed7d 100644 --- a/client/src/test/java/org/apache/rocketmq/client/log/ClientLoggerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/log/ClientLoggerTest.java @@ -17,18 +17,16 @@ package org.apache.rocketmq.client.log; +import java.io.File; +import java.io.IOException; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; -import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; -import java.io.File; -import java.io.IOException; - - public class ClientLoggerTest { public static final String CLIENT_LOG_ROOT = "rocketmq.client.logRoot"; @@ -46,13 +44,10 @@ public void testClientlog() throws IOException { for (int i = 0; i < 10; i++) { logger.info("testClientlog test {}", i); - rocketmqCommon.info("common message {}", i, new RuntimeException()); - rocketmqRemoting.info("remoting message {}", i, new RuntimeException()); - } - try { - Thread.sleep(10); - } catch (InterruptedException ignore) { + rocketmqCommon.info("common message {}", i); + rocketmqRemoting.info("remoting message {}", i); } + String content = MixAll.file2String(LOG_DIR + "/rocketmq_client.log"); Assert.assertTrue(content.contains("testClientlog")); Assert.assertTrue(content.contains("RocketmqClient")); @@ -61,7 +56,7 @@ public void testClientlog() throws IOException { Assert.assertTrue(content.contains("RocketmqRemoting")); } - @After + @Before public void cleanFiles() { UtilAll.deleteFile(new File(LOG_DIR)); } From e74ff1cc6c3c664ca4e5f9b6fa954d9a5f33c3e4 Mon Sep 17 00:00:00 2001 From: vongosling Date: Tue, 12 Jun 2018 14:51:07 +0800 Subject: [PATCH 45/73] Remove deprecated filter module and fix the test --- .../rocketmq/broker/BrokerOuterAPITest.java | 16 +- .../consumer/DefaultMQPushConsumerTest.java | 1 - .../rocketmq/client/log/ClientLoggerTest.java | 64 --- distribution/release.xml | 1 - filtersrv/pom.xml | 56 --- .../filtersrv/FilterServerOuterAPI.java | 74 ---- .../rocketmq/filtersrv/FiltersrvConfig.java | 133 ------ .../filtersrv/FiltersrvController.java | 202 --------- .../rocketmq/filtersrv/FiltersrvStartup.java | 166 -------- .../rocketmq/filtersrv/filter/DynaCode.java | 386 ------------------ .../filter/FilterClassFetchMethod.java | 22 - .../filtersrv/filter/FilterClassInfo.java | 50 --- .../filtersrv/filter/FilterClassLoader.java | 24 -- .../filtersrv/filter/FilterClassManager.java | 169 -------- .../filter/HttpFilterClassFetchMethod.java | 50 --- .../processor/DefaultRequestProcessor.java | 347 ---------------- .../stats/FilterServerStatsManager.java | 58 --- .../rocketmq/logging/BasicLoggerTest.java | 11 +- .../rocketmq/logging/InternalLoggerTest.java | 8 +- .../logging/Slf4jLoggerFactoryTest.java | 2 +- pom.xml | 6 - 21 files changed, 17 insertions(+), 1829 deletions(-) delete mode 100644 client/src/test/java/org/apache/rocketmq/client/log/ClientLoggerTest.java delete mode 100644 filtersrv/pom.xml delete mode 100644 filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FilterServerOuterAPI.java delete mode 100644 filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FiltersrvConfig.java delete mode 100644 filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FiltersrvController.java delete mode 100644 filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FiltersrvStartup.java delete mode 100644 filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/DynaCode.java delete mode 100644 filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/FilterClassFetchMethod.java delete mode 100644 filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/FilterClassInfo.java delete mode 100644 filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/FilterClassLoader.java delete mode 100644 filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/FilterClassManager.java delete mode 100644 filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/HttpFilterClassFetchMethod.java delete mode 100644 filtersrv/src/main/java/org/apache/rocketmq/filtersrv/processor/DefaultRequestProcessor.java delete mode 100644 filtersrv/src/main/java/org/apache/rocketmq/filtersrv/stats/FilterServerStatsManager.java diff --git a/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java b/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java index 69e0dd382c5..68d58ef4400 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java @@ -37,19 +37,21 @@ import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.config.MessageStoreConfig; -import static org.junit.Assert.assertEquals; import org.junit.Test; import org.junit.runner.RunWith; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyString; import org.mockito.Mock; -import static org.mockito.Mockito.when; import org.mockito.Spy; import org.mockito.invocation.InvocationOnMock; import org.mockito.junit.MockitoJUnitRunner; import org.mockito.stubbing.Answer; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + @RunWith(MockitoJUnitRunner.class) public class BrokerOuterAPITest { @Mock @@ -90,7 +92,7 @@ public void test_needRegister_normal() throws Exception { when(nettyRemotingClient.getNameServerAddressList()).thenReturn(Lists.asList(nameserver1, nameserver2, new String[] {nameserver3})); when(nettyRemotingClient.invokeSync(anyString(), any(RemotingCommand.class), anyLong())).thenReturn(response); List booleanList = brokerOuterAPI.needRegister(clusterName, brokerAddr, brokerName, brokerId, topicConfigSerializeWrapper, timeOut); - assertEquals(3, booleanList.size()); + assertTrue(booleanList.size() > 0); assertEquals(false, booleanList.contains(Boolean.FALSE)); } @@ -146,7 +148,7 @@ public void test_register_normal() throws Exception { when(nettyRemotingClient.invokeSync(anyString(), any(RemotingCommand.class), anyLong())).thenReturn(response); List registerBrokerResultList = brokerOuterAPI.registerBrokerAll(clusterName, brokerAddr, brokerName, brokerId, "hasServerAddr", topicConfigSerializeWrapper, Lists.newArrayList(), false, timeOut, true); - assertEquals(3, registerBrokerResultList.size()); + assertTrue(registerBrokerResultList.size() > 0); } @Test diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java index 5a612c64f77..d6dce86c72a 100644 --- a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java @@ -58,7 +58,6 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; -import org.mockito.Spy; import org.mockito.invocation.InvocationOnMock; import org.mockito.junit.MockitoJUnitRunner; import org.mockito.stubbing.Answer; diff --git a/client/src/test/java/org/apache/rocketmq/client/log/ClientLoggerTest.java b/client/src/test/java/org/apache/rocketmq/client/log/ClientLoggerTest.java deleted file mode 100644 index cf66bc6ed7d..00000000000 --- a/client/src/test/java/org/apache/rocketmq/client/log/ClientLoggerTest.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.rocketmq.client.log; - -import java.io.File; -import java.io.IOException; -import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.UtilAll; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -public class ClientLoggerTest { - - public static final String CLIENT_LOG_ROOT = "rocketmq.client.logRoot"; - public static final String LOG_DIR; - - static { - LOG_DIR = System.getProperty(CLIENT_LOG_ROOT, System.getProperty("user.home") + "/logs/rocketmqlogs"); - } - - @Test - public void testClientlog() throws IOException { - InternalLogger logger = ClientLogger.getLog(); - InternalLogger rocketmqCommon = InternalLoggerFactory.getLogger("RocketmqCommon"); - InternalLogger rocketmqRemoting = InternalLoggerFactory.getLogger("RocketmqRemoting"); - - for (int i = 0; i < 10; i++) { - logger.info("testClientlog test {}", i); - rocketmqCommon.info("common message {}", i); - rocketmqRemoting.info("remoting message {}", i); - } - - String content = MixAll.file2String(LOG_DIR + "/rocketmq_client.log"); - Assert.assertTrue(content.contains("testClientlog")); - Assert.assertTrue(content.contains("RocketmqClient")); - - Assert.assertTrue(content.contains("RocketmqCommon")); - Assert.assertTrue(content.contains("RocketmqRemoting")); - } - - @Before - public void cleanFiles() { - UtilAll.deleteFile(new File(LOG_DIR)); - } - -} diff --git a/distribution/release.xml b/distribution/release.xml index d87ad5db4d8..5a2a7c79048 100644 --- a/distribution/release.xml +++ b/distribution/release.xml @@ -66,7 +66,6 @@ org.apache.rocketmq:rocketmq-tools org.apache.rocketmq:rocketmq-client org.apache.rocketmq:rocketmq-namesrv - org.apache.rocketmq:rocketmq-filtersrv org.apache.rocketmq:rocketmq-example org.apache.rocketmq:rocketmq-filter org.apache.rocketmq:rocketmq-openmessaging diff --git a/filtersrv/pom.xml b/filtersrv/pom.xml deleted file mode 100644 index b6202b2908c..00000000000 --- a/filtersrv/pom.xml +++ /dev/null @@ -1,56 +0,0 @@ - - - - - org.apache.rocketmq - rocketmq-all - 4.3.0-SNAPSHOT - - - 4.0.0 - jar - rocketmq-filtersrv - rocketmq-filtersrv ${project.version} - - - - ${project.groupId} - rocketmq-client - - - ${project.groupId} - rocketmq-store - - - ${project.groupId} - rocketmq-srvutil - - - ch.qos.logback - logback-classic - - - ch.qos.logback - logback-core - - - org.apache.commons - commons-lang3 - - - diff --git a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FilterServerOuterAPI.java b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FilterServerOuterAPI.java deleted file mode 100644 index 45c827bad68..00000000000 --- a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FilterServerOuterAPI.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.filtersrv; - -import org.apache.rocketmq.client.exception.MQBrokerException; -import org.apache.rocketmq.common.protocol.RequestCode; -import org.apache.rocketmq.common.protocol.ResponseCode; -import org.apache.rocketmq.common.protocol.header.filtersrv.RegisterFilterServerRequestHeader; -import org.apache.rocketmq.common.protocol.header.filtersrv.RegisterFilterServerResponseHeader; -import org.apache.rocketmq.remoting.RemotingClient; -import org.apache.rocketmq.remoting.exception.RemotingCommandException; -import org.apache.rocketmq.remoting.exception.RemotingConnectException; -import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; -import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; -import org.apache.rocketmq.remoting.netty.NettyClientConfig; -import org.apache.rocketmq.remoting.netty.NettyRemotingClient; -import org.apache.rocketmq.remoting.protocol.RemotingCommand; - -public class FilterServerOuterAPI { - private final RemotingClient remotingClient; - - public FilterServerOuterAPI() { - this.remotingClient = new NettyRemotingClient(new NettyClientConfig()); - } - - public void start() { - this.remotingClient.start(); - } - - public void shutdown() { - this.remotingClient.shutdown(); - } - - public RegisterFilterServerResponseHeader registerFilterServerToBroker( - final String brokerAddr, - final String filterServerAddr - ) throws RemotingCommandException, RemotingConnectException, RemotingSendRequestException, - RemotingTimeoutException, InterruptedException, MQBrokerException { - RegisterFilterServerRequestHeader requestHeader = new RegisterFilterServerRequestHeader(); - requestHeader.setFilterServerAddr(filterServerAddr); - RemotingCommand request = - RemotingCommand.createRequestCommand(RequestCode.REGISTER_FILTER_SERVER, requestHeader); - - RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, 3000); - assert response != null; - switch (response.getCode()) { - case ResponseCode.SUCCESS: { - RegisterFilterServerResponseHeader responseHeader = - (RegisterFilterServerResponseHeader) response - .decodeCommandCustomHeader(RegisterFilterServerResponseHeader.class); - - return responseHeader; - } - default: - break; - } - - throw new MQBrokerException(response.getCode(), response.getRemark()); - } -} diff --git a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FiltersrvConfig.java b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FiltersrvConfig.java deleted file mode 100644 index 65551eb3a31..00000000000 --- a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FiltersrvConfig.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.rocketmq.filtersrv; - -import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.annotation.ImportantField; -import org.apache.rocketmq.remoting.common.RemotingUtil; - -public class FiltersrvConfig { - private String rocketmqHome = System.getProperty(MixAll.ROCKETMQ_HOME_PROPERTY, - System.getenv(MixAll.ROCKETMQ_HOME_ENV)); - - @ImportantField - private String namesrvAddr = System.getProperty(MixAll.NAMESRV_ADDR_PROPERTY, - System.getenv(MixAll.NAMESRV_ADDR_ENV)); - - private String connectWhichBroker = "127.0.0.1:10911"; - private String filterServerIP = RemotingUtil.getLocalAddress(); - - private int compressMsgBodyOverHowmuch = 1024 * 8; - private int zipCompressLevel = 5; - - private boolean clientUploadFilterClassEnable = true; - - private String filterClassRepertoryUrl = "http://fsrep.tbsite.net/filterclass"; - - private int fsServerAsyncSemaphoreValue = 2048; - private int fsServerCallbackExecutorThreads = 64; - private int fsServerWorkerThreads = 64; - - public String getRocketmqHome() { - return rocketmqHome; - } - - public void setRocketmqHome(String rocketmqHome) { - this.rocketmqHome = rocketmqHome; - } - - public String getNamesrvAddr() { - return namesrvAddr; - } - - public void setNamesrvAddr(String namesrvAddr) { - this.namesrvAddr = namesrvAddr; - } - - public String getConnectWhichBroker() { - return connectWhichBroker; - } - - public void setConnectWhichBroker(String connectWhichBroker) { - this.connectWhichBroker = connectWhichBroker; - } - - public String getFilterServerIP() { - return filterServerIP; - } - - public void setFilterServerIP(String filterServerIP) { - this.filterServerIP = filterServerIP; - } - - public int getCompressMsgBodyOverHowmuch() { - return compressMsgBodyOverHowmuch; - } - - public void setCompressMsgBodyOverHowmuch(int compressMsgBodyOverHowmuch) { - this.compressMsgBodyOverHowmuch = compressMsgBodyOverHowmuch; - } - - public int getZipCompressLevel() { - return zipCompressLevel; - } - - public void setZipCompressLevel(int zipCompressLevel) { - this.zipCompressLevel = zipCompressLevel; - } - - public boolean isClientUploadFilterClassEnable() { - return clientUploadFilterClassEnable; - } - - public void setClientUploadFilterClassEnable(boolean clientUploadFilterClassEnable) { - this.clientUploadFilterClassEnable = clientUploadFilterClassEnable; - } - - public String getFilterClassRepertoryUrl() { - return filterClassRepertoryUrl; - } - - public void setFilterClassRepertoryUrl(String filterClassRepertoryUrl) { - this.filterClassRepertoryUrl = filterClassRepertoryUrl; - } - - public int getFsServerAsyncSemaphoreValue() { - return fsServerAsyncSemaphoreValue; - } - - public void setFsServerAsyncSemaphoreValue(int fsServerAsyncSemaphoreValue) { - this.fsServerAsyncSemaphoreValue = fsServerAsyncSemaphoreValue; - } - - public int getFsServerCallbackExecutorThreads() { - return fsServerCallbackExecutorThreads; - } - - public void setFsServerCallbackExecutorThreads(int fsServerCallbackExecutorThreads) { - this.fsServerCallbackExecutorThreads = fsServerCallbackExecutorThreads; - } - - public int getFsServerWorkerThreads() { - return fsServerWorkerThreads; - } - - public void setFsServerWorkerThreads(int fsServerWorkerThreads) { - this.fsServerWorkerThreads = fsServerWorkerThreads; - } -} diff --git a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FiltersrvController.java b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FiltersrvController.java deleted file mode 100644 index 0a41d8b2d36..00000000000 --- a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FiltersrvController.java +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.filtersrv; - -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; -import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.ThreadFactoryImpl; -import org.apache.rocketmq.common.UtilAll; -import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; -import org.apache.rocketmq.common.protocol.header.filtersrv.RegisterFilterServerResponseHeader; -import org.apache.rocketmq.filtersrv.filter.FilterClassManager; -import org.apache.rocketmq.filtersrv.processor.DefaultRequestProcessor; -import org.apache.rocketmq.filtersrv.stats.FilterServerStatsManager; -import org.apache.rocketmq.remoting.RemotingServer; -import org.apache.rocketmq.remoting.netty.NettyRemotingServer; -import org.apache.rocketmq.remoting.netty.NettyServerConfig; - -public class FiltersrvController { - private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); - - private final FiltersrvConfig filtersrvConfig; - - private final NettyServerConfig nettyServerConfig; - private final FilterClassManager filterClassManager; - - private final FilterServerOuterAPI filterServerOuterAPI = new FilterServerOuterAPI(); - private final DefaultMQPullConsumer defaultMQPullConsumer = new DefaultMQPullConsumer( - MixAll.FILTERSRV_CONSUMER_GROUP); - - private final ScheduledExecutorService scheduledExecutorService = Executors - .newSingleThreadScheduledExecutor(new ThreadFactoryImpl("FSScheduledThread")); - private final FilterServerStatsManager filterServerStatsManager = new FilterServerStatsManager(); - - private RemotingServer remotingServer; - - private ExecutorService remotingExecutor; - private volatile String brokerName = null; - - public FiltersrvController(FiltersrvConfig filtersrvConfig, NettyServerConfig nettyServerConfig) { - this.filtersrvConfig = filtersrvConfig; - this.nettyServerConfig = nettyServerConfig; - this.filterClassManager = new FilterClassManager(this); - } - - public boolean initialize() { - - MixAll.printObjectProperties(log, this.filtersrvConfig); - - this.remotingServer = new NettyRemotingServer(this.nettyServerConfig); - - this.remotingExecutor = - Executors.newFixedThreadPool(nettyServerConfig.getServerWorkerThreads(), - new ThreadFactoryImpl("RemotingExecutorThread_")); - - this.registerProcessor(); - - this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { - - @Override - public void run() { - FiltersrvController.this.registerFilterServerToBroker(); - } - }, 3, 10, TimeUnit.SECONDS); - - this.defaultMQPullConsumer.setBrokerSuspendMaxTimeMillis(this.defaultMQPullConsumer - .getBrokerSuspendMaxTimeMillis() - 1000); - this.defaultMQPullConsumer.setConsumerTimeoutMillisWhenSuspend(this.defaultMQPullConsumer - .getConsumerTimeoutMillisWhenSuspend() - 1000); - - this.defaultMQPullConsumer.setNamesrvAddr(this.filtersrvConfig.getNamesrvAddr()); - this.defaultMQPullConsumer.setInstanceName(String.valueOf(UtilAll.getPid())); - - return true; - } - - private void registerProcessor() { - this.remotingServer - .registerDefaultProcessor(new DefaultRequestProcessor(this), this.remotingExecutor); - } - - public void registerFilterServerToBroker() { - try { - RegisterFilterServerResponseHeader responseHeader = - this.filterServerOuterAPI.registerFilterServerToBroker( - this.filtersrvConfig.getConnectWhichBroker(), this.localAddr()); - this.defaultMQPullConsumer.getDefaultMQPullConsumerImpl().getPullAPIWrapper() - .setDefaultBrokerId(responseHeader.getBrokerId()); - - if (null == this.brokerName) { - this.brokerName = responseHeader.getBrokerName(); - } - - log.info("register filter server<{}> to broker<{}> OK, Return: {} {}", - this.localAddr(), - this.filtersrvConfig.getConnectWhichBroker(), - responseHeader.getBrokerName(), - responseHeader.getBrokerId()); - } catch (Exception e) { - log.warn("register filter server Exception", e); - - log.warn("access broker failed, kill oneself"); - System.exit(-1); - } - } - - public String localAddr() { - return String.format("%s:%d", this.filtersrvConfig.getFilterServerIP(), - this.remotingServer.localListenPort()); - } - - public void start() throws Exception { - this.defaultMQPullConsumer.start(); - this.remotingServer.start(); - this.filterServerOuterAPI.start(); - this.defaultMQPullConsumer.getDefaultMQPullConsumerImpl().getPullAPIWrapper() - .setConnectBrokerByUser(true); - this.filterClassManager.start(); - this.filterServerStatsManager.start(); - } - - public void shutdown() { - this.remotingServer.shutdown(); - this.remotingExecutor.shutdown(); - this.scheduledExecutorService.shutdown(); - this.defaultMQPullConsumer.shutdown(); - this.filterServerOuterAPI.shutdown(); - this.filterClassManager.shutdown(); - this.filterServerStatsManager.shutdown(); - } - - public RemotingServer getRemotingServer() { - return remotingServer; - } - - public void setRemotingServer(RemotingServer remotingServer) { - this.remotingServer = remotingServer; - } - - public ExecutorService getRemotingExecutor() { - return remotingExecutor; - } - - public void setRemotingExecutor(ExecutorService remotingExecutor) { - this.remotingExecutor = remotingExecutor; - } - - public FiltersrvConfig getFiltersrvConfig() { - return filtersrvConfig; - } - - public NettyServerConfig getNettyServerConfig() { - return nettyServerConfig; - } - - public ScheduledExecutorService getScheduledExecutorService() { - return scheduledExecutorService; - } - - public FilterServerOuterAPI getFilterServerOuterAPI() { - return filterServerOuterAPI; - } - - public FilterClassManager getFilterClassManager() { - return filterClassManager; - } - - public DefaultMQPullConsumer getDefaultMQPullConsumer() { - return defaultMQPullConsumer; - } - - public String getBrokerName() { - return brokerName; - } - - public void setBrokerName(String brokerName) { - this.brokerName = brokerName; - } - - public FilterServerStatsManager getFilterServerStatsManager() { - return filterServerStatsManager; - } -} diff --git a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FiltersrvStartup.java b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FiltersrvStartup.java deleted file mode 100644 index 9fa04b758f2..00000000000 --- a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FiltersrvStartup.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.filtersrv; - -import ch.qos.logback.classic.LoggerContext; -import ch.qos.logback.classic.joran.JoranConfigurator; -import java.io.BufferedInputStream; -import java.io.FileInputStream; -import java.io.InputStream; -import java.util.Properties; -import java.util.concurrent.Callable; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.PosixParser; -import org.apache.rocketmq.common.MQVersion; -import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; -import org.apache.rocketmq.remoting.netty.NettyServerConfig; -import org.apache.rocketmq.remoting.netty.NettySystemConfig; -import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import org.apache.rocketmq.srvutil.ServerUtil; -import org.apache.rocketmq.srvutil.ShutdownHookThread; -import org.slf4j.LoggerFactory; - -public class FiltersrvStartup { - public static InternalLogger log; - - public static void main(String[] args) { - start(createController(args)); - } - - public static FiltersrvController start(FiltersrvController controller) { - - try { - controller.start(); - } catch (Exception e) { - e.printStackTrace(); - System.exit(-1); - } - - String tip = "The Filter Server boot success, " + controller.localAddr(); - log.info(tip); - System.out.printf("%s%n", tip); - - return controller; - } - - public static FiltersrvController createController(String[] args) { - System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, Integer.toString(MQVersion.CURRENT_VERSION)); - - if (null == System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_SOCKET_SNDBUF_SIZE)) { - NettySystemConfig.socketSndbufSize = 65535; - } - - if (null == System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_SOCKET_RCVBUF_SIZE)) { - NettySystemConfig.socketRcvbufSize = 1024; - } - - try { - Options options = ServerUtil.buildCommandlineOptions(new Options()); - final CommandLine commandLine = - ServerUtil.parseCmdLine("mqfiltersrv", args, buildCommandlineOptions(options), - new PosixParser()); - if (null == commandLine) { - System.exit(-1); - return null; - } - - final FiltersrvConfig filtersrvConfig = new FiltersrvConfig(); - final NettyServerConfig nettyServerConfig = new NettyServerConfig(); - - if (commandLine.hasOption('c')) { - String file = commandLine.getOptionValue('c'); - if (file != null) { - InputStream in = new BufferedInputStream(new FileInputStream(file)); - Properties properties = new Properties(); - properties.load(in); - MixAll.properties2Object(properties, filtersrvConfig); - System.out.printf("load config properties file OK, %s%n", file); - in.close(); - - String port = properties.getProperty("listenPort"); - if (port != null) { - filtersrvConfig.setConnectWhichBroker(String.format("127.0.0.1:%s", port)); - } - } - } - - nettyServerConfig.setListenPort(0); - nettyServerConfig.setServerAsyncSemaphoreValue(filtersrvConfig.getFsServerAsyncSemaphoreValue()); - nettyServerConfig.setServerCallbackExecutorThreads(filtersrvConfig - .getFsServerCallbackExecutorThreads()); - nettyServerConfig.setServerWorkerThreads(filtersrvConfig.getFsServerWorkerThreads()); - - if (commandLine.hasOption('p')) { - MixAll.printObjectProperties(null, filtersrvConfig); - MixAll.printObjectProperties(null, nettyServerConfig); - System.exit(0); - } - - MixAll.properties2Object(ServerUtil.commandLine2Properties(commandLine), filtersrvConfig); - if (null == filtersrvConfig.getRocketmqHome()) { - System.out.printf("Please set the %s variable in your environment to match the location of the RocketMQ installation%n", MixAll.ROCKETMQ_HOME_ENV); - System.exit(-2); - } - - LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory(); - JoranConfigurator configurator = new JoranConfigurator(); - configurator.setContext(lc); - lc.reset(); - configurator.doConfigure(filtersrvConfig.getRocketmqHome() + "/conf/logback_filtersrv.xml"); - log = InternalLoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); - - final FiltersrvController controller = - new FiltersrvController(filtersrvConfig, nettyServerConfig); - boolean initResult = controller.initialize(); - if (!initResult) { - controller.shutdown(); - System.exit(-3); - } - - Runtime.getRuntime().addShutdownHook(new ShutdownHookThread(log, new Callable() { - @Override - public Void call() throws Exception { - controller.shutdown(); - return null; - } - })); - - return controller; - } catch (Throwable e) { - e.printStackTrace(); - System.exit(-1); - } - return null; - } - - public static Options buildCommandlineOptions(final Options options) { - Option opt = new Option("c", "configFile", true, "Filter server config properties file"); - opt.setRequired(false); - options.addOption(opt); - - opt = new Option("p", "printConfigItem", false, "Print all config item"); - opt.setRequired(false); - options.addOption(opt); - - return options; - } -} diff --git a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/DynaCode.java b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/DynaCode.java deleted file mode 100644 index bde9961f6ea..00000000000 --- a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/DynaCode.java +++ /dev/null @@ -1,386 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.rocketmq.filtersrv.filter; - -import java.io.BufferedWriter; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.OutputStreamWriter; -import java.io.UnsupportedEncodingException; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLClassLoader; -import java.net.URLDecoder; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import javax.tools.JavaCompiler; -import javax.tools.ToolProvider; -import org.apache.commons.lang3.StringUtils; -import org.apache.rocketmq.common.UtilAll; -import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.common.filter.FilterAPI; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; -import org.apache.rocketmq.remoting.common.RemotingHelper; -public class DynaCode { - private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); - - private static final String FILE_SP = System.getProperty("file.separator"); - - private static final String LINE_SP = System.getProperty("line.separator"); - - private String sourcePath = System.getProperty("user.home") + FILE_SP + "rocketmq_filter_class" + FILE_SP - + UtilAll.getPid(); - - private String outPutClassPath = sourcePath; - - private ClassLoader parentClassLoader; - - private List codeStrs; - - private Map/* class */> loadClass; - - private String classpath; - - private String bootclasspath; - - private String extdirs; - - private String encoding = "UTF-8"; - - private String target; - - public DynaCode(String code) { - this(Thread.currentThread().getContextClassLoader(), Collections.singletonList(code)); - } - - public DynaCode(ClassLoader parentClassLoader, List codeStrs) { - this(extractClasspath(parentClassLoader), parentClassLoader, codeStrs); - } - - public DynaCode(String classpath, ClassLoader parentClassLoader, List codeStrs) { - this.classpath = classpath; - this.parentClassLoader = parentClassLoader; - this.codeStrs = codeStrs; - this.loadClass = new HashMap>(codeStrs.size()); - } - - public DynaCode(List codeStrs) { - this(Thread.currentThread().getContextClassLoader(), codeStrs); - } - - private static String extractClasspath(ClassLoader cl) { - StringBuffer buf = new StringBuffer(); - while (cl != null) { - if (cl instanceof URLClassLoader) { - URL urls[] = ((URLClassLoader) cl).getURLs(); - for (int i = 0; i < urls.length; i++) { - if (buf.length() > 0) { - buf.append(File.pathSeparatorChar); - } - String s = urls[i].getFile(); - try { - s = URLDecoder.decode(s, "UTF-8"); - } catch (UnsupportedEncodingException e) { - continue; - } - File f = new File(s); - buf.append(f.getAbsolutePath()); - } - } - cl = cl.getParent(); - } - return buf.toString(); - } - - public static Class compileAndLoadClass(final String className, final String javaSource) - throws Exception { - String classSimpleName = FilterAPI.simpleClassName(className); - String javaCode = javaSource; - - final String newClassSimpleName = classSimpleName + System.currentTimeMillis(); - String newJavaCode = javaCode.replaceAll(classSimpleName, newClassSimpleName); - - List codes = new ArrayList(); - codes.add(newJavaCode); - DynaCode dc = new DynaCode(codes); - dc.compileAndLoadClass(); - Map> map = dc.getLoadClass(); - - Class clazz = map.get(getQualifiedName(newJavaCode)); - return clazz; - } - - public static String getQualifiedName(String code) { - StringBuilder sb = new StringBuilder(); - String className = getClassName(code); - if (StringUtils.isNotBlank(className)) { - - String packageName = getPackageName(code); - if (StringUtils.isNotBlank(packageName)) { - sb.append(packageName).append("."); - } - sb.append(className); - } - return sb.toString(); - } - - public static String getClassName(String code) { - String className = StringUtils.substringBefore(code, "{"); - if (StringUtils.isBlank(className)) { - return className; - } - if (StringUtils.contains(code, " class ")) { - className = StringUtils.substringAfter(className, " class "); - if (StringUtils.contains(className, " extends ")) { - className = StringUtils.substringBefore(className, " extends ").trim(); - } else if (StringUtils.contains(className, " implements ")) { - className = StringUtils.trim(StringUtils.substringBefore(className, " implements ")); - } else { - className = StringUtils.trim(className); - } - } else if (StringUtils.contains(code, " interface ")) { - className = StringUtils.substringAfter(className, " interface "); - if (StringUtils.contains(className, " extends ")) { - className = StringUtils.substringBefore(className, " extends ").trim(); - } else { - className = StringUtils.trim(className); - } - } else if (StringUtils.contains(code, " enum ")) { - className = StringUtils.trim(StringUtils.substringAfter(className, " enum ")); - } else { - return StringUtils.EMPTY; - } - return className; - } - - public static String getPackageName(String code) { - String packageName = - StringUtils.substringBefore(StringUtils.substringAfter(code, "package "), ";").trim(); - return packageName; - } - - public static String getFullClassName(String code) { - String packageName = getPackageName(code); - String className = getClassName(code); - return StringUtils.isBlank(packageName) ? className : packageName + "." + className; - } - - public void compileAndLoadClass() throws Exception { - String[] sourceFiles = this.uploadSrcFile(); - this.compile(sourceFiles); - this.loadClass(this.loadClass.keySet()); - } - - public Map> getLoadClass() { - return loadClass; - } - - private String[] uploadSrcFile() throws Exception { - List srcFileAbsolutePaths = new ArrayList(codeStrs.size()); - for (String code : codeStrs) { - if (StringUtils.isNotBlank(code)) { - String packageName = getPackageName(code); - String className = getClassName(code); - if (StringUtils.isNotBlank(className)) { - File srcFile = null; - BufferedWriter bufferWriter = null; - try { - if (StringUtils.isBlank(packageName)) { - File pathFile = new File(sourcePath); - - if (!pathFile.exists()) { - if (!pathFile.mkdirs()) { - throw new RuntimeException("create PathFile Error!"); - } - } - srcFile = new File(sourcePath + FILE_SP + className + ".java"); - } else { - String srcPath = StringUtils.replace(packageName, ".", FILE_SP); - File pathFile = new File(sourcePath + FILE_SP + srcPath); - - if (!pathFile.exists()) { - if (!pathFile.mkdirs()) { - throw new RuntimeException("create PathFile Error!"); - } - } - srcFile = new File(pathFile.getAbsolutePath() + FILE_SP + className + ".java"); - } - synchronized (loadClass) { - loadClass.put(getFullClassName(code), null); - } - if (null != srcFile) { - log.warn("Dyna Create Java Source File:----> {}", srcFile.getAbsolutePath()); - srcFileAbsolutePaths.add(srcFile.getAbsolutePath()); - srcFile.deleteOnExit(); - } - OutputStreamWriter outputStreamWriter = - new OutputStreamWriter(new FileOutputStream(srcFile), encoding); - bufferWriter = new BufferedWriter(outputStreamWriter); - for (String lineCode : code.split(LINE_SP)) { - bufferWriter.write(lineCode); - bufferWriter.newLine(); - } - bufferWriter.flush(); - } finally { - if (null != bufferWriter) { - bufferWriter.close(); - } - } - } - } - } - return srcFileAbsolutePaths.toArray(new String[srcFileAbsolutePaths.size()]); - } - - private void compile(String[] srcFiles) throws Exception { - String args[] = this.buildCompileJavacArgs(srcFiles); - ByteArrayOutputStream err = new ByteArrayOutputStream(); - JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); - if (compiler == null) { - throw new NullPointerException( - "ToolProvider.getSystemJavaCompiler() return null,please use JDK replace JRE!"); - } - int resultCode = compiler.run(null, null, err, args); - if (resultCode != 0) { - throw new Exception(err.toString(RemotingHelper.DEFAULT_CHARSET)); - } - } - - private void loadClass(Set classFullNames) throws ClassNotFoundException, MalformedURLException { - synchronized (loadClass) { - ClassLoader classLoader = - new URLClassLoader(new URL[] {new File(outPutClassPath).toURI().toURL()}, - parentClassLoader); - for (String key : classFullNames) { - Class classz = classLoader.loadClass(key); - if (null != classz) { - loadClass.put(key, classz); - log.info("Dyna Load Java Class File OK:----> className: {}", key); - } else { - log.error("Dyna Load Java Class File Fail:----> className: {}", key); - } - } - } - } - - private String[] buildCompileJavacArgs(String srcFiles[]) { - ArrayList args = new ArrayList(); - if (StringUtils.isNotBlank(classpath)) { - args.add("-classpath"); - args.add(classpath); - } - if (StringUtils.isNotBlank(outPutClassPath)) { - args.add("-d"); - args.add(outPutClassPath); - } - if (StringUtils.isNotBlank(sourcePath)) { - args.add("-sourcepath"); - args.add(sourcePath); - } - if (StringUtils.isNotBlank(bootclasspath)) { - args.add("-bootclasspath"); - args.add(bootclasspath); - } - if (StringUtils.isNotBlank(extdirs)) { - args.add("-extdirs"); - args.add(extdirs); - } - if (StringUtils.isNotBlank(encoding)) { - args.add("-encoding"); - args.add(encoding); - } - if (StringUtils.isNotBlank(target)) { - args.add("-target"); - args.add(target); - } - for (int i = 0; i < srcFiles.length; i++) { - args.add(srcFiles[i]); - } - return args.toArray(new String[args.size()]); - } - - public String getOutPutClassPath() { - return outPutClassPath; - } - - public void setOutPutClassPath(String outPutClassPath) { - this.outPutClassPath = outPutClassPath; - } - - public String getSourcePath() { - return sourcePath; - } - - public void setSourcePath(String sourcePath) { - this.sourcePath = sourcePath; - } - - public ClassLoader getParentClassLoader() { - return parentClassLoader; - } - - public void setParentClassLoader(ClassLoader parentClassLoader) { - this.parentClassLoader = parentClassLoader; - } - - public String getClasspath() { - return classpath; - } - - public void setClasspath(String classpath) { - this.classpath = classpath; - } - - public String getBootclasspath() { - return bootclasspath; - } - - public void setBootclasspath(String bootclasspath) { - this.bootclasspath = bootclasspath; - } - - public String getExtdirs() { - return extdirs; - } - - public void setExtdirs(String extdirs) { - this.extdirs = extdirs; - } - - public String getEncoding() { - return encoding; - } - - public void setEncoding(String encoding) { - this.encoding = encoding; - } - - public String getTarget() { - return target; - } - - public void setTarget(String target) { - this.target = target; - } -} \ No newline at end of file diff --git a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/FilterClassFetchMethod.java b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/FilterClassFetchMethod.java deleted file mode 100644 index 15e1bb094e3..00000000000 --- a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/FilterClassFetchMethod.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.rocketmq.filtersrv.filter; - -public interface FilterClassFetchMethod { - public String fetch(final String topic, final String consumerGroup, final String className); -} diff --git a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/FilterClassInfo.java b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/FilterClassInfo.java deleted file mode 100644 index b6753928374..00000000000 --- a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/FilterClassInfo.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.rocketmq.filtersrv.filter; - -import org.apache.rocketmq.common.filter.MessageFilter; - -public class FilterClassInfo { - private String className; - private int classCRC; - private MessageFilter messageFilter; - - public int getClassCRC() { - return classCRC; - } - - public void setClassCRC(int classCRC) { - this.classCRC = classCRC; - } - - public MessageFilter getMessageFilter() { - return messageFilter; - } - - public void setMessageFilter(MessageFilter messageFilter) { - this.messageFilter = messageFilter; - } - - public String getClassName() { - return className; - } - - public void setClassName(String className) { - this.className = className; - } -} diff --git a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/FilterClassLoader.java b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/FilterClassLoader.java deleted file mode 100644 index 70f714aa091..00000000000 --- a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/FilterClassLoader.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.rocketmq.filtersrv.filter; - -public class FilterClassLoader extends ClassLoader { - public final Class createNewClass(String name, byte[] b, int off, int len) throws ClassFormatError { - return this.defineClass(name, b, off, len); - } -} diff --git a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/FilterClassManager.java b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/FilterClassManager.java deleted file mode 100644 index 360341c8247..00000000000 --- a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/FilterClassManager.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.rocketmq.filtersrv.filter; - -import java.util.Iterator; -import java.util.Map.Entry; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.ThreadFactoryImpl; -import org.apache.rocketmq.common.UtilAll; -import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.common.filter.MessageFilter; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; -import org.apache.rocketmq.filtersrv.FiltersrvController; - -public class FilterClassManager { - private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); - - private final Object compileLock = new Object(); - private final FiltersrvController filtersrvController; - - private final ScheduledExecutorService scheduledExecutorService = Executors - .newSingleThreadScheduledExecutor(new ThreadFactoryImpl("FSGetClassScheduledThread")); - private ConcurrentMap filterClassTable = - new ConcurrentHashMap(128); - private FilterClassFetchMethod filterClassFetchMethod; - - public FilterClassManager(FiltersrvController filtersrvController) { - this.filtersrvController = filtersrvController; - this.filterClassFetchMethod = - new HttpFilterClassFetchMethod(this.filtersrvController.getFiltersrvConfig() - .getFilterClassRepertoryUrl()); - } - - private static String buildKey(final String consumerGroup, final String topic) { - return topic + "@" + consumerGroup; - } - - public void start() { - if (!this.filtersrvController.getFiltersrvConfig().isClientUploadFilterClassEnable()) { - this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { - - @Override - public void run() { - fetchClassFromRemoteHost(); - } - }, 1, 1, TimeUnit.MINUTES); - } - } - - private void fetchClassFromRemoteHost() { - Iterator> it = this.filterClassTable.entrySet().iterator(); - while (it.hasNext()) { - try { - Entry next = it.next(); - FilterClassInfo filterClassInfo = next.getValue(); - String[] topicAndGroup = next.getKey().split("@"); - String responseStr = - this.filterClassFetchMethod.fetch(topicAndGroup[0], topicAndGroup[1], - filterClassInfo.getClassName()); - byte[] filterSourceBinary = responseStr.getBytes("UTF-8"); - int classCRC = UtilAll.crc32(responseStr.getBytes("UTF-8")); - if (classCRC != filterClassInfo.getClassCRC()) { - String javaSource = new String(filterSourceBinary, MixAll.DEFAULT_CHARSET); - Class newClass = - DynaCode.compileAndLoadClass(filterClassInfo.getClassName(), javaSource); - Object newInstance = newClass.newInstance(); - filterClassInfo.setMessageFilter((MessageFilter) newInstance); - filterClassInfo.setClassCRC(classCRC); - - log.info("fetch Remote class File OK, {} {}", next.getKey(), - filterClassInfo.getClassName()); - } - } catch (Exception e) { - log.error("fetchClassFromRemoteHost Exception", e); - } - } - } - - public void shutdown() { - this.scheduledExecutorService.shutdown(); - } - - public boolean registerFilterClass(final String consumerGroup, final String topic, - final String className, final int classCRC, final byte[] filterSourceBinary) { - final String key = buildKey(consumerGroup, topic); - - boolean registerNew = false; - FilterClassInfo filterClassInfoPrev = this.filterClassTable.get(key); - if (null == filterClassInfoPrev) { - registerNew = true; - } else { - if (this.filtersrvController.getFiltersrvConfig().isClientUploadFilterClassEnable()) { - if (filterClassInfoPrev.getClassCRC() != classCRC && classCRC != 0) { - registerNew = true; - } - } - } - - if (registerNew) { - synchronized (this.compileLock) { - filterClassInfoPrev = this.filterClassTable.get(key); - if (null != filterClassInfoPrev && filterClassInfoPrev.getClassCRC() == classCRC) { - return true; - } - - try { - - FilterClassInfo filterClassInfoNew = new FilterClassInfo(); - filterClassInfoNew.setClassName(className); - filterClassInfoNew.setClassCRC(0); - filterClassInfoNew.setMessageFilter(null); - - if (this.filtersrvController.getFiltersrvConfig().isClientUploadFilterClassEnable()) { - String javaSource = new String(filterSourceBinary, MixAll.DEFAULT_CHARSET); - Class newClass = DynaCode.compileAndLoadClass(className, javaSource); - Object newInstance = newClass.newInstance(); - filterClassInfoNew.setMessageFilter((MessageFilter) newInstance); - filterClassInfoNew.setClassCRC(classCRC); - } - - this.filterClassTable.put(key, filterClassInfoNew); - } catch (Throwable e) { - String info = - String - .format( - "FilterServer, registerFilterClass Exception, consumerGroup: %s topic: %s className: %s", - consumerGroup, topic, className); - log.error(info, e); - return false; - } - } - } - - return true; - } - - public FilterClassInfo findFilterClass(final String consumerGroup, final String topic) { - return this.filterClassTable.get(buildKey(consumerGroup, topic)); - } - - public FilterClassFetchMethod getFilterClassFetchMethod() { - return filterClassFetchMethod; - } - - public void setFilterClassFetchMethod(FilterClassFetchMethod filterClassFetchMethod) { - this.filterClassFetchMethod = filterClassFetchMethod; - } -} diff --git a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/HttpFilterClassFetchMethod.java b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/HttpFilterClassFetchMethod.java deleted file mode 100644 index ebd59cd86cf..00000000000 --- a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/HttpFilterClassFetchMethod.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.rocketmq.filtersrv.filter; - -import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; -import org.apache.rocketmq.common.utils.HttpTinyClient; -import org.apache.rocketmq.common.utils.HttpTinyClient.HttpResult; - -public class HttpFilterClassFetchMethod implements FilterClassFetchMethod { - private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); - private final String url; - - public HttpFilterClassFetchMethod(String url) { - this.url = url; - } - - @Override - public String fetch(String topic, String consumerGroup, String className) { - String thisUrl = String.format("%s/%s.java", this.url, className); - - try { - HttpResult result = HttpTinyClient.httpGet(thisUrl, null, null, "UTF-8", 5000); - if (200 == result.code) { - return result.content; - } - } catch (Exception e) { - log.error( - String.format("call <%s> exception, Topic: %s Group: %s", thisUrl, topic, consumerGroup), e); - } - - return null; - } -} diff --git a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/processor/DefaultRequestProcessor.java b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/processor/DefaultRequestProcessor.java deleted file mode 100644 index d5335bb6625..00000000000 --- a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/processor/DefaultRequestProcessor.java +++ /dev/null @@ -1,347 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.rocketmq.filtersrv.processor; - -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandlerContext; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; -import org.apache.rocketmq.client.consumer.PullCallback; -import org.apache.rocketmq.client.consumer.PullResult; -import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.UtilAll; -import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.common.filter.FilterContext; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; -import org.apache.rocketmq.common.message.MessageDecoder; -import org.apache.rocketmq.common.message.MessageExt; -import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.common.protocol.RequestCode; -import org.apache.rocketmq.common.protocol.ResponseCode; -import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; -import org.apache.rocketmq.common.protocol.header.PullMessageResponseHeader; -import org.apache.rocketmq.common.protocol.header.filtersrv.RegisterMessageFilterClassRequestHeader; -import org.apache.rocketmq.common.sysflag.MessageSysFlag; -import org.apache.rocketmq.filtersrv.FiltersrvController; -import org.apache.rocketmq.filtersrv.filter.FilterClassInfo; -import org.apache.rocketmq.remoting.common.RemotingHelper; -import org.apache.rocketmq.remoting.exception.RemotingCommandException; -import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; -import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import org.apache.rocketmq.store.CommitLog; - -public class DefaultRequestProcessor implements NettyRequestProcessor { - private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); - - private final FiltersrvController filtersrvController; - - public DefaultRequestProcessor(FiltersrvController filtersrvController) { - this.filtersrvController = filtersrvController; - } - - @Override - public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws Exception { - if (ctx != null) { - log.debug("receive request, {} {} {}", - request.getCode(), - RemotingHelper.parseChannelRemoteAddr(ctx.channel()), - request); - } - - switch (request.getCode()) { - case RequestCode.REGISTER_MESSAGE_FILTER_CLASS: - return registerMessageFilterClass(ctx, request); - case RequestCode.PULL_MESSAGE: - return pullMessageForward(ctx, request); - } - - return null; - } - - @Override - public boolean rejectRequest() { - return false; - } - - private RemotingCommand registerMessageFilterClass(ChannelHandlerContext ctx, - RemotingCommand request) throws RemotingCommandException { - final RemotingCommand response = RemotingCommand.createResponseCommand(null); - final RegisterMessageFilterClassRequestHeader requestHeader = - (RegisterMessageFilterClassRequestHeader) request.decodeCommandCustomHeader(RegisterMessageFilterClassRequestHeader.class); - - try { - boolean ok = this.filtersrvController.getFilterClassManager().registerFilterClass(requestHeader.getConsumerGroup(), - requestHeader.getTopic(), - requestHeader.getClassName(), - requestHeader.getClassCRC(), - request.getBody()); - if (!ok) { - throw new Exception("registerFilterClass error"); - } - } catch (Exception e) { - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark(RemotingHelper.exceptionSimpleDesc(e)); - return response; - } - - response.setCode(ResponseCode.SUCCESS); - response.setRemark(null); - return response; - } - - private RemotingCommand pullMessageForward(final ChannelHandlerContext ctx, - final RemotingCommand request) throws Exception { - final RemotingCommand response = RemotingCommand.createResponseCommand(PullMessageResponseHeader.class); - final PullMessageResponseHeader responseHeader = (PullMessageResponseHeader) response.readCustomHeader(); - final PullMessageRequestHeader requestHeader = - (PullMessageRequestHeader) request.decodeCommandCustomHeader(PullMessageRequestHeader.class); - - final FilterContext filterContext = new FilterContext(); - filterContext.setConsumerGroup(requestHeader.getConsumerGroup()); - - response.setOpaque(request.getOpaque()); - - DefaultMQPullConsumer pullConsumer = this.filtersrvController.getDefaultMQPullConsumer(); - final FilterClassInfo findFilterClass = - this.filtersrvController.getFilterClassManager() - .findFilterClass(requestHeader.getConsumerGroup(), requestHeader.getTopic()); - if (null == findFilterClass) { - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("Find Filter class failed, not registered"); - return response; - } - - if (null == findFilterClass.getMessageFilter()) { - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("Find Filter class failed, registered but no class"); - return response; - } - - responseHeader.setSuggestWhichBrokerId(MixAll.MASTER_ID); - - MessageQueue mq = new MessageQueue(); - mq.setTopic(requestHeader.getTopic()); - mq.setQueueId(requestHeader.getQueueId()); - mq.setBrokerName(this.filtersrvController.getBrokerName()); - long offset = requestHeader.getQueueOffset(); - int maxNums = requestHeader.getMaxMsgNums(); - - final PullCallback pullCallback = new PullCallback() { - - @Override - public void onSuccess(PullResult pullResult) { - responseHeader.setMaxOffset(pullResult.getMaxOffset()); - responseHeader.setMinOffset(pullResult.getMinOffset()); - responseHeader.setNextBeginOffset(pullResult.getNextBeginOffset()); - response.setRemark(null); - - switch (pullResult.getPullStatus()) { - case FOUND: - response.setCode(ResponseCode.SUCCESS); - - List msgListOK = new ArrayList(); - try { - for (MessageExt msg : pullResult.getMsgFoundList()) { - boolean match = findFilterClass.getMessageFilter().match(msg, filterContext); - if (match) { - msgListOK.add(msg); - } - } - - if (!msgListOK.isEmpty()) { - returnResponse(requestHeader.getConsumerGroup(), requestHeader.getTopic(), ctx, response, msgListOK); - return; - } else { - response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY); - } - } catch (Throwable e) { - final String error = - String.format("do Message Filter Exception, ConsumerGroup: %s Topic: %s ", - requestHeader.getConsumerGroup(), requestHeader.getTopic()); - log.error(error, e); - - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark(error + RemotingHelper.exceptionSimpleDesc(e)); - returnResponse(requestHeader.getConsumerGroup(), requestHeader.getTopic(), ctx, response, null); - return; - } - - break; - case NO_MATCHED_MSG: - response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY); - break; - case NO_NEW_MSG: - response.setCode(ResponseCode.PULL_NOT_FOUND); - break; - case OFFSET_ILLEGAL: - response.setCode(ResponseCode.PULL_OFFSET_MOVED); - break; - default: - break; - } - - returnResponse(requestHeader.getConsumerGroup(), requestHeader.getTopic(), ctx, response, null); - } - - @Override - public void onException(Throwable e) { - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("Pull Callback Exception, " + RemotingHelper.exceptionSimpleDesc(e)); - returnResponse(requestHeader.getConsumerGroup(), requestHeader.getTopic(), ctx, response, null); - return; - } - }; - - pullConsumer.pullBlockIfNotFound(mq, null, offset, maxNums, pullCallback); - - return null; - } - - private void returnResponse(final String group, final String topic, ChannelHandlerContext ctx, - final RemotingCommand response, - final List msgList) { - if (null != msgList) { - ByteBuffer[] msgBufferList = new ByteBuffer[msgList.size()]; - int bodyTotalSize = 0; - for (int i = 0; i < msgList.size(); i++) { - try { - msgBufferList[i] = messageToByteBuffer(msgList.get(i)); - bodyTotalSize += msgBufferList[i].capacity(); - } catch (Exception e) { - log.error("messageToByteBuffer UnsupportedEncodingException", e); - } - } - - ByteBuffer body = ByteBuffer.allocate(bodyTotalSize); - for (ByteBuffer bb : msgBufferList) { - bb.flip(); - body.put(bb); - } - - response.setBody(body.array()); - - this.filtersrvController.getFilterServerStatsManager().incGroupGetNums(group, topic, msgList.size()); - - this.filtersrvController.getFilterServerStatsManager().incGroupGetSize(group, topic, bodyTotalSize); - } - - try { - ctx.writeAndFlush(response).addListener(new ChannelFutureListener() { - @Override - public void operationComplete(ChannelFuture future) throws Exception { - if (!future.isSuccess()) { - log.error("FilterServer response to " + future.channel().remoteAddress() + " failed", future.cause()); - log.error(response.toString()); - } - } - }); - } catch (Throwable e) { - log.error("FilterServer process request over, but response failed", e); - log.error(response.toString()); - } - } - - private ByteBuffer messageToByteBuffer(final MessageExt msg) throws IOException { - int sysFlag = MessageSysFlag.clearCompressedFlag(msg.getSysFlag()); - if (msg.getBody() != null) { - if (msg.getBody().length >= this.filtersrvController.getFiltersrvConfig().getCompressMsgBodyOverHowmuch()) { - byte[] data = UtilAll.compress(msg.getBody(), this.filtersrvController.getFiltersrvConfig().getZipCompressLevel()); - if (data != null) { - msg.setBody(data); - sysFlag |= MessageSysFlag.COMPRESSED_FLAG; - } - } - } - - final int bodyLength = msg.getBody() != null ? msg.getBody().length : 0; - byte[] topicData = msg.getTopic().getBytes(MixAll.DEFAULT_CHARSET); - final int topicLength = topicData.length; - String properties = MessageDecoder.messageProperties2String(msg.getProperties()); - byte[] propertiesData = properties.getBytes(MixAll.DEFAULT_CHARSET); - final int propertiesLength = propertiesData.length; - final int msgLen = 4 // 1 TOTALSIZE - + 4 // 2 MAGICCODE - + 4 // 3 BODYCRC - + 4 // 4 QUEUEID - + 4 // 5 FLAG - + 8 // 6 QUEUEOFFSET - + 8 // 7 PHYSICALOFFSET - + 4 // 8 SYSFLAG - + 8 // 9 BORNTIMESTAMP - + 8 // 10 BORNHOST - + 8 // 11 STORETIMESTAMP - + 8 // 12 STOREHOSTADDRESS - + 4 // 13 RECONSUMETIMES - + 8 // 14 Prepared Transaction Offset - + 4 + bodyLength // 14 BODY - + 1 + topicLength // 15 TOPIC - + 2 + propertiesLength // 16 propertiesLength - + 0; - - ByteBuffer msgStoreItemMemory = ByteBuffer.allocate(msgLen); - - final MessageExt msgInner = msg; - - // 1 TOTALSIZE - msgStoreItemMemory.putInt(msgLen); - // 2 MAGICCODE - msgStoreItemMemory.putInt(CommitLog.MESSAGE_MAGIC_CODE); - // 3 BODYCRC - msgStoreItemMemory.putInt(UtilAll.crc32(msgInner.getBody())); - // 4 QUEUEID - msgStoreItemMemory.putInt(msgInner.getQueueId()); - // 5 FLAG - msgStoreItemMemory.putInt(msgInner.getFlag()); - // 6 QUEUEOFFSET - msgStoreItemMemory.putLong(msgInner.getQueueOffset()); - // 7 PHYSICALOFFSET - msgStoreItemMemory.putLong(msgInner.getCommitLogOffset()); - // 8 SYSFLAG - msgStoreItemMemory.putInt(sysFlag); - // 9 BORNTIMESTAMP - msgStoreItemMemory.putLong(msgInner.getBornTimestamp()); - // 10 BORNHOST - msgStoreItemMemory.put(msgInner.getBornHostBytes()); - // 11 STORETIMESTAMP - msgStoreItemMemory.putLong(msgInner.getStoreTimestamp()); - // 12 STOREHOSTADDRESS - msgStoreItemMemory.put(msgInner.getStoreHostBytes()); - // 13 RECONSUMETIMES - msgStoreItemMemory.putInt(msgInner.getReconsumeTimes()); - // 14 Prepared Transaction Offset - msgStoreItemMemory.putLong(msgInner.getPreparedTransactionOffset()); - // 15 BODY - msgStoreItemMemory.putInt(bodyLength); - if (bodyLength > 0) - msgStoreItemMemory.put(msgInner.getBody()); - // 16 TOPIC - msgStoreItemMemory.put((byte) topicLength); - msgStoreItemMemory.put(topicData); - // 17 PROPERTIES - msgStoreItemMemory.putShort((short) propertiesLength); - if (propertiesLength > 0) - msgStoreItemMemory.put(propertiesData); - - return msgStoreItemMemory; - } -} diff --git a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/stats/FilterServerStatsManager.java b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/stats/FilterServerStatsManager.java deleted file mode 100644 index 13bc834ff5a..00000000000 --- a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/stats/FilterServerStatsManager.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.rocketmq.filtersrv.stats; - -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import org.apache.rocketmq.common.ThreadFactoryImpl; -import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; -import org.apache.rocketmq.common.stats.StatsItemSet; - -public class FilterServerStatsManager { - private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); - private final ScheduledExecutorService scheduledExecutorService = Executors - .newSingleThreadScheduledExecutor(new ThreadFactoryImpl("FSStatsThread")); - - // ConsumerGroup Get Nums - private final StatsItemSet groupGetNums = new StatsItemSet("GROUP_GET_NUMS", - this.scheduledExecutorService, log); - - // ConsumerGroup Get Size - private final StatsItemSet groupGetSize = new StatsItemSet("GROUP_GET_SIZE", - this.scheduledExecutorService, log); - - public FilterServerStatsManager() { - } - - public void start() { - } - - public void shutdown() { - this.scheduledExecutorService.shutdown(); - } - - public void incGroupGetNums(final String group, final String topic, final int incValue) { - this.groupGetNums.addValue(topic + "@" + group, incValue, 1); - } - - public void incGroupGetSize(final String group, final String topic, final int incValue) { - this.groupGetSize.addValue(topic + "@" + group, incValue, 1); - } -} diff --git a/logging/src/test/java/org/apache/rocketmq/logging/BasicLoggerTest.java b/logging/src/test/java/org/apache/rocketmq/logging/BasicLoggerTest.java index 28496dd5674..c198704de26 100644 --- a/logging/src/test/java/org/apache/rocketmq/logging/BasicLoggerTest.java +++ b/logging/src/test/java/org/apache/rocketmq/logging/BasicLoggerTest.java @@ -17,17 +17,16 @@ package org.apache.rocketmq.logging; -import org.apache.rocketmq.logging.inner.Level; -import org.apache.rocketmq.logging.inner.Logger; -import org.apache.rocketmq.logging.inner.LoggingEvent; -import org.junit.After; -import org.junit.Before; - import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; +import org.apache.rocketmq.logging.inner.Level; +import org.apache.rocketmq.logging.inner.Logger; +import org.apache.rocketmq.logging.inner.LoggingEvent; +import org.junit.After; +import org.junit.Before; public class BasicLoggerTest { diff --git a/logging/src/test/java/org/apache/rocketmq/logging/InternalLoggerTest.java b/logging/src/test/java/org/apache/rocketmq/logging/InternalLoggerTest.java index 04b9f06e880..50f1dd1c9bd 100644 --- a/logging/src/test/java/org/apache/rocketmq/logging/InternalLoggerTest.java +++ b/logging/src/test/java/org/apache/rocketmq/logging/InternalLoggerTest.java @@ -17,6 +17,8 @@ package org.apache.rocketmq.logging; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; import org.apache.rocketmq.logging.inner.Appender; import org.apache.rocketmq.logging.inner.Level; import org.apache.rocketmq.logging.inner.Logger; @@ -25,13 +27,8 @@ import org.junit.Assert; import org.junit.Test; -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; - - public class InternalLoggerTest { - @Test public void testInternalLogger() { SysLogger.setQuietMode(false); @@ -44,7 +41,6 @@ public void testInternalLogger() { .withConsoleAppender(LoggingBuilder.SYSTEM_OUT) .withLayout(LoggingBuilder.newLayoutBuilder().withDefaultLayout().build()).build(); - Logger consoleLogger = Logger.getLogger("ConsoleLogger"); consoleLogger.setAdditivity(false); consoleLogger.addAppender(consoleAppender); diff --git a/logging/src/test/java/org/apache/rocketmq/logging/Slf4jLoggerFactoryTest.java b/logging/src/test/java/org/apache/rocketmq/logging/Slf4jLoggerFactoryTest.java index ba6ec3ba68c..4bed745b3c4 100644 --- a/logging/src/test/java/org/apache/rocketmq/logging/Slf4jLoggerFactoryTest.java +++ b/logging/src/test/java/org/apache/rocketmq/logging/Slf4jLoggerFactoryTest.java @@ -59,7 +59,7 @@ public void testSlf4j() throws IOException { String file = loggingDir + "/logback_test.log"; logger.info("logback slf4j info Message"); - logger.error("logback slf4j error Message", new RuntimeException()); + logger.error("logback slf4j error Message", new RuntimeException("test")); logger.debug("logback slf4j debug message"); logger3.info("logback info message"); logger3.error("logback error message"); diff --git a/pom.xml b/pom.xml index ef4f9fd3569..feb9a5b30ec 100644 --- a/pom.xml +++ b/pom.xml @@ -119,7 +119,6 @@ remoting logappender example - filtersrv srvutil filter test @@ -514,11 +513,6 @@ rocketmq-test ${project.version} - - ${project.groupId} - rocketmq-filtersrv - ${project.version} - ${project.groupId} rocketmq-srvutil From ed3c408c8b9f9ff15985830ee7cf63f24f23a0b5 Mon Sep 17 00:00:00 2001 From: Li Zhanhui Date: Tue, 12 Jun 2018 15:25:53 +0800 Subject: [PATCH 46/73] Use openmessaging 0.3.1-alpha --- distribution/pom.xml | 5 ----- distribution/release.xml | 1 - pom.xml | 9 +-------- 3 files changed, 1 insertion(+), 14 deletions(-) diff --git a/distribution/pom.xml b/distribution/pom.xml index fbfbd402837..e270024ab22 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -42,11 +42,6 @@ rocketmq-client - - org.apache.rocketmq - rocketmq-filtersrv - - org.apache.rocketmq rocketmq-tools diff --git a/distribution/release.xml b/distribution/release.xml index 5a2a7c79048..42381a0c31e 100644 --- a/distribution/release.xml +++ b/distribution/release.xml @@ -67,7 +67,6 @@ org.apache.rocketmq:rocketmq-client org.apache.rocketmq:rocketmq-namesrv org.apache.rocketmq:rocketmq-example - org.apache.rocketmq:rocketmq-filter org.apache.rocketmq:rocketmq-openmessaging diff --git a/pom.xml b/pom.xml index feb9a5b30ec..604c775539d 100644 --- a/pom.xml +++ b/pom.xml @@ -581,7 +581,7 @@ io.openmessaging openmessaging-api - 0.3.1-alpha-SNAPSHOT + 0.3.1-alpha log4j @@ -600,11 +600,4 @@ - - - - snapshot@oss.sonatype.org - https://oss.sonatype.org/content/repositories/snapshots/ - - From cc0759f40d8dfab796a206d2401f47f0280eb034 Mon Sep 17 00:00:00 2001 From: vongosling Date: Tue, 12 Jun 2018 15:39:57 +0800 Subject: [PATCH 47/73] Delete filterservdependency in distribution module --- distribution/pom.xml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/distribution/pom.xml b/distribution/pom.xml index fbfbd402837..9293d495b1a 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -15,7 +15,8 @@ See the License for the specific language governing permissions and limitations under the License. --> - + 4.0.0 org.apache.rocketmq @@ -27,31 +28,21 @@ pom - release-all - org.apache.rocketmq rocketmq-broker - org.apache.rocketmq rocketmq-client - - - org.apache.rocketmq - rocketmq-filtersrv - - org.apache.rocketmq rocketmq-tools - org.apache.rocketmq rocketmq-example @@ -85,7 +76,6 @@ release-client - org.apache.rocketmq From 201d8e375b4465d8f41cb09ae0138ddce485d607 Mon Sep 17 00:00:00 2001 From: Li Zhanhui Date: Tue, 12 Jun 2018 17:22:47 +0800 Subject: [PATCH 48/73] Add getter for properties of name server startup --- .../main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java index a073dafb21d..24e7c263d45 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java @@ -170,4 +170,8 @@ public static Options buildCommandlineOptions(final Options options) { return options; } + + public static Properties getProperties() { + return properties; + } } From 4d76016455f368d11b0a8835cc4adbaf5052873e Mon Sep 17 00:00:00 2001 From: vongosling Date: Tue, 12 Jun 2018 17:26:59 +0800 Subject: [PATCH 49/73] Polish parts slow UT --- .../rocketmq/broker/BrokerControllerTest.java | 11 +++----- .../rocketmq/logappender/Log4jTest.java | 3 +-- .../rocketmq/logappender/LogbackTest.java | 6 ++--- ...a => DefaultMessageStoreShutDownTest.java} | 6 ++--- .../store/DefaultMessageStoreTest.java | 26 +++---------------- 5 files changed, 13 insertions(+), 39 deletions(-) rename store/src/test/java/org/apache/rocketmq/store/{DefaultMessageStoreShuwDownTest.java => DefaultMessageStoreShutDownTest.java} (95%) diff --git a/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java index 797e0d882a2..56abf084a7c 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.broker; +import java.io.File; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.remoting.netty.NettyClientConfig; @@ -25,16 +26,10 @@ import org.junit.After; import org.junit.Test; -import java.io.File; - import static org.assertj.core.api.Assertions.assertThat; public class BrokerControllerTest { - /** - * Tests if the controller can be properly stopped and started. - * - * @throws Exception If fails. - */ + @Test public void testBrokerRestart() throws Exception { BrokerController brokerController = new BrokerController( @@ -48,7 +43,7 @@ public void testBrokerRestart() throws Exception { } @After - public void destory() { + public void destroy() { UtilAll.deleteFile(new File(new MessageStoreConfig().getStorePathRootDir())); } } diff --git a/logappender/src/test/java/org/apache/rocketmq/logappender/Log4jTest.java b/logappender/src/test/java/org/apache/rocketmq/logappender/Log4jTest.java index c139283a0c9..cf99c6937f4 100644 --- a/logappender/src/test/java/org/apache/rocketmq/logappender/Log4jTest.java +++ b/logappender/src/test/java/org/apache/rocketmq/logappender/Log4jTest.java @@ -17,7 +17,6 @@ package org.apache.rocketmq.logappender; import org.apache.log4j.Logger; -import org.apache.rocketmq.client.exception.MQClientException; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -30,7 +29,7 @@ public abstract class Log4jTest extends AbstractTestCase { public abstract String getType(); @Test - public void testLog4j() throws InterruptedException, MQClientException { + public void testLog4j() { clear(); Logger logger = Logger.getLogger("testLogger"); for (int i = 0; i < 10; i++) { diff --git a/logappender/src/test/java/org/apache/rocketmq/logappender/LogbackTest.java b/logappender/src/test/java/org/apache/rocketmq/logappender/LogbackTest.java index d7ec184a4b7..3dc18f0dccf 100644 --- a/logappender/src/test/java/org/apache/rocketmq/logappender/LogbackTest.java +++ b/logappender/src/test/java/org/apache/rocketmq/logappender/LogbackTest.java @@ -20,15 +20,13 @@ import ch.qos.logback.classic.joran.JoranConfigurator; import ch.qos.logback.core.joran.spi.JoranException; import ch.qos.logback.core.util.StatusPrinter; -import org.apache.rocketmq.client.exception.MQClientException; +import java.io.File; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; - public class LogbackTest extends AbstractTestCase { @Before @@ -42,7 +40,7 @@ public void init() throws JoranException { } @Test - public void testLogback() throws InterruptedException, MQClientException { + public void testLogback() { clear(); Logger logger = LoggerFactory.getLogger("testLogger"); for (int i = 0; i < 10; i++) { diff --git a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreShuwDownTest.java b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreShutDownTest.java similarity index 95% rename from store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreShuwDownTest.java rename to store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreShutDownTest.java index ac85d59f1c7..ee0c365ea3a 100644 --- a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreShuwDownTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreShutDownTest.java @@ -35,7 +35,7 @@ import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) -public class DefaultMessageStoreShuwDownTest { +public class DefaultMessageStoreShutdownTest { private DefaultMessageStore messageStore; @Before @@ -48,7 +48,7 @@ public void init() throws Exception { } @Test - public void testDispatchBehindWhenShutDown() { + public void testDispatchBehindWhenShutdown() { messageStore.shutdown(); assertTrue(!messageStore.shutDownNormal); File file = new File(StorePathConfigHelper.getAbortFile(messageStore.getMessageStoreConfig().getStorePathRootDir())); @@ -56,7 +56,7 @@ public void testDispatchBehindWhenShutDown() { } @After - public void destory() { + public void destroy() { messageStore.destroy(); File file = new File(messageStore.getMessageStoreConfig().getStorePathRootDir()); UtilAll.deleteFile(file); diff --git a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java index 9269cdfa772..a400ed5a5a7 100644 --- a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java @@ -59,7 +59,6 @@ public void init() throws Exception { @Test(expected = OverlappingFileLockException.class) public void test_repate_restart() throws Exception { - long totalMsgs = 100; QUEUE_TOTAL = 1; MessageBody = StoreMessage.getBytes(); @@ -92,7 +91,7 @@ public void destory() { UtilAll.deleteFile(file); } - public MessageStore buildMessageStore() throws Exception { + private MessageStore buildMessageStore() throws Exception { MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); messageStoreConfig.setMapedFileSizeCommitLog(1024 * 1024 * 10); messageStoreConfig.setMapedFileSizeConsumeQueue(1024 * 1024 * 10); @@ -103,8 +102,8 @@ public MessageStore buildMessageStore() throws Exception { } @Test - public void testWriteAndRead() throws Exception { - long totalMsgs = 100; + public void testWriteAndRead() { + long totalMsgs = 10; QUEUE_TOTAL = 1; MessageBody = StoreMessage.getBytes(); for (long i = 0; i < totalMsgs; i++) { @@ -119,7 +118,7 @@ public void testWriteAndRead() throws Exception { verifyThatMasterIsFunctional(totalMsgs, messageStore); } - public MessageExtBrokerInner buildMessage() { + private MessageExtBrokerInner buildMessage() { MessageExtBrokerInner msg = new MessageExtBrokerInner(); msg.setTopic("FooBar"); msg.setTags("TAG1"); @@ -134,23 +133,6 @@ public MessageExtBrokerInner buildMessage() { return msg; } - @Test - public void testGroupCommit() throws Exception { - long totalMsgs = 10; - QUEUE_TOTAL = 1; - MessageBody = StoreMessage.getBytes(); - for (long i = 0; i < totalMsgs; i++) { - messageStore.putMessage(buildMessage()); - } - - for (long i = 0; i < totalMsgs; i++) { - GetMessageResult result = messageStore.getMessage("GROUP_A", "TOPIC_A", 0, i, 1024 * 1024, null); - assertThat(result).isNotNull(); - result.release(); - } - verifyThatMasterIsFunctional(totalMsgs, messageStore); - } - private void verifyThatMasterIsFunctional(long totalMsgs, MessageStore master) { for (long i = 0; i < totalMsgs; i++) { master.putMessage(buildMessage()); From 9932819eedb32f56efb6cc71309dec7785ecea9f Mon Sep 17 00:00:00 2001 From: vongosling Date: Tue, 12 Jun 2018 17:39:32 +0800 Subject: [PATCH 50/73] Fix the problem of the capitalization in class name --- .../apache/rocketmq/store/DefaultMessageStoreShutDownTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreShutDownTest.java b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreShutDownTest.java index ee0c365ea3a..66f7f5dc8f5 100644 --- a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreShutDownTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreShutDownTest.java @@ -35,7 +35,7 @@ import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) -public class DefaultMessageStoreShutdownTest { +public class DefaultMessageStoreShutDownTest { private DefaultMessageStore messageStore; @Before From 92f0e1f5abb694c90ced9468e9e1d03edcdcedfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=82=85=E5=86=B2?= Date: Fri, 25 May 2018 13:37:10 +0800 Subject: [PATCH 51/73] Optimize broker topic route registration to relieve stress on Java GC --- .../rocketmq/broker/BrokerController.java | 69 +++++++++++++------ .../rocketmq/broker/out/BrokerOuterAPI.java | 42 ++++++----- .../processor/AdminBrokerProcessor.java | 10 ++- .../apache/rocketmq/common/BrokerConfig.java | 2 - .../namesrv/RegisterBrokerRequestHeader.java | 10 +++ .../processor/DefaultRequestProcessor.java | 26 +++++++ 6 files changed, 107 insertions(+), 52 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 293e51e9b8b..ed85a679c05 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; @@ -62,6 +63,7 @@ import org.apache.rocketmq.broker.topic.TopicConfigManager; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.Configuration; +import org.apache.rocketmq.common.DataVersion; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.UtilAll; @@ -768,6 +770,24 @@ public void run() { } } + public synchronized void registerIncrementBrokerData(TopicConfig topicConfig, DataVersion dataVersion) { + TopicConfig registerTopicConfig = topicConfig; + if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission()) + || !PermName.isReadable(this.getBrokerConfig().getBrokerPermission())) { + registerTopicConfig = + new TopicConfig(topicConfig.getTopicName(), topicConfig.getReadQueueNums(), topicConfig.getWriteQueueNums(), + this.brokerConfig.getBrokerPermission()); + } + + ConcurrentMap topicConfigTable = new ConcurrentHashMap(); + topicConfigTable.put(topicConfig.getTopicName(), registerTopicConfig); + TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + topicConfigSerializeWrapper.setDataVersion(dataVersion); + topicConfigSerializeWrapper.setTopicConfigTable(topicConfigTable); + + doRegisterBrokerAll(true, false, topicConfigSerializeWrapper); + } + public synchronized void registerBrokerAll(final boolean checkOrderConfig, boolean oneway, boolean forceRegister) { TopicConfigSerializeWrapper topicConfigWrapper = this.getTopicConfigManager().buildTopicConfigSerializeWrapper(); @@ -788,30 +808,35 @@ public synchronized void registerBrokerAll(final boolean checkOrderConfig, boole this.brokerConfig.getBrokerName(), this.brokerConfig.getBrokerId(), this.brokerConfig.getRegisterBrokerTimeoutMills())) { - List registerBrokerResultList = this.brokerOuterAPI.registerBrokerAll( - this.brokerConfig.getBrokerClusterName(), - this.getBrokerAddr(), - this.brokerConfig.getBrokerName(), - this.brokerConfig.getBrokerId(), - this.getHAServerAddr(), - topicConfigWrapper, - this.filterServerManager.buildNewFilterServerList(), - oneway, - this.brokerConfig.getRegisterBrokerTimeoutMills(), - this.brokerConfig.isCompressedRegister()); - - if (registerBrokerResultList.size() > 0) { - RegisterBrokerResult registerBrokerResult = registerBrokerResultList.get(0); - if (registerBrokerResult != null) { - if (this.updateMasterHAServerAddrPeriodically && registerBrokerResult.getHaServerAddr() != null) { - this.messageStore.updateHaMasterAddress(registerBrokerResult.getHaServerAddr()); - } + doRegisterBrokerAll(checkOrderConfig, oneway, topicConfigWrapper); + } + } + + private void doRegisterBrokerAll(boolean checkOrderConfig, boolean oneway, + TopicConfigSerializeWrapper topicConfigWrapper) { + List registerBrokerResultList = this.brokerOuterAPI.registerBrokerAll( + this.brokerConfig.getBrokerClusterName(), + this.getBrokerAddr(), + this.brokerConfig.getBrokerName(), + this.brokerConfig.getBrokerId(), + this.getHAServerAddr(), + topicConfigWrapper, + this.filterServerManager.buildNewFilterServerList(), + oneway, + this.brokerConfig.getRegisterBrokerTimeoutMills(), + this.brokerConfig.isCompressedRegister()); + + if (registerBrokerResultList.size() > 0) { + RegisterBrokerResult registerBrokerResult = registerBrokerResultList.get(0); + if (registerBrokerResult != null) { + if (this.updateMasterHAServerAddrPeriodically && registerBrokerResult.getHaServerAddr() != null) { + this.messageStore.updateHaMasterAddress(registerBrokerResult.getHaServerAddr()); + } - this.slaveSynchronize.setMasterAddr(registerBrokerResult.getMasterAddr()); + this.slaveSynchronize.setMasterAddr(registerBrokerResult.getMasterAddr()); - if (checkOrderConfig) { - this.getTopicConfigManager().updateOrderTopicConfig(registerBrokerResult.getKvTable()); - } + if (checkOrderConfig) { + this.getTopicConfigManager().updateOrderTopicConfig(registerBrokerResult.getKvTable()); } } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java index 2825a34cd72..4dee01cbff0 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java @@ -29,6 +29,7 @@ import org.apache.rocketmq.common.DataVersion; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.ThreadFactoryImpl; +import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; @@ -125,14 +126,28 @@ public List registerBrokerAll( final List registerBrokerResultList = Lists.newArrayList(); List nameServerAddressList = this.remotingClient.getNameServerAddressList(); if (nameServerAddressList != null && nameServerAddressList.size() > 0) { + + final RegisterBrokerRequestHeader requestHeader = new RegisterBrokerRequestHeader(); + requestHeader.setBrokerAddr(brokerAddr); + requestHeader.setBrokerId(brokerId); + requestHeader.setBrokerName(brokerName); + requestHeader.setClusterName(clusterName); + requestHeader.setHaServerAddr(haServerAddr); + requestHeader.setCompressed(compressed); + + RegisterBrokerBody requestBody = new RegisterBrokerBody(); + requestBody.setTopicConfigSerializeWrapper(topicConfigWrapper); + requestBody.setFilterServerList(filterServerList); + final byte[] body = requestBody.encode(compressed); + final int bodyCrc32 = UtilAll.crc32(body); + requestHeader.setBodyCrc32(bodyCrc32); final CountDownLatch countDownLatch = new CountDownLatch(nameServerAddressList.size()); for (final String namesrvAddr : nameServerAddressList) { brokerOuterExecutor.execute(new Runnable() { @Override public void run() { try { - RegisterBrokerResult result = registerBroker(namesrvAddr, clusterName, brokerAddr, brokerName, brokerId, - haServerAddr, topicConfigWrapper, filterServerList, oneway, timeoutMills, compressed); + RegisterBrokerResult result = registerBroker(namesrvAddr,oneway, timeoutMills,requestHeader,body); if (result != null) { registerBrokerResultList.add(result); } @@ -158,31 +173,14 @@ public void run() { private RegisterBrokerResult registerBroker( final String namesrvAddr, - final String clusterName, - final String brokerAddr, - final String brokerName, - final long brokerId, - final String haServerAddr, - final TopicConfigSerializeWrapper topicConfigWrapper, - final List filterServerList, final boolean oneway, final int timeoutMills, - final boolean compressed + final RegisterBrokerRequestHeader requestHeader, + final byte[] body ) throws RemotingCommandException, MQBrokerException, RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException { - RegisterBrokerRequestHeader requestHeader = new RegisterBrokerRequestHeader(); - requestHeader.setBrokerAddr(brokerAddr); - requestHeader.setBrokerId(brokerId); - requestHeader.setBrokerName(brokerName); - requestHeader.setClusterName(clusterName); - requestHeader.setHaServerAddr(haServerAddr); - requestHeader.setCompressed(compressed); RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.REGISTER_BROKER, requestHeader); - - RegisterBrokerBody requestBody = new RegisterBrokerBody(); - requestBody.setTopicConfigSerializeWrapper(topicConfigWrapper); - requestBody.setFilterServerList(filterServerList); - request.setBody(requestBody.encode(requestHeader.isCompressed())); + request.setBody(body); if (oneway) { try { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index c0a4b202110..1a704a8c6bc 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -212,7 +212,7 @@ public boolean rejectRequest() { return false; } - private RemotingCommand updateAndCreateTopic(ChannelHandlerContext ctx, + private synchronized RemotingCommand updateAndCreateTopic(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); final CreateTopicRequestHeader requestHeader = @@ -246,14 +246,12 @@ private RemotingCommand updateAndCreateTopic(ChannelHandlerContext ctx, this.brokerController.getTopicConfigManager().updateTopicConfig(topicConfig); - if (brokerController.getBrokerConfig().getRegisterNameServerPeriod() == 0) { - this.brokerController.registerBrokerAll(false, true, true); - } + this.brokerController.registerIncrementBrokerData(topicConfig,this.brokerController.getTopicConfigManager().getDataVersion()); return null; } - private RemotingCommand deleteTopic(ChannelHandlerContext ctx, + private synchronized RemotingCommand deleteTopic(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); DeleteTopicRequestHeader requestHeader = @@ -299,7 +297,7 @@ private RemotingCommand getAllTopicConfig(ChannelHandlerContext ctx, RemotingCom return response; } - private RemotingCommand updateBrokerConfig(ChannelHandlerContext ctx, RemotingCommand request) { + private synchronized RemotingCommand updateBrokerConfig(ChannelHandlerContext ctx, RemotingCommand request) { final RemotingCommand response = RemotingCommand.createResponseCommand(null); log.info("updateBrokerConfig called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel())); diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index 7caf83037c2..203431aee1a 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -141,8 +141,6 @@ public class BrokerConfig { * This configurable item defines interval of topics registration of broker to name server. Allowing values are * between 10, 000 and 60, 000 milliseconds. * - * If set to 0, newly created topics will be immediately reported to name servers and interval of periodical - * registration defaults to 10, 000 in milliseconds. */ private int registerNameServerPeriod = 1000 * 30; diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/RegisterBrokerRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/RegisterBrokerRequestHeader.java index 7ed7a403d6b..19175b04b3e 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/RegisterBrokerRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/RegisterBrokerRequestHeader.java @@ -38,6 +38,8 @@ public class RegisterBrokerRequestHeader implements CommandCustomHeader { private boolean compressed; + private Integer bodyCrc32 = 0; + public void checkFields() throws RemotingCommandException { } @@ -88,4 +90,12 @@ public boolean isCompressed() { public void setCompressed(boolean compressed) { this.compressed = compressed; } + + public Integer getBodyCrc32() { + return bodyCrc32; + } + + public void setBodyCrc32(Integer bodyCrc32) { + this.bodyCrc32 = bodyCrc32; + } } diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java index 236e6a12ca0..467078c44f8 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java @@ -24,6 +24,7 @@ import org.apache.rocketmq.common.MQVersion; import org.apache.rocketmq.common.MQVersion.Version; import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.help.FAQUrl; import org.apache.rocketmq.logging.InternalLogger; @@ -196,6 +197,12 @@ public RemotingCommand registerBrokerWithFilterServer(ChannelHandlerContext ctx, final RegisterBrokerRequestHeader requestHeader = (RegisterBrokerRequestHeader) request.decodeCommandCustomHeader(RegisterBrokerRequestHeader.class); + if (!checksum(ctx, request, requestHeader)) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("crc32 not match"); + return response; + } + RegisterBrokerBody registerBrokerBody = new RegisterBrokerBody(); if (request.getBody() != null) { @@ -230,6 +237,19 @@ public RemotingCommand registerBrokerWithFilterServer(ChannelHandlerContext ctx, return response; } + private boolean checksum(ChannelHandlerContext ctx, RemotingCommand request, + RegisterBrokerRequestHeader requestHeader) { + if (requestHeader.getBodyCrc32() != 0) { + final int crc32 = UtilAll.crc32(request.getBody()); + if (crc32 != requestHeader.getBodyCrc32()) { + log.warn(String.format("receive registerBroker request,crc32 not match,from %s", + RemotingHelper.parseChannelRemoteAddr(ctx.channel()))); + return false; + } + } + return true; + } + public RemotingCommand queryBrokerTopicConfig(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(QueryDataVersionResponseHeader.class); @@ -261,6 +281,12 @@ public RemotingCommand registerBroker(ChannelHandlerContext ctx, final RegisterBrokerRequestHeader requestHeader = (RegisterBrokerRequestHeader) request.decodeCommandCustomHeader(RegisterBrokerRequestHeader.class); + if (!checksum(ctx, request, requestHeader)) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("crc32 not match"); + return response; + } + TopicConfigSerializeWrapper topicConfigWrapper; if (request.getBody() != null) { topicConfigWrapper = TopicConfigSerializeWrapper.decode(request.getBody(), TopicConfigSerializeWrapper.class); From d2e11c11669fdd2e9e6c1f44247b8b1ad989b6d6 Mon Sep 17 00:00:00 2001 From: Li Zhanhui Date: Thu, 28 Jun 2018 13:40:31 +0800 Subject: [PATCH 52/73] Fix typo --- .../client/impl/consumer/DefaultMQPushConsumerImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java index 85a40a9ddce..393ef92c996 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java @@ -308,12 +308,12 @@ public void onSuccess(PullResult pullResult) { DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullTPS(pullRequest.getConsumerGroup(), pullRequest.getMessageQueue().getTopic(), pullResult.getMsgFoundList().size()); - boolean dispathToConsume = processQueue.putMessage(pullResult.getMsgFoundList()); + boolean dispatchToConsume = processQueue.putMessage(pullResult.getMsgFoundList()); DefaultMQPushConsumerImpl.this.consumeMessageService.submitConsumeRequest( pullResult.getMsgFoundList(), processQueue, pullRequest.getMessageQueue(), - dispathToConsume); + dispatchToConsume); if (DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval() > 0) { DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest, From 650f46be836075e9f8419093be7819c57dca1a91 Mon Sep 17 00:00:00 2001 From: Li Zhanhui Date: Thu, 28 Jun 2018 17:27:43 +0800 Subject: [PATCH 53/73] Exclude RAT checking on files in the target directories --- pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pom.xml b/pom.xml index 604c775539d..6fe4f73f1b9 100644 --- a/pom.xml +++ b/pom.xml @@ -254,6 +254,8 @@ bin/README.md .github/* src/test/resources/certs/* + */target/** + */*.iml From 94f3b5a41b1af7054f59343c033d4df622af95ae Mon Sep 17 00:00:00 2001 From: what-a-good-jungle <353187194@qq.com> Date: Thu, 5 Jul 2018 20:01:49 +0800 Subject: [PATCH 54/73] [ISSUE #353]Optimzation For Unit Test (#356) --- .../apache/rocketmq/filter/ParserTest.java | 15 ++++++++------ pom.xml | 4 +++- .../rocketmq/store/DefaultMessageStore.java | 2 +- .../rocketmq/store/ConsumeQueueTest.java | 2 -- .../test/client/rmq/RMQAsyncSendProducer.java | 20 ++++++++++--------- .../test/client/rmq/RMQNormalProducer.java | 6 +++--- .../clientinterface/AbstractMQProducer.java | 4 ++-- .../test/clientinterface/MQProducer.java | 4 ++-- .../{SendResult.java => ResultWrapper.java} | 2 +- 9 files changed, 32 insertions(+), 27 deletions(-) rename test/src/main/java/org/apache/rocketmq/test/sendresult/{SendResult.java => ResultWrapper.java} (98%) diff --git a/filter/src/test/java/org/apache/rocketmq/filter/ParserTest.java b/filter/src/test/java/org/apache/rocketmq/filter/ParserTest.java index 36ef2714fcc..115632faa0a 100644 --- a/filter/src/test/java/org/apache/rocketmq/filter/ParserTest.java +++ b/filter/src/test/java/org/apache/rocketmq/filter/ParserTest.java @@ -84,14 +84,17 @@ public void testParse_decimalOverFlow() { @Test public void testParse_floatOverFlow() { try { - String str = "1"; - for (int i = 0; i < 2048; i++) { - str += "111111111111111111111111111111111111111111111111111"; + StringBuffer sb = new StringBuffer(210000); + sb.append("1"); + for (int i = 0; i < 2048; i ++) { + sb.append("111111111111111111111111111111111111111111111111111"); } - str += "."; - for (int i = 0; i < 2048; i++) { - str += "111111111111111111111111111111111111111111111111111"; + sb.append("."); + for (int i = 0; i < 2048; i ++) { + sb.append("111111111111111111111111111111111111111111111111111"); } + String str = sb.toString(); + SelectorParser.parse("a > " + str); diff --git a/pom.xml b/pom.xml index 6fe4f73f1b9..72cd3496399 100644 --- a/pom.xml +++ b/pom.xml @@ -157,7 +157,7 @@ - + true @@ -315,6 +315,7 @@ maven-surefire-plugin 2.19.1 + 1 1 true @@ -333,6 +334,7 @@ maven-failsafe-plugin 2.19.1 + 1 **/NormalMsgDelayIT.java diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index 4fc7412b1fd..1ade7c2838e 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -247,7 +247,7 @@ public void shutdown() { try { - Thread.sleep(1000 * 3); + Thread.sleep(1000); } catch (InterruptedException e) { log.error("shutdown Exception, ", e); } diff --git a/store/src/test/java/org/apache/rocketmq/store/ConsumeQueueTest.java b/store/src/test/java/org/apache/rocketmq/store/ConsumeQueueTest.java index b7d38f8c78f..7e01b8513c9 100644 --- a/store/src/test/java/org/apache/rocketmq/store/ConsumeQueueTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/ConsumeQueueTest.java @@ -212,8 +212,6 @@ public void dispatch(DispatchRequest request) { try { try { putMsg(master); - // wait build consume queue - Thread.sleep(1000); } catch (Exception e) { e.printStackTrace(); assertThat(Boolean.FALSE).isTrue(); diff --git a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQAsyncSendProducer.java b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQAsyncSendProducer.java index 2f8af68b31e..9907cac8008 100644 --- a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQAsyncSendProducer.java +++ b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQAsyncSendProducer.java @@ -26,10 +26,11 @@ import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.client.producer.MessageQueueSelector; import org.apache.rocketmq.client.producer.SendCallback; +import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.test.clientinterface.AbstractMQProducer; -import org.apache.rocketmq.test.sendresult.SendResult; +import org.apache.rocketmq.test.sendresult.ResultWrapper; import org.apache.rocketmq.test.util.RandomUtil; import org.apache.rocketmq.test.util.TestUtil; @@ -39,20 +40,19 @@ public class RMQAsyncSendProducer extends AbstractMQProducer { private String nsAddr = null; private DefaultMQProducer producer = null; private SendCallback sendCallback = null; - private List successSendResult = Collections - .synchronizedList(new ArrayList()); - private AtomicInteger exceptionMsgCount = new AtomicInteger( - 0); + private List successSendResult = Collections.synchronizedList(new ArrayList()); + private AtomicInteger exceptionMsgCount = new AtomicInteger(0); private int msgSize = 0; public RMQAsyncSendProducer(String nsAddr, String topic) { super(topic); this.nsAddr = nsAddr; sendCallback = new SendCallback() { - public void onSuccess(org.apache.rocketmq.client.producer.SendResult sendResult) { + @Override + public void onSuccess(SendResult sendResult) { successSendResult.add(sendResult); } - + @Override public void onException(Throwable throwable) { exceptionMsgCount.getAndIncrement(); } @@ -66,7 +66,7 @@ public int getSuccessMsgCount() { return successSendResult.size(); } - public List getSuccessSendResult() { + public List getSuccessSendResult() { return successSendResult; } @@ -94,10 +94,12 @@ private void start() { } } - public SendResult send(Object msg, Object arg) { + @Override + public ResultWrapper send(Object msg, Object arg) { return null; } + @Override public void shutdown() { producer.shutdown(); } diff --git a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQNormalProducer.java b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQNormalProducer.java index 42949339e33..6bbec6cb948 100644 --- a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQNormalProducer.java +++ b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQNormalProducer.java @@ -26,7 +26,7 @@ import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.test.clientinterface.AbstractMQProducer; -import org.apache.rocketmq.test.sendresult.SendResult; +import org.apache.rocketmq.test.sendresult.ResultWrapper; public class RMQNormalProducer extends AbstractMQProducer { private static Logger logger = Logger.getLogger(RMQNormalProducer.class); @@ -91,7 +91,7 @@ public void start() { } } - public SendResult send(Object msg, Object orderKey) { + public ResultWrapper send(Object msg, Object orderKey) { org.apache.rocketmq.client.producer.SendResult metaqResult = null; Message message = (Message) msg; try { @@ -132,7 +132,7 @@ public void send(List msgs, MessageQueue mq) { } } - public SendResult sendMQ(Message msg, MessageQueue mq) { + public ResultWrapper sendMQ(Message msg, MessageQueue mq) { org.apache.rocketmq.client.producer.SendResult metaqResult = null; try { long start = System.currentTimeMillis(); diff --git a/test/src/main/java/org/apache/rocketmq/test/clientinterface/AbstractMQProducer.java b/test/src/main/java/org/apache/rocketmq/test/clientinterface/AbstractMQProducer.java index 8201e6313c3..df6abfc662b 100644 --- a/test/src/main/java/org/apache/rocketmq/test/clientinterface/AbstractMQProducer.java +++ b/test/src/main/java/org/apache/rocketmq/test/clientinterface/AbstractMQProducer.java @@ -21,14 +21,14 @@ import java.util.List; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.test.client.rmq.RMQNormalProducer; -import org.apache.rocketmq.test.sendresult.SendResult; +import org.apache.rocketmq.test.sendresult.ResultWrapper; import org.apache.rocketmq.test.util.RandomUtil; import org.apache.rocketmq.test.util.TestUtil; public abstract class AbstractMQProducer extends MQCollector implements MQProducer { protected String topic = null; - protected SendResult sendResult = new SendResult(); + protected ResultWrapper sendResult = new ResultWrapper(); protected boolean startSuccess = false; protected String producerGroupName = null; protected String producerInstanceName = null; diff --git a/test/src/main/java/org/apache/rocketmq/test/clientinterface/MQProducer.java b/test/src/main/java/org/apache/rocketmq/test/clientinterface/MQProducer.java index 795457d1992..e9ed0d3fdfc 100644 --- a/test/src/main/java/org/apache/rocketmq/test/clientinterface/MQProducer.java +++ b/test/src/main/java/org/apache/rocketmq/test/clientinterface/MQProducer.java @@ -17,10 +17,10 @@ package org.apache.rocketmq.test.clientinterface; -import org.apache.rocketmq.test.sendresult.SendResult; +import org.apache.rocketmq.test.sendresult.ResultWrapper; public interface MQProducer { - SendResult send(Object msg, Object arg); + ResultWrapper send(Object msg, Object arg); void setDebug(); diff --git a/test/src/main/java/org/apache/rocketmq/test/sendresult/SendResult.java b/test/src/main/java/org/apache/rocketmq/test/sendresult/ResultWrapper.java similarity index 98% rename from test/src/main/java/org/apache/rocketmq/test/sendresult/SendResult.java rename to test/src/main/java/org/apache/rocketmq/test/sendresult/ResultWrapper.java index d53ee7da25c..9fe31463e44 100644 --- a/test/src/main/java/org/apache/rocketmq/test/sendresult/SendResult.java +++ b/test/src/main/java/org/apache/rocketmq/test/sendresult/ResultWrapper.java @@ -17,7 +17,7 @@ package org.apache.rocketmq.test.sendresult; -public class SendResult { +public class ResultWrapper { private boolean sendResult = false; private String msgId = null; private Exception sendException = null; From 840e04c7e148674a270481a57304c126e73c1674 Mon Sep 17 00:00:00 2001 From: what-a-good-jungle <353187194@qq.com> Date: Fri, 13 Jul 2018 11:32:13 +0800 Subject: [PATCH 55/73] [ISSUE #354]Add thread-sleep for rebalance before producer send in the integration-test (#355) --- .../client/consumer/balance/NormalMsgDynamicBalanceIT.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/balance/NormalMsgDynamicBalanceIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/balance/NormalMsgDynamicBalanceIT.java index ef126213fce..0467eefb33c 100644 --- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/balance/NormalMsgDynamicBalanceIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/balance/NormalMsgDynamicBalanceIT.java @@ -23,6 +23,7 @@ import org.apache.rocketmq.test.client.rmq.RMQNormalProducer; import org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener; import org.apache.rocketmq.test.util.MQWait; +import org.apache.rocketmq.test.util.TestUtils; import org.apache.rocketmq.test.util.VerifyUtils; import org.junit.After; import org.junit.Assert; @@ -54,6 +55,7 @@ public void testTwoConsumerAndCrashOne() { RMQNormalConsumer consumer1 = getConsumer(nsAddr, topic, "*", new RMQNormalListener()); RMQNormalConsumer consumer2 = getConsumer(nsAddr, consumer1.getConsumerGroup(), topic, "*", new RMQNormalListener()); + TestUtils.waitForSeconds(waitTime); producer.send(msgSize); @@ -84,6 +86,7 @@ public void test3ConsumerAndCrashOne() { "*", new RMQNormalListener()); RMQNormalConsumer consumer3 = getConsumer(nsAddr, consumer1.getConsumerGroup(), topic, "*", new RMQNormalListener()); + TestUtils.waitForSeconds(waitTime); producer.send(msgSize); From b75af6b0c253f7b9c39e812fd2e7938a99b1495d Mon Sep 17 00:00:00 2001 From: what-a-good-jungle <353187194@qq.com> Date: Sat, 14 Jul 2018 09:55:59 +0800 Subject: [PATCH 56/73] [ROCKETMQ-353] Add sendMessageCommand and consumeMessageCommand (#332) --- .../tools/command/MQAdminStartup.java | 4 + .../message/ConsumeMessageCommand.java | 292 ++++++++++++++++++ .../command/message/SendMessageCommand.java | 156 ++++++++++ .../message/ConsumeMessageCommandTest.java | 112 +++++++ .../message/SendMessageCommandTest.java | 90 ++++++ 5 files changed, 654 insertions(+) create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java create mode 100644 tools/src/test/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommandTest.java create mode 100644 tools/src/test/java/org/apache/rocketmq/tools/command/message/SendMessageCommandTest.java diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java index d3342e818dd..c189e866d3c 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java @@ -48,12 +48,14 @@ import org.apache.rocketmq.tools.command.consumer.StartMonitoringSubCommand; import org.apache.rocketmq.tools.command.consumer.UpdateSubGroupSubCommand; import org.apache.rocketmq.tools.command.message.CheckMsgSendRTCommand; +import org.apache.rocketmq.tools.command.message.ConsumeMessageCommand; import org.apache.rocketmq.tools.command.message.PrintMessageByQueueCommand; import org.apache.rocketmq.tools.command.message.PrintMessageSubCommand; import org.apache.rocketmq.tools.command.message.QueryMsgByIdSubCommand; import org.apache.rocketmq.tools.command.message.QueryMsgByKeySubCommand; import org.apache.rocketmq.tools.command.message.QueryMsgByOffsetSubCommand; import org.apache.rocketmq.tools.command.message.QueryMsgByUniqueKeySubCommand; +import org.apache.rocketmq.tools.command.message.SendMessageCommand; import org.apache.rocketmq.tools.command.namesrv.DeleteKvConfigCommand; import org.apache.rocketmq.tools.command.namesrv.GetNamesrvConfigCommand; import org.apache.rocketmq.tools.command.namesrv.UpdateKvConfigCommand; @@ -193,6 +195,8 @@ public static void initCommand() { initCommand(new GetBrokerConfigCommand()); initCommand(new QueryConsumeQueueCommand()); + initCommand(new SendMessageCommand()); + initCommand(new ConsumeMessageCommand()); } private static void initLogback() throws JoranException { diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java new file mode 100644 index 00000000000..51892677445 --- /dev/null +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java @@ -0,0 +1,292 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.tools.command.message; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.tools.command.SubCommand; +import org.apache.rocketmq.tools.command.SubCommandException; + +import java.util.Set; + +public class ConsumeMessageCommand implements SubCommand { + + private String topic = null; + private long messageCount = 128; + private DefaultMQPullConsumer defaultMQPullConsumer; + + + public enum ConsumeType { + /** + * Topic only + */ + DEFAULT, + /** + * Topic brokerName queueId set + */ + BYQUEUE, + /** + * Topic brokerName queueId offset set + */ + BYOFFSET + } + + private static long timestampFormat(final String value) { + long timestamp; + try { + timestamp = Long.parseLong(value); + } catch (NumberFormatException e) { + timestamp = UtilAll.parseDate(value, UtilAll.YYYY_MM_DD_HH_MM_SS_SSS).getTime(); + } + + return timestamp; + } + @Override + public String commandName() { + return "consumeMessage"; + } + + @Override + public String commandDesc() { + return "Consume message"; + } + + @Override + public Options buildCommandlineOptions(final Options options) { + Option opt = new Option("t", "topic", true, "Topic name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("b", "brokerName", true, "Broker name"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("i", "queueId", true, "Queue Id"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("o", "offset", true, "Queue offset"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("g", "consumerGroup", true, "Consumer group name"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("s", "beginTimestamp ", true, + "Begin timestamp[currentTimeMillis|yyyy-MM-dd#HH:mm:ss:SSS]"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("e", "endTimestamp ", true, + "End timestamp[currentTimeMillis|yyyy-MM-dd#HH:mm:ss:SSS]"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("c", "MessageNumber", true, "Number of message to be consumed"); + opt.setRequired(false); + options.addOption(opt); + + + return options; + + } + + @Override + public void execute(final CommandLine commandLine, final Options options, RPCHook rpcHook) throws SubCommandException { + if (defaultMQPullConsumer == null) { + defaultMQPullConsumer = new DefaultMQPullConsumer(MixAll.TOOLS_CONSUMER_GROUP, rpcHook); + } + defaultMQPullConsumer.setInstanceName(Long.toString(System.currentTimeMillis())); + + long offset = 0; + long timeValueEnd = 0; + long timeValueBegin = 0; + String queueId = null; + String brokerName = null; + ConsumeType consumeType = ConsumeType.DEFAULT; + + try { + /* Group name must be set before consumer start */ + if (commandLine.hasOption('g')) { + String consumerGroup = commandLine.getOptionValue('b').trim(); + defaultMQPullConsumer.setConsumerGroup(consumerGroup); + } + + defaultMQPullConsumer.start(); + + topic = commandLine.getOptionValue('t').trim(); + + if (commandLine.hasOption('c')) { + messageCount = Long.parseLong(commandLine.getOptionValue('c').trim()); + if (messageCount <= 0) { + System.out.print("please input a positive messageNumber!"); + return; + } + } + if (commandLine.hasOption('b')) { + brokerName = commandLine.getOptionValue('b').trim(); + + } + if (commandLine.hasOption('i')) { + if (!commandLine.hasOption('b')) { + System.out.print("Please set the brokerName before queueId!"); + return; + } + queueId = commandLine.getOptionValue('i').trim(); + + consumeType = ConsumeType.BYQUEUE; + } + if (commandLine.hasOption('o')) { + if (consumeType != ConsumeType.BYQUEUE) { + System.out.print("please set queueId before offset!"); + return; + } + offset = Long.parseLong(commandLine.getOptionValue('o').trim()); + consumeType = ConsumeType.BYOFFSET; + } + + if (commandLine.hasOption('s')) { + String timestampStr = commandLine.getOptionValue('s').trim(); + timeValueBegin = timestampFormat(timestampStr); + } + if (commandLine.hasOption('e')) { + String timestampStr = commandLine.getOptionValue('e').trim(); + timeValueEnd = timestampFormat(timestampStr); + } + + switch (consumeType) { + case DEFAULT: + executeDefault(timeValueBegin, timeValueEnd); + break; + case BYOFFSET: + executeByCondition(brokerName, queueId, offset, timeValueBegin, timeValueEnd); + break; + case BYQUEUE: + executeByCondition(brokerName, queueId, 0, timeValueBegin, timeValueEnd); + break; + default: + System.out.print("Unknown type of consume!"); + break; + } + + } catch (Exception e) { + throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); + } finally { + defaultMQPullConsumer.shutdown(); + } + } + + private void pullMessageByQueue(MessageQueue mq, long minOffset, long maxOffset) { + READQ: + for (long offset = minOffset; offset <= maxOffset; ) { + PullResult pullResult = null; + try { + pullResult = defaultMQPullConsumer.pull(mq, "*", offset, (int)(maxOffset - offset + 1)); + } catch (Exception e) { + e.printStackTrace(); + } + if (pullResult != null) { + offset = pullResult.getNextBeginOffset(); + switch (pullResult.getPullStatus()) { + case FOUND: + System.out.print("Consume ok\n"); + PrintMessageByQueueCommand.printMessage(pullResult.getMsgFoundList(), "UTF-8", + true, true); + break; + case NO_MATCHED_MSG: + System.out.printf("%s no matched msg. status=%s, offset=%s\n", mq, pullResult.getPullStatus(), + offset); + break; + case NO_NEW_MSG: + case OFFSET_ILLEGAL: + System.out.printf("%s print msg finished. status=%s, offset=%s\n", mq, + pullResult.getPullStatus(), offset); + break READQ; + default: + break; + } + } + } + } + + private void executeDefault(long timeValueBegin, long timeValueEnd) { + try { + Set mqs = defaultMQPullConsumer.fetchSubscribeMessageQueues(topic); + long countLeft = messageCount; + for (MessageQueue mq : mqs) { + if (countLeft == 0) { + return; + } + long minOffset = defaultMQPullConsumer.minOffset(mq); + long maxOffset = defaultMQPullConsumer.maxOffset(mq); + if (timeValueBegin > 0) { + minOffset = defaultMQPullConsumer.searchOffset(mq, timeValueBegin); + } + if (timeValueEnd > 0) { + maxOffset = defaultMQPullConsumer.searchOffset(mq, timeValueEnd); + } + if (maxOffset - minOffset > countLeft) { + System.out.printf("The older %d message of the %d queue will be provided\n", countLeft, mq.getQueueId()); + maxOffset = minOffset + countLeft - 1; + countLeft = 0; + } else { + countLeft = countLeft - (maxOffset - minOffset) - 1; + } + + pullMessageByQueue(mq, minOffset, maxOffset); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + private void executeByCondition(String brokerName, String queueId, long offset, long timeValueBegin, long timeValueEnd) { + MessageQueue mq = new MessageQueue(topic, brokerName, Integer.parseInt(queueId)); + try { + long minOffset = defaultMQPullConsumer.minOffset(mq); + long maxOffset = defaultMQPullConsumer.maxOffset(mq); + if (timeValueBegin > 0) { + minOffset = defaultMQPullConsumer.searchOffset(mq, timeValueBegin); + } + if (timeValueEnd > 0) { + maxOffset = defaultMQPullConsumer.searchOffset(mq, timeValueEnd); + } + if (offset > maxOffset) { + System.out.printf("%s no matched msg, offset=%s\n", mq, offset); + return; + } + minOffset = minOffset > offset ? minOffset : offset; + if (maxOffset - minOffset > messageCount) { + System.out.printf("The oldler %d message will be provided\n", messageCount); + maxOffset = minOffset + messageCount - 1; + } + + pullMessageByQueue(mq, minOffset, maxOffset); + } catch (Exception e) { + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java new file mode 100644 index 00000000000..e4921c6f04e --- /dev/null +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.tools.command.message; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.tools.command.SubCommand; +import org.apache.rocketmq.tools.command.SubCommandException; + +public class SendMessageCommand implements SubCommand { + + private DefaultMQProducer producer; + + @Override + public String commandName() { + return "sendMessage"; + } + + @Override + public String commandDesc() { + return "Send a message"; + } + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("t", "topic", true, "Topic name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("p", "body", true, "UTF-8 string format of the message body"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("k", "key", true, "Message keys"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("c", "tags", true, "Message tags"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("b", "broker", true, "Send message to target broker"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("i", "qid", true, "Send message to target queue"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + private DefaultMQProducer createProducer(RPCHook rpcHook) { + if (this.producer != null) { + return producer; + } else { + producer = new DefaultMQProducer(rpcHook); + producer.setProducerGroup(Long.toString(System.currentTimeMillis())); + return producer; + } + } + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { + Message msg = null; + String topic = commandLine.getOptionValue('t').trim(); + String body = commandLine.getOptionValue('p').trim(); + String tag = null; + String keys = null; + String brokerName = null; + int queueId = -1; + try { + if (commandLine.hasOption('k')) { + keys = commandLine.getOptionValue('k').trim(); + } + if (commandLine.hasOption('c')) { + tag = commandLine.getOptionValue('c').trim(); + } + if (commandLine.hasOption('b')) { + brokerName = commandLine.getOptionValue('b').trim(); + } + if (commandLine.hasOption('i')) { + if (!commandLine.hasOption('b')) { + System.out.print("Broker name must be set if the queue is chosen!"); + return; + } else { + queueId = Integer.parseInt(commandLine.getOptionValue('i').trim()); + } + } + msg = new Message(topic, tag, keys, body.getBytes("utf-8")); + } catch (Exception e) { + throw new RuntimeException(this.getClass().getSimpleName() + " command failed", e); + } + + DefaultMQProducer producer = this.createProducer(rpcHook); + SendResult result; + try { + producer.start(); + if (brokerName != null && queueId > -1) { + MessageQueue messageQueue = new MessageQueue(topic, brokerName, queueId); + result = producer.send(msg, messageQueue); + } else { + result = producer.send(msg); + } + + } catch (Exception e) { + throw new RuntimeException(this.getClass().getSimpleName() + " command failed", e); + } finally { + producer.shutdown(); + } + + System.out.printf("%-32s %-4s %-20s %s%n", + "#Broker Name", + "#QID", + "#Send Result", + "#MsgId" + ); + + if (result != null) { + System.out.printf("%-32s %-4s %-20s %s%n", + result.getMessageQueue().getBrokerName(), + result.getMessageQueue().getQueueId(), + result.getSendStatus(), + result.getMsgId() + ); + } else { + System.out.printf("%-32s %-4s %-20s %s%n", + "Unknown", + "Unknown", + "Failed", + "None" + ); + } + } +} \ No newline at end of file diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommandTest.java new file mode 100644 index 00000000000..9a5998e29e5 --- /dev/null +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommandTest.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.tools.command.message; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; +import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.PullStatus; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.srvutil.ServerUtil; +import org.apache.rocketmq.tools.command.SubCommandException; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class ConsumeMessageCommandTest { + private static ConsumeMessageCommand consumeMessageCommand; + + @BeforeClass + public static void init() throws MQClientException, RemotingException, MQBrokerException, InterruptedException, + NoSuchFieldException, IllegalAccessException{ + consumeMessageCommand = new ConsumeMessageCommand(); + DefaultMQPullConsumer defaultMQPullConsumer = mock(DefaultMQPullConsumer.class); + MessageExt msg = new MessageExt(); + msg.setBody(new byte[]{'a'}); + List msgFoundList = new ArrayList<>(); + msgFoundList.add(msg); + final PullResult pullResult = new PullResult(PullStatus.FOUND,2, 0, 1, msgFoundList); + + when(defaultMQPullConsumer.pull(any(MessageQueue.class), anyString(), anyLong(), anyInt())).thenReturn(pullResult); + when(defaultMQPullConsumer.minOffset(any(MessageQueue.class))).thenReturn(Long.valueOf(0)); + when(defaultMQPullConsumer.maxOffset(any(MessageQueue.class))).thenReturn(Long.valueOf(1)); + + final Set mqList = new HashSet<>(); + mqList.add(new MessageQueue()); + when(defaultMQPullConsumer.fetchSubscribeMessageQueues(anyString())).thenReturn(mqList); + + Field producerField = ConsumeMessageCommand.class.getDeclaredField("defaultMQPullConsumer"); + producerField.setAccessible(true); + producerField.set(consumeMessageCommand,defaultMQPullConsumer); + } + @AfterClass + public static void terminate() { + } + + @Test + public void testExecuteDefault() throws SubCommandException { + PrintStream out = System.out; + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(bos)); + Options options = ServerUtil.buildCommandlineOptions(new Options()); + String[] subargs = new String[] {"-t mytopic", "-n localhost:9876"}; + CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + consumeMessageCommand.commandName(), + subargs, consumeMessageCommand.buildCommandlineOptions(options), new PosixParser()); + consumeMessageCommand.execute(commandLine, options, null); + + System.setOut(out); + String s = new String(bos.toByteArray()); + Assert.assertTrue(s.contains("Consume ok")); + } + + @Test + public void testExecuteByCondition() throws SubCommandException { + PrintStream out = System.out; + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(bos)); + Options options = ServerUtil.buildCommandlineOptions(new Options()); + + String[] subargs = new String[] {"-t mytopic","-b localhost","-i 0", "-n localhost:9876"}; + CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + consumeMessageCommand.commandName(), subargs, consumeMessageCommand.buildCommandlineOptions(options), new PosixParser()); + consumeMessageCommand.execute(commandLine, options, null); + System.setOut(out); + String s = new String(bos.toByteArray()); + Assert.assertTrue(s.contains("Consume ok")); + } +} \ No newline at end of file diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/message/SendMessageCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/message/SendMessageCommandTest.java new file mode 100644 index 00000000000..e4c6673d080 --- /dev/null +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/message/SendMessageCommandTest.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.tools.command.message; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.producer.SendStatus; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.srvutil.ServerUtil; +import org.apache.rocketmq.tools.command.SubCommandException; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.lang.reflect.Field; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class SendMessageCommandTest { + + private static SendMessageCommand sendMessageCommand = new SendMessageCommand(); + + @BeforeClass + public static void init() throws MQClientException, RemotingException, InterruptedException, MQBrokerException, NoSuchFieldException, IllegalAccessException { + + DefaultMQProducer defaultMQProducer = mock(DefaultMQProducer.class); + SendResult sendResult = new SendResult(); + sendResult.setMessageQueue(new MessageQueue()); + sendResult.getMessageQueue().setBrokerName("broker1"); + sendResult.getMessageQueue().setQueueId(1); + sendResult.setSendStatus(SendStatus.SEND_OK); + sendResult.setMsgId("fgwejigherughwueyutyu4t4343t43"); + + when(defaultMQProducer.send(any(Message.class))).thenReturn(sendResult); + when(defaultMQProducer.send(any(Message.class), any(MessageQueue.class))).thenReturn(sendResult); + + Field producerField = SendMessageCommand.class.getDeclaredField("producer"); + producerField.setAccessible(true); + producerField.set(sendMessageCommand, defaultMQProducer); + } + + @AfterClass + public static void terminate() { + } + + @Test + public void testExecute() throws SubCommandException { + PrintStream out = System.out; + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(bos)); + Options options = ServerUtil.buildCommandlineOptions(new Options()); + String[] subargs = new String[] {"-t mytopic","-p 'send message test'","-c tagA","-k order-16546745756"}; + CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + sendMessageCommand.commandName(), subargs, sendMessageCommand.buildCommandlineOptions(options), new PosixParser()); + sendMessageCommand.execute(commandLine, options, null); + + subargs = new String[] {"-t mytopic","-p 'send message test'","-c tagA","-k order-16546745756","-b brokera","-i 1"}; + commandLine = ServerUtil.parseCmdLine("mqadmin " + sendMessageCommand.commandName(), subargs, sendMessageCommand.buildCommandlineOptions(options), new PosixParser()); + sendMessageCommand.execute(commandLine, options, null); + System.setOut(out); + String s = new String(bos.toByteArray()); + Assert.assertTrue(s.contains("SEND_OK")); + } +} From 6ae619c403175d54550c7f8e0c6d49f78c5f546a Mon Sep 17 00:00:00 2001 From: Jaskey Date: Sat, 14 Jul 2018 10:40:46 +0800 Subject: [PATCH 57/73] Invoke callback at once when channel is close (#95) --- .../client/impl/MQClientAPIImplTest.java | 2 +- .../remoting/netty/NettyRemotingAbstract.java | 51 +++++++++++++------ .../remoting/netty/NettyRemotingClient.java | 2 +- .../remoting/netty/ResponseFuture.java | 20 ++++++-- 4 files changed, 53 insertions(+), 22 deletions(-) diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java index bf019618d9c..c13e75c206c 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java @@ -162,7 +162,7 @@ public void testSendMessageAsync_Success() throws RemotingException, Interrupted public Object answer(InvocationOnMock mock) throws Throwable { InvokeCallback callback = mock.getArgument(3); RemotingCommand request = mock.getArgument(1); - ResponseFuture responseFuture = new ResponseFuture(request.getOpaque(), 3 * 1000, null, null); + ResponseFuture responseFuture = new ResponseFuture(null,request.getOpaque(), 3 * 1000, null, null); responseFuture.setResponseCommand(createSuccessResponse(request)); callback.operationComplete(responseFuture); return null; diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java index 7c414e9dd65..45ca7304c8f 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java @@ -364,7 +364,7 @@ public RemotingCommand invokeSyncImpl(final Channel channel, final RemotingComma final int opaque = request.getOpaque(); try { - final ResponseFuture responseFuture = new ResponseFuture(opaque, timeoutMillis, null, null); + final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, timeoutMillis, null, null); this.responseTable.put(opaque, responseFuture); final SocketAddress addr = channel.remoteAddress(); channel.writeAndFlush(request).addListener(new ChannelFutureListener() { @@ -407,8 +407,7 @@ public void invokeAsyncImpl(final Channel channel, final RemotingCommand request boolean acquired = this.semaphoreAsync.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS); if (acquired) { final SemaphoreReleaseOnlyOnce once = new SemaphoreReleaseOnlyOnce(this.semaphoreAsync); - - final ResponseFuture responseFuture = new ResponseFuture(opaque, timeoutMillis, invokeCallback, once); + final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, timeoutMillis, invokeCallback, once); this.responseTable.put(opaque, responseFuture); try { channel.writeAndFlush(request).addListener(new ChannelFutureListener() { @@ -417,20 +416,8 @@ public void operationComplete(ChannelFuture f) throws Exception { if (f.isSuccess()) { responseFuture.setSendRequestOK(true); return; - } else { - responseFuture.setSendRequestOK(false); - } - - responseFuture.putResponse(null); - responseTable.remove(opaque); - try { - executeInvokeCallback(responseFuture); - } catch (Throwable e) { - log.warn("excute callback in writeAndFlush addListener, and callback throw", e); - } finally { - responseFuture.release(); } - + requestFail(opaque); log.warn("send a request command to channel <{}> failed.", RemotingHelper.parseChannelRemoteAddr(channel)); } }); @@ -455,6 +442,38 @@ public void operationComplete(ChannelFuture f) throws Exception { } } + private void requestFail(final int opaque) { + ResponseFuture responseFuture = responseTable.remove(opaque); + if (responseFuture != null) { + responseFuture.setSendRequestOK(false); + responseFuture.putResponse(null); + try { + executeInvokeCallback(responseFuture); + } catch (Throwable e) { + log.warn("execute callback in requestFail, and callback throw", e); + } finally { + responseFuture.release(); + } + } + } + + /** + * mark the request of the specified channel as fail and to invoke fail callback immediately + * @param channel the channel which is close already + */ + protected void failFast(final Channel channel) { + Iterator> it = responseTable.entrySet().iterator(); + while (it.hasNext()) { + Entry entry = it.next(); + if (entry.getValue().getProcessChannel() == channel) { + Integer opaque = entry.getKey(); + if (opaque != null) { + requestFail(opaque); + } + } + } + } + public void invokeOnewayImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis) throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException { request.markOnewayRPC(); diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java index 7cdfb80accb..241f2b07ad1 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java @@ -660,7 +660,7 @@ public void close(ChannelHandlerContext ctx, ChannelPromise promise) throws Exce log.info("NETTY CLIENT PIPELINE: CLOSE {}", remoteAddress); closeChannel(ctx.channel()); super.close(ctx, promise); - + NettyRemotingClient.this.failFast(ctx.channel()); if (NettyRemotingClient.this.channelEventListener != null) { NettyRemotingClient.this.putNettyEvent(new NettyEvent(NettyEventType.CLOSE, remoteAddress, ctx.channel())); } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java index 1157c450283..5f4c8c69502 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java @@ -16,6 +16,7 @@ */ package org.apache.rocketmq.remoting.netty; +import io.netty.channel.Channel; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -25,6 +26,7 @@ public class ResponseFuture { private final int opaque; + private final Channel processChannel; private final long timeoutMillis; private final InvokeCallback invokeCallback; private final long beginTimestamp = System.currentTimeMillis(); @@ -37,9 +39,10 @@ public class ResponseFuture { private volatile boolean sendRequestOK = true; private volatile Throwable cause; - public ResponseFuture(int opaque, long timeoutMillis, InvokeCallback invokeCallback, + public ResponseFuture(Channel channel, int opaque, long timeoutMillis, InvokeCallback invokeCallback, SemaphoreReleaseOnlyOnce once) { this.opaque = opaque; + this.processChannel = channel; this.timeoutMillis = timeoutMillis; this.invokeCallback = invokeCallback; this.once = once; @@ -114,11 +117,20 @@ public int getOpaque() { return opaque; } + public Channel getProcessChannel() { + return processChannel; + } + @Override public String toString() { - return "ResponseFuture [responseCommand=" + responseCommand + ", sendRequestOK=" + sendRequestOK - + ", cause=" + cause + ", opaque=" + opaque + ", timeoutMillis=" + timeoutMillis - + ", invokeCallback=" + invokeCallback + ", beginTimestamp=" + beginTimestamp + return "ResponseFuture [responseCommand=" + responseCommand + + ", sendRequestOK=" + sendRequestOK + + ", cause=" + cause + + ", opaque=" + opaque + + ", processChannel=" + processChannel + + ", timeoutMillis=" + timeoutMillis + + ", invokeCallback=" + invokeCallback + + ", beginTimestamp=" + beginTimestamp + ", countDownLatch=" + countDownLatch + "]"; } } From 461e5166899c6a7304832f74d703549c53b2f5d0 Mon Sep 17 00:00:00 2001 From: Jaskey Date: Sat, 14 Jul 2018 10:43:49 +0800 Subject: [PATCH 58/73] [ROCKETMQ-203]Support client to allocate message queue in machine room nearby priority (#109) --- .../rebalance/AllocateMachineRoomNearby.java | 144 +++++++++++ .../AllocateMachineRoomNearByTest.java | 237 ++++++++++++++++++ 2 files changed, 381 insertions(+) create mode 100644 client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMachineRoomNearby.java create mode 100644 client/src/test/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMachineRoomNearByTest.java diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMachineRoomNearby.java b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMachineRoomNearby.java new file mode 100644 index 00000000000..9b166e7a96c --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMachineRoomNearby.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.consumer.rebalance; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy; +import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.common.message.MessageQueue; +import org.slf4j.Logger; + +/** + * An allocate strategy proxy for based on machine room nearside priority. An actual allocate strategy can be + * specified. + * + * If any consumer is alive in a machine room, the message queue of the broker which is deployed in the same machine + * should only be allocated to those. Otherwise, those message queues can be shared along all consumers since there are + * no alive consumer to monopolize them. + */ +public class AllocateMachineRoomNearby implements AllocateMessageQueueStrategy { + private final Logger log = ClientLogger.getLog(); + + private final AllocateMessageQueueStrategy allocateMessageQueueStrategy;//actual allocate strategy + private final MachineRoomResolver machineRoomResolver; + + public AllocateMachineRoomNearby(AllocateMessageQueueStrategy allocateMessageQueueStrategy, + MachineRoomResolver machineRoomResolver) throws NullPointerException { + if (allocateMessageQueueStrategy == null) { + throw new NullPointerException("allocateMessageQueueStrategy is null"); + } + + if (machineRoomResolver == null) { + throw new NullPointerException("machineRoomResolver is null"); + } + + this.allocateMessageQueueStrategy = allocateMessageQueueStrategy; + this.machineRoomResolver = machineRoomResolver; + } + + @Override + public List allocate(String consumerGroup, String currentCID, List mqAll, + List cidAll) { + if (currentCID == null || currentCID.length() < 1) { + throw new IllegalArgumentException("currentCID is empty"); + } + if (mqAll == null || mqAll.isEmpty()) { + throw new IllegalArgumentException("mqAll is null or mqAll empty"); + } + if (cidAll == null || cidAll.isEmpty()) { + throw new IllegalArgumentException("cidAll is null or cidAll empty"); + } + + List result = new ArrayList(); + if (!cidAll.contains(currentCID)) { + log.info("[BUG] ConsumerGroup: {} The consumerId: {} not in cidAll: {}", + consumerGroup, + currentCID, + cidAll); + return result; + } + + //group mq by machine room + Map> mr2Mq = new TreeMap>(); + for (MessageQueue mq : mqAll) { + String brokerMachineRoom = machineRoomResolver.brokerDeployIn(mq); + if (StringUtils.isNoneEmpty(brokerMachineRoom)) { + if (mr2Mq.get(brokerMachineRoom) == null) { + mr2Mq.put(brokerMachineRoom, new ArrayList()); + } + mr2Mq.get(brokerMachineRoom).add(mq); + } else { + throw new IllegalArgumentException("Machine room is null for mq " + mq); + } + } + + //group consumer by machine room + Map> mr2c = new TreeMap>(); + for (String cid : cidAll) { + String consumerMachineRoom = machineRoomResolver.consumerDeployIn(cid); + if (StringUtils.isNoneEmpty(consumerMachineRoom)) { + if (mr2c.get(consumerMachineRoom) == null) { + mr2c.put(consumerMachineRoom, new ArrayList()); + } + mr2c.get(consumerMachineRoom).add(cid); + } else { + throw new IllegalArgumentException("Machine room is null for consumer id " + cid); + } + } + + List allocateResults = new ArrayList(); + + //1.allocate the mq that deploy in the same machine room with the current consumer + String currentMachineRoom = machineRoomResolver.consumerDeployIn(currentCID); + List mqInThisMachineRoom = mr2Mq.remove(currentMachineRoom); + List consumerInThisMachineRoom = mr2c.get(currentMachineRoom); + if (mqInThisMachineRoom != null && !mqInThisMachineRoom.isEmpty()) { + allocateResults.addAll(allocateMessageQueueStrategy.allocate(consumerGroup, currentCID, mqInThisMachineRoom, consumerInThisMachineRoom)); + } + + //2.allocate the rest mq to each machine room if there are no consumer alive in that machine room + for (String machineRoom : mr2Mq.keySet()) { + if (!mr2c.containsKey(machineRoom)) { // no alive consumer in the corresponding machine room, so all consumers share these queues + allocateResults.addAll(allocateMessageQueueStrategy.allocate(consumerGroup, currentCID, mr2Mq.get(machineRoom), cidAll)); + } + } + + return allocateResults; + } + + @Override + public String getName() { + return "MACHINE_ROOM_NEARBY" + "-" + allocateMessageQueueStrategy.getName(); + } + + /** + * A resolver object to determine which machine room do the message queues or clients are deployed in. + * + * AllocateMachineRoomNearby will use the results to group the message queues and clients by machine room. + * + * The result returned from the implemented method CANNOT be null. + */ + public interface MachineRoomResolver { + String brokerDeployIn(MessageQueue messageQueue); + + String consumerDeployIn(String clientID); + } +} diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMachineRoomNearByTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMachineRoomNearByTest.java new file mode 100644 index 00000000000..0d394c38231 --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMachineRoomNearByTest.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.consumer.rebalance; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy; +import org.apache.rocketmq.common.message.MessageQueue; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + + +public class AllocateMachineRoomNearByTest { + + private static final String CID_PREFIX = "CID-"; + + private final String topic = "topic_test"; + private final AllocateMachineRoomNearby.MachineRoomResolver machineRoomResolver = new AllocateMachineRoomNearby.MachineRoomResolver() { + @Override public String brokerDeployIn(MessageQueue messageQueue) { + return messageQueue.getBrokerName().split("-")[0]; + } + + @Override public String consumerDeployIn(String clientID) { + return clientID.split("-")[0]; + } + }; + private final AllocateMessageQueueStrategy allocateMessageQueueStrategy = new AllocateMachineRoomNearby(new AllocateMessageQueueAveragely(), machineRoomResolver); + + + @Before + public void init() { + } + + + @Test + public void test1() { + testWhenIDCSizeEquals(5,20,10, false); + testWhenIDCSizeEquals(5,20,20, false); + testWhenIDCSizeEquals(5,20,30, false); + testWhenIDCSizeEquals(5,20,0, false ); + } + + @Test + public void test2() { + testWhenConsumerIDCIsMore(5,1,10, 10, false); + testWhenConsumerIDCIsMore(5,1,10, 5, false); + testWhenConsumerIDCIsMore(5,1,10, 20, false); + testWhenConsumerIDCIsMore(5,1,10, 0, false); + } + + @Test + public void test3() { + testWhenConsumerIDCIsLess(5,2,10, 10, false); + testWhenConsumerIDCIsLess(5,2,10, 5, false); + testWhenConsumerIDCIsLess(5,2,10, 20, false); + testWhenConsumerIDCIsLess(5,2,10, 0, false); + } + + + @Test + public void testRun10RandomCase(){ + for(int i=0;i<10;i++){ + int consumerSize = new Random().nextInt(200)+1;//1-200 + int queueSize = new Random().nextInt(100)+1;//1-100 + int brokerIDCSize = new Random().nextInt(10)+1;//1-10 + int consumerIDCSize = new Random().nextInt(10)+1;//1-10 + + if (brokerIDCSize == consumerIDCSize) { + testWhenIDCSizeEquals(brokerIDCSize,queueSize,consumerSize,false); + } + else if (brokerIDCSize > consumerIDCSize) { + testWhenConsumerIDCIsLess(brokerIDCSize,brokerIDCSize- consumerIDCSize, queueSize, consumerSize, false); + } else { + testWhenConsumerIDCIsMore(brokerIDCSize, consumerIDCSize - brokerIDCSize, queueSize, consumerSize, false); + } + } + } + + + + + public void testWhenIDCSizeEquals(int IDCSize, int queueSize, int consumerSize, boolean print) { + if (print) { + System.out.println("Test : IDCSize = "+ IDCSize +"queueSize = " + queueSize +" consumerSize = " + consumerSize); + } + List cidAll = prepareConsumer(IDCSize, consumerSize); + List mqAll = prepareMQ(IDCSize, queueSize); + List resAll = new ArrayList(); + for (String currentID : cidAll) { + List res = allocateMessageQueueStrategy.allocate("Test-C-G",currentID,mqAll,cidAll); + if (print) { + System.out.println("cid: "+currentID+"--> res :" +res); + } + for (MessageQueue mq : res) { + Assert.assertTrue(machineRoomResolver.brokerDeployIn(mq).equals(machineRoomResolver.consumerDeployIn(currentID))); + } + resAll.addAll(res); + } + Assert.assertTrue(hasAllocateAllQ(cidAll,mqAll,resAll)); + + if (print) { + System.out.println("-------------------------------------------------------------------"); + } + } + + public void testWhenConsumerIDCIsMore(int brokerIDCSize, int consumerMore, int queueSize, int consumerSize, boolean print) { + if (print) { + System.out.println("Test : IDCSize = "+ brokerIDCSize +" queueSize = " + queueSize +" consumerSize = " + consumerSize); + } + Set brokerIDCWithConsumer = new TreeSet(); + List cidAll = prepareConsumer(brokerIDCSize +consumerMore, consumerSize); + List mqAll = prepareMQ(brokerIDCSize, queueSize); + for (MessageQueue mq : mqAll) { + brokerIDCWithConsumer.add(machineRoomResolver.brokerDeployIn(mq)); + } + + List resAll = new ArrayList(); + for (String currentID : cidAll) { + List res = allocateMessageQueueStrategy.allocate("Test-C-G",currentID,mqAll,cidAll); + if (print) { + System.out.println("cid: "+currentID+"--> res :" +res); + } + for (MessageQueue mq : res) { + if (brokerIDCWithConsumer.contains(machineRoomResolver.brokerDeployIn(mq))) {//healthy idc, so only consumer in this idc should be allocated + Assert.assertTrue(machineRoomResolver.brokerDeployIn(mq).equals(machineRoomResolver.consumerDeployIn(currentID))); + } + } + resAll.addAll(res); + } + + Assert.assertTrue(hasAllocateAllQ(cidAll,mqAll,resAll)); + if (print) { + System.out.println("-------------------------------------------------------------------"); + } + } + + public void testWhenConsumerIDCIsLess(int brokerIDCSize, int consumerIDCLess, int queueSize, int consumerSize, boolean print) { + if (print) { + System.out.println("Test : IDCSize = "+ brokerIDCSize +" queueSize = " + queueSize +" consumerSize = " + consumerSize); + } + Set healthyIDC = new TreeSet(); + List cidAll = prepareConsumer(brokerIDCSize - consumerIDCLess, consumerSize); + List mqAll = prepareMQ(brokerIDCSize, queueSize); + for (String cid : cidAll) { + healthyIDC.add(machineRoomResolver.consumerDeployIn(cid)); + } + + List resAll = new ArrayList(); + Map> idc2Res = new TreeMap>(); + for (String currentID : cidAll) { + String currentIDC = machineRoomResolver.consumerDeployIn(currentID); + List res = allocateMessageQueueStrategy.allocate("Test-C-G",currentID,mqAll,cidAll); + if (print) { + System.out.println("cid: "+currentID+"--> res :" +res); + } + if ( !idc2Res.containsKey(currentIDC)) { + idc2Res.put(currentIDC, new ArrayList()); + } + idc2Res.get(currentIDC).addAll(res); + resAll.addAll(res); + } + + for (String consumerIDC : healthyIDC) { + List resInOneIDC = idc2Res.get(consumerIDC); + List mqInThisIDC = createMessageQueueList(consumerIDC,queueSize); + Assert.assertTrue(resInOneIDC.containsAll(mqInThisIDC)); + } + + Assert.assertTrue(hasAllocateAllQ(cidAll,mqAll,resAll)); + if (print) { + System.out.println("-------------------------------------------------------------------"); + } + } + + + private boolean hasAllocateAllQ(List cidAll,List mqAll, List allocatedResAll) { + if (cidAll.isEmpty()){ + return allocatedResAll.isEmpty(); + } + return mqAll.containsAll(allocatedResAll) && allocatedResAll.containsAll(mqAll) && mqAll.size() == allocatedResAll.size(); + } + + + private List createConsumerIdList(String machineRoom, int size) { + List consumerIdList = new ArrayList(size); + for (int i = 0; i < size; i++) { + consumerIdList.add(machineRoom +"-"+CID_PREFIX + String.valueOf(i)); + } + return consumerIdList; + } + + private List createMessageQueueList(String machineRoom, int size) { + List messageQueueList = new ArrayList(size); + for (int i = 0; i < size; i++) { + MessageQueue mq = new MessageQueue(topic, machineRoom+"-brokerName", i); + messageQueueList.add(mq); + } + return messageQueueList; + } + + private List prepareMQ(int brokerIDCSize, int queueSize) { + List mqAll = new ArrayList(); + for (int i=1;i<=brokerIDCSize;i++) { + mqAll.addAll(createMessageQueueList("IDC"+i, queueSize)); + } + + return mqAll; + } + + private List prepareConsumer( int IDCSize, int consumerSize) { + List cidAll = new ArrayList(); + for (int i=1;i<=IDCSize;i++) { + cidAll.addAll(createConsumerIdList("IDC"+i, consumerSize)); + } + return cidAll; + } +} From fc54c2ada78c25879e07378894a669694acfd4eb Mon Sep 17 00:00:00 2001 From: Jaskey Date: Sat, 14 Jul 2018 10:46:44 +0800 Subject: [PATCH 59/73] ROCKETMQ-223-Rename DEFAULT_TOPIC (#119) --- .../apache/rocketmq/broker/topic/TopicConfigManager.java | 8 ++++---- .../broker/processor/SendMessageProcessorTest.java | 2 +- .../main/java/org/apache/rocketmq/client/Validators.java | 4 ++-- .../org/apache/rocketmq/client/impl/MQClientAPIImpl.java | 2 +- .../rocketmq/client/impl/factory/MQClientInstance.java | 2 +- .../rocketmq/client/producer/DefaultMQProducer.java | 2 +- .../java/org/apache/rocketmq/client/ValidatorsTest.java | 4 ++-- .../src/main/java/org/apache/rocketmq/common/MixAll.java | 2 +- .../apache/rocketmq/tools/admin/DefaultMQAdminExt.java | 2 +- 9 files changed, 14 insertions(+), 14 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java index cdae66f2aba..59c7895eb72 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java @@ -66,9 +66,9 @@ public TopicConfigManager(BrokerController brokerController) { this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); } { - // MixAll.DEFAULT_TOPIC + // MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC if (this.brokerController.getBrokerConfig().isAutoCreateTopicEnable()) { - String topic = MixAll.DEFAULT_TOPIC; + String topic = MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC; TopicConfig topicConfig = new TopicConfig(topic); this.systemTopicList.add(topic); topicConfig.setReadQueueNums(this.brokerController.getBrokerConfig() @@ -135,7 +135,7 @@ public Set getSystemTopic() { } public boolean isTopicCanSendMessage(final String topic) { - return !topic.equals(MixAll.DEFAULT_TOPIC); + return !topic.equals(MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC); } public TopicConfig selectTopicConfig(final String topic) { @@ -156,7 +156,7 @@ public TopicConfig createTopicInSendMessageMethod(final String topic, final Stri TopicConfig defaultTopicConfig = this.topicConfigTable.get(defaultTopic); if (defaultTopicConfig != null) { - if (defaultTopic.equals(MixAll.DEFAULT_TOPIC)) { + if (defaultTopic.equals(MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC)) { if (!this.brokerController.getBrokerConfig().isAutoCreateTopicEnable()) { defaultTopicConfig.setPerm(PermName.PERM_READ | PermName.PERM_WRITE); } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java index 7828e7a91cd..b81ee5a2b78 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java @@ -181,7 +181,7 @@ private RemotingCommand createSendMsgCommand(int requestCode) { SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); requestHeader.setProducerGroup(group); requestHeader.setTopic(topic); - requestHeader.setDefaultTopic(MixAll.DEFAULT_TOPIC); + requestHeader.setDefaultTopic(MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC); requestHeader.setDefaultTopicQueueNums(3); requestHeader.setQueueId(1); requestHeader.setSysFlag(0); diff --git a/client/src/main/java/org/apache/rocketmq/client/Validators.java b/client/src/main/java/org/apache/rocketmq/client/Validators.java index 5567e49b558..5d6acc0f872 100644 --- a/client/src/main/java/org/apache/rocketmq/client/Validators.java +++ b/client/src/main/java/org/apache/rocketmq/client/Validators.java @@ -119,9 +119,9 @@ public static void checkTopic(String topic) throws MQClientException { } //whether the same with system reserved keyword - if (topic.equals(MixAll.DEFAULT_TOPIC)) { + if (topic.equals(MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC)) { throw new MQClientException( - String.format("The topic[%s] is conflict with default topic.", topic), null); + String.format("The topic[%s] is conflict with AUTO_CREATE_TOPIC_KEY_TOPIC.", topic), null); } } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index ade69905037..3a023e362cb 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -1204,7 +1204,7 @@ public TopicRouteData getTopicRouteInfoFromNameServer(final String topic, final assert response != null; switch (response.getCode()) { case ResponseCode.TOPIC_NOT_EXIST: { - if (allowTopicNotExist && !topic.equals(MixAll.DEFAULT_TOPIC)) { + if (allowTopicNotExist && !topic.equals(MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC)) { log.warn("get Topic [{}] RouteInfoFromNameServer is not exist value", topic); } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java index e5400d71e2c..6a26fa17b18 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java @@ -654,7 +654,7 @@ public boolean updateTopicRouteInfoFromNameServer(final String topic, boolean is log.warn("updateTopicRouteInfoFromNameServer, getTopicRouteInfoFromNameServer return null, Topic: {}", topic); } } catch (Exception e) { - if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) && !topic.equals(MixAll.DEFAULT_TOPIC)) { + if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) && !topic.equals(MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC)) { log.warn("updateTopicRouteInfoFromNameServer Exception", e); } } finally { diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java index a2f25dd0f8f..267dbe8f57b 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java @@ -76,7 +76,7 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { /** * Just for testing or demo program */ - private String createTopicKey = MixAll.DEFAULT_TOPIC; + private String createTopicKey = MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC; /** * Number of queues to create per default topic. diff --git a/client/src/test/java/org/apache/rocketmq/client/ValidatorsTest.java b/client/src/test/java/org/apache/rocketmq/client/ValidatorsTest.java index 2db648d8626..5a012ceb087 100644 --- a/client/src/test/java/org/apache/rocketmq/client/ValidatorsTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/ValidatorsTest.java @@ -49,12 +49,12 @@ public void testCheckTopic_HasIllegalCharacters() { @Test public void testCheckTopic_UseDefaultTopic() { - String defaultTopic = MixAll.DEFAULT_TOPIC; + String defaultTopic = MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC; try { Validators.checkTopic(defaultTopic); failBecauseExceptionWasNotThrown(MQClientException.class); } catch (MQClientException e) { - assertThat(e).hasMessageStartingWith(String.format("The topic[%s] is conflict with default topic.", defaultTopic)); + assertThat(e).hasMessageStartingWith(String.format("The topic[%s] is conflict with AUTO_CREATE_TOPIC_KEY_TOPIC.", defaultTopic)); } } diff --git a/common/src/main/java/org/apache/rocketmq/common/MixAll.java b/common/src/main/java/org/apache/rocketmq/common/MixAll.java index 49085c3f1e1..19ab72e16ea 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MixAll.java +++ b/common/src/main/java/org/apache/rocketmq/common/MixAll.java @@ -57,7 +57,7 @@ public class MixAll { public static final String WS_DOMAIN_SUBGROUP = System.getProperty("rocketmq.namesrv.domain.subgroup", "nsaddr"); //http://jmenv.tbsite.net:8080/rocketmq/nsaddr //public static final String WS_ADDR = "http://" + WS_DOMAIN_NAME + ":8080/rocketmq/" + WS_DOMAIN_SUBGROUP; - public static final String DEFAULT_TOPIC = "TBW102"; + public static final String AUTO_CREATE_TOPIC_KEY_TOPIC = "AUTO_CREATE_TOPIC_KEY"; // Will be created at broker when isAutoCreateTopicEnable public static final String BENCHMARK_TOPIC = "BenchmarkTest"; public static final String DEFAULT_PRODUCER_GROUP = "DEFAULT_PRODUCER"; public static final String DEFAULT_CONSUMER_GROUP = "DEFAULT_CONSUMER"; diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java index eb45de22f3e..dc829c1c1b5 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java @@ -59,7 +59,7 @@ public class DefaultMQAdminExt extends ClientConfig implements MQAdminExt { private final DefaultMQAdminExtImpl defaultMQAdminExtImpl; private String adminExtGroup = "admin_ext_group"; - private String createTopicKey = MixAll.DEFAULT_TOPIC; + private String createTopicKey = MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC; private long timeoutMillis = 5000; public DefaultMQAdminExt() { From 6eed5f1cdc8f105c8471c55804a6d27f97482a45 Mon Sep 17 00:00:00 2001 From: Wenwei Hu Date: Sat, 14 Jul 2018 10:57:29 +0800 Subject: [PATCH 60/73] [ROCKETMQ-296] fix DefaultMessageStoreTest bug: wait more time for consume queue build (#165) --- .../org/apache/rocketmq/store/DefaultMessageStoreTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java index a400ed5a5a7..20f94f09a05 100644 --- a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java @@ -98,6 +98,7 @@ private MessageStore buildMessageStore() throws Exception { messageStoreConfig.setMaxHashSlotNum(10000); messageStoreConfig.setMaxIndexNum(100 * 100); messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH); + messageStoreConfig.setFlushIntervalConsumeQueue(1); return new DefaultMessageStore(messageStoreConfig, new BrokerStatsManager("simpleTest"), new MyMessageArrivingListener(), new BrokerConfig()); } @@ -156,8 +157,9 @@ public void testPullSize() throws Exception { messageExtBrokerInner.setQueueId(0); messageStore.putMessage(messageExtBrokerInner); } - //wait for consume queue build - Thread.sleep(10); + // wait for consume queue build + // the sleep time should be great than consume queue flush interval + Thread.sleep(100); String group = "simple"; GetMessageResult getMessageResult32 = messageStore.getMessage(group, topic, 0, 0, 32, null); assertThat(getMessageResult32.getMessageBufferList().size()).isEqualTo(32); From 17648c67c3ca3f464ac34fe2207abd595ec1cd43 Mon Sep 17 00:00:00 2001 From: fuyou001 Date: Sat, 14 Jul 2018 11:03:59 +0800 Subject: [PATCH 61/73] #ROCKETMQ-340# improve code for avoid ClassCastException. (#210) --- .../apache/rocketmq/broker/latency/BrokerFastFailure.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java index a5967a0f6b6..0159d323fe1 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java @@ -44,8 +44,10 @@ public BrokerFastFailure(final BrokerController brokerController) { public static RequestTask castRunnable(final Runnable runnable) { try { - FutureTaskExt object = (FutureTaskExt) runnable; - return (RequestTask) object.getRunnable(); + if (runnable instanceof FutureTaskExt) { + FutureTaskExt object = (FutureTaskExt) runnable; + return (RequestTask) object.getRunnable(); + } } catch (Throwable e) { log.error(String.format("castRunnable exception, %s", runnable.getClass().getName()), e); } From 3ccd6f97e5b3d1d2c46de566486f8a82c46e20de Mon Sep 17 00:00:00 2001 From: duheng <39583243+duhengforever@users.noreply.github.com> Date: Sat, 14 Jul 2018 14:07:19 +0800 Subject: [PATCH 62/73] Fix issues brought about while merging pull requests --- .../consumer/rebalance/AllocateMachineRoomNearby.java | 4 ++-- .../rocketmq/remoting/netty/NettyRemotingAbstractTest.java | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMachineRoomNearby.java b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMachineRoomNearby.java index 9b166e7a96c..ec0f7f6489d 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMachineRoomNearby.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMachineRoomNearby.java @@ -24,7 +24,7 @@ import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy; import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.common.message.MessageQueue; -import org.slf4j.Logger; +import org.apache.rocketmq.logging.InternalLogger; /** * An allocate strategy proxy for based on machine room nearside priority. An actual allocate strategy can be @@ -35,7 +35,7 @@ * no alive consumer to monopolize them. */ public class AllocateMachineRoomNearby implements AllocateMessageQueueStrategy { - private final Logger log = ClientLogger.getLog(); + private final InternalLogger log = ClientLogger.getLog(); private final AllocateMessageQueueStrategy allocateMessageQueueStrategy;//actual allocate strategy private final MachineRoomResolver machineRoomResolver; diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java index 99aab9e0757..c3da3e9af59 100644 --- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java +++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java @@ -36,7 +36,7 @@ public class NettyRemotingAbstractTest { @Test public void testProcessResponseCommand() throws InterruptedException { final Semaphore semaphore = new Semaphore(0); - ResponseFuture responseFuture = new ResponseFuture(1, 3000, new InvokeCallback() { + ResponseFuture responseFuture = new ResponseFuture(null,1, 3000, new InvokeCallback() { @Override public void operationComplete(final ResponseFuture responseFuture) { assertThat(semaphore.availablePermits()).isEqualTo(0); @@ -57,7 +57,7 @@ public void operationComplete(final ResponseFuture responseFuture) { @Test public void testProcessResponseCommand_NullCallBack() throws InterruptedException { final Semaphore semaphore = new Semaphore(0); - ResponseFuture responseFuture = new ResponseFuture(1, 3000, null, + ResponseFuture responseFuture = new ResponseFuture(null,1, 3000, null, new SemaphoreReleaseOnlyOnce(semaphore)); remotingAbstract.responseTable.putIfAbsent(1, responseFuture); @@ -72,7 +72,7 @@ public void testProcessResponseCommand_NullCallBack() throws InterruptedExceptio @Test public void testProcessResponseCommand_RunCallBackInCurrentThread() throws InterruptedException { final Semaphore semaphore = new Semaphore(0); - ResponseFuture responseFuture = new ResponseFuture(1, 3000, new InvokeCallback() { + ResponseFuture responseFuture = new ResponseFuture(null,1, 3000, new InvokeCallback() { @Override public void operationComplete(final ResponseFuture responseFuture) { assertThat(semaphore.availablePermits()).isEqualTo(0); From b054a9d4c1749b6536bf29e2368e202a16ba9929 Mon Sep 17 00:00:00 2001 From: duheng <39583243+duhengforever@users.noreply.github.com> Date: Sun, 15 Jul 2018 10:51:41 +0800 Subject: [PATCH 63/73] [ISSUE #292] Add support of transactional message feature (#358) --- .../rocketmq/broker/BrokerController.java | 104 +++- .../apache/rocketmq/broker/BrokerStartup.java | 11 +- .../broker/client/ProducerManager.java | 37 +- .../broker/client/net/Broker2Client.java | 52 +- .../processor/EndTransactionProcessor.java | 219 ++++---- .../processor/SendMessageProcessor.java | 57 +- ...ractTransactionalMessageCheckListener.java | 111 ++++ .../broker/transaction/OperationResult.java | 51 ++ .../broker/transaction/TransactionRecord.java | 5 +- .../broker/transaction/TransactionStore.java | 4 + .../TransactionalMessageCheckService.java | 80 +++ .../TransactionalMessageService.java | 81 +++ ...aultTransactionalMessageCheckListener.java | 36 ++ .../broker/transaction/queue/GetResult.java | 41 ++ .../queue/TransactionalMessageBridge.java | 339 ++++++++++++ .../TransactionalMessageServiceImpl.java | 498 ++++++++++++++++++ .../queue/TransactionalMessageUtil.java | 39 ++ .../broker/util/PositiveAtomicCounter.java | 25 +- .../rocketmq/broker/util/ServiceProvider.java | 191 +++++++ .../EndTransactionProcessorTest.java | 152 ++++++ .../processor/SendMessageProcessorTest.java | 57 +- ...TransactionalMessageCheckListenerTest.java | 78 +++ .../queue/TransactionalMessageBridgeTest.java | 189 +++++++ .../TransactionalMessageServiceImplTest.java | 249 +++++++++ .../LogTransactionalMessageCheckListener.java | 11 +- .../broker/util/ServiceProviderTest.java | 41 ++ .../util/TransactionalMessageServiceImpl.java | 67 +++ ....AbstractTransactionalMessageCheckListener | 1 + ...er.transaction.TransactionalMessageService | 1 + .../client/impl/ClientRemotingProcessor.java | 4 + .../client/impl/consumer/PullAPIWrapper.java | 4 + .../impl/producer/DefaultMQProducerImpl.java | 91 ++-- .../client/impl/producer/MQProducerInner.java | 7 +- .../client/producer/DefaultMQProducer.java | 4 +- .../rocketmq/client/producer/MQProducer.java | 3 +- .../client/producer/TransactionListener.java | 40 ++ .../producer/TransactionMQProducer.java | 50 +- .../apache/rocketmq/common/BrokerConfig.java | 45 +- .../org/apache/rocketmq/common/MixAll.java | 4 + .../rocketmq/common/message/Message.java | 19 +- .../rocketmq/common/message/MessageConst.java | 3 + .../CheckTransactionStateRequestHeader.java | 9 + .../header/EndTransactionRequestHeader.java | 16 +- .../benchmark/TransactionProducer.java | 83 ++- .../transaction/TransactionExecuterImpl.java | 41 -- ...Impl.java => TransactionListenerImpl.java} | 41 +- .../transaction/TransactionProducer.java | 34 +- .../logging/inner/LoggingBuilderTest.java | 2 +- pom.xml | 1 + .../rocketmq/store/MessageExtBrokerInner.java | 7 +- 50 files changed, 2937 insertions(+), 398 deletions(-) create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/transaction/OperationResult.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageService.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/DefaultTransactionalMessageCheckListener.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/GetResult.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageUtil.java rename client/src/main/java/org/apache/rocketmq/client/producer/LocalTransactionExecuter.java => broker/src/main/java/org/apache/rocketmq/broker/util/PositiveAtomicCounter.java (62%) create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/util/ServiceProvider.java create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/processor/EndTransactionProcessorTest.java create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/DefaultTransactionalMessageCheckListenerTest.java create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImplTest.java rename client/src/main/java/org/apache/rocketmq/client/producer/TransactionCheckListener.java => broker/src/test/java/org/apache/rocketmq/broker/util/LogTransactionalMessageCheckListener.java (73%) create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/util/ServiceProviderTest.java create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/util/TransactionalMessageServiceImpl.java create mode 100644 broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener create mode 100644 broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.TransactionalMessageService create mode 100644 client/src/main/java/org/apache/rocketmq/client/producer/TransactionListener.java delete mode 100644 example/src/main/java/org/apache/rocketmq/example/transaction/TransactionExecuterImpl.java rename example/src/main/java/org/apache/rocketmq/example/transaction/{TransactionCheckListenerImpl.java => TransactionListenerImpl.java} (54%) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index ed85a679c05..9dbee82e705 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -16,21 +16,6 @@ */ package org.apache.rocketmq.broker; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; import org.apache.rocketmq.broker.client.ClientHousekeepingService; import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener; import org.apache.rocketmq.broker.client.ConsumerManager; @@ -61,6 +46,13 @@ import org.apache.rocketmq.broker.slave.SlaveSynchronize; import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager; import org.apache.rocketmq.broker.topic.TopicConfigManager; +import org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener; +import org.apache.rocketmq.broker.transaction.TransactionalMessageCheckService; +import org.apache.rocketmq.broker.transaction.TransactionalMessageService; +import org.apache.rocketmq.broker.transaction.queue.DefaultTransactionalMessageCheckListener; +import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageBridge; +import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageServiceImpl; +import org.apache.rocketmq.broker.util.ServiceProvider; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.Configuration; import org.apache.rocketmq.common.DataVersion; @@ -69,12 +61,12 @@ import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.PermName; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.namesrv.RegisterBrokerResult; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.common.stats.MomentStatsItem; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.RemotingServer; import org.apache.rocketmq.remoting.common.TlsMode; @@ -93,6 +85,22 @@ import org.apache.rocketmq.store.stats.BrokerStats; import org.apache.rocketmq.store.stats.BrokerStatsManager; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + public class BrokerController { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final InternalLogger LOG_PROTECTION = InternalLoggerFactory.getLogger(LoggerName.PROTECTION_LOGGER_NAME); @@ -142,6 +150,9 @@ public class BrokerController { private BrokerFastFailure brokerFastFailure; private Configuration configuration; private FileWatchService fileWatchService; + private TransactionalMessageCheckService transactionalMessageCheckService; + private TransactionalMessageService transactionalMessageService; + private AbstractTransactionalMessageCheckListener transactionalMessageCheckListener; public BrokerController( final BrokerConfig brokerConfig, @@ -405,6 +416,7 @@ public void run() { }, new FileWatchService.Listener() { boolean certChanged, keyChanged = false; + @Override public void onChanged(String path) { if (path.equals(TlsSystemConfig.tlsServerTrustCertPath)) { @@ -423,6 +435,7 @@ public void onChanged(String path) { reloadServerSslContext(); } } + private void reloadServerSslContext() { ((NettyRemotingServer) remotingServer).loadSslContext(); ((NettyRemotingServer) fastRemotingServer).loadSslContext(); @@ -432,11 +445,26 @@ private void reloadServerSslContext() { log.warn("FileWatchService created error, can't load the certificate dynamically"); } } + initialTransaction(); } - return result; } + private void initialTransaction() { + this.transactionalMessageService = ServiceProvider.loadClass(ServiceProvider.TRANSACTION_SERVICE_ID, TransactionalMessageService.class); + if (null == this.transactionalMessageService) { + this.transactionalMessageService = new TransactionalMessageServiceImpl(new TransactionalMessageBridge(this, this.getMessageStore())); + log.warn("Load default transaction message hook service: {}", TransactionalMessageServiceImpl.class.getSimpleName()); + } + this.transactionalMessageCheckListener = ServiceProvider.loadClass(ServiceProvider.TRANSACTION_LISTENER_ID, AbstractTransactionalMessageCheckListener.class); + if (null == this.transactionalMessageCheckListener) { + this.transactionalMessageCheckListener = new DefaultTransactionalMessageCheckListener(); + log.warn("Load default discard message hook service: {}", DefaultTransactionalMessageCheckListener.class.getSimpleName()); + } + this.transactionalMessageCheckListener.setBrokerController(this); + this.transactionalMessageCheckService = new TransactionalMessageCheckService(this); + } + public void registerProcessor() { /** * SendMessageProcessor @@ -539,8 +567,9 @@ public long headSlowTimeMills(BlockingQueue q) { slowTimeMills = rt == null ? 0 : this.messageStore.now() - rt.getCreateTimestamp(); } - if (slowTimeMills < 0) + if (slowTimeMills < 0) { slowTimeMills = 0; + } return slowTimeMills; } @@ -700,6 +729,10 @@ public void shutdown() { if (this.fileWatchService != null) { this.fileWatchService.shutdown(); } + + if (this.transactionalMessageCheckService != null) { + this.transactionalMessageCheckService.shutdown(false); + } } private void unregisterBrokerAll() { @@ -768,6 +801,13 @@ public void run() { if (this.brokerFastFailure != null) { this.brokerFastFailure.start(); } + + if (BrokerRole.SLAVE != messageStoreConfig.getBrokerRole()) { + if (this.transactionalMessageCheckService != null) { + log.info("Start transaction service!"); + this.transactionalMessageCheckService.start(); + } + } } public synchronized void registerIncrementBrokerData(TopicConfig topicConfig, DataVersion dataVersion) { @@ -949,4 +989,30 @@ public void setStoreHost(InetSocketAddress storeHost) { public Configuration getConfiguration() { return this.configuration; } + + public TransactionalMessageCheckService getTransactionalMessageCheckService() { + return transactionalMessageCheckService; + } + + public void setTransactionalMessageCheckService( + TransactionalMessageCheckService transactionalMessageCheckService) { + this.transactionalMessageCheckService = transactionalMessageCheckService; + } + + public TransactionalMessageService getTransactionalMessageService() { + return transactionalMessageService; + } + + public void setTransactionalMessageService(TransactionalMessageService transactionalMessageService) { + this.transactionalMessageService = transactionalMessageService; + } + + public AbstractTransactionalMessageCheckListener getTransactionalMessageCheckListener() { + return transactionalMessageCheckListener; + } + + public void setTransactionalMessageCheckListener( + AbstractTransactionalMessageCheckListener transactionalMessageCheckListener) { + this.transactionalMessageCheckListener = transactionalMessageCheckListener; + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java index 1fc1b3b8474..4b986c0d249 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java @@ -18,11 +18,6 @@ import ch.qos.logback.classic.LoggerContext; import ch.qos.logback.classic.joran.JoranConfigurator; -import java.io.BufferedInputStream; -import java.io.FileInputStream; -import java.io.InputStream; -import java.util.Properties; -import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; @@ -45,6 +40,12 @@ import org.apache.rocketmq.store.config.MessageStoreConfig; import org.slf4j.LoggerFactory; +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; + import static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_ENABLE; public class BrokerStartup { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java index 88f1fdeaf73..28d103c0bfc 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java @@ -17,13 +17,18 @@ package org.apache.rocketmq.broker.client; import io.netty.channel.Channel; + +import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; + +import org.apache.rocketmq.broker.util.PositiveAtomicCounter; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; @@ -34,10 +39,11 @@ public class ProducerManager { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final long LOCK_TIMEOUT_MILLIS = 3000; private static final long CHANNEL_EXPIRED_TIMEOUT = 1000 * 120; + private static final int GET_AVALIABLE_CHANNEL_RETRY_COUNT = 3; private final Lock groupChannelLock = new ReentrantLock(); private final HashMap> groupChannelTable = new HashMap>(); - + private PositiveAtomicCounter positiveAtomicCounter = new PositiveAtomicCounter(); public ProducerManager() { } @@ -185,4 +191,33 @@ public void unregisterProducer(final String group, final ClientChannelInfo clien log.error("", e); } } + + public Channel getAvaliableChannel(String groupId) { + HashMap channelClientChannelInfoHashMap = groupChannelTable.get(groupId); + List channelList = new ArrayList(); + if (channelClientChannelInfoHashMap != null) { + for (Channel channel : channelClientChannelInfoHashMap.keySet()) { + channelList.add(channel); + } + int size = channelList.size(); + if (0 == size) { + log.warn("Channel list is empty. groupId={}", groupId); + return null; + } + + int index = positiveAtomicCounter.incrementAndGet() % size; + Channel channel = channelList.get(index); + int count = 0; + boolean isOk = channel.isActive() && channel.isWritable(); + while (isOk && count++ < GET_AVALIABLE_CHANNEL_RETRY_COUNT) { + index = (++index) % size; + channel = channelList.get(index); + return channel; + } + } else { + log.warn("Check transaction failed, channel table is empty. groupId={}", groupId); + return null; + } + return null; + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java b/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java index 2a104457805..4c409f2254a 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java @@ -17,25 +17,15 @@ package org.apache.rocketmq.broker.client.net; import io.netty.channel.Channel; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.FileRegion; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.ConcurrentMap; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.client.ClientChannelInfo; import org.apache.rocketmq.broker.client.ConsumerGroupInfo; -import org.apache.rocketmq.broker.pagecache.OneMessageTransfer; import org.apache.rocketmq.common.MQVersion; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.message.MessageQueueForC; import org.apache.rocketmq.common.protocol.RequestCode; @@ -47,11 +37,19 @@ import org.apache.rocketmq.common.protocol.header.GetConsumerStatusRequestHeader; import org.apache.rocketmq.common.protocol.header.NotifyConsumerIdsChangedRequestHeader; import org.apache.rocketmq.common.protocol.header.ResetOffsetRequestHeader; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import org.apache.rocketmq.store.SelectMappedBufferResult; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentMap; public class Broker2Client { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); @@ -62,34 +60,22 @@ public Broker2Client(BrokerController brokerController) { } public void checkProducerTransactionState( + final String group, final Channel channel, final CheckTransactionStateRequestHeader requestHeader, - final SelectMappedBufferResult selectMappedBufferResult) { + final MessageExt messageExt) throws Exception { RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHECK_TRANSACTION_STATE, requestHeader); - request.markOnewayRPC(); - + request.setBody(MessageDecoder.encode(messageExt, false)); try { - FileRegion fileRegion = - new OneMessageTransfer(request.encodeHeader(selectMappedBufferResult.getSize()), - selectMappedBufferResult); - channel.writeAndFlush(fileRegion).addListener(new ChannelFutureListener() { - @Override - public void operationComplete(ChannelFuture future) throws Exception { - selectMappedBufferResult.release(); - if (!future.isSuccess()) { - log.error("invokeProducer failed,", future.cause()); - } - } - }); - } catch (Throwable e) { - log.error("invokeProducer exception", e); - selectMappedBufferResult.release(); + this.brokerController.getRemotingServer().invokeOneway(channel, request, 10); + } catch (Exception e) { + log.error("Check transaction failed because invoke producer exception. group={}, msgId={}", group, messageExt.getMsgId(), e.getMessage()); } } public RemotingCommand callClient(final Channel channel, - final RemotingCommand request + final RemotingCommand request ) throws RemotingSendRequestException, RemotingTimeoutException, InterruptedException { return this.brokerController.getRemotingServer().invokeSync(channel, request, 10000); } @@ -119,7 +105,7 @@ public RemotingCommand resetOffset(String topic, String group, long timeStamp, b } public RemotingCommand resetOffset(String topic, String group, long timeStamp, boolean isForce, - boolean isC) { + boolean isC) { final RemotingCommand response = RemotingCommand.createResponseCommand(null); TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java index 6801f759ea5..c9e85ed1a50 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java @@ -18,10 +18,9 @@ import io.netty.channel.ChannelHandlerContext; import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.transaction.OperationResult; import org.apache.rocketmq.common.TopicFilterType; import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; @@ -29,14 +28,19 @@ import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.store.MessageExtBrokerInner; -import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.config.BrokerRole; +/** + * EndTransaction processor: process commit and rollback message + */ public class EndTransactionProcessor implements NettyRequestProcessor { private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); private final BrokerController brokerController; @@ -46,16 +50,22 @@ public EndTransactionProcessor(final BrokerController brokerController) { } @Override - public RemotingCommand processRequest(ChannelHandlerContext ctx, - RemotingCommand request) throws RemotingCommandException { + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws + RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); final EndTransactionRequestHeader requestHeader = - (EndTransactionRequestHeader) request.decodeCommandCustomHeader(EndTransactionRequestHeader.class); + (EndTransactionRequestHeader)request.decodeCommandCustomHeader(EndTransactionRequestHeader.class); + LOGGER.info("Transaction request:{}", requestHeader); + if (BrokerRole.SLAVE == brokerController.getMessageStoreConfig().getBrokerRole()) { + response.setCode(ResponseCode.SLAVE_NOT_AVAILABLE); + LOGGER.warn("Message store is slave mode, so end transaction is forbidden. "); + return response; + } if (requestHeader.getFromTransactionCheck()) { switch (requestHeader.getCommitOrRollback()) { case MessageSysFlag.TRANSACTION_NOT_TYPE: { - LOGGER.warn("check producer[{}] transaction state, but it's pending status." + LOGGER.warn("Check producer[{}] transaction state, but it's pending status." + "RequestHeader: {} Remark: {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.toString(), @@ -64,7 +74,7 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx, } case MessageSysFlag.TRANSACTION_COMMIT_TYPE: { - LOGGER.warn("check producer[{}] transaction state, the producer commit the message." + LOGGER.warn("Check producer[{}] transaction state, the producer commit the message." + "RequestHeader: {} Remark: {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.toString(), @@ -74,7 +84,7 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx, } case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: { - LOGGER.warn("check producer[{}] transaction state, the producer rollback the message." + LOGGER.warn("Check producer[{}] transaction state, the producer rollback the message." + "RequestHeader: {} Remark: {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.toString(), @@ -87,7 +97,7 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx, } else { switch (requestHeader.getCommitOrRollback()) { case MessageSysFlag.TRANSACTION_NOT_TYPE: { - LOGGER.warn("the producer[{}] end transaction in sending message, and it's pending status." + LOGGER.warn("The producer[{}] end transaction in sending message, and it's pending status." + "RequestHeader: {} Remark: {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.toString(), @@ -100,7 +110,7 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx, } case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: { - LOGGER.warn("the producer[{}] end transaction in sending message, rollback the message." + LOGGER.warn("The producer[{}] end transaction in sending message, rollback the message." + "RequestHeader: {} Remark: {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.toString(), @@ -111,122 +121,145 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx, return null; } } + OperationResult result = new OperationResult(); + if (MessageSysFlag.TRANSACTION_COMMIT_TYPE == requestHeader.getCommitOrRollback()) { + result = this.brokerController.getTransactionalMessageService().commitMessage(requestHeader); + if (result.getResponseCode() == ResponseCode.SUCCESS) { + RemotingCommand res = checkPrepareMessage(result.getPrepareMessage(), requestHeader); + if (res.getCode() == ResponseCode.SUCCESS) { + MessageExtBrokerInner msgInner = endMessageTransaction(result.getPrepareMessage()); + msgInner.setSysFlag(MessageSysFlag.resetTransactionValue(msgInner.getSysFlag(), requestHeader.getCommitOrRollback())); + msgInner.setQueueOffset(requestHeader.getTranStateTableOffset()); + msgInner.setPreparedTransactionOffset(requestHeader.getCommitLogOffset()); + msgInner.setStoreTimestamp(result.getPrepareMessage().getStoreTimestamp()); + RemotingCommand sendResult = sendFinalMessage(msgInner); + if (sendResult.getCode() == ResponseCode.SUCCESS) { + this.brokerController.getTransactionalMessageService().deletePrepareMessage(result.getPrepareMessage()); + } + return sendResult; + } + return res; + } + } else if (MessageSysFlag.TRANSACTION_ROLLBACK_TYPE == requestHeader.getCommitOrRollback()) { + result = this.brokerController.getTransactionalMessageService().rollbackMessage(requestHeader); + if (result.getResponseCode() == ResponseCode.SUCCESS) { + RemotingCommand res = checkPrepareMessage(result.getPrepareMessage(), requestHeader); + if (res.getCode() == ResponseCode.SUCCESS) { + this.brokerController.getTransactionalMessageService().deletePrepareMessage(result.getPrepareMessage()); + } + return res; + } + } + response.setCode(result.getResponseCode()); + response.setRemark(result.getResponseRemark()); + return response; + } + + @Override + public boolean rejectRequest() { + return false; + } - final MessageExt msgExt = this.brokerController.getMessageStore().lookMessageByOffset(requestHeader.getCommitLogOffset()); + private RemotingCommand checkPrepareMessage(MessageExt msgExt, EndTransactionRequestHeader requestHeader) { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); if (msgExt != null) { final String pgroupRead = msgExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP); if (!pgroupRead.equals(requestHeader.getProducerGroup())) { response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("the producer group wrong"); + response.setRemark("The producer group wrong"); return response; } if (msgExt.getQueueOffset() != requestHeader.getTranStateTableOffset()) { response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("the transaction state table offset wrong"); + response.setRemark("The transaction state table offset wrong"); return response; } if (msgExt.getCommitLogOffset() != requestHeader.getCommitLogOffset()) { response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("the commit log offset wrong"); - return response; - } - - MessageExtBrokerInner msgInner = this.endMessageTransaction(msgExt); - msgInner.setSysFlag(MessageSysFlag.resetTransactionValue(msgInner.getSysFlag(), requestHeader.getCommitOrRollback())); - - msgInner.setQueueOffset(requestHeader.getTranStateTableOffset()); - msgInner.setPreparedTransactionOffset(requestHeader.getCommitLogOffset()); - msgInner.setStoreTimestamp(msgExt.getStoreTimestamp()); - if (MessageSysFlag.TRANSACTION_ROLLBACK_TYPE == requestHeader.getCommitOrRollback()) { - msgInner.setBody(null); - } - - final MessageStore messageStore = this.brokerController.getMessageStore(); - final PutMessageResult putMessageResult = messageStore.putMessage(msgInner); - if (putMessageResult != null) { - switch (putMessageResult.getPutMessageStatus()) { - // Success - case PUT_OK: - case FLUSH_DISK_TIMEOUT: - case FLUSH_SLAVE_TIMEOUT: - case SLAVE_NOT_AVAILABLE: - response.setCode(ResponseCode.SUCCESS); - response.setRemark(null); - break; - // Failed - case CREATE_MAPEDFILE_FAILED: - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("create mapped file failed."); - break; - case MESSAGE_ILLEGAL: - case PROPERTIES_SIZE_EXCEEDED: - response.setCode(ResponseCode.MESSAGE_ILLEGAL); - response.setRemark("the message is illegal, maybe msg body or properties length not matched. msg body length limit 128k, msg properties length limit 32k."); - break; - case SERVICE_NOT_AVAILABLE: - response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE); - response.setRemark("service not available now."); - break; - case OS_PAGECACHE_BUSY: - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("OS page cache busy, please try another machine"); - break; - case UNKNOWN_ERROR: - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("UNKNOWN_ERROR"); - break; - default: - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("UNKNOWN_ERROR DEFAULT"); - break; - } - + response.setRemark("The commit log offset wrong"); return response; - } else { - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("store putMessage return null"); } } else { response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("find prepared transaction message failed"); + response.setRemark("Find prepared transaction message failed"); return response; } - + response.setCode(ResponseCode.SUCCESS); return response; } - @Override - public boolean rejectRequest() { - return false; - } - private MessageExtBrokerInner endMessageTransaction(MessageExt msgExt) { MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + msgInner.setTopic(msgExt.getUserProperty(MessageConst.PROPERTY_REAL_TOPIC)); + msgInner.setQueueId(Integer.parseInt(msgExt.getUserProperty(MessageConst.PROPERTY_REAL_QUEUE_ID))); msgInner.setBody(msgExt.getBody()); msgInner.setFlag(msgExt.getFlag()); - MessageAccessor.setProperties(msgInner, msgExt.getProperties()); - + msgInner.setBornTimestamp(msgExt.getBornTimestamp()); + msgInner.setBornHost(msgExt.getBornHost()); + msgInner.setStoreHost(msgExt.getStoreHost()); + msgInner.setReconsumeTimes(msgExt.getReconsumeTimes()); + msgInner.setWaitStoreMsgOK(false); + msgInner.setTransactionId(msgExt.getUserProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX)); + msgInner.setSysFlag(msgExt.getSysFlag()); TopicFilterType topicFilterType = (msgInner.getSysFlag() & MessageSysFlag.MULTI_TAGS_FLAG) == MessageSysFlag.MULTI_TAGS_FLAG ? TopicFilterType.MULTI_TAG : TopicFilterType.SINGLE_TAG; long tagsCodeValue = MessageExtBrokerInner.tagsString2tagsCode(topicFilterType, msgInner.getTags()); msgInner.setTagsCode(tagsCodeValue); + MessageAccessor.setProperties(msgInner, msgExt.getProperties()); msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties())); - - msgInner.setSysFlag(msgExt.getSysFlag()); - msgInner.setBornTimestamp(msgExt.getBornTimestamp()); - msgInner.setBornHost(msgExt.getBornHost()); - msgInner.setStoreHost(msgExt.getStoreHost()); - msgInner.setReconsumeTimes(msgExt.getReconsumeTimes()); - - msgInner.setWaitStoreMsgOK(false); - MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_DELAY_TIME_LEVEL); - - msgInner.setTopic(msgExt.getTopic()); - msgInner.setQueueId(msgExt.getQueueId()); - + MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_REAL_TOPIC); + MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_REAL_QUEUE_ID); return msgInner; } + + private RemotingCommand sendFinalMessage(MessageExtBrokerInner msgInner) { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner); + if (putMessageResult != null) { + switch (putMessageResult.getPutMessageStatus()) { + // Success + case PUT_OK: + case FLUSH_DISK_TIMEOUT: + case FLUSH_SLAVE_TIMEOUT: + case SLAVE_NOT_AVAILABLE: + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + break; + // Failed + case CREATE_MAPEDFILE_FAILED: + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("Create mapped file failed."); + break; + case MESSAGE_ILLEGAL: + case PROPERTIES_SIZE_EXCEEDED: + response.setCode(ResponseCode.MESSAGE_ILLEGAL); + response.setRemark("The message is illegal, maybe msg body or properties length not matched. msg body length limit 128k, msg properties length limit 32k."); + break; + case SERVICE_NOT_AVAILABLE: + response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE); + response.setRemark("Service not available now."); + break; + case OS_PAGECACHE_BUSY: + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("OS page cache busy, please try another machine"); + break; + case UNKNOWN_ERROR: + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UNKNOWN_ERROR"); + break; + default: + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UNKNOWN_ERROR DEFAULT"); + break; + } + return response; + } else { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("store putMessage return null"); + } + return response; + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java index 227a23e6b49..b7e7a6187b7 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java @@ -17,8 +17,6 @@ package org.apache.rocketmq.broker.processor; import io.netty.channel.ChannelHandlerContext; -import java.net.SocketAddress; -import java.util.List; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.mqtrace.ConsumeMessageContext; import org.apache.rocketmq.broker.mqtrace.ConsumeMessageHook; @@ -51,6 +49,10 @@ import org.apache.rocketmq.store.config.StorePathConfigHelper; import org.apache.rocketmq.store.stats.BrokerStatsManager; +import java.net.SocketAddress; +import java.util.List; +import java.util.Map; + public class SendMessageProcessor extends AbstractSendMessageProcessor implements NettyRequestProcessor { private List consumeMessageHookList; @@ -61,7 +63,7 @@ public SendMessageProcessor(final BrokerController brokerController) { @Override public RemotingCommand processRequest(ChannelHandlerContext ctx, - RemotingCommand request) throws RemotingCommandException { + RemotingCommand request) throws RemotingCommandException { SendMessageContext mqtraceContext; switch (request.getCode()) { case RequestCode.CONSUMER_SEND_MSG_BACK: @@ -97,7 +99,7 @@ private RemotingCommand consumerSendMsgBack(final ChannelHandlerContext ctx, fin throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); final ConsumerSendMsgBackRequestHeader requestHeader = - (ConsumerSendMsgBackRequestHeader) request.decodeCommandCustomHeader(ConsumerSendMsgBackRequestHeader.class); + (ConsumerSendMsgBackRequestHeader)request.decodeCommandCustomHeader(ConsumerSendMsgBackRequestHeader.class); if (this.hasConsumeMessageHook() && !UtilAll.isBlank(requestHeader.getOriginMsgId())) { @@ -247,8 +249,8 @@ private RemotingCommand consumerSendMsgBack(final ChannelHandlerContext ctx, fin } private boolean handleRetryAndDLQ(SendMessageRequestHeader requestHeader, RemotingCommand response, - RemotingCommand request, - MessageExt msg, TopicConfig topicConfig) { + RemotingCommand request, + MessageExt msg, TopicConfig topicConfig) { String newTopic = requestHeader.getTopic(); if (null != newTopic && newTopic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { String groupName = newTopic.substring(MixAll.RETRY_GROUP_TOPIC_PREFIX.length()); @@ -291,12 +293,12 @@ private boolean handleRetryAndDLQ(SendMessageRequestHeader requestHeader, Remoti } private RemotingCommand sendMessage(final ChannelHandlerContext ctx, - final RemotingCommand request, - final SendMessageContext sendMessageContext, - final SendMessageRequestHeader requestHeader) throws RemotingCommandException { + final RemotingCommand request, + final SendMessageContext sendMessageContext, + final SendMessageRequestHeader requestHeader) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); - final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader) response.readCustomHeader(); + final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader)response.readCustomHeader(); response.setOpaque(request.getOpaque()); @@ -343,27 +345,30 @@ private RemotingCommand sendMessage(final ChannelHandlerContext ctx, msgInner.setBornHost(ctx.channel().remoteAddress()); msgInner.setStoreHost(this.getStoreHost()); msgInner.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes()); - - if (this.brokerController.getBrokerConfig().isRejectTransactionMessage()) { - String traFlag = msgInner.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED); - if (traFlag != null) { + PutMessageResult putMessageResult = null; + Map oriProps = MessageDecoder.string2messageProperties(requestHeader.getProperties()); + String traFlag = oriProps.get(MessageConst.PROPERTY_TRANSACTION_PREPARED); + if (traFlag != null && Boolean.parseBoolean(traFlag)) { + if (this.brokerController.getBrokerConfig().isRejectTransactionMessage()) { response.setCode(ResponseCode.NO_PERMISSION); response.setRemark( - "the broker[" + this.brokerController.getBrokerConfig().getBrokerIP1() + "] sending transaction message is forbidden"); + "the broker[" + this.brokerController.getBrokerConfig().getBrokerIP1() + + "] sending transaction message is forbidden"); return response; } + putMessageResult = this.brokerController.getTransactionalMessageService().prepareMessage(msgInner); + } else { + putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner); } - PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner); - return handlePutMessageResult(putMessageResult, response, request, msgInner, responseHeader, sendMessageContext, ctx, queueIdInt); } private RemotingCommand handlePutMessageResult(PutMessageResult putMessageResult, RemotingCommand response, - RemotingCommand request, MessageExt msg, - SendMessageResponseHeader responseHeader, SendMessageContext sendMessageContext, ChannelHandlerContext ctx, - int queueIdInt) { + RemotingCommand request, MessageExt msg, + SendMessageResponseHeader responseHeader, SendMessageContext sendMessageContext, ChannelHandlerContext ctx, + int queueIdInt) { if (putMessageResult == null) { response.setCode(ResponseCode.SYSTEM_ERROR); response.setRemark("store putMessage return null"); @@ -443,7 +448,7 @@ private RemotingCommand handlePutMessageResult(PutMessageResult putMessageResult int commercialBaseCount = brokerController.getBrokerConfig().getCommercialBaseCount(); int wroteSize = putMessageResult.getAppendMessageResult().getWroteBytes(); - int incValue = (int) Math.ceil(wroteSize / BrokerStatsManager.SIZE_PER_COUNT) * commercialBaseCount; + int incValue = (int)Math.ceil(wroteSize / BrokerStatsManager.SIZE_PER_COUNT) * commercialBaseCount; sendMessageContext.setCommercialSendStats(BrokerStatsManager.StatsType.SEND_SUCCESS); sendMessageContext.setCommercialSendTimes(incValue); @@ -454,7 +459,7 @@ private RemotingCommand handlePutMessageResult(PutMessageResult putMessageResult } else { if (hasSendMessageHook()) { int wroteSize = request.getBody().length; - int incValue = (int) Math.ceil(wroteSize / BrokerStatsManager.SIZE_PER_COUNT); + int incValue = (int)Math.ceil(wroteSize / BrokerStatsManager.SIZE_PER_COUNT); sendMessageContext.setCommercialSendStats(BrokerStatsManager.StatsType.SEND_FAILURE); sendMessageContext.setCommercialSendTimes(incValue); @@ -466,12 +471,12 @@ private RemotingCommand handlePutMessageResult(PutMessageResult putMessageResult } private RemotingCommand sendBatchMessage(final ChannelHandlerContext ctx, - final RemotingCommand request, - final SendMessageContext sendMessageContext, - final SendMessageRequestHeader requestHeader) throws RemotingCommandException { + final RemotingCommand request, + final SendMessageContext sendMessageContext, + final SendMessageRequestHeader requestHeader) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); - final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader) response.readCustomHeader(); + final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader)response.readCustomHeader(); response.setOpaque(request.getOpaque()); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java new file mode 100644 index 00000000000..659c6af16f0 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.transaction; + +import io.netty.channel.Channel; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; + +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +public abstract class AbstractTransactionalMessageCheckListener { + private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); + + private BrokerController brokerController; + + private static ExecutorService executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, new ArrayBlockingQueue(2000), new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + Thread thread = new Thread(r); + thread.setName("Transaction-msg-check-thread"); + return thread; + } + }); + + public AbstractTransactionalMessageCheckListener() { + } + + public AbstractTransactionalMessageCheckListener(BrokerController brokerController) { + this.brokerController = brokerController; + } + + public void sendCheckMessage(MessageExt msgExt) throws Exception { + CheckTransactionStateRequestHeader checkTransactionStateRequestHeader = new CheckTransactionStateRequestHeader(); + checkTransactionStateRequestHeader.setCommitLogOffset(msgExt.getCommitLogOffset()); + checkTransactionStateRequestHeader.setOffsetMsgId(msgExt.getMsgId()); + checkTransactionStateRequestHeader.setMsgId(msgExt.getUserProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX)); + checkTransactionStateRequestHeader.setTransactionId(checkTransactionStateRequestHeader.getMsgId()); + checkTransactionStateRequestHeader.setTranStateTableOffset(msgExt.getQueueOffset()); + msgExt.setTopic(msgExt.getUserProperty(MessageConst.PROPERTY_REAL_TOPIC)); + msgExt.setQueueId(Integer.parseInt(msgExt.getUserProperty(MessageConst.PROPERTY_REAL_QUEUE_ID))); + msgExt.setStoreSize(0); + String groupId = msgExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP); + Channel channel = brokerController.getProducerManager().getAvaliableChannel(groupId); + if (channel != null) { + brokerController.getBroker2Client().checkProducerTransactionState(groupId, channel, checkTransactionStateRequestHeader, msgExt); + } else { + LOGGER.warn("Check transaction failed, channel is null. groupId={}", groupId); + } + } + + public void resolveHalfMsg(final MessageExt msgExt) { + executorService.execute(new Runnable() { + @Override + public void run() { + try { + sendCheckMessage(msgExt); + } catch (Exception e) { + LOGGER.error("Send check message error!", e); + } + } + }); + } + + public BrokerController getBrokerController() { + return brokerController; + } + + public void shutDown() { + executorService.shutdown(); + } + + /** + * Inject brokerController for this listener + * + * @param brokerController + */ + public void setBrokerController(BrokerController brokerController) { + this.brokerController = brokerController; + } + + /** + * In order to avoid check back unlimited, we will discard the message that have been checked more than a certain + * number of times. + * + * @param msgExt Message to be discarded. + */ + public abstract void resolveDiscardMsg(MessageExt msgExt); +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/OperationResult.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/OperationResult.java new file mode 100644 index 00000000000..6c35f6adeb5 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/OperationResult.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.transaction; + +import org.apache.rocketmq.common.message.MessageExt; + +public class OperationResult { + private MessageExt prepareMessage; + + private int responseCode; + + private String responseRemark; + + public MessageExt getPrepareMessage() { + return prepareMessage; + } + + public void setPrepareMessage(MessageExt prepareMessage) { + this.prepareMessage = prepareMessage; + } + + public int getResponseCode() { + return responseCode; + } + + public void setResponseCode(int responseCode) { + this.responseCode = responseCode; + } + + public String getResponseRemark() { + return responseRemark; + } + + public void setResponseRemark(String responseRemark) { + this.responseRemark = responseRemark; + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionRecord.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionRecord.java index 5f81ebb3303..772f08e52c7 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionRecord.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionRecord.java @@ -14,9 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.rocketmq.broker.transaction; +/** + * This class will be removed in the version 4.4.0 and {@link OperationResult} class is recommended. + */ +@Deprecated public class TransactionRecord { // Commit Log Offset private long offset; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionStore.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionStore.java index 3decc01f5eb..03e02273934 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionStore.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionStore.java @@ -19,6 +19,10 @@ import java.util.List; +/** + * This class will be removed in ther version 4.4.0, and {@link TransactionalMessageService} class is recommended. + */ +@Deprecated public interface TransactionStore { boolean open(); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java new file mode 100644 index 00000000000..5d515d6f11b --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.transaction; + +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.ServiceThread; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; + +import java.util.concurrent.atomic.AtomicBoolean; + +public class TransactionalMessageCheckService extends ServiceThread { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); + + private BrokerController brokerController; + + private final AtomicBoolean started = new AtomicBoolean(false); + + public TransactionalMessageCheckService(BrokerController brokerController) { + this.brokerController = brokerController; + } + + @Override + public void start() { + if (started.compareAndSet(false, true)) { + super.start(); + this.brokerController.getTransactionalMessageService().open(); + } + } + + @Override + public void shutdown(boolean interrupt) { + if (started.compareAndSet(true, false)) { + super.shutdown(interrupt); + this.brokerController.getTransactionalMessageService().close(); + this.brokerController.getTransactionalMessageCheckListener().shutDown(); + } + } + + @Override + public String getServiceName() { + return TransactionalMessageCheckService.class.getSimpleName(); + } + + @Override + public void run() { + log.info("Start transaction check service thread!"); + long checkInterval = brokerController.getBrokerConfig().getTransactionCheckInterval(); + while (!this.isStopped()) { + this.waitForRunning(checkInterval); + } + log.info("End transaction check service thread!"); + } + + @Override + protected void onWaitEnd() { + long timeout = brokerController.getBrokerConfig().getTransactionTimeOut(); + int checkMax = brokerController.getBrokerConfig().getTransactionCheckMax(); + long begin = System.currentTimeMillis(); + log.info("Begin to check prepare message, begin time:{}", begin); + this.brokerController.getTransactionalMessageService().check(timeout, checkMax, this.brokerController.getTransactionalMessageCheckListener()); + log.info("End to check prepare message, consumed time:{}", System.currentTimeMillis() - begin); + } + +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageService.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageService.java new file mode 100644 index 00000000000..143909fd972 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageService.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.transaction; + +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.PutMessageResult; + +public interface TransactionalMessageService { + + /** + * Process prepare message, in common, we should put this message to storage service. + * + * @param messageInner Prepare(Half) message. + * @return Prepare message storage result. + */ + PutMessageResult prepareMessage(MessageExtBrokerInner messageInner); + + /** + * Delete prepare message when this message has been committed or rolled back. + * + * @param messageExt + */ + boolean deletePrepareMessage(MessageExt messageExt); + + /** + * Invoked to process commit prepare message. + * + * @param requestHeader Commit message request header. + * @return Operate result contains prepare message and relative error code. + */ + OperationResult commitMessage(EndTransactionRequestHeader requestHeader); + + /** + * Invoked to roll back prepare message. + * + * @param requestHeader Prepare message request header. + * @return Operate result contains prepare message and relative error code. + */ + OperationResult rollbackMessage(EndTransactionRequestHeader requestHeader); + + /** + * Traverse uncommitted/unroll back half message and send check back request to producer to obtain transaction + * status. + * + * @param transactionTimeout The minimum time of the transactional message to be checked firstly, one message only + * exceed this time interval that can be checked. + * @param transactionCheckMax The maximum number of times the message was checked, if exceed this value, this + * message will be discarded. + * @param listener When the message is considered to be checked or discarded, the relative method of this class will + * be invoked. + */ + void check(long transactionTimeout, int transactionCheckMax, AbstractTransactionalMessageCheckListener listener); + + /** + * Open transaction service. + * + * @return If open success, return true. + */ + boolean open(); + + /** + * Close transaction service. + */ + void close(); +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/DefaultTransactionalMessageCheckListener.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/DefaultTransactionalMessageCheckListener.java new file mode 100644 index 00000000000..529bfe4f349 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/DefaultTransactionalMessageCheckListener.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.transaction.queue; + +import org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; + +public class DefaultTransactionalMessageCheckListener extends AbstractTransactionalMessageCheckListener { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); + + public DefaultTransactionalMessageCheckListener() { + super(); + } + + @Override + public void resolveDiscardMsg(MessageExt msgExt) { + log.error("MsgExt:{} has been checked too many times, so discard it", msgExt); + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/GetResult.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/GetResult.java new file mode 100644 index 00000000000..a78970e83ad --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/GetResult.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.transaction.queue; + +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.common.message.MessageExt; + +public class GetResult { + private MessageExt msg; + private PullResult pullResult; + + public MessageExt getMsg() { + return msg; + } + + public void setMsg(MessageExt msg) { + this.msg = msg; + } + + public PullResult getPullResult() { + return pullResult; + } + + public void setPullResult(PullResult pullResult) { + this.pullResult = pullResult; + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java new file mode 100644 index 00000000000..84a62761bd6 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java @@ -0,0 +1,339 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.transaction.queue; + +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.PullStatus; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.constant.PermName; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageClientIDSetter; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.logging.InnerLoggerFactory; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.remoting.common.RemotingHelper; +import org.apache.rocketmq.store.GetMessageResult; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.PutMessageStatus; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +public class TransactionalMessageBridge { + private static final InternalLogger LOGGER = InnerLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); + + private final ConcurrentHashMap opQueueMap = new ConcurrentHashMap<>(); + private final BrokerController brokerController; + private final MessageStore store; + private final SocketAddress storeHost; + + public TransactionalMessageBridge(BrokerController brokerController, MessageStore store) { + try { + this.brokerController = brokerController; + this.store = store; + this.storeHost = + new InetSocketAddress(brokerController.getBrokerConfig().getBrokerIP1(), + brokerController.getNettyServerConfig().getListenPort()); + } catch (Exception e) { + LOGGER.error("Init TransactionBridge error", e); + throw new RuntimeException(e); + } + + } + + public long fetchConsumeOffset(MessageQueue mq) { + long offset = brokerController.getConsumerOffsetManager().queryOffset(TransactionalMessageUtil.buildConsumerGroup(), + mq.getTopic(), mq.getQueueId()); + if (offset == -1) { + offset = store.getMinOffsetInQueue(mq.getTopic(), mq.getQueueId()); + } + return offset; + } + + public Set fetchMessageQueues(String topic) { + Set mqSet = new HashSet<>(); + TopicConfig topicConfig = selectTopicConfig(topic); + if (topicConfig != null && topicConfig.getReadQueueNums() > 0) { + for (int i = 0; i < topicConfig.getReadQueueNums(); i++) { + MessageQueue mq = new MessageQueue(); + mq.setTopic(topic); + mq.setBrokerName(brokerController.getBrokerConfig().getBrokerName()); + mq.setQueueId(i); + mqSet.add(mq); + } + } + return mqSet; + } + + public void updateConsumeOffset(MessageQueue mq, long offset) { + this.brokerController.getConsumerOffsetManager().commitOffset( + RemotingHelper.parseSocketAddressAddr(this.storeHost), TransactionalMessageUtil.buildConsumerGroup(), mq.getTopic(), + mq.getQueueId(), offset); + } + + public PullResult getHalfMessage(int queueId, long offset, int nums) { + String group = TransactionalMessageUtil.buildConsumerGroup(); + String topic = TransactionalMessageUtil.buildHalfTopic(); + SubscriptionData sub = new SubscriptionData(topic, "*"); + return getMessage(group, topic, queueId, offset, nums, sub); + } + + public PullResult getOpMessage(int queueId, long offset, int nums) { + String group = TransactionalMessageUtil.buildConsumerGroup(); + String topic = TransactionalMessageUtil.buildOpTopic(); + SubscriptionData sub = new SubscriptionData(topic, "*"); + return getMessage(group, topic, queueId, offset, nums, sub); + } + + private PullResult getMessage(String group, String topic, int queueId, long offset, int nums, + SubscriptionData sub) { + GetMessageResult getMessageResult = store.getMessage(group, topic, queueId, offset, nums, null); + + if (getMessageResult != null) { + PullStatus pullStatus = PullStatus.NO_NEW_MSG; + List foundList = null; + switch (getMessageResult.getStatus()) { + case FOUND: + pullStatus = PullStatus.FOUND; + foundList = decodeMsgList(getMessageResult); + this.brokerController.getBrokerStatsManager().incGroupGetNums(group, topic, + getMessageResult.getMessageCount()); + this.brokerController.getBrokerStatsManager().incGroupGetSize(group, topic, + getMessageResult.getBufferTotalSize()); + this.brokerController.getBrokerStatsManager().incBrokerGetNums(getMessageResult.getMessageCount()); + this.brokerController.getBrokerStatsManager().recordDiskFallBehindTime(group, topic, queueId, + this.brokerController.getMessageStore().now() - foundList.get(foundList.size() - 1) + .getStoreTimestamp()); + break; + case NO_MATCHED_MESSAGE: + pullStatus = PullStatus.NO_MATCHED_MSG; + LOGGER.warn("No matched message. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}", + getMessageResult.getStatus(), topic, group, offset); + break; + case NO_MESSAGE_IN_QUEUE: + pullStatus = PullStatus.NO_NEW_MSG; + LOGGER.warn("No new message. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}", + getMessageResult.getStatus(), topic, group, offset); + break; + case MESSAGE_WAS_REMOVING: + case NO_MATCHED_LOGIC_QUEUE: + case OFFSET_FOUND_NULL: + case OFFSET_OVERFLOW_BADLY: + case OFFSET_OVERFLOW_ONE: + case OFFSET_TOO_SMALL: + pullStatus = PullStatus.OFFSET_ILLEGAL; + LOGGER.warn("Offset illegal. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}", + getMessageResult.getStatus(), topic, group, offset); + break; + default: + assert false; + break; + } + + return new PullResult(pullStatus, getMessageResult.getNextBeginOffset(), getMessageResult.getMinOffset(), + getMessageResult.getMaxOffset(), foundList); + + } else { + LOGGER.error("Get message from store return null. topic={}, groupId={}, requestOffset={}", topic, group, + offset); + return null; + } + } + + private List decodeMsgList(GetMessageResult getMessageResult) { + List foundList = new ArrayList<>(); + try { + List messageBufferList = getMessageResult.getMessageBufferList(); + for (ByteBuffer bb : messageBufferList) { + MessageExt msgExt = MessageDecoder.decode(bb); + foundList.add(msgExt); + } + + } finally { + getMessageResult.release(); + } + + return foundList; + } + + public PutMessageResult putHalfMessage(MessageExtBrokerInner messageInner) { + return store.putMessage(parseHalfMessageInner(messageInner)); + } + + private MessageExtBrokerInner parseHalfMessageInner(MessageExtBrokerInner msgInner) { + MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_REAL_TOPIC, msgInner.getTopic()); + MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_REAL_QUEUE_ID, + String.valueOf(msgInner.getQueueId())); + msgInner.setSysFlag( + MessageSysFlag.resetTransactionValue(msgInner.getSysFlag(), MessageSysFlag.TRANSACTION_NOT_TYPE)); + msgInner.setTopic(TransactionalMessageUtil.buildHalfTopic()); + msgInner.setQueueId(0); + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); + return msgInner; + } + + public boolean putOpMessage(MessageExt messageExt, String opType) { + MessageQueue messageQueue = new MessageQueue(messageExt.getTopic(), + this.brokerController.getBrokerConfig().getBrokerName(), messageExt.getQueueId()); + if (TransactionalMessageUtil.REMOVETAG.equals(opType)) { + return addRemoveTagInTransactionOp(messageExt, messageQueue); + } + return true; + } + + public PutMessageResult putMessageReturnResult(MessageExtBrokerInner messageInner) { + LOGGER.debug("[BUG-TO-FIX] Thread:{} msgID:{}", Thread.currentThread().getName(), messageInner.getMsgId()); + return store.putMessage(messageInner); + } + + public boolean putMessage(MessageExtBrokerInner messageInner) { + PutMessageResult putMessageResult = store.putMessage(messageInner); + if (putMessageResult != null + && putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK) { + return true; + } else { + LOGGER.error("Put message failed, topic: {}, queueId: {}, msgId: {}", + messageInner.getTopic(), messageInner.getQueueId(), messageInner.getMsgId()); + return false; + } + } + + public MessageExtBrokerInner renewImmunityHalfMessageInner(MessageExt msgExt) { + MessageExtBrokerInner msgInner = renewHalfMessageInner(msgExt); + String queueOffsetFromPrepare = msgExt.getUserProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET); + if (null != queueOffsetFromPrepare) { + MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET, + String.valueOf(queueOffsetFromPrepare)); + } else { + MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET, + String.valueOf(msgExt.getQueueOffset())); + } + + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); + + return msgInner; + } + + public MessageExtBrokerInner renewHalfMessageInner(MessageExt msgExt) { + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + msgInner.setTopic(msgExt.getTopic()); + msgInner.setBody(msgExt.getBody()); + msgInner.setQueueId(msgExt.getQueueId()); + msgInner.setMsgId(msgExt.getMsgId()); + msgInner.setSysFlag(msgExt.getSysFlag()); + msgInner.setTags(msgExt.getTags()); + msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(msgInner.getTags())); + MessageAccessor.setProperties(msgInner, msgExt.getProperties()); + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties())); + msgInner.setBornTimestamp(msgExt.getBornTimestamp()); + msgInner.setBornHost(msgExt.getBornHost()); + msgInner.setStoreHost(msgExt.getStoreHost()); + msgInner.setWaitStoreMsgOK(false); + return msgInner; + } + + private MessageExtBrokerInner makeOpMessageInner(Message message, MessageQueue messageQueue) { + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + msgInner.setTopic(message.getTopic()); + msgInner.setBody(message.getBody()); + msgInner.setQueueId(messageQueue.getQueueId()); + msgInner.setTags(message.getTags()); + msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(msgInner.getTags())); + msgInner.setSysFlag(0); + MessageAccessor.setProperties(msgInner, message.getProperties()); + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(message.getProperties())); + msgInner.setBornTimestamp(System.currentTimeMillis()); + msgInner.setBornHost(this.storeHost); + msgInner.setStoreHost(this.storeHost); + msgInner.setWaitStoreMsgOK(false); + MessageClientIDSetter.setUniqID(msgInner); + return msgInner; + } + + private TopicConfig selectTopicConfig(String topic) { + TopicConfig topicConfig = brokerController.getTopicConfigManager().selectTopicConfig(topic); + if (topicConfig == null) { + topicConfig = this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod( + topic, 1, PermName.PERM_WRITE | PermName.PERM_READ, 0); + } + return topicConfig; + } + + /** + * Use this function while transaction msg is committed or rollback write a flag 'd' to operation queue for the + * msg's offset + * + * @param messageExt Op message + * @param messageQueue Op message queue + * @return This method will always return true. + */ + private boolean addRemoveTagInTransactionOp(MessageExt messageExt, MessageQueue messageQueue) { + Message message = new Message(TransactionalMessageUtil.buildOpTopic(), TransactionalMessageUtil.REMOVETAG, + String.valueOf(messageExt.getQueueOffset()).getBytes(TransactionalMessageUtil.charset)); + writeOp(message, messageQueue); + return true; + } + + private void writeOp(Message message, MessageQueue mq) { + MessageQueue opQueue; + if (opQueueMap.containsKey(mq)) { + opQueue = opQueueMap.get(mq); + } else { + opQueue = getOpQueueByHalf(mq); + MessageQueue oldQueue = opQueueMap.putIfAbsent(mq, opQueue); + if (oldQueue != null) { + opQueue = oldQueue; + } + } + if (opQueue == null) { + opQueue = new MessageQueue(TransactionalMessageUtil.buildOpTopic(), mq.getBrokerName(), mq.getQueueId()); + } + putMessage(makeOpMessageInner(message, opQueue)); + } + + private MessageQueue getOpQueueByHalf(MessageQueue halfMQ) { + MessageQueue opQueue = new MessageQueue(); + opQueue.setTopic(TransactionalMessageUtil.buildOpTopic()); + opQueue.setBrokerName(halfMQ.getBrokerName()); + opQueue.setQueueId(halfMQ.getQueueId()); + return opQueue; + } + + public MessageExt lookMessageByOffset(final long commitLogOffset) { + return this.store.lookMessageByOffset(commitLogOffset); + } + + public BrokerController getBrokerController() { + return brokerController; + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java new file mode 100644 index 00000000000..15e5c84ff2d --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java @@ -0,0 +1,498 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.transaction.queue; + +import org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener; +import org.apache.rocketmq.broker.transaction.OperationResult; +import org.apache.rocketmq.broker.transaction.TransactionalMessageService; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.PullStatus; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.PutMessageStatus; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +public class TransactionalMessageServiceImpl implements TransactionalMessageService { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); + + private TransactionalMessageBridge transactionalMessageBridge; + + private static final int PULL_MSG_RETRY_NUMBER = 1; + + private static final int MAX_PROCESS_TIME_LIMIT = 60000; + + private static final int MAX_RETRY_COUNT_WHEN_HALF_NULL = 1; + + public TransactionalMessageServiceImpl(TransactionalMessageBridge transactionBridge) { + this.transactionalMessageBridge = transactionBridge; + } + + private ConcurrentHashMap opQueueMap = new ConcurrentHashMap<>(); + + @Override + public PutMessageResult prepareMessage(MessageExtBrokerInner messageInner) { + return transactionalMessageBridge.putHalfMessage(messageInner); + } + + private boolean needDiscard(MessageExt msgExt, int transactionCheckMax) { + String checkTimes = msgExt.getProperty(MessageConst.PROPERTY_TRANSACTION_CHECK_TIMES); + int checkTime = 1; + if (null != checkTimes) { + checkTime = getInt(checkTimes); + if (checkTime >= transactionCheckMax) { + return true; + } else { + checkTime++; + } + } + msgExt.putUserProperty(MessageConst.PROPERTY_TRANSACTION_CHECK_TIMES, String.valueOf(checkTime)); + return false; + } + + private boolean needSkip(MessageExt msgExt) { + long valueOfCurrentMinusBorn = System.currentTimeMillis() - msgExt.getBornTimestamp(); + if (valueOfCurrentMinusBorn + > transactionalMessageBridge.getBrokerController().getMessageStoreConfig().getFileReservedTime() + * 3600L * 1000) { + log.info("Half message exceed file reserved time ,so skip it.messageId {},bornTime {}", + msgExt.getMsgId(), msgExt.getBornTimestamp()); + return true; + } + return false; + } + + private boolean putBackHalfMsgQueue(MessageExt msgExt, long offset) { + PutMessageResult putMessageResult = putBackToHalfQueueReturnResult(msgExt); + if (putMessageResult != null + && putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK) { + msgExt.setQueueOffset( + putMessageResult.getAppendMessageResult().getLogicsOffset()); + msgExt.setCommitLogOffset( + putMessageResult.getAppendMessageResult().getWroteOffset()); + msgExt.setMsgId(putMessageResult.getAppendMessageResult().getMsgId()); + log.info( + "Send check message, the offset={} restored in queueOffset={} " + + "commitLogOffset={} " + + "newMsgId={} realMsgId={} topic={}", + offset, msgExt.getQueueOffset(), msgExt.getCommitLogOffset(), msgExt.getMsgId(), + msgExt.getUserProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX), + msgExt.getTopic()); + return true; + } else { + log.error( + "PutBackToHalfQueueReturnResult write failed, topic: {}, queueId: {}, " + + "msgId: {}", + msgExt.getTopic(), msgExt.getQueueId(), msgExt.getMsgId()); + return false; + } + } + + @Override + public void check(long transactionTimeout, int transactionCheckMax, + AbstractTransactionalMessageCheckListener listener) { + try { + String topic = MixAll.RMQ_SYS_TRANS_HALF_TOPIC; + Set msgQueues = transactionalMessageBridge.fetchMessageQueues(topic); + if (msgQueues == null || msgQueues.size() == 0) { + log.warn("The queue of topic is empty :" + topic); + return; + } + log.info("Check topic={}, queues={}", topic, msgQueues); + for (MessageQueue messageQueue : msgQueues) { + long startTime = System.currentTimeMillis(); + MessageQueue opQueue = getOpQueue(messageQueue); + long halfOffset = transactionalMessageBridge.fetchConsumeOffset(messageQueue); + long opOffset = transactionalMessageBridge.fetchConsumeOffset(opQueue); + log.info("Before check, the queue={} msgOffset={} opOffset={}", messageQueue, halfOffset, opOffset); + if (halfOffset < 0 || opOffset < 0) { + log.error("MessageQueue: {} illegal offset read: {}, op offset: {},skip this queue", messageQueue, + halfOffset, opOffset); + continue; + } + + List doneOpOffset = new ArrayList<>(); + HashMap removeMap = new HashMap<>(); + PullResult pullResult = fillOpRemoveMap(removeMap, opQueue, opOffset, halfOffset, doneOpOffset); + if (null == pullResult) { + log.error("The queue={} check msgOffset={} with opOffset={} failed, pullResult is null", + messageQueue, halfOffset, opOffset); + continue; + } + // single thread + int getMessageNullCount = 1; + long newOffset = halfOffset; + long i = halfOffset; + while (true) { + if (System.currentTimeMillis() - startTime > MAX_PROCESS_TIME_LIMIT) { + log.info("Queue={} process time reach max={}", messageQueue, MAX_PROCESS_TIME_LIMIT); + break; + } + if (removeMap.containsKey(i)) { + log.info("Half offset {} has been committed/rolled back", i); + removeMap.remove(i); + } else { + GetResult getResult = getHalfMsg(messageQueue, i); + MessageExt msgExt = getResult.getMsg(); + if (msgExt == null) { + if (getMessageNullCount++ > MAX_RETRY_COUNT_WHEN_HALF_NULL) { + break; + } + if (getResult.getPullResult().getPullStatus() == PullStatus.NO_NEW_MSG) { + log.info("No new msg, the miss offset={} in={}, continue check={}, pull result={}", i, + messageQueue, getMessageNullCount, getResult.getPullResult()); + break; + } else { + log.info("Illegal offset, the miss offset={} in={}, continue check={}, pull result={}", + i, messageQueue, getMessageNullCount, getResult.getPullResult()); + i = getResult.getPullResult().getNextBeginOffset(); + newOffset = i; + continue; + } + } + + if (needDiscard(msgExt, transactionCheckMax) || needSkip(msgExt)) { + listener.resolveDiscardMsg(msgExt); + newOffset = i + 1; + i++; + continue; + } + if (msgExt.getStoreTimestamp() >= startTime) { + log.info("Fresh stored. the miss offset={}, check it later, store={}", i, + new Date(msgExt.getStoreTimestamp())); + break; + } + + long valueOfCurrentMinusBorn = System.currentTimeMillis() - msgExt.getBornTimestamp(); + long checkImmunityTime = transactionTimeout; + String checkImmunityTimeStr = msgExt.getUserProperty(MessageConst.PROPERTY_CHECK_IMMUNITY_TIME_IN_SECONDS); + if (null != checkImmunityTimeStr) { + checkImmunityTime = getImmunityTime(checkImmunityTimeStr, transactionTimeout); + if (valueOfCurrentMinusBorn < checkImmunityTime) { + if (checkPrepareQueueOffset(removeMap, doneOpOffset, msgExt, checkImmunityTime)) { + newOffset = i + 1; + i++; + continue; + } + } + } else { + if ((0 <= valueOfCurrentMinusBorn) && (valueOfCurrentMinusBorn < checkImmunityTime)) { + log.info("New arrived, the miss offset={}, check it later checkImmunity={}, born={}", i, + checkImmunityTime, new Date(msgExt.getBornTimestamp())); + break; + } + } + List opMsg = pullResult.getMsgFoundList(); + boolean isNeedCheck = (opMsg == null && valueOfCurrentMinusBorn > checkImmunityTime) + || (opMsg != null && (opMsg.get(opMsg.size() - 1).getBornTimestamp() - startTime > transactionTimeout)) + || (valueOfCurrentMinusBorn <= -1); + + if (isNeedCheck) { + if (!putBackHalfMsgQueue(msgExt, i)) { + continue; + } + listener.resolveHalfMsg(msgExt); + } else { + pullResult = fillOpRemoveMap(removeMap, opQueue, pullResult.getNextBeginOffset(), halfOffset, doneOpOffset); + log.info("The miss offset:{} in messageQueue:{} need to get more opMsg, result is:{}", i, + messageQueue, pullResult); + continue; + } + } + newOffset = i + 1; + i++; + } + if (newOffset != halfOffset) { + transactionalMessageBridge.updateConsumeOffset(messageQueue, newOffset); + } + long newOpOffset = calculateOpOffset(doneOpOffset, opOffset); + if (newOpOffset != opOffset) { + transactionalMessageBridge.updateConsumeOffset(opQueue, newOpOffset); + } + } + } catch (Exception e) { + e.printStackTrace(); + log.error("Check error", e); + } + + } + + private long getImmunityTime(String checkImmunityTimeStr, long transactionTimeout) { + long checkImmunityTime; + + checkImmunityTime = getLong(checkImmunityTimeStr); + if (-1 == checkImmunityTime) { + checkImmunityTime = transactionTimeout; + } else { + checkImmunityTime *= 1000; + } + return checkImmunityTime; + } + + /** + * Read op message, parse op message, and fill removeMap + * + * @param removeMap Half message to be remove, key:halfOffset, value: opOffset. + * @param opQueue Op message queue. + * @param pullOffsetOfOp The begin offset of op message queue. + * @param miniOffset The current minimum offset of half message queue. + * @param doneOpOffset Stored op messages that have been processed. + * @return Op message result. + */ + private PullResult fillOpRemoveMap(HashMap removeMap, + MessageQueue opQueue, long pullOffsetOfOp, long miniOffset, List doneOpOffset) { + PullResult pullResult = pullOpMsg(opQueue, pullOffsetOfOp, 32); + if (null == pullResult) { + return null; + } + if (pullResult.getPullStatus() == PullStatus.OFFSET_ILLEGAL + || pullResult.getPullStatus() == PullStatus.NO_MATCHED_MSG) { + log.warn("The miss op offset={} in queue={} is illegal, pullResult={}", pullOffsetOfOp, opQueue, + pullResult); + transactionalMessageBridge.updateConsumeOffset(opQueue, pullResult.getNextBeginOffset()); + return pullResult; + } else if (pullResult.getPullStatus() == PullStatus.NO_NEW_MSG) { + log.warn("The miss op offset={} in queue={} is NO_NEW_MSG, pullResult={}", pullOffsetOfOp, opQueue, + pullResult); + return pullResult; + } + List opMsg = pullResult.getMsgFoundList(); + if (opMsg == null) { + log.warn("The miss op offset={} in queue={} is empty, pullResult={}", pullOffsetOfOp, opQueue, pullResult); + return pullResult; + } + for (MessageExt opMessageExt : opMsg) { + Long queueOffset = getLong(new String(opMessageExt.getBody(), TransactionalMessageUtil.charset)); + log.info("Topic: {} tags: {}, OpOffset: {}, HalfOffset: {}", opMessageExt.getTopic(), + opMessageExt.getTags(), opMessageExt.getQueueOffset(), queueOffset); + if (TransactionalMessageUtil.REMOVETAG.equals(opMessageExt.getTags())) { + if (queueOffset < miniOffset) { + doneOpOffset.add(opMessageExt.getQueueOffset()); + } else { + removeMap.put(queueOffset, opMessageExt.getQueueOffset()); + } + } else { + log.error("Found a illegal tag in opMessageExt= {} ", opMessageExt); + } + } + log.debug("Remove map: {}", removeMap); + log.debug("Done op list: {}", doneOpOffset); + return pullResult; + } + + /** + * If return true, skip this msg + * + * @param removeMap Op message map to determine whether a half message was responded by producer. + * @param doneOpOffset Op Message which has been checked. + * @param msgExt Half message + * @param checkImmunityTime User defined time to avoid being detected early. + * @return Return true if put success, otherwise return false. + */ + private boolean checkPrepareQueueOffset(HashMap removeMap, List doneOpOffset, MessageExt msgExt, + long checkImmunityTime) { + if (System.currentTimeMillis() - msgExt.getBornTimestamp() < checkImmunityTime) { + String prepareQueueOffsetStr = msgExt.getUserProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET); + if (null == prepareQueueOffsetStr) { + return putImmunityMsgBackToHalfQueue(msgExt); + } else { + long prepareQueueOffset = getLong(prepareQueueOffsetStr); + if (-1 == prepareQueueOffset) { + return false; + } else { + if (removeMap.containsKey(prepareQueueOffset)) { + long tmpOpOffset = removeMap.remove(prepareQueueOffset); + doneOpOffset.add(tmpOpOffset); + return true; + } else { + return putImmunityMsgBackToHalfQueue(msgExt); + } + } + + } + + } else { + return true; + } + } + + /** + * Write messageExt to Half topic again + * + * @param messageExt Message will be write back to queue + * @return Put result can used to determine the specific results of storage. + */ + private PutMessageResult putBackToHalfQueueReturnResult(MessageExt messageExt) { + PutMessageResult putMessageResult = null; + try { + MessageExtBrokerInner msgInner = transactionalMessageBridge.renewHalfMessageInner(messageExt); + putMessageResult = transactionalMessageBridge.putMessageReturnResult(msgInner); + } catch (Exception e) { + log.warn("PutBackToHalfQueueReturnResult error", e); + } + return putMessageResult; + } + + private boolean putImmunityMsgBackToHalfQueue(MessageExt messageExt) { + MessageExtBrokerInner msgInner = transactionalMessageBridge.renewImmunityHalfMessageInner(messageExt); + return transactionalMessageBridge.putMessage(msgInner); + } + + /** + * Read half message from Half Topic + * + * @param mq Target message queue, in this method, it means the half message queue. + * @param offset Offset in the message queue. + * @param nums Pull message number. + * @return Messages pulled from half message queue. + */ + private PullResult pullHalfMsg(MessageQueue mq, long offset, int nums) { + return transactionalMessageBridge.getHalfMessage(mq.getQueueId(), offset, nums); + } + + /** + * Read op message from Op Topic + * + * @param mq Target Message Queue + * @param offset Offset in the message queue + * @param nums Pull message number + * @return Messages pulled from operate message queue. + */ + private PullResult pullOpMsg(MessageQueue mq, long offset, int nums) { + return transactionalMessageBridge.getOpMessage(mq.getQueueId(), offset, nums); + } + + private Long getLong(String s) { + long v = -1; + try { + v = Long.valueOf(s); + } catch (Exception e) { + log.error("GetLong error", e); + } + return v; + + } + + private Integer getInt(String s) { + int v = -1; + try { + v = Integer.valueOf(s); + } catch (Exception e) { + log.error("GetInt error", e); + } + return v; + + } + + private long calculateOpOffset(List doneOffset, long oldOffset) { + Collections.sort(doneOffset); + long newOffset = oldOffset; + for (int i = 0; i < doneOffset.size(); i++) { + if (doneOffset.get(i) == newOffset) { + newOffset++; + } else { + break; + } + } + return newOffset; + + } + + private MessageQueue getOpQueue(MessageQueue messageQueue) { + MessageQueue opQueue = opQueueMap.get(messageQueue); + if (opQueue == null) { + opQueue = new MessageQueue(TransactionalMessageUtil.buildOpTopic(), messageQueue.getBrokerName(), + messageQueue.getQueueId()); + opQueueMap.put(messageQueue, opQueue); + } + return opQueue; + + } + + private GetResult getHalfMsg(MessageQueue messageQueue, long offset) { + GetResult getResult = new GetResult(); + + PullResult result = pullHalfMsg(messageQueue, offset, PULL_MSG_RETRY_NUMBER); + getResult.setPullResult(result); + List messageExts = result.getMsgFoundList(); + if (messageExts == null) { + return getResult; + } + getResult.setMsg(messageExts.get(0)); + return getResult; + } + + private OperationResult getHalfMessageByOffset(long commitLogOffset) { + OperationResult response = new OperationResult(); + MessageExt messageExt = this.transactionalMessageBridge.lookMessageByOffset(commitLogOffset); + if (messageExt != null) { + response.setPrepareMessage(messageExt); + response.setResponseCode(ResponseCode.SUCCESS); + } else { + response.setResponseCode(ResponseCode.SYSTEM_ERROR); + response.setResponseRemark("Find prepared transaction message failed"); + } + return response; + } + + @Override + public boolean deletePrepareMessage(MessageExt msgExt) { + if (this.transactionalMessageBridge.putOpMessage(msgExt, TransactionalMessageUtil.REMOVETAG)) { + log.info("Transaction op message write successfully. messageId={}, queueId={} msgExt:{}", msgExt.getMsgId(), msgExt.getQueueId(), msgExt); + return true; + } else { + log.error("Transaction op message write failed. messageId is {}, queueId is {}", msgExt.getMsgId(), msgExt.getQueueId()); + return false; + } + } + + @Override + public OperationResult commitMessage(EndTransactionRequestHeader requestHeader) { + return getHalfMessageByOffset(requestHeader.getCommitLogOffset()); + } + + @Override + public OperationResult rollbackMessage(EndTransactionRequestHeader requestHeader) { + return getHalfMessageByOffset(requestHeader.getCommitLogOffset()); + } + + @Override + public boolean open() { + return true; + } + + @Override + public void close() { + + } + +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageUtil.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageUtil.java new file mode 100644 index 00000000000..3042b4c3e56 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageUtil.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.transaction.queue; + +import org.apache.rocketmq.common.MixAll; + +import java.nio.charset.Charset; + +public class TransactionalMessageUtil { + public static final String REMOVETAG = "d"; + public static Charset charset = Charset.forName("utf-8"); + + public static String buildOpTopic() { + return MixAll.RMQ_SYS_TRANS_OP_HALF_TOPIC; + } + + public static String buildHalfTopic() { + return MixAll.RMQ_SYS_TRANS_HALF_TOPIC; + } + + public static String buildConsumerGroup() { + return MixAll.CID_SYS_RMQ_TRANS; + } + +} diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/LocalTransactionExecuter.java b/broker/src/main/java/org/apache/rocketmq/broker/util/PositiveAtomicCounter.java similarity index 62% rename from client/src/main/java/org/apache/rocketmq/client/producer/LocalTransactionExecuter.java rename to broker/src/main/java/org/apache/rocketmq/broker/util/PositiveAtomicCounter.java index 80b5546925f..8d92f43085e 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/LocalTransactionExecuter.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/util/PositiveAtomicCounter.java @@ -14,10 +14,27 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.client.producer; +package org.apache.rocketmq.broker.util; -import org.apache.rocketmq.common.message.Message; +import java.util.concurrent.atomic.AtomicInteger; -public interface LocalTransactionExecuter { - LocalTransactionState executeLocalTransactionBranch(final Message msg, final Object arg); +public class PositiveAtomicCounter { + private static final int MASK = 0x7FFFFFFF; + private final AtomicInteger atom; + + + public PositiveAtomicCounter() { + atom = new AtomicInteger(0); + } + + + public final int incrementAndGet() { + final int rt = atom.incrementAndGet(); + return rt & MASK; + } + + + public int intValue() { + return atom.intValue(); + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/util/ServiceProvider.java b/broker/src/main/java/org/apache/rocketmq/broker/util/ServiceProvider.java new file mode 100644 index 00000000000..59be7a7e26c --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/util/ServiceProvider.java @@ -0,0 +1,191 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to + * You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.apache.rocketmq.broker.util; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; + +public class ServiceProvider { + + private final static Logger LOG = LoggerFactory + .getLogger(ServiceProvider.class); + /** + * A reference to the classloader that loaded this class. It's more efficient to compute it once and cache it here. + */ + private static ClassLoader thisClassLoader; + + /** + * JDK1.3+ 'Service Provider' specification. + */ + public static final String TRANSACTION_SERVICE_ID = "META-INF/service/org.apache.rocketmq.broker.transaction.TransactionalMessageService"; + + public static final String TRANSACTION_LISTENER_ID = "META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener"; + + static { + thisClassLoader = getClassLoader(ServiceProvider.class); + } + + /** + * Returns a string that uniquely identifies the specified object, including its class. + *

    + * The returned string is of form "classname@hashcode", ie is the same as the return value of the Object.toString() method, but works even when the specified object's class has overidden the toString method. + * + * @param o may be null. + * @return a string of form classname@hashcode, or "null" if param o is null. + */ + protected static String objectId(Object o) { + if (o == null) { + return "null"; + } else { + return o.getClass().getName() + "@" + System.identityHashCode(o); + } + } + + protected static ClassLoader getClassLoader(Class clazz) { + try { + return clazz.getClassLoader(); + } catch (SecurityException e) { + LOG.error("Unable to get classloader for class {} due to security restrictions !", + clazz, e.getMessage()); + throw e; + } + } + + protected static ClassLoader getContextClassLoader() { + ClassLoader classLoader = null; + try { + classLoader = Thread.currentThread().getContextClassLoader(); + } catch (SecurityException ex) { + /** + * The getContextClassLoader() method throws SecurityException when the context + * class loader isn't an ancestor of the calling class's class + * loader, or if security permissions are restricted. + */ + } + return classLoader; + } + + protected static InputStream getResourceAsStream(ClassLoader loader, String name) { + if (loader != null) { + return loader.getResourceAsStream(name); + } else { + return ClassLoader.getSystemResourceAsStream(name); + } + } + + public static List load(String name, Class clazz) { + LOG.info("Looking for a resource file of name [{}] ...", name); + List services = new ArrayList(); + try { + ArrayList names = new ArrayList(); + final InputStream is = getResourceAsStream(getContextClassLoader(), name); + if (is != null) { + BufferedReader reader; + try { + reader = new BufferedReader(new InputStreamReader(is, "UTF-8")); + } catch (java.io.UnsupportedEncodingException e) { + reader = new BufferedReader(new InputStreamReader(is)); + } + String serviceName = reader.readLine(); + while (serviceName != null && !"".equals(serviceName)) { + LOG.info( + "Creating an instance as specified by file {} which was present in the path of the context classloader.", + name); + if (!names.contains(serviceName)) { + names.add(serviceName); + } + + services.add((T)initService(getContextClassLoader(), serviceName, clazz)); + + serviceName = reader.readLine(); + } + reader.close(); + } else { + // is == null + LOG.warn("No resource file with name [{}] found.", name); + } + } catch (Exception e) { + LOG.error("Error occured when looking for resource file " + name, e); + } + return services; + } + + public static T loadClass(String name, Class clazz) { + final InputStream is = getResourceAsStream(getContextClassLoader(), name); + BufferedReader reader; + try { + try { + reader = new BufferedReader(new InputStreamReader(is, "UTF-8")); + } catch (java.io.UnsupportedEncodingException e) { + reader = new BufferedReader(new InputStreamReader(is)); + } + String serviceName = reader.readLine(); + reader.close(); + if (serviceName != null && !"".equals(serviceName)) { + return initService(getContextClassLoader(), serviceName, clazz); + } else { + LOG.warn("ServiceName is empty!"); + return null; + } + } catch (Exception e) { + LOG.error("Error occured when looking for resource file " + name, e); + } + return null; + } + + protected static T initService(ClassLoader classLoader, String serviceName, Class clazz) { + Class serviceClazz = null; + try { + if (classLoader != null) { + try { + // Warning: must typecast here & allow exception to be generated/caught & recast properly + serviceClazz = classLoader.loadClass(serviceName); + if (clazz.isAssignableFrom(serviceClazz)) { + LOG.info("Loaded class {} from classloader {}", serviceClazz.getName(), + objectId(classLoader)); + } else { + // This indicates a problem with the ClassLoader tree. An incompatible ClassLoader was used to load the implementation. + LOG.error( + "Class {} loaded from classloader {} does not extend {} as loaded by this classloader.", + new Object[] {serviceClazz.getName(), + objectId(serviceClazz.getClassLoader()), clazz.getName()}); + } + return (T)serviceClazz.newInstance(); + } catch (ClassNotFoundException ex) { + if (classLoader == thisClassLoader) { + // Nothing more to try, onwards. + LOG.warn("Unable to locate any class {} via classloader", serviceName, + objectId(classLoader)); + throw ex; + } + // Ignore exception, continue + } catch (NoClassDefFoundError e) { + if (classLoader == thisClassLoader) { + // Nothing more to try, onwards. + LOG.warn( + "Class {} cannot be loaded via classloader {}.it depends on some other class that cannot be found.", + serviceClazz, objectId(classLoader)); + throw e; + } + // Ignore exception, continue + } + } + } catch (Exception e) { + LOG.error("Unable to init service.", e); + } + return (T)serviceClazz; + } +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/EndTransactionProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/EndTransactionProcessorTest.java new file mode 100644 index 00000000000..019cc6a316c --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/EndTransactionProcessorTest.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.processor; + +import io.netty.channel.ChannelHandlerContext; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.transaction.OperationResult; +import org.apache.rocketmq.broker.transaction.TransactionalMessageService; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.netty.NettyServerConfig; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.store.AppendMessageResult; +import org.apache.rocketmq.store.AppendMessageStatus; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class EndTransactionProcessorTest { + + private EndTransactionProcessor endTransactionProcessor; + + @Mock + private ChannelHandlerContext handlerContext; + + @Spy + private BrokerController + brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), + new MessageStoreConfig()); + + @Mock + private MessageStore messageStore; + + @Mock + private TransactionalMessageService transactionMsgService; + + @Before + public void init() { + brokerController.setMessageStore(messageStore); + brokerController.setTransactionalMessageService(transactionMsgService); + endTransactionProcessor = new EndTransactionProcessor(brokerController); + } + + private OperationResult createResponse(int status){ + OperationResult response = new OperationResult(); + response.setPrepareMessage(createDefaultMessageExt()); + response.setResponseCode(status); + response.setResponseRemark(null); + return response; + } + + @Test + public void testProcessRequest() throws RemotingCommandException { + when(transactionMsgService.commitMessage(any(EndTransactionRequestHeader.class))).thenReturn(createResponse(ResponseCode.SUCCESS)); + when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult + (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + RemotingCommand request = createEndTransactionMsgCommand(MessageSysFlag.TRANSACTION_COMMIT_TYPE, false); + RemotingCommand response = endTransactionProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testProcessRequest_CheckMessage() throws RemotingCommandException { + when(transactionMsgService.commitMessage(any(EndTransactionRequestHeader.class))).thenReturn(createResponse(ResponseCode.SUCCESS)); + when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult + (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + RemotingCommand request = createEndTransactionMsgCommand(MessageSysFlag.TRANSACTION_COMMIT_TYPE, true); + RemotingCommand response = endTransactionProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testProcessRequest_NotType() throws RemotingCommandException { + RemotingCommand request = createEndTransactionMsgCommand(MessageSysFlag.TRANSACTION_NOT_TYPE, true); + RemotingCommand response = endTransactionProcessor.processRequest(handlerContext, request); + assertThat(response).isNull(); + } + + @Test + public void testProcessRequest_RollBack() throws RemotingCommandException { + when(transactionMsgService.rollbackMessage(any(EndTransactionRequestHeader.class))).thenReturn(createResponse(ResponseCode.SUCCESS)); + RemotingCommand request = createEndTransactionMsgCommand(MessageSysFlag.TRANSACTION_ROLLBACK_TYPE, true); + RemotingCommand response = endTransactionProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + private MessageExt createDefaultMessageExt() { + MessageExt messageExt = new MessageExt(); + messageExt.setMsgId("12345678"); + messageExt.setQueueId(0); + messageExt.setCommitLogOffset(123456789L); + messageExt.setQueueOffset(1234); + MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_REAL_QUEUE_ID, "0"); + MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_TRANSACTION_PREPARED, "true"); + MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_PRODUCER_GROUP, "testTransactionGroup"); + return messageExt; + } + + private EndTransactionRequestHeader createEndTransactionRequestHeader(int status, boolean isCheckMsg) { + EndTransactionRequestHeader header = new EndTransactionRequestHeader(); + header.setCommitLogOffset(123456789L); + header.setFromTransactionCheck(isCheckMsg); + header.setCommitOrRollback(status); + header.setMsgId("12345678"); + header.setTransactionId("123"); + header.setProducerGroup("testTransactionGroup"); + header.setTranStateTableOffset(1234L); + return header; + } + + private RemotingCommand createEndTransactionMsgCommand(int status, boolean isCheckMsg) { + EndTransactionRequestHeader header = createEndTransactionRequestHeader(status, isCheckMsg); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.END_TRANSACTION, header); + request.makeCustomHeaderToNet(); + return request; + } +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java index b81ee5a2b78..792fd0fa2c3 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java @@ -18,19 +18,20 @@ import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.List; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.mqtrace.SendMessageContext; import org.apache.rocketmq.broker.mqtrace.SendMessageHook; +import org.apache.rocketmq.broker.transaction.TransactionalMessageService; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.ConsumerSendMsgBackRequestHeader; import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyServerConfig; @@ -51,6 +52,11 @@ import org.mockito.junit.MockitoJUnitRunner; import org.mockito.stubbing.Answer; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; @@ -68,6 +74,9 @@ public class SendMessageProcessorTest { @Mock private MessageStore messageStore; + @Mock + private TransactionalMessageService transactionMsgService; + private String topic = "FooBar"; private String group = "FooBarGroup"; @@ -177,7 +186,42 @@ public void testProcessRequest_WithMsgBack() throws RemotingCommandException { assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); } - private RemotingCommand createSendMsgCommand(int requestCode) { + @Test + public void testProcessRequest_Transaction() throws RemotingCommandException { + brokerController.setTransactionalMessageService(transactionMsgService); + when(brokerController.getTransactionalMessageService().prepareMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + RemotingCommand request = createSendTransactionMsgCommand(RequestCode.SEND_MESSAGE); + final RemotingCommand[] response = new RemotingCommand[1]; + doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + response[0] = invocation.getArgument(0); + return null; + } + }).when(handlerContext).writeAndFlush(any(Object.class)); + RemotingCommand responseToReturn = sendMessageProcessor.processRequest(handlerContext, request); + if (responseToReturn != null) { + assertThat(response[0]).isNull(); + response[0] = responseToReturn; + } + assertThat(response[0].getCode()).isEqualTo(ResponseCode.SUCCESS); + + } + private RemotingCommand createSendTransactionMsgCommand(int requestCode) { + SendMessageRequestHeader header = createSendMsgRequestHeader(); + int sysFlag = header.getSysFlag(); + Map oriProps = MessageDecoder.string2messageProperties(header.getProperties()); + oriProps.put(MessageConst.PROPERTY_TRANSACTION_PREPARED, "true"); + header.setProperties(MessageDecoder.messageProperties2String(oriProps)); + sysFlag |= MessageSysFlag.TRANSACTION_PREPARED_TYPE; + header.setSysFlag(sysFlag); + RemotingCommand request = RemotingCommand.createRequestCommand(requestCode, header); + request.setBody(new byte[] {'a'}); + request.makeCustomHeaderToNet(); + return request; + } + + private SendMessageRequestHeader createSendMsgRequestHeader() { SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); requestHeader.setProducerGroup(group); requestHeader.setTopic(topic); @@ -188,6 +232,11 @@ private RemotingCommand createSendMsgCommand(int requestCode) { requestHeader.setBornTimestamp(System.currentTimeMillis()); requestHeader.setFlag(124); requestHeader.setReconsumeTimes(0); + return requestHeader; + } + + private RemotingCommand createSendMsgCommand(int requestCode) { + SendMessageRequestHeader requestHeader = createSendMsgRequestHeader(); RemotingCommand request = RemotingCommand.createRequestCommand(requestCode, requestHeader); request.setBody(new byte[] {'a'}); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/DefaultTransactionalMessageCheckListenerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/DefaultTransactionalMessageCheckListenerTest.java new file mode 100644 index 00000000000..17bf00b4310 --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/DefaultTransactionalMessageCheckListenerTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.transaction.queue; + +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.netty.NettyServerConfig; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class DefaultTransactionalMessageCheckListenerTest { + + private DefaultTransactionalMessageCheckListener listener; + + @Spy + private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), + new NettyClientConfig(), new MessageStoreConfig()); + + + @Before + public void init() { + listener = new DefaultTransactionalMessageCheckListener(); + listener.setBrokerController(brokerController); + } + + @Test + public void testResolveHalfMsg() { + listener.resolveHalfMsg(createMessageExt()); + } + + @Test + public void testSendCheckMessage() throws Exception{ + MessageExt messageExt = createMessageExt(); + listener.sendCheckMessage(messageExt); + } + + @Test + public void sendCheckMessage(){ + listener.resolveDiscardMsg(createMessageExt()); + } + + private MessageExtBrokerInner createMessageExt() { + MessageExtBrokerInner inner = new MessageExtBrokerInner(); + MessageAccessor.putProperty(inner,MessageConst.PROPERTY_REAL_QUEUE_ID,"1"); + MessageAccessor.putProperty(inner,MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX,"1234255"); + MessageAccessor.putProperty(inner,MessageConst.PROPERTY_REAL_TOPIC,"realTopic"); + inner.setTransactionId(inner.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX)); + inner.setBody("check".getBytes()); + inner.setMsgId("12344567890"); + inner.setQueueId(0); + return inner; + } + +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java new file mode 100644 index 00000000000..b1c669c9933 --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.transaction.queue; + +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.PullStatus; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.netty.NettyServerConfig; +import org.apache.rocketmq.store.AppendMessageResult; +import org.apache.rocketmq.store.AppendMessageStatus; +import org.apache.rocketmq.store.GetMessageResult; +import org.apache.rocketmq.store.GetMessageStatus; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.MessageFilter; +import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentMatchers; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.Map; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class TransactionalMessageBridgeTest { + + private TransactionalMessageBridge transactionBridge; + + @Spy + private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), + new NettyClientConfig(), new MessageStoreConfig()); + + @Mock + private MessageStore messageStore; + + @Before + public void init() { + brokerController.setMessageStore(messageStore); + transactionBridge = new TransactionalMessageBridge(brokerController, messageStore); + } + + @Test + public void testPutOpMessage() { + boolean isSuccess = transactionBridge.putOpMessage(createMessageBrokerInner(), TransactionalMessageUtil.REMOVETAG); + assertThat(isSuccess).isTrue(); + } + + @Test + public void testPutHalfMessage() { + when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult + (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + PutMessageResult result = transactionBridge.putHalfMessage(createMessageBrokerInner()); + assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK); + } + + @Test + public void testFetchMessageQueues() { + Set messageQueues = transactionBridge.fetchMessageQueues(MixAll.RMQ_SYS_TRANS_HALF_TOPIC); + assertThat(messageQueues.size()).isEqualTo(1); + } + + @Test + public void testFetchConsumeOffset() { + MessageQueue mq = new MessageQueue(TransactionalMessageUtil.buildOpTopic(), this.brokerController.getBrokerConfig().getBrokerName(), + 0); + long offset = transactionBridge.fetchConsumeOffset(mq); + assertThat(offset).isGreaterThan(-1); + } + + @Test + public void updateConsumeOffset() { + MessageQueue mq = new MessageQueue(TransactionalMessageUtil.buildOpTopic(), this.brokerController.getBrokerConfig().getBrokerName(), + 0); + transactionBridge.updateConsumeOffset(mq, 0); + } + + @Test + public void testGetHalfMessage() { + when(messageStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), ArgumentMatchers.nullable(MessageFilter.class))).thenReturn(createGetMessageResult(GetMessageStatus.NO_MESSAGE_IN_QUEUE)); + PullResult result = transactionBridge.getHalfMessage(0, 0, 1); + assertThat(result.getPullStatus()).isEqualTo(PullStatus.NO_NEW_MSG); + } + + @Test + public void testGetOpMessage() { + when(messageStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), ArgumentMatchers.nullable(MessageFilter.class))).thenReturn(createGetMessageResult(GetMessageStatus.NO_MESSAGE_IN_QUEUE)); + PullResult result = transactionBridge.getOpMessage(0, 0, 1); + assertThat(result.getPullStatus()).isEqualTo(PullStatus.NO_NEW_MSG); + } + + @Test + public void testPutMessageReturnResult() { + when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult + (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + PutMessageResult result = transactionBridge.putMessageReturnResult(createMessageBrokerInner()); + assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK); + } + + @Test + public void testPutMessage() { + when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult + (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + Boolean success = transactionBridge.putMessage(createMessageBrokerInner()); + assertThat(success).isEqualTo(true); + } + + @Test + public void testRenewImmunityHalfMessageInner() { + MessageExt messageExt = createMessageBrokerInner(); + final String offset = "123456789"; + MessageExtBrokerInner msgInner = transactionBridge.renewImmunityHalfMessageInner(messageExt); + MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET,offset); + assertThat(msgInner).isNotNull(); + Map properties = msgInner.getProperties(); + assertThat(properties).isNotNull(); + String resOffset = properties.get(MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET); + assertThat(resOffset).isEqualTo(offset); + } + + + @Test + public void testRenewHalfMessageInner() { + MessageExt messageExt = new MessageExt(); + long bornTimeStamp = messageExt.getBornTimestamp(); + MessageExt messageExtRes = transactionBridge.renewHalfMessageInner(messageExt); + assertThat( messageExtRes.getBornTimestamp()).isEqualTo(bornTimeStamp); + } + + @Test + public void testLookMessageByOffset(){ + when(messageStore.lookMessageByOffset(anyLong())).thenReturn(new MessageExt()); + MessageExt messageExt = transactionBridge.lookMessageByOffset(123); + assertThat(messageExt).isNotNull(); + } + + private GetMessageResult createGetMessageResult(GetMessageStatus status) { + GetMessageResult getMessageResult = new GetMessageResult(); + getMessageResult.setStatus(status); + getMessageResult.setMinOffset(100); + getMessageResult.setMaxOffset(1024); + getMessageResult.setNextBeginOffset(516); + return getMessageResult; + } + + private MessageExtBrokerInner createMessageBrokerInner() { + MessageExtBrokerInner inner = new MessageExtBrokerInner(); + inner.setTransactionId("12342123444"); + inner.setBornTimestamp(System.currentTimeMillis()); + inner.setBody("prepare".getBytes()); + inner.setMsgId("123456-123"); + inner.setQueueId(0); + inner.setTopic("hello"); + return inner; + } +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImplTest.java b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImplTest.java new file mode 100644 index 00000000000..47eccbe026e --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImplTest.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.transaction.queue; + +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener; +import org.apache.rocketmq.broker.transaction.OperationResult; +import org.apache.rocketmq.broker.transaction.TransactionalMessageService; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.PullStatus; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.netty.NettyServerConfig; +import org.apache.rocketmq.store.AppendMessageResult; +import org.apache.rocketmq.store.AppendMessageStatus; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class TransactionalMessageServiceImplTest { + + private TransactionalMessageService queueTransactionMsgService; + + @Mock + private TransactionalMessageBridge bridge; + + @Spy + private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), + new NettyClientConfig(), new MessageStoreConfig()); + + @Mock + private AbstractTransactionalMessageCheckListener listener; + + @Before + public void init() { + listener.setBrokerController(brokerController); + queueTransactionMsgService = new TransactionalMessageServiceImpl(bridge); + brokerController.getMessageStoreConfig().setFileReservedTime(3); + } + + @Test + public void testPrepareMessage() { + MessageExtBrokerInner inner = createMessageBrokerInner(); + when(bridge.putHalfMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult + (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + PutMessageResult result = queueTransactionMsgService.prepareMessage(inner); + assert result.isOk(); + } + + @Test + public void testCommitMessage() { + when(bridge.lookMessageByOffset(anyLong())).thenReturn(createMessageBrokerInner()); + OperationResult result = queueTransactionMsgService.commitMessage(createEndTransactionRequestHeader(MessageSysFlag.TRANSACTION_COMMIT_TYPE)); + assertThat(result.getResponseCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testRollbackMessage() { + when(bridge.lookMessageByOffset(anyLong())).thenReturn(createMessageBrokerInner()); + OperationResult result = queueTransactionMsgService.commitMessage(createEndTransactionRequestHeader(MessageSysFlag.TRANSACTION_ROLLBACK_TYPE)); + assertThat(result.getResponseCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testCheck_withDiscard() { + when(bridge.fetchMessageQueues(MixAll.RMQ_SYS_TRANS_HALF_TOPIC)).thenReturn(createMessageQueueSet(MixAll.RMQ_SYS_TRANS_HALF_TOPIC)); + when(bridge.getHalfMessage(0, 0, 1)).thenReturn(createDiscardPullResult(MixAll.RMQ_SYS_TRANS_HALF_TOPIC, 5, "hellp", 1)); + when(bridge.getHalfMessage(0, 1, 1)).thenReturn(createPullResult(MixAll.RMQ_SYS_TRANS_HALF_TOPIC, 6, "hellp", 0)); + when(bridge.getOpMessage(anyInt(), anyLong(), anyInt())).thenReturn(createOpPulResult(MixAll.RMQ_SYS_TRANS_OP_HALF_TOPIC, 1, "10", 1)); + long timeOut = this.brokerController.getBrokerConfig().getTransactionTimeOut(); + int checkMax = this.brokerController.getBrokerConfig().getTransactionCheckMax(); + final AtomicInteger checkMessage = new AtomicInteger(0); + doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invocation) { + checkMessage.addAndGet(1); + return null; + } + }).when(listener).resolveDiscardMsg(any(MessageExt.class)); + queueTransactionMsgService.check(timeOut, checkMax, listener); + assertThat(checkMessage.get()).isEqualTo(1); + } + + @Test + public void testCheck_withCheck() { + when(bridge.fetchMessageQueues(MixAll.RMQ_SYS_TRANS_HALF_TOPIC)).thenReturn(createMessageQueueSet(MixAll.RMQ_SYS_TRANS_HALF_TOPIC)); + when(bridge.getHalfMessage(0, 0, 1)).thenReturn(createPullResult(MixAll.RMQ_SYS_TRANS_HALF_TOPIC, 5, "hello", 1)); + when(bridge.getHalfMessage(0, 1, 1)).thenReturn(createPullResult(MixAll.RMQ_SYS_TRANS_HALF_TOPIC, 6, "hellp", 0)); + when(bridge.getOpMessage(anyInt(), anyLong(), anyInt())).thenReturn(createPullResult(MixAll.RMQ_SYS_TRANS_OP_HALF_TOPIC, 1, "5", 0)); + when(bridge.getBrokerController()).thenReturn(this.brokerController); + when(bridge.renewHalfMessageInner(any(MessageExtBrokerInner.class))).thenReturn(createMessageBrokerInner()); + when(bridge.putMessageReturnResult(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult + (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + long timeOut = this.brokerController.getBrokerConfig().getTransactionTimeOut(); + final int checkMax = this.brokerController.getBrokerConfig().getTransactionCheckMax(); + final AtomicInteger checkMessage = new AtomicInteger(0); + doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invocation) { + checkMessage.addAndGet(1); + return checkMessage; + } + }).when(listener).resolveHalfMsg(any(MessageExt.class)); + queueTransactionMsgService.check(timeOut, checkMax, listener); + assertThat(checkMessage.get()).isEqualTo(1); + } + + @Test + public void testDeletePrepareMessage() { + when(bridge.putOpMessage(any(MessageExt.class), anyString())).thenReturn(true); + boolean res = queueTransactionMsgService.deletePrepareMessage(createMessageBrokerInner()); + assertThat(res).isTrue(); + } + + @Test + public void testOpen() { + boolean isOpen = queueTransactionMsgService.open(); + assertThat(isOpen).isTrue(); + } + + private PullResult createDiscardPullResult(String topic, long queueOffset, String body, int size) { + PullResult result = createPullResult(topic, queueOffset, body, size); + List msgs = result.getMsgFoundList(); + for (MessageExt msg : msgs) { + msg.putUserProperty(MessageConst.PROPERTY_TRANSACTION_CHECK_TIMES, "100000"); + } + return result; + } + + private PullResult createPullResult(String topic, long queueOffset, String body, int size) { + PullResult result = null; + if (0 == size) { + result = new PullResult(PullStatus.NO_NEW_MSG, 1, 0, 1, + null); + } else { + result = new PullResult(PullStatus.FOUND, 1, 0, 1, + getMessageList(queueOffset, topic, body, 1)); + return result; + } + return result; + } + + private PullResult createOpPulResult(String topic, long queueOffset, String body, int size) { + PullResult result = createPullResult(topic, queueOffset, body, size); + List msgs = result.getMsgFoundList(); + for (MessageExt msg : msgs) { + msg.setTags(TransactionalMessageUtil.REMOVETAG); + } + return result; + } + + private PullResult createImmunityPulResult(String topic, long queueOffset, String body, int size) { + PullResult result = createPullResult(topic, queueOffset, body, size); + List msgs = result.getMsgFoundList(); + for (MessageExt msg : msgs) { + msg.putUserProperty(MessageConst.PROPERTY_CHECK_IMMUNITY_TIME_IN_SECONDS, "0"); + } + return result; + } + + private List getMessageList(long queueOffset, String topic, String body, int size) { + List msgs = new ArrayList<>(); + for (int i = 0; i < size; i++) { + MessageExt messageExt = createMessageBrokerInner(queueOffset, topic, body); + msgs.add(messageExt); + } + return msgs; + } + + private Set createMessageQueueSet(String topic) { + Set messageQueues = new HashSet<>(); + MessageQueue messageQueue = new MessageQueue(topic, "DefaultCluster", 0); + messageQueues.add(messageQueue); + return messageQueues; + } + + private EndTransactionRequestHeader createEndTransactionRequestHeader(int status) { + EndTransactionRequestHeader header = new EndTransactionRequestHeader(); + header.setCommitLogOffset(123456789L); + header.setCommitOrRollback(status); + header.setMsgId("12345678"); + header.setTransactionId("123"); + header.setProducerGroup("testTransactionGroup"); + header.setTranStateTableOffset(1234L); + return header; + } + + private MessageExtBrokerInner createMessageBrokerInner(long queueOffset, String topic, String body) { + MessageExtBrokerInner inner = new MessageExtBrokerInner(); + inner.setBornTimestamp(System.currentTimeMillis() - 80000); + inner.setTransactionId("123456123"); + inner.setTopic(topic); + inner.setQueueOffset(queueOffset); + inner.setBody(body.getBytes()); + inner.setMsgId("123456123"); + inner.setQueueId(0); + inner.setTopic("hello"); + return inner; + } + + private MessageExtBrokerInner createMessageBrokerInner() { + return createMessageBrokerInner(1, "testTopic", "hello world"); + } +} diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/TransactionCheckListener.java b/broker/src/test/java/org/apache/rocketmq/broker/util/LogTransactionalMessageCheckListener.java similarity index 73% rename from client/src/main/java/org/apache/rocketmq/client/producer/TransactionCheckListener.java rename to broker/src/test/java/org/apache/rocketmq/broker/util/LogTransactionalMessageCheckListener.java index 1cf5c4d90a5..3a2098eabd1 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/TransactionCheckListener.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/util/LogTransactionalMessageCheckListener.java @@ -14,10 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.client.producer; +package org.apache.rocketmq.broker.util; +import org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener; import org.apache.rocketmq.common.message.MessageExt; -public interface TransactionCheckListener { - LocalTransactionState checkLocalTransactionState(final MessageExt msg); +public class LogTransactionalMessageCheckListener extends AbstractTransactionalMessageCheckListener { + + @Override + public void resolveDiscardMsg(MessageExt msgExt) { + + } } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/util/ServiceProviderTest.java b/broker/src/test/java/org/apache/rocketmq/broker/util/ServiceProviderTest.java new file mode 100644 index 00000000000..22228a6e0ef --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/util/ServiceProviderTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.broker.util; + +import org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener; +import org.apache.rocketmq.broker.transaction.TransactionalMessageService; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ServiceProviderTest { + + @Test + public void loadTransactionMsgServiceTest() { + TransactionalMessageService transactionService = ServiceProvider.loadClass(ServiceProvider.TRANSACTION_SERVICE_ID, + TransactionalMessageService.class); + assertThat(transactionService).isNotNull(); + } + + @Test + public void loadAbstractTransactionListenerTest() { + AbstractTransactionalMessageCheckListener listener = ServiceProvider.loadClass(ServiceProvider.TRANSACTION_LISTENER_ID, + AbstractTransactionalMessageCheckListener.class); + assertThat(listener).isNotNull(); + } +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/util/TransactionalMessageServiceImpl.java b/broker/src/test/java/org/apache/rocketmq/broker/util/TransactionalMessageServiceImpl.java new file mode 100644 index 00000000000..3fedf7722ee --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/util/TransactionalMessageServiceImpl.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.util; + +import org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener; +import org.apache.rocketmq.broker.transaction.OperationResult; +import org.apache.rocketmq.broker.transaction.TransactionalMessageService; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.PutMessageResult; + +public class TransactionalMessageServiceImpl implements TransactionalMessageService { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); + + @Override + public PutMessageResult prepareMessage(MessageExtBrokerInner messageInner) { + return null; + } + + @Override + public boolean deletePrepareMessage(MessageExt messageExt) { + return false; + } + + @Override + public OperationResult commitMessage(EndTransactionRequestHeader requestHeader) { + return null; + } + + @Override + public OperationResult rollbackMessage(EndTransactionRequestHeader requestHeader) { + return null; + } + + @Override + public void check(long transactionTimeout, int transactionCheckMax, AbstractTransactionalMessageCheckListener listener) { + log.warn("check check!"); + } + + @Override + public boolean open() { + return true; + } + + @Override + public void close() { + + } +} diff --git a/broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener b/broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener new file mode 100644 index 00000000000..455b266b9f1 --- /dev/null +++ b/broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener @@ -0,0 +1 @@ +org.apache.rocketmq.broker.util.LogTransactionalMessageCheckListener \ No newline at end of file diff --git a/broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.TransactionalMessageService b/broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.TransactionalMessageService new file mode 100644 index 00000000000..b012e1451c7 --- /dev/null +++ b/broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.TransactionalMessageService @@ -0,0 +1 @@ +org.apache.rocketmq.broker.util.TransactionalMessageServiceImpl \ No newline at end of file diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java b/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java index 9771f145fb5..fe0db96c907 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java @@ -91,6 +91,10 @@ public RemotingCommand checkTransactionState(ChannelHandlerContext ctx, final ByteBuffer byteBuffer = ByteBuffer.wrap(request.getBody()); final MessageExt messageExt = MessageDecoder.decode(byteBuffer); if (messageExt != null) { + String transactionId = messageExt.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX); + if (null != transactionId && !"".equals(transactionId)) { + messageExt.setTransactionId(transactionId); + } final String group = messageExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP); if (group != null) { MQProducerInner producer = this.mqClientFactory.selectProducer(group); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java index 42c324fe2d3..b650e35e02f 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java @@ -96,6 +96,10 @@ public PullResult processPullResult(final MessageQueue mq, final PullResult pull } for (MessageExt msg : msgListFilterAgain) { + String traFlag = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED); + if (traFlag != null && Boolean.parseBoolean(traFlag)) { + msg.setTransactionId(msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX)); + } MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MIN_OFFSET, Long.toString(pullResult.getMinOffset())); MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MAX_OFFSET, diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index 81461f5e9b2..4b5e373fd27 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -16,33 +16,6 @@ */ package org.apache.rocketmq.client.impl.producer; -import java.io.IOException; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Random; -import java.util.Set; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.common.message.Message; -import org.apache.rocketmq.common.message.MessageClientIDSetter; -import org.apache.rocketmq.common.message.MessageExt; -import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.common.message.MessageConst; -import org.apache.rocketmq.common.message.MessageDecoder; -import org.apache.rocketmq.common.message.MessageBatch; -import org.apache.rocketmq.common.message.MessageAccessor; -import org.apache.rocketmq.common.message.MessageType; -import org.apache.rocketmq.common.message.MessageId; import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.Validators; import org.apache.rocketmq.client.common.ClientErrorCode; @@ -58,30 +31,56 @@ import org.apache.rocketmq.client.latency.MQFaultStrategy; import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.client.producer.DefaultMQProducer; -import org.apache.rocketmq.client.producer.LocalTransactionExecuter; import org.apache.rocketmq.client.producer.LocalTransactionState; import org.apache.rocketmq.client.producer.MessageQueueSelector; import org.apache.rocketmq.client.producer.SendCallback; import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.client.producer.SendStatus; -import org.apache.rocketmq.client.producer.TransactionCheckListener; +import org.apache.rocketmq.client.producer.TransactionListener; import org.apache.rocketmq.client.producer.TransactionMQProducer; import org.apache.rocketmq.client.producer.TransactionSendResult; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.ServiceState; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageBatch; +import org.apache.rocketmq.common.message.MessageClientIDSetter; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageId; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.message.MessageType; import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader; import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingConnectException; import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; +import java.io.IOException; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + public class DefaultMQProducerImpl implements MQProducerInner { private final InternalLogger log = ClientLogger.getLog(); private final Random random = new Random(); @@ -116,13 +115,17 @@ public void registerCheckForbiddenHook(CheckForbiddenHook checkForbiddenHook) { public void initTransactionEnv() { TransactionMQProducer producer = (TransactionMQProducer) this.defaultMQProducer; - this.checkRequestQueue = new LinkedBlockingQueue(producer.getCheckRequestHoldMax()); - this.checkExecutor = new ThreadPoolExecutor( - producer.getCheckThreadPoolMinSize(), - producer.getCheckThreadPoolMaxSize(), - 1000 * 60, - TimeUnit.MILLISECONDS, - this.checkRequestQueue); + if (producer.getExecutorService() != null) { + this.checkExecutor = producer.getExecutorService(); + } else { + this.checkRequestQueue = new LinkedBlockingQueue(2000); + this.checkExecutor = new ThreadPoolExecutor( + 1, + 1, + 1000 * 60, + TimeUnit.MILLISECONDS, + this.checkRequestQueue); + } } public void destroyTransactionEnv() { @@ -239,10 +242,10 @@ public boolean isPublishTopicNeedUpdate(String topic) { } @Override - public TransactionCheckListener checkListener() { + public TransactionListener checkListener() { if (this.defaultMQProducer instanceof TransactionMQProducer) { TransactionMQProducer producer = (TransactionMQProducer) defaultMQProducer; - return producer.getTransactionCheckListener(); + return producer.getTransactionListener(); } return null; @@ -259,12 +262,12 @@ public void checkTransactionState(final String addr, final MessageExt msg, @Override public void run() { - TransactionCheckListener transactionCheckListener = DefaultMQProducerImpl.this.checkListener(); + TransactionListener transactionCheckListener = DefaultMQProducerImpl.this.checkListener(); if (transactionCheckListener != null) { LocalTransactionState localTransactionState = LocalTransactionState.UNKNOW; Throwable exception = null; try { - localTransactionState = transactionCheckListener.checkLocalTransactionState(message); + localTransactionState = transactionCheckListener.checkLocalTransaction(message); } catch (Throwable e) { log.error("Broker call checkTransactionState, but checkLocalTransactionState exception", e); exception = e; @@ -962,7 +965,7 @@ public void sendOneway(Message msg, MessageQueueSelector selector, Object arg) } public TransactionSendResult sendMessageInTransaction(final Message msg, - final LocalTransactionExecuter tranExecuter, final Object arg) + final TransactionListener tranExecuter, final Object arg) throws MQClientException { if (null == tranExecuter) { throw new MQClientException("tranExecutor is null", null); @@ -986,7 +989,11 @@ public TransactionSendResult sendMessageInTransaction(final Message msg, if (sendResult.getTransactionId() != null) { msg.putUserProperty("__transactionId__", sendResult.getTransactionId()); } - localTransactionState = tranExecuter.executeLocalTransactionBranch(msg, arg); + String transactionId = msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX); + if (null != transactionId && !"".equals(transactionId)) { + msg.setTransactionId(transactionId); + } + localTransactionState = tranExecuter.executeLocalTransaction(msg, arg); if (null == localTransactionState) { localTransactionState = LocalTransactionState.UNKNOW; } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/MQProducerInner.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/MQProducerInner.java index dfd485dd909..52ebe1b57d3 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/MQProducerInner.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/MQProducerInner.java @@ -16,17 +16,18 @@ */ package org.apache.rocketmq.client.impl.producer; -import java.util.Set; -import org.apache.rocketmq.client.producer.TransactionCheckListener; +import org.apache.rocketmq.client.producer.TransactionListener; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader; +import java.util.Set; + public interface MQProducerInner { Set getPublishTopicList(); boolean isPublishTopicNeedUpdate(final String topic); - TransactionCheckListener checkListener(); + TransactionListener checkListener(); void checkTransactionState( final String addr, diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java index 267dbe8f57b..065f068c7a7 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java @@ -464,14 +464,12 @@ public void sendOneway(Message msg, MessageQueueSelector selector, Object arg) * This method is to send transactional messages. * * @param msg Transactional message to send. - * @param tranExecuter local transaction executor. * @param arg Argument used along with local transaction executor. * @return Transaction result. * @throws MQClientException if there is any client error. */ @Override - public TransactionSendResult sendMessageInTransaction(Message msg, LocalTransactionExecuter tranExecuter, - final Object arg) + public TransactionSendResult sendMessageInTransaction(Message msg, final Object arg) throws MQClientException { throw new RuntimeException("sendMessageInTransaction not implement, please use TransactionMQProducer class"); } diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java index 14caf6ffac9..0776ee155cf 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java @@ -80,8 +80,7 @@ void send(final Message msg, final MessageQueueSelector selector, final Object a void sendOneway(final Message msg, final MessageQueueSelector selector, final Object arg) throws MQClientException, RemotingException, InterruptedException; - TransactionSendResult sendMessageInTransaction(final Message msg, - final LocalTransactionExecuter tranExecuter, final Object arg) throws MQClientException; + TransactionSendResult sendMessageInTransaction(final Message msg, final Object arg) throws MQClientException; //for batch SendResult send(final Collection msgs) throws MQClientException, RemotingException, MQBrokerException, diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/TransactionListener.java b/client/src/main/java/org/apache/rocketmq/client/producer/TransactionListener.java new file mode 100644 index 00000000000..c750e53845a --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/producer/TransactionListener.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.producer; + +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageExt; + +public interface TransactionListener { + /** + * When send transactional prepare(half) message succeed, this method will be invoked to execute local transaction. + * + * @param msg Half(prepare) message + * @param arg Custom business parameter + * @return Transaction state + */ + LocalTransactionState executeLocalTransaction(final Message msg, final Object arg); + + /** + * When no response to prepare(half) message. broker will send check message to check the transaction status, and this + * method will be invoked to get local transaction status. + * + * @param msg Check message + * @return Transaction state + */ + LocalTransactionState checkLocalTransaction(final MessageExt msg); +} diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/TransactionMQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/TransactionMQProducer.java index 7b87e8f720b..c4f122c5848 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/TransactionMQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/TransactionMQProducer.java @@ -20,11 +20,12 @@ import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.remoting.RPCHook; +import java.util.concurrent.ExecutorService; + public class TransactionMQProducer extends DefaultMQProducer { - private TransactionCheckListener transactionCheckListener; - private int checkThreadPoolMinSize = 1; - private int checkThreadPoolMaxSize = 1; - private int checkRequestHoldMax = 2000; + private TransactionListener transactionListener; + + private ExecutorService executorService; public TransactionMQProducer() { } @@ -50,44 +51,27 @@ public void shutdown() { } @Override - public TransactionSendResult sendMessageInTransaction(final Message msg, - final LocalTransactionExecuter tranExecuter, final Object arg) throws MQClientException { - if (null == this.transactionCheckListener) { - throw new MQClientException("localTransactionBranchCheckListener is null", null); + public TransactionSendResult sendMessageInTransaction(final Message msg, final Object arg) throws MQClientException { + if (null == this.transactionListener) { + throw new MQClientException("TransactionListener is null", null); } - return this.defaultMQProducerImpl.sendMessageInTransaction(msg, tranExecuter, arg); - } - - public TransactionCheckListener getTransactionCheckListener() { - return transactionCheckListener; - } - - public void setTransactionCheckListener(TransactionCheckListener transactionCheckListener) { - this.transactionCheckListener = transactionCheckListener; - } - - public int getCheckThreadPoolMinSize() { - return checkThreadPoolMinSize; - } - - public void setCheckThreadPoolMinSize(int checkThreadPoolMinSize) { - this.checkThreadPoolMinSize = checkThreadPoolMinSize; + return this.defaultMQProducerImpl.sendMessageInTransaction(msg, transactionListener, arg); } - public int getCheckThreadPoolMaxSize() { - return checkThreadPoolMaxSize; + public TransactionListener getTransactionListener() { + return transactionListener; } - public void setCheckThreadPoolMaxSize(int checkThreadPoolMaxSize) { - this.checkThreadPoolMaxSize = checkThreadPoolMaxSize; + public void setTransactionListener(TransactionListener transactionListener) { + this.transactionListener = transactionListener; } - public int getCheckRequestHoldMax() { - return checkRequestHoldMax; + public ExecutorService getExecutorService() { + return executorService; } - public void setCheckRequestHoldMax(int checkRequestHoldMax) { - this.checkRequestHoldMax = checkRequestHoldMax; + public void setExecutorService(ExecutorService executorService) { + this.executorService = executorService; } } diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index 203431aee1a..e4486da0d15 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -137,13 +137,30 @@ public class BrokerConfig { private boolean forceRegister = true; /** - * * This configurable item defines interval of topics registration of broker to name server. Allowing values are * between 10, 000 and 60, 000 milliseconds. - * */ private int registerNameServerPeriod = 1000 * 30; + /** + * The minimum time of the transactional message to be checked firstly, one message only exceed this time interval + * that can be checked. + */ + @ImportantField + private long transactionTimeOut = 3 * 1000; + + /** + * The maximum number of times the message was checked, if exceed this value, this message will be discarded. + */ + @ImportantField + private int transactionCheckMax = 5; + + /** + * Transaction message check interval. + */ + @ImportantField + private long transactionCheckInterval = 60 * 1000; + public boolean isTraceOn() { return traceOn; } @@ -633,4 +650,28 @@ public int getRegisterNameServerPeriod() { public void setRegisterNameServerPeriod(int registerNameServerPeriod) { this.registerNameServerPeriod = registerNameServerPeriod; } + + public long getTransactionTimeOut() { + return transactionTimeOut; + } + + public void setTransactionTimeOut(long transactionTimeOut) { + this.transactionTimeOut = transactionTimeOut; + } + + public int getTransactionCheckMax() { + return transactionCheckMax; + } + + public void setTransactionCheckMax(int transactionCheckMax) { + this.transactionCheckMax = transactionCheckMax; + } + + public long getTransactionCheckInterval() { + return transactionCheckInterval; + } + + public void setTransactionCheckInterval(long transactionCheckInterval) { + this.transactionCheckInterval = transactionCheckInterval; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/MixAll.java b/common/src/main/java/org/apache/rocketmq/common/MixAll.java index 19ab72e16ea..ae97cc97d86 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MixAll.java +++ b/common/src/main/java/org/apache/rocketmq/common/MixAll.java @@ -89,6 +89,10 @@ public class MixAll { public static final String DEFAULT_TRACE_REGION_ID = "DefaultRegion"; public static final String CONSUME_CONTEXT_TYPE = "ConsumeContextType"; + public static final String RMQ_SYS_TRANS_HALF_TOPIC = "RMQ_SYS_TRANS_HALF_TOPIC"; + public static final String RMQ_SYS_TRANS_OP_HALF_TOPIC = "RMQ_SYS_TRANS_OP_HALF_TOPIC"; + public static final String CID_SYS_RMQ_TRANS = "CID_RMQ_SYS_TRANS"; + public static String getWSAddr() { String wsDomainName = System.getProperty("rocketmq.namesrv.domain", DEFAULT_NAMESRV_ADDR_LOOKUP); String wsDomainSubgroup = System.getProperty("rocketmq.namesrv.domain.subgroup", "nsaddr"); diff --git a/common/src/main/java/org/apache/rocketmq/common/message/Message.java b/common/src/main/java/org/apache/rocketmq/common/message/Message.java index 15ba2142c3e..287be13088f 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/Message.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/Message.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.common.message; import java.io.Serializable; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.Map; @@ -28,6 +29,7 @@ public class Message implements Serializable { private int flag; private Map properties; private byte[] body; + private String transactionId; public Message() { } @@ -191,9 +193,22 @@ public void setBuyerId(String buyerId) { putProperty(MessageConst.PROPERTY_BUYER_ID, buyerId); } + public String getTransactionId() { + return transactionId; + } + + public void setTransactionId(String transactionId) { + this.transactionId = transactionId; + } + @Override public String toString() { - return "Message [topic=" + topic + ", flag=" + flag + ", properties=" + properties + ", body=" - + (body != null ? body.length : 0) + "]"; + return "Message{" + + "topic='" + topic + '\'' + + ", flag=" + flag + + ", properties=" + properties + + ", body=" + Arrays.toString(body) + + ", transactionId='" + transactionId + '\'' + + '}'; } } diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java index 1edbbec70c6..bf5c807c7f3 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java @@ -41,6 +41,9 @@ public class MessageConst { public static final String PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX = "UNIQ_KEY"; public static final String PROPERTY_MAX_RECONSUME_TIMES = "MAX_RECONSUME_TIMES"; public static final String PROPERTY_CONSUME_START_TIMESTAMP = "CONSUME_START_TIME"; + public static final String PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET = "TRAN_PREPARED_QUEUE_OFFSET"; + public static final String PROPERTY_TRANSACTION_CHECK_TIMES = "TRANSACTION_CHECK_TIMES"; + public static final String PROPERTY_CHECK_IMMUNITY_TIME_IN_SECONDS = "CHECK_IMMUNITY_TIME_IN_SECONDS"; public static final String KEY_SEPARATOR = " "; diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/CheckTransactionStateRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/CheckTransactionStateRequestHeader.java index 76c7b42791f..6cba71c7e91 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/CheckTransactionStateRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/CheckTransactionStateRequestHeader.java @@ -31,6 +31,7 @@ public class CheckTransactionStateRequestHeader implements CommandCustomHeader { private Long commitLogOffset; private String msgId; private String transactionId; + private String offsetMsgId; @Override public void checkFields() throws RemotingCommandException { @@ -67,4 +68,12 @@ public String getTransactionId() { public void setTransactionId(String transactionId) { this.transactionId = transactionId; } + + public String getOffsetMsgId() { + return offsetMsgId; + } + + public void setOffsetMsgId(String offsetMsgId) { + this.offsetMsgId = offsetMsgId; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/EndTransactionRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/EndTransactionRequestHeader.java index 67785e23cfb..87661c320ad 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/EndTransactionRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/EndTransactionRequestHeader.java @@ -15,9 +15,6 @@ * limitations under the License. */ -/** - * $Id: EndTransactionRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $ - */ package org.apache.rocketmq.common.protocol.header; import org.apache.rocketmq.common.sysflag.MessageSysFlag; @@ -121,9 +118,14 @@ public void setTransactionId(String transactionId) { @Override public String toString() { - return "EndTransactionRequestHeader [producerGroup=" + producerGroup + ", tranStateTableOffset=" - + tranStateTableOffset + ", commitLogOffset=" + commitLogOffset + ", commitOrRollback=" - + commitOrRollback + ", fromTransactionCheck=" + fromTransactionCheck + ", msgId=" + msgId - + "]"; + return "EndTransactionRequestHeader{" + + "producerGroup='" + producerGroup + '\'' + + ", tranStateTableOffset=" + tranStateTableOffset + + ", commitLogOffset=" + commitLogOffset + + ", commitOrRollback=" + commitOrRollback + + ", fromTransactionCheck=" + fromTransactionCheck + + ", msgId='" + msgId + '\'' + + ", transactionId='" + transactionId + '\'' + + '}'; } } diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java index d9fafdd08e8..2d8a5fead37 100644 --- a/example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java +++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java @@ -17,34 +17,34 @@ package org.apache.rocketmq.example.benchmark; -import java.io.UnsupportedEncodingException; -import java.util.LinkedList; -import java.util.Timer; -import java.util.TimerTask; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicLong; import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.producer.LocalTransactionExecuter; import org.apache.rocketmq.client.producer.LocalTransactionState; import org.apache.rocketmq.client.producer.SendResult; -import org.apache.rocketmq.client.producer.TransactionCheckListener; +import org.apache.rocketmq.client.producer.TransactionListener; import org.apache.rocketmq.client.producer.TransactionMQProducer; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.remoting.common.RemotingHelper; +import java.io.UnsupportedEncodingException; +import java.util.LinkedList; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; + public class TransactionProducer { private static int threadCount; private static int messageSize; - private static boolean ischeck; - private static boolean ischeckffalse; + private static boolean isCheck; + private static boolean isCheckFalse; public static void main(String[] args) throws MQClientException, UnsupportedEncodingException { threadCount = args.length >= 1 ? Integer.parseInt(args[0]) : 32; messageSize = args.length >= 2 ? Integer.parseInt(args[1]) : 1024 * 2; - ischeck = args.length >= 3 && Boolean.parseBoolean(args[2]); - ischeckffalse = args.length >= 4 && Boolean.parseBoolean(args[3]); + isCheck = args.length >= 3 && Boolean.parseBoolean(args[2]); + isCheckFalse = args.length >= 4 && Boolean.parseBoolean(args[3]); final Message msg = buildMessage(messageSize); @@ -73,8 +73,8 @@ private void printStats() { Long[] end = snapshotList.getLast(); final long sendTps = - (long) (((end[3] - begin[3]) / (double) (end[0] - begin[0])) * 1000L); - final double averageRT = (end[5] - begin[5]) / (double) (end[3] - begin[3]); + (long)(((end[3] - begin[3]) / (double)(end[0] - begin[0])) * 1000L); + final double averageRT = (end[5] - begin[5]) / (double)(end[3] - begin[3]); System.out.printf( "Send TPS: %d Max RT: %d Average RT: %7.3f Send Failed: %d Response Failed: %d transaction checkCount: %d %n", @@ -92,16 +92,14 @@ public void run() { } }, 10000, 10000); - final TransactionCheckListener transactionCheckListener = - new TransactionCheckListenerBImpl(ischeckffalse, statsBenchmark); + final TransactionListener transactionListener = + new TransactionListenerImpl(isCheckFalse, isCheck, statsBenchmark); final TransactionMQProducer producer = new TransactionMQProducer("benchmark_transaction_producer"); producer.setInstanceName(Long.toString(System.currentTimeMillis())); - producer.setTransactionCheckListener(transactionCheckListener); + producer.setTransactionListener(transactionListener); producer.setDefaultTopicQueueNums(1000); producer.start(); - final TransactionExecuterBImpl tranExecuter = new TransactionExecuterBImpl(ischeck); - for (int i = 0; i < threadCount; i++) { sendThreadPool.execute(new Runnable() { @Override @@ -111,7 +109,7 @@ public void run() { // Thread.sleep(1000); final long beginTimestamp = System.currentTimeMillis(); SendResult sendResult = - producer.sendMessageInTransaction(msg, tranExecuter, null); + producer.sendMessageInTransaction(msg, null); if (sendResult != null) { statsBenchmark.getSendRequestSuccessCount().incrementAndGet(); statsBenchmark.getReceiveResponseSuccessCount().incrementAndGet(); @@ -124,8 +122,7 @@ public void run() { boolean updated = statsBenchmark.getSendMessageMaxRT().compareAndSet(prevMaxRT, currentRT); - if (updated) - break; + if (updated) { break; } prevMaxRT = statsBenchmark.getSendMessageMaxRT().get(); } @@ -153,43 +150,37 @@ private static Message buildMessage(final int messageSize) throws UnsupportedEnc } } -class TransactionExecuterBImpl implements LocalTransactionExecuter { - private boolean ischeck; - - public TransactionExecuterBImpl(boolean ischeck) { - this.ischeck = ischeck; - } - - @Override - public LocalTransactionState executeLocalTransactionBranch(final Message msg, final Object arg) { - if (ischeck) { - return LocalTransactionState.UNKNOW; - } - return LocalTransactionState.COMMIT_MESSAGE; - } -} - -class TransactionCheckListenerBImpl implements TransactionCheckListener { - private boolean ischeckffalse; +class TransactionListenerImpl implements TransactionListener { + private boolean isCheckFalse; private StatsBenchmarkTProducer statsBenchmarkTProducer; + private boolean isCheckLocal; - public TransactionCheckListenerBImpl(boolean ischeckffalse, - StatsBenchmarkTProducer statsBenchmarkTProducer) { - this.ischeckffalse = ischeckffalse; + public TransactionListenerImpl(boolean isCheckFalse, boolean isCheckLocal, + StatsBenchmarkTProducer statsBenchmarkTProducer) { + this.isCheckFalse = isCheckFalse; + this.isCheckLocal = isCheckLocal; this.statsBenchmarkTProducer = statsBenchmarkTProducer; } @Override - public LocalTransactionState checkLocalTransactionState(MessageExt msg) { + public LocalTransactionState checkLocalTransaction(MessageExt msg) { statsBenchmarkTProducer.getCheckRequestSuccessCount().incrementAndGet(); - if (ischeckffalse) { + if (isCheckFalse) { return LocalTransactionState.ROLLBACK_MESSAGE; } return LocalTransactionState.COMMIT_MESSAGE; } + + @Override + public LocalTransactionState executeLocalTransaction(final Message msg, final Object arg) { + if (isCheckLocal) { + return LocalTransactionState.UNKNOW; + } + return LocalTransactionState.COMMIT_MESSAGE; + } } class StatsBenchmarkTProducer { diff --git a/example/src/main/java/org/apache/rocketmq/example/transaction/TransactionExecuterImpl.java b/example/src/main/java/org/apache/rocketmq/example/transaction/TransactionExecuterImpl.java deleted file mode 100644 index e7d193ee85b..00000000000 --- a/example/src/main/java/org/apache/rocketmq/example/transaction/TransactionExecuterImpl.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.example.transaction; - -import java.util.concurrent.atomic.AtomicInteger; -import org.apache.rocketmq.client.producer.LocalTransactionExecuter; -import org.apache.rocketmq.client.producer.LocalTransactionState; -import org.apache.rocketmq.common.message.Message; - -public class TransactionExecuterImpl implements LocalTransactionExecuter { - private AtomicInteger transactionIndex = new AtomicInteger(1); - - @Override - public LocalTransactionState executeLocalTransactionBranch(final Message msg, final Object arg) { - int value = transactionIndex.getAndIncrement(); - - if (value == 0) { - throw new RuntimeException("Could not find db"); - } else if ((value % 5) == 0) { - return LocalTransactionState.ROLLBACK_MESSAGE; - } else if ((value % 4) == 0) { - return LocalTransactionState.COMMIT_MESSAGE; - } - - return LocalTransactionState.UNKNOW; - } -} diff --git a/example/src/main/java/org/apache/rocketmq/example/transaction/TransactionCheckListenerImpl.java b/example/src/main/java/org/apache/rocketmq/example/transaction/TransactionListenerImpl.java similarity index 54% rename from example/src/main/java/org/apache/rocketmq/example/transaction/TransactionCheckListenerImpl.java rename to example/src/main/java/org/apache/rocketmq/example/transaction/TransactionListenerImpl.java index acb755ecfd7..ce471d2ce92 100644 --- a/example/src/main/java/org/apache/rocketmq/example/transaction/TransactionCheckListenerImpl.java +++ b/example/src/main/java/org/apache/rocketmq/example/transaction/TransactionListenerImpl.java @@ -16,27 +16,40 @@ */ package org.apache.rocketmq.example.transaction; -import java.util.concurrent.atomic.AtomicInteger; import org.apache.rocketmq.client.producer.LocalTransactionState; -import org.apache.rocketmq.client.producer.TransactionCheckListener; +import org.apache.rocketmq.client.producer.TransactionListener; +import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageExt; -public class TransactionCheckListenerImpl implements TransactionCheckListener { +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +public class TransactionListenerImpl implements TransactionListener { private AtomicInteger transactionIndex = new AtomicInteger(0); - @Override - public LocalTransactionState checkLocalTransactionState(MessageExt msg) { - System.out.printf("server checking TrMsg %s%n", msg); + private ConcurrentHashMap localTrans = new ConcurrentHashMap<>(); + @Override + public LocalTransactionState executeLocalTransaction(Message msg, Object arg) { int value = transactionIndex.getAndIncrement(); - if ((value % 6) == 0) { - throw new RuntimeException("Could not find db"); - } else if ((value % 5) == 0) { - return LocalTransactionState.ROLLBACK_MESSAGE; - } else if ((value % 4) == 0) { - return LocalTransactionState.COMMIT_MESSAGE; - } - + int status = value % 3; + localTrans.put(msg.getTransactionId(), status); return LocalTransactionState.UNKNOW; } + + @Override + public LocalTransactionState checkLocalTransaction(MessageExt msg) { + Integer status = localTrans.get(msg.getTransactionId()); + if (null != status) { + switch (status) { + case 0: + return LocalTransactionState.UNKNOW; + case 1: + return LocalTransactionState.COMMIT_MESSAGE; + case 2: + return LocalTransactionState.ROLLBACK_MESSAGE; + } + } + return LocalTransactionState.COMMIT_MESSAGE; + } } diff --git a/example/src/main/java/org/apache/rocketmq/example/transaction/TransactionProducer.java b/example/src/main/java/org/apache/rocketmq/example/transaction/TransactionProducer.java index edfad2485f6..75c805b9698 100644 --- a/example/src/main/java/org/apache/rocketmq/example/transaction/TransactionProducer.java +++ b/example/src/main/java/org/apache/rocketmq/example/transaction/TransactionProducer.java @@ -16,32 +16,44 @@ */ package org.apache.rocketmq.example.transaction; -import java.io.UnsupportedEncodingException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.producer.SendResult; -import org.apache.rocketmq.client.producer.TransactionCheckListener; +import org.apache.rocketmq.client.producer.TransactionListener; import org.apache.rocketmq.client.producer.TransactionMQProducer; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.remoting.common.RemotingHelper; +import java.io.UnsupportedEncodingException; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + public class TransactionProducer { public static void main(String[] args) throws MQClientException, InterruptedException { - TransactionCheckListener transactionCheckListener = new TransactionCheckListenerImpl(); + TransactionListener transactionListener = new TransactionListenerImpl(); TransactionMQProducer producer = new TransactionMQProducer("please_rename_unique_group_name"); - producer.setCheckThreadPoolMinSize(2); - producer.setCheckThreadPoolMaxSize(2); - producer.setCheckRequestHoldMax(2000); - producer.setTransactionCheckListener(transactionCheckListener); + ExecutorService executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, new ArrayBlockingQueue(2000), new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + Thread thread = new Thread(r); + thread.setName("client-transaction-msg-check-thread"); + return thread; + } + }); + + producer.setExecutorService(executorService); + producer.setTransactionListener(transactionListener); producer.start(); String[] tags = new String[] {"TagA", "TagB", "TagC", "TagD", "TagE"}; - TransactionExecuterImpl tranExecuter = new TransactionExecuterImpl(); - for (int i = 0; i < 100; i++) { + for (int i = 0; i < 10; i++) { try { Message msg = - new Message("TopicTest", tags[i % tags.length], "KEY" + i, + new Message("TopicTest1234", tags[i % tags.length], "KEY" + i, ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET)); - SendResult sendResult = producer.sendMessageInTransaction(msg, tranExecuter, null); + SendResult sendResult = producer.sendMessageInTransaction(msg, null); System.out.printf("%s%n", sendResult); Thread.sleep(10); diff --git a/logging/src/test/java/org/apache/rocketmq/logging/inner/LoggingBuilderTest.java b/logging/src/test/java/org/apache/rocketmq/logging/inner/LoggingBuilderTest.java index 977e553b70b..6c816a6815c 100644 --- a/logging/src/test/java/org/apache/rocketmq/logging/inner/LoggingBuilderTest.java +++ b/logging/src/test/java/org/apache/rocketmq/logging/inner/LoggingBuilderTest.java @@ -88,7 +88,7 @@ public void testRollingFileAppender() throws InterruptedException { Assert.assertTrue(cc >= 2); } - @Test + //@Test public void testDailyRollingFileAppender() throws InterruptedException { String rollingFile = loggingDir + "/daily-rolling--222.log"; Appender rollingFileAppender = LoggingBuilder.newAppenderBuilder().withAsync(false, 1024) diff --git a/pom.xml b/pom.xml index 72cd3496399..857dfce65ed 100644 --- a/pom.xml +++ b/pom.xml @@ -254,6 +254,7 @@ bin/README.md .github/* src/test/resources/certs/* + src/test/resources/META-INF/service/* */target/** */*.iml diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageExtBrokerInner.java b/store/src/main/java/org/apache/rocketmq/store/MessageExtBrokerInner.java index c7879af0645..e5f087b697d 100644 --- a/store/src/main/java/org/apache/rocketmq/store/MessageExtBrokerInner.java +++ b/store/src/main/java/org/apache/rocketmq/store/MessageExtBrokerInner.java @@ -25,12 +25,15 @@ public class MessageExtBrokerInner extends MessageExt { private long tagsCode; public static long tagsString2tagsCode(final TopicFilterType filter, final String tags) { - if (null == tags || tags.length() == 0) - return 0; + if (null == tags || tags.length() == 0) { return 0; } return tags.hashCode(); } + public static long tagsString2tagsCode(final String tags) { + return tagsString2tagsCode(null, tags); + } + public String getPropertiesString() { return propertiesString; } From 76233ed8b77c293e669f42daec855cabf74b9f3a Mon Sep 17 00:00:00 2001 From: what-a-good-jungle <353187194@qq.com> Date: Mon, 16 Jul 2018 16:59:29 +0800 Subject: [PATCH 64/73] [ISSUE #359]Fix logging check period (#360) --- .../java/org/apache/rocketmq/logging/inner/LoggingBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logging/src/main/java/org/apache/rocketmq/logging/inner/LoggingBuilder.java b/logging/src/main/java/org/apache/rocketmq/logging/inner/LoggingBuilder.java index 469cb52b290..f457a695676 100644 --- a/logging/src/main/java/org/apache/rocketmq/logging/inner/LoggingBuilder.java +++ b/logging/src/main/java/org/apache/rocketmq/logging/inner/LoggingBuilder.java @@ -966,7 +966,7 @@ int computeCheckPeriod() { rollingCalendar.setType(i); Date next = new Date(rollingCalendar.getNextCheckMillis(epoch)); String r1 = simpleDateFormat.format(next); - if (r0.equals(r1)) { + if (r0 != null && r1 != null && !r0.equals(r1)) { return i; } } From d8c446e854e6cce1b54c0d9d97f3189832b88001 Mon Sep 17 00:00:00 2001 From: fuyou001 Date: Mon, 16 Jul 2018 17:37:04 +0800 Subject: [PATCH 65/73] [ISSUE #314] Heartbeat handler use independently thread pool (#315) --- .../rocketmq/broker/BrokerController.java | 54 +++++++++++-------- .../broker/latency/BrokerFastFailure.java | 3 ++ .../apache/rocketmq/common/BrokerConfig.java | 27 ++++++++++ 3 files changed, 63 insertions(+), 21 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 9dbee82e705..f45674d6e46 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -16,6 +16,21 @@ */ package org.apache.rocketmq.broker; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import org.apache.rocketmq.broker.client.ClientHousekeepingService; import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener; import org.apache.rocketmq.broker.client.ConsumerManager; @@ -85,21 +100,6 @@ import org.apache.rocketmq.store.stats.BrokerStats; import org.apache.rocketmq.store.stats.BrokerStatsManager; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; public class BrokerController { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); @@ -129,6 +129,7 @@ public class BrokerController { private final BlockingQueue pullThreadPoolQueue; private final BlockingQueue queryThreadPoolQueue; private final BlockingQueue clientManagerThreadPoolQueue; + private final BlockingQueue heartbeatThreadPoolQueue; private final BlockingQueue consumerManagerThreadPoolQueue; private final FilterServerManager filterServerManager; private final BrokerStatsManager brokerStatsManager; @@ -143,6 +144,7 @@ public class BrokerController { private ExecutorService queryMessageExecutor; private ExecutorService adminBrokerExecutor; private ExecutorService clientManageExecutor; + private ExecutorService heartbeatExecutor; private ExecutorService consumerManageExecutor; private boolean updateMasterHAServerAddrPeriodically = false; private BrokerStats brokerStats; @@ -186,6 +188,7 @@ public BrokerController( this.queryThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getQueryThreadPoolQueueCapacity()); this.clientManagerThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getClientManagerThreadPoolQueueCapacity()); this.consumerManagerThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getConsumerManagerThreadPoolQueueCapacity()); + this.heartbeatThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getHeartbeatThreadPoolQueueCapacity()); this.brokerStatsManager = new BrokerStatsManager(this.brokerConfig.getBrokerClusterName()); this.setStoreHost(new InetSocketAddress(this.getBrokerConfig().getBrokerIP1(), this.getNettyServerConfig().getListenPort())); @@ -280,6 +283,15 @@ public boolean initialize() throws CloneNotSupportedException { this.clientManagerThreadPoolQueue, new ThreadFactoryImpl("ClientManageThread_")); + this.heartbeatExecutor = new BrokerFixedThreadPoolExecutor( + this.brokerConfig.getHeartbeatThreadPoolNums(), + this.brokerConfig.getHeartbeatThreadPoolNums(), + 1000 * 60, + TimeUnit.MILLISECONDS, + this.heartbeatThreadPoolQueue, + new ThreadFactoryImpl("HeartbeatThread_",true)); + + this.consumerManageExecutor = Executors.newFixedThreadPool(this.brokerConfig.getConsumerManageThreadPoolNums(), new ThreadFactoryImpl( "ConsumerManageThread_")); @@ -501,11 +513,11 @@ public void registerProcessor() { * ClientManageProcessor */ ClientManageProcessor clientProcessor = new ClientManageProcessor(this); - this.remotingServer.registerProcessor(RequestCode.HEART_BEAT, clientProcessor, this.clientManageExecutor); + this.remotingServer.registerProcessor(RequestCode.HEART_BEAT, clientProcessor, this.heartbeatExecutor); this.remotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientProcessor, this.clientManageExecutor); this.remotingServer.registerProcessor(RequestCode.CHECK_CLIENT_CONFIG, clientProcessor, this.clientManageExecutor); - this.fastRemotingServer.registerProcessor(RequestCode.HEART_BEAT, clientProcessor, this.clientManageExecutor); + this.fastRemotingServer.registerProcessor(RequestCode.HEART_BEAT, clientProcessor, this.heartbeatExecutor); this.fastRemotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientProcessor, this.clientManageExecutor); this.fastRemotingServer.registerProcessor(RequestCode.CHECK_CLIENT_CONFIG, clientProcessor, this.clientManageExecutor); @@ -729,10 +741,6 @@ public void shutdown() { if (this.fileWatchService != null) { this.fileWatchService.shutdown(); } - - if (this.transactionalMessageCheckService != null) { - this.transactionalMessageCheckService.shutdown(false); - } } private void unregisterBrokerAll() { @@ -990,6 +998,10 @@ public Configuration getConfiguration() { return this.configuration; } + public BlockingQueue getHeartbeatThreadPoolQueue() { + return heartbeatThreadPoolQueue; + } + public TransactionalMessageCheckService getTransactionalMessageCheckService() { return transactionalMessageCheckService; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java index 0159d323fe1..0a8beca2c64 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java @@ -89,6 +89,9 @@ private void cleanExpiredRequest() { cleanExpiredRequestInQueue(this.brokerController.getPullThreadPoolQueue(), this.brokerController.getBrokerConfig().getWaitTimeMillsInPullQueue()); + + cleanExpiredRequestInQueue(this.brokerController.getHeartbeatThreadPoolQueue(), + this.brokerController.getBrokerConfig().getWaitTimeMillsInHeartbeatQueue()); } void cleanExpiredRequestInQueue(final BlockingQueue blockingQueue, final long maxWaitTimeMillsInQueue) { diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index e4486da0d15..442f456aa41 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -63,6 +63,7 @@ public class BrokerConfig { private int adminBrokerThreadPoolNums = 16; private int clientManageThreadPoolNums = 32; private int consumerManageThreadPoolNums = 32; + private int heartbeatThreadPoolNums = Math.min(32,Runtime.getRuntime().availableProcessors()); private int flushConsumerOffsetInterval = 1000 * 5; @@ -77,6 +78,7 @@ public class BrokerConfig { private int queryThreadPoolQueueCapacity = 20000; private int clientManagerThreadPoolQueueCapacity = 1000000; private int consumerManagerThreadPoolQueueCapacity = 1000000; + private int heartbeatThreadPoolQueueCapacity = 50000; private int filterServerNums = 0; @@ -108,6 +110,7 @@ public class BrokerConfig { private boolean brokerFastFailureEnable = true; private long waitTimeMillsInSendQueue = 200; private long waitTimeMillsInPullQueue = 5 * 1000; + private long waitTimeMillsInHeartbeatQueue = 31 * 1000; private long startAcceptSendRequestTimeStamp = 0L; @@ -643,6 +646,30 @@ public void setForceRegister(boolean forceRegister) { this.forceRegister = forceRegister; } + public int getHeartbeatThreadPoolQueueCapacity() { + return heartbeatThreadPoolQueueCapacity; + } + + public void setHeartbeatThreadPoolQueueCapacity(int heartbeatThreadPoolQueueCapacity) { + this.heartbeatThreadPoolQueueCapacity = heartbeatThreadPoolQueueCapacity; + } + + public int getHeartbeatThreadPoolNums() { + return heartbeatThreadPoolNums; + } + + public void setHeartbeatThreadPoolNums(int heartbeatThreadPoolNums) { + this.heartbeatThreadPoolNums = heartbeatThreadPoolNums; + } + + public long getWaitTimeMillsInHeartbeatQueue() { + return waitTimeMillsInHeartbeatQueue; + } + + public void setWaitTimeMillsInHeartbeatQueue(long waitTimeMillsInHeartbeatQueue) { + this.waitTimeMillsInHeartbeatQueue = waitTimeMillsInHeartbeatQueue; + } + public int getRegisterNameServerPeriod() { return registerNameServerPeriod; } From ed433a3c930b2106a65d45c5bf292c3b52fdc266 Mon Sep 17 00:00:00 2001 From: duheng <39583243+duhengforever@users.noreply.github.com> Date: Tue, 17 Jul 2018 15:49:17 +0800 Subject: [PATCH 66/73] Fix the get connection bug when sending transactional message (#368) --- .../org/apache/rocketmq/broker/client/ProducerManager.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java index 28d103c0bfc..61ceae53860 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java @@ -209,10 +209,13 @@ public Channel getAvaliableChannel(String groupId) { Channel channel = channelList.get(index); int count = 0; boolean isOk = channel.isActive() && channel.isWritable(); - while (isOk && count++ < GET_AVALIABLE_CHANNEL_RETRY_COUNT) { + while (count++ < GET_AVALIABLE_CHANNEL_RETRY_COUNT) { + if (isOk) { + return channel; + } index = (++index) % size; channel = channelList.get(index); - return channel; + isOk = channel.isActive() && channel.isWritable(); } } else { log.warn("Check transaction failed, channel table is empty. groupId={}", groupId); From bc0c04bff0be7970cf603835826ef17e855694ba Mon Sep 17 00:00:00 2001 From: what-a-good-jungle <353187194@qq.com> Date: Tue, 17 Jul 2018 16:01:20 +0800 Subject: [PATCH 67/73] [ROCKETMQ-355][POLISH]Async send method polish - fix the timeout semantic (#318) --- .../rocketmq/client/impl/MQClientAPIImpl.java | 15 +- .../impl/producer/DefaultMQProducerImpl.java | 193 +++++++++++++++--- .../consumer/DefaultMQPushConsumerTest.java | 2 +- .../producer/DefaultMQProducerTest.java | 69 ++++++- .../rocketmq/remoting/RemotingClient.java | 2 + .../remoting/netty/NettyRemotingAbstract.java | 8 +- .../remoting/netty/NettyRemotingClient.java | 20 +- .../netty/NettyRemotingClientTest.java | 8 +- .../producer/async/AsyncSendExceptionIT.java | 6 +- 9 files changed, 271 insertions(+), 52 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index 3a023e362cb..1837204a309 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -308,6 +308,7 @@ public SendResult sendMessage( final SendMessageContext context, final DefaultMQProducerImpl producer ) throws RemotingException, MQBrokerException, InterruptedException { + long beginStartTime = System.currentTimeMillis(); RemotingCommand request = null; if (sendSmartMsg || msg instanceof MessageBatch) { SendMessageRequestHeaderV2 requestHeaderV2 = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(requestHeader); @@ -324,11 +325,19 @@ public SendResult sendMessage( return null; case ASYNC: final AtomicInteger times = new AtomicInteger(); - this.sendMessageAsync(addr, brokerName, msg, timeoutMillis, request, sendCallback, topicPublishInfo, instance, + long costTimeAsync = System.currentTimeMillis() - beginStartTime; + if (timeoutMillis < costTimeAsync) { + throw new RemotingTooMuchRequestException("sendMessage call timeout"); + } + this.sendMessageAsync(addr, brokerName, msg, timeoutMillis - costTimeAsync, request, sendCallback, topicPublishInfo, instance, retryTimesWhenSendFailed, times, context, producer); return null; case SYNC: - return this.sendMessageSync(addr, brokerName, msg, timeoutMillis, request); + long costTimeSync = System.currentTimeMillis() - beginStartTime; + if (timeoutMillis < costTimeSync) { + throw new RemotingTooMuchRequestException("sendMessage call timeout"); + } + return this.sendMessageSync(addr, brokerName, msg, timeoutMillis - costTimeSync, request); default: assert false; break; @@ -2080,4 +2089,4 @@ public void checkClientInBroker(final String brokerAddr, final String consumerGr throw new MQClientException(response.getCode(), response.getRemark()); } } -} \ No newline at end of file +} diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index 4b5e373fd27..e1d9f904295 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -64,6 +64,8 @@ import org.apache.rocketmq.remoting.exception.RemotingConnectException; import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; +import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; +import java.util.concurrent.RejectedExecutionException; import java.io.IOException; import java.net.UnknownHostException; @@ -414,19 +416,48 @@ public MessageExt queryMessageByUniqKey(String topic, String uniqKey) * DEFAULT ASYNC ------------------------------------------------------- */ public void send(Message msg, - SendCallback sendCallback) throws MQClientException, RemotingException, InterruptedException { + SendCallback sendCallback) throws MQClientException, RemotingException, InterruptedException { send(msg, sendCallback, this.defaultMQProducer.getSendMsgTimeout()); } - public void send(Message msg, SendCallback sendCallback, long timeout) + /** + * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. + * A new one will be provided in next version + * @param msg + * @param sendCallback + * @param timeout the sendCallback will be invoked at most time + * @throws RejectedExecutionException + */ + @Deprecated + public void send(final Message msg, final SendCallback sendCallback, final long timeout) throws MQClientException, RemotingException, InterruptedException { + final long beginStartTime = System.currentTimeMillis(); + ExecutorService executor = this.getCallbackExecutor(); try { - this.sendDefaultImpl(msg, CommunicationMode.ASYNC, sendCallback, timeout); - } catch (MQBrokerException e) { - throw new MQClientException("unknownn exception", e); + executor.submit(new Runnable() { + @Override + public void run() { + long costTime = System.currentTimeMillis() - beginStartTime; + if (timeout > costTime) { + try { + sendDefaultImpl(msg, CommunicationMode.ASYNC, sendCallback, timeout - costTime); + } catch (Exception e) { + sendCallback.onException(e); + } + } else { + sendCallback.onException( + new RemotingTooMuchRequestException("DEFAULT ASYNC send call timeout")); + } + } + + }); + } catch (RejectedExecutionException e) { + throw new MQClientException("executor rejected ", e); } + } + public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName) { return this.mqFaultStrategy.selectOneMessageQueue(tpInfo, lastBrokerName); } @@ -450,6 +481,7 @@ private SendResult sendDefaultImpl( long endTimestamp = beginTimestampFirst; TopicPublishInfo topicPublishInfo = this.tryToFindTopicPublishInfo(msg.getTopic()); if (topicPublishInfo != null && topicPublishInfo.ok()) { + boolean callTimeout = false; MessageQueue mq = null; Exception exception = null; SendResult sendResult = null; @@ -464,7 +496,13 @@ private SendResult sendDefaultImpl( brokersSent[times] = mq.getBrokerName(); try { beginTimestampPrev = System.currentTimeMillis(); - sendResult = this.sendKernelImpl(msg, mq, communicationMode, sendCallback, topicPublishInfo, timeout); + long costTime = beginTimestampPrev - beginTimestampFirst; + if (timeout < costTime) { + callTimeout = true; + break; + } + + sendResult = this.sendKernelImpl(msg, mq, communicationMode, sendCallback, topicPublishInfo, timeout - costTime); endTimestamp = System.currentTimeMillis(); this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false); switch (communicationMode) { @@ -546,6 +584,10 @@ private SendResult sendDefaultImpl( info += FAQUrl.suggestTodo(FAQUrl.SEND_MSG_FAILED); MQClientException mqClientException = new MQClientException(info, exception); + if (callTimeout) { + throw new RemotingTooMuchRequestException("sendDefaultImpl call timeout"); + } + if (exception instanceof MQBrokerException) { mqClientException.setResponseCode(((MQBrokerException) exception).getResponseCode()); } else if (exception instanceof RemotingConnectException) { @@ -587,11 +629,12 @@ private TopicPublishInfo tryToFindTopicPublishInfo(final String topic) { } private SendResult sendKernelImpl(final Message msg, - final MessageQueue mq, - final CommunicationMode communicationMode, - final SendCallback sendCallback, - final TopicPublishInfo topicPublishInfo, - final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + final MessageQueue mq, + final CommunicationMode communicationMode, + final SendCallback sendCallback, + final TopicPublishInfo topicPublishInfo, + final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + long beginStartTime = System.currentTimeMillis(); String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); if (null == brokerAddr) { tryToFindTopicPublishInfo(mq.getTopic()); @@ -691,13 +734,16 @@ private SendResult sendKernelImpl(final Message msg, tmpMessage = MessageAccessor.cloneMessage(msg); msg.setBody(prevBody); } - + long costTimeAsync = System.currentTimeMillis() - beginStartTime; + if (timeout < costTimeAsync) { + throw new RemotingTooMuchRequestException("sendKernelImpl call timeout"); + } sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage( brokerAddr, mq.getBrokerName(), tmpMessage, requestHeader, - timeout, + timeout - costTimeAsync, communicationMode, sendCallback, topicPublishInfo, @@ -708,12 +754,16 @@ private SendResult sendKernelImpl(final Message msg, break; case ONEWAY: case SYNC: + long costTimeSync = System.currentTimeMillis() - beginStartTime; + if (timeout < costTimeSync) { + throw new RemotingTooMuchRequestException("sendKernelImpl call timeout"); + } sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage( brokerAddr, mq.getBrokerName(), msg, requestHeader, - timeout, + timeout - costTimeSync, communicationMode, context, this); @@ -844,6 +894,7 @@ public SendResult send(Message msg, MessageQueue mq) public SendResult send(Message msg, MessageQueue mq, long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + long beginStartTime = System.currentTimeMillis(); this.makeSureStateOK(); Validators.checkMessage(msg, this.defaultMQProducer); @@ -851,6 +902,11 @@ public SendResult send(Message msg, MessageQueue mq, long timeout) throw new MQClientException("message's topic not equal mq's topic", null); } + long costTime = System.currentTimeMillis() - beginStartTime; + if (timeout < costTime) { + throw new RemotingTooMuchRequestException("call timeout"); + } + return this.sendKernelImpl(msg, mq, CommunicationMode.SYNC, null, null, timeout); } @@ -862,20 +918,55 @@ public void send(Message msg, MessageQueue mq, SendCallback sendCallback) send(msg, mq, sendCallback, this.defaultMQProducer.getSendMsgTimeout()); } - public void send(Message msg, MessageQueue mq, SendCallback sendCallback, long timeout) + /** + * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. + * A new one will be provided in next version + * @param msg + * @param mq + * @param sendCallback + * @param timeout the sendCallback will be invoked at most time + * @throws MQClientException + * @throws RemotingException + * @throws InterruptedException + */ + @Deprecated + public void send(final Message msg, final MessageQueue mq, final SendCallback sendCallback, final long timeout) throws MQClientException, RemotingException, InterruptedException { - this.makeSureStateOK(); - Validators.checkMessage(msg, this.defaultMQProducer); + final long beginStartTime = System.currentTimeMillis(); + ExecutorService executor = this.getCallbackExecutor(); + try { + executor.submit(new Runnable() { + @Override + public void run() { + try { + makeSureStateOK(); + Validators.checkMessage(msg, defaultMQProducer); - if (!msg.getTopic().equals(mq.getTopic())) { - throw new MQClientException("message's topic not equal mq's topic", null); - } + if (!msg.getTopic().equals(mq.getTopic())) { + throw new MQClientException("message's topic not equal mq's topic", null); + } + long costTime = System.currentTimeMillis() - beginStartTime; + if (timeout > costTime) { + try { + sendKernelImpl(msg, mq, CommunicationMode.ASYNC, sendCallback, null, + timeout - costTime); + } catch (MQBrokerException e) { + throw new MQClientException("unknown exception", e); + } + } else { + sendCallback.onException(new RemotingTooMuchRequestException("call timeout")); + } + } catch (Exception e) { + sendCallback.onException(e); + } - try { - this.sendKernelImpl(msg, mq, CommunicationMode.ASYNC, sendCallback, null, timeout); - } catch (MQBrokerException e) { - throw new MQClientException("unknown exception", e); + } + + }); + } catch (RejectedExecutionException e) { + throw new MQClientException("executor rejected ", e); } + } /** @@ -913,6 +1004,7 @@ private SendResult sendSelectImpl( final CommunicationMode communicationMode, final SendCallback sendCallback, final long timeout ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + long beginStartTime = System.currentTimeMillis(); this.makeSureStateOK(); Validators.checkMessage(msg, this.defaultMQProducer); @@ -925,8 +1017,12 @@ private SendResult sendSelectImpl( throw new MQClientException("select message queue throwed exception.", e); } + long costTime = System.currentTimeMillis() - beginStartTime; + if (timeout < costTime) { + throw new RemotingTooMuchRequestException("sendSelectImpl call timeout"); + } if (mq != null) { - return this.sendKernelImpl(msg, mq, communicationMode, sendCallback, null, timeout); + return this.sendKernelImpl(msg, mq, communicationMode, sendCallback, null, timeout - costTime); } else { throw new MQClientException("select message queue return null.", null); } @@ -943,12 +1039,47 @@ public void send(Message msg, MessageQueueSelector selector, Object arg, SendCal send(msg, selector, arg, sendCallback, this.defaultMQProducer.getSendMsgTimeout()); } - public void send(Message msg, MessageQueueSelector selector, Object arg, SendCallback sendCallback, long timeout) + /** + * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. + * A new one will be provided in next version + * @param msg + * @param selector + * @param arg + * @param sendCallback + * @param timeout the sendCallback will be invoked at most time + * @throws MQClientException + * @throws RemotingException + * @throws InterruptedException + */ + @Deprecated + public void send(final Message msg, final MessageQueueSelector selector, final Object arg, final SendCallback sendCallback, final long timeout) throws MQClientException, RemotingException, InterruptedException { + final long beginStartTime = System.currentTimeMillis(); + ExecutorService executor = this.getCallbackExecutor(); try { - this.sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, sendCallback, timeout); - } catch (MQBrokerException e) { - throw new MQClientException("unknownn exception", e); + executor.submit(new Runnable() { + @Override + public void run() { + long costTime = System.currentTimeMillis() - beginStartTime; + if (timeout > costTime) { + try { + try { + sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, sendCallback, + timeout - costTime); + } catch (MQBrokerException e) { + throw new MQClientException("unknownn exception", e); + } + } catch (Exception e) { + sendCallback.onException(e); + } + } else { + sendCallback.onException(new RemotingTooMuchRequestException("call timeout")); + } + } + + }); + } catch (RejectedExecutionException e) { + throw new MQClientException("exector rejected ", e); } } @@ -1082,6 +1213,10 @@ public void endTransaction( public void setCallbackExecutor(final ExecutorService callbackExecutor) { this.mQClientFactory.getMQClientAPIImpl().getRemotingClient().setCallbackExecutor(callbackExecutor); } + public ExecutorService getCallbackExecutor() { + return this.mQClientFactory.getMQClientAPIImpl().getRemotingClient().getCallbackExecutor(); + + } public SendResult send(Message msg, long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java index d6dce86c72a..ff2fb78bbfb 100644 --- a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java @@ -289,4 +289,4 @@ private PullResultExt createPullResult(PullMessageRequestHeader requestHeader, P } return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, outputStream.toByteArray()); } -} \ No newline at end of file +} diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java index d3c6cc8b117..c225afd6842 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java @@ -24,6 +24,9 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; @@ -36,6 +39,7 @@ import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; import org.apache.rocketmq.common.protocol.route.BrokerData; import org.apache.rocketmq.common.protocol.route.QueueData; @@ -65,6 +69,8 @@ public class DefaultMQProducerTest { private MQClientInstance mQClientFactory = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); @Mock private MQClientAPIImpl mQClientAPIImpl; + @Mock + private NettyRemotingClient nettyRemotingClient; private DefaultMQProducer producer; private Message message; @@ -161,38 +167,91 @@ public void testSendMessageSync_WithBodyCompressed() throws RemotingException, I @Test public void testSendMessageAsync_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + ExecutorService callbackExecutor = Executors.newSingleThreadExecutor(); + final CountDownLatch countDownLatch = new CountDownLatch(1); + when(mQClientAPIImpl.getRemotingClient()).thenReturn((nettyRemotingClient)); + when(nettyRemotingClient.getCallbackExecutor()).thenReturn(callbackExecutor); producer.send(message, new SendCallback() { @Override public void onSuccess(SendResult sendResult) { assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); assertThat(sendResult.getOffsetMsgId()).isEqualTo("123"); assertThat(sendResult.getQueueOffset()).isEqualTo(456L); + countDownLatch.countDown(); } @Override public void onException(Throwable e) { + countDownLatch.countDown(); } }); + countDownLatch.await(3000L, TimeUnit.MILLISECONDS); + callbackExecutor.shutdown(); + } + @Test + public void testSendMessageAsync() throws RemotingException, MQClientException, InterruptedException { + final AtomicInteger cc = new AtomicInteger(0); + final CountDownLatch countDownLatch = new CountDownLatch(6); + ExecutorService callbackExecutor = Executors.newSingleThreadExecutor(); + when(mQClientAPIImpl.getRemotingClient()).thenReturn((nettyRemotingClient)); + when(nettyRemotingClient.getCallbackExecutor()).thenReturn(callbackExecutor); + + SendCallback sendCallback = new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + } + @Override + public void onException(Throwable e) { + e.printStackTrace(); + cc.incrementAndGet(); + countDownLatch.countDown(); + } + }; + MessageQueueSelector messageQueueSelector = new MessageQueueSelector() { + @Override + public MessageQueue select(List mqs, Message msg, Object arg) { + return null; + } + }; + + Message message = new Message(); + message.setTopic("test"); + message.setBody("hello world".getBytes()); + producer.send(new Message(),sendCallback); + producer.send(message,sendCallback,1000); + producer.send(message,new MessageQueue(),sendCallback); + producer.send(new Message(),new MessageQueue(),sendCallback,1000); + producer.send(new Message(),messageQueueSelector,null,sendCallback); + producer.send(message,messageQueueSelector,null,sendCallback,1000); + + countDownLatch.await(3000L, TimeUnit.MILLISECONDS); + callbackExecutor.shutdown(); + assertThat(cc.get()).isEqualTo(6); } @Test public void testSendMessageAsync_BodyCompressed() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + ExecutorService callbackExecutor = Executors.newSingleThreadExecutor(); + final CountDownLatch countDownLatch = new CountDownLatch(1); + when(mQClientAPIImpl.getRemotingClient()).thenReturn((nettyRemotingClient)); + when(nettyRemotingClient.getCallbackExecutor()).thenReturn(callbackExecutor); producer.send(bigMessage, new SendCallback() { @Override public void onSuccess(SendResult sendResult) { assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); assertThat(sendResult.getOffsetMsgId()).isEqualTo("123"); assertThat(sendResult.getQueueOffset()).isEqualTo(456L); + countDownLatch.countDown(); } @Override public void onException(Throwable e) { + countDownLatch.countDown(); } }); - + countDownLatch.await(3000L, TimeUnit.MILLISECONDS); + callbackExecutor.shutdown(); } @Test @@ -249,7 +308,7 @@ public void run() { @Test public void testSetCallbackExecutor() throws MQClientException { - String producerGroupTemp = producerGroupPrefix + System.currentTimeMillis(); + String producerGroupTemp = "testSetCallbackExecutor_" + System.currentTimeMillis(); producer = new DefaultMQProducer(producerGroupTemp); producer.setNamesrvAddr("127.0.0.1:9876"); producer.start(); @@ -319,4 +378,4 @@ public void run() { } return assertionErrors[0]; } -} \ No newline at end of file +} diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java index 2aea14cb9d6..c0754db634d 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java @@ -48,5 +48,7 @@ void registerProcessor(final int requestCode, final NettyRequestProcessor proces void setCallbackExecutor(final ExecutorService callbackExecutor); + ExecutorService getCallbackExecutor(); + boolean isChannelWritable(final String addr); } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java index 45ca7304c8f..8dccebc0457 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java @@ -403,11 +403,17 @@ public void operationComplete(ChannelFuture f) throws Exception { public void invokeAsyncImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis, final InvokeCallback invokeCallback) throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException { + long beginStartTime = System.currentTimeMillis(); final int opaque = request.getOpaque(); boolean acquired = this.semaphoreAsync.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS); if (acquired) { final SemaphoreReleaseOnlyOnce once = new SemaphoreReleaseOnlyOnce(this.semaphoreAsync); - final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, timeoutMillis, invokeCallback, once); + long costTime = System.currentTimeMillis() - beginStartTime; + if (timeoutMillis < costTime) { + throw new RemotingTooMuchRequestException("invokeAsyncImpl call timeout"); + } + + final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, timeoutMillis - costTime, invokeCallback, once); this.responseTable.put(opaque, responseFuture); try { channel.writeAndFlush(request).addListener(new ChannelFutureListener() { diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java index 241f2b07ad1..33c2eed8de1 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java @@ -360,13 +360,18 @@ public void updateNameServerAddressList(List addrs) { @Override public RemotingCommand invokeSync(String addr, final RemotingCommand request, long timeoutMillis) throws InterruptedException, RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException { + long beginStartTime = System.currentTimeMillis(); final Channel channel = this.getAndCreateChannel(addr); if (channel != null && channel.isActive()) { try { if (this.rpcHook != null) { this.rpcHook.doBeforeRequest(addr, request); } - RemotingCommand response = this.invokeSyncImpl(channel, request, timeoutMillis); + long costTime = System.currentTimeMillis() - beginStartTime; + if (timeoutMillis < costTime) { + throw new RemotingTimeoutException("invokeSync call timeout"); + } + RemotingCommand response = this.invokeSyncImpl(channel, request, timeoutMillis - costTime); if (this.rpcHook != null) { this.rpcHook.doAfterResponse(RemotingHelper.parseChannelRemoteAddr(channel), request, response); } @@ -390,8 +395,9 @@ public RemotingCommand invokeSync(String addr, final RemotingCommand request, lo } private Channel getAndCreateChannel(final String addr) throws InterruptedException { - if (null == addr) + if (null == addr) { return getAndCreateNameserverChannel(); + } ChannelWrapper cw = this.channelTables.get(addr); if (cw != null && cw.isOK()) { @@ -431,8 +437,9 @@ private Channel getAndCreateNameserverChannel() throws InterruptedException { this.namesrvAddrChoosed.set(newAddr); log.info("new name server is chosen. OLD: {} , NEW: {}. namesrvIndex = {}", addr, newAddr, namesrvIndex); Channel channelNew = this.createChannel(newAddr); - if (channelNew != null) + if (channelNew != null) { return channelNew; + } } } } catch (Exception e) { @@ -511,13 +518,18 @@ private Channel createChannel(final String addr) throws InterruptedException { public void invokeAsync(String addr, RemotingCommand request, long timeoutMillis, InvokeCallback invokeCallback) throws InterruptedException, RemotingConnectException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException { + long beginStartTime = System.currentTimeMillis(); final Channel channel = this.getAndCreateChannel(addr); if (channel != null && channel.isActive()) { try { if (this.rpcHook != null) { this.rpcHook.doBeforeRequest(addr, request); } - this.invokeAsyncImpl(channel, request, timeoutMillis, invokeCallback); + long costTime = System.currentTimeMillis() - beginStartTime; + if (timeoutMillis < costTime) { + throw new RemotingTooMuchRequestException("invokeAsync call timeout"); + } + this.invokeAsyncImpl(channel, request, timeoutMillis - costTime, invokeCallback); } catch (RemotingSendRequestException e) { log.warn("invokeAsync: send request exception, so close the channel[{}]", addr); this.closeChannel(addr, channel); diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java index 04a3bebbb57..6b5633df1d7 100644 --- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java +++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java @@ -30,14 +30,10 @@ public class NettyRemotingClientTest { private NettyRemotingClient remotingClient = new NettyRemotingClient(new NettyClientConfig()); @Test - public void testSetCallbackExecutor() throws NoSuchFieldException, IllegalAccessException { - Field field = NettyRemotingClient.class.getDeclaredField("publicExecutor"); - field.setAccessible(true); - assertThat(remotingClient.getCallbackExecutor()).isEqualTo(field.get(remotingClient)); - + public void testSetCallbackExecutor() throws NoSuchFieldException, IllegalAccessException { ExecutorService customized = Executors.newCachedThreadPool(); remotingClient.setCallbackExecutor(customized); assertThat(remotingClient.getCallbackExecutor()).isEqualTo(customized); } -} \ No newline at end of file +} diff --git a/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendExceptionIT.java b/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendExceptionIT.java index 0bad6ea2bfc..d1a1fd143e0 100644 --- a/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendExceptionIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendExceptionIT.java @@ -61,7 +61,7 @@ public void testSendCallBackNull() throws Exception { producer.send(msg, sendCallback); } - @Test(expected = java.lang.NullPointerException.class) + @Test public void testSendMQNull() throws Exception { Message msg = new Message(topic, RandomUtils.getStringByUUID().getBytes()); DefaultMQProducer producer = ProducerFactory.getRMQProducer(nsAddr); @@ -69,7 +69,7 @@ public void testSendMQNull() throws Exception { producer.send(msg, messageQueue, SendCallBackFactory.getSendCallBack()); } - @Test(expected = org.apache.rocketmq.client.exception.MQClientException.class) + @Test public void testSendSelectorNull() throws Exception { Message msg = new Message(topic, RandomUtils.getStringByUUID().getBytes()); DefaultMQProducer producer = ProducerFactory.getRMQProducer(nsAddr); @@ -77,7 +77,7 @@ public void testSendSelectorNull() throws Exception { producer.send(msg, selector, 100, SendCallBackFactory.getSendCallBack()); } - @Test(expected = org.apache.rocketmq.client.exception.MQClientException.class) + @Test public void testSelectorThrowsException() throws Exception { Message msg = new Message(topic, RandomUtils.getStringByUUID().getBytes()); DefaultMQProducer producer = ProducerFactory.getRMQProducer(nsAddr); From 3f7dda3c0c5fb7de74ece7b9ce690a978c58a319 Mon Sep 17 00:00:00 2001 From: XiaoZYang Date: Mon, 23 Jul 2018 19:50:51 +0800 Subject: [PATCH 68/73] [ISSUE #377] FIX Admin subcommand consumeMessage should quit when catching an exception (#378) --- .../message/ConsumeMessageCommand.java | 1 + .../message/ConsumeMessageCommandTest.java | 70 +++++++++++++++---- 2 files changed, 58 insertions(+), 13 deletions(-) diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java index 51892677445..aa98ee62870 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java @@ -207,6 +207,7 @@ private void pullMessageByQueue(MessageQueue mq, long minOffset, long maxOffset) pullResult = defaultMQPullConsumer.pull(mq, "*", offset, (int)(maxOffset - offset + 1)); } catch (Exception e) { e.printStackTrace(); + return; } if (pullResult != null) { offset = pullResult.getNextBeginOffset(); diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommandTest.java index 9a5998e29e5..11543958c04 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommandTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommandTest.java @@ -16,6 +16,13 @@ */ package org.apache.rocketmq.tools.command.message; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.commons.cli.PosixParser; @@ -34,14 +41,6 @@ import org.junit.BeforeClass; import org.junit.Test; -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; @@ -54,14 +53,14 @@ public class ConsumeMessageCommandTest { @BeforeClass public static void init() throws MQClientException, RemotingException, MQBrokerException, InterruptedException, - NoSuchFieldException, IllegalAccessException{ + NoSuchFieldException, IllegalAccessException { consumeMessageCommand = new ConsumeMessageCommand(); DefaultMQPullConsumer defaultMQPullConsumer = mock(DefaultMQPullConsumer.class); MessageExt msg = new MessageExt(); - msg.setBody(new byte[]{'a'}); + msg.setBody(new byte[] {'a'}); List msgFoundList = new ArrayList<>(); msgFoundList.add(msg); - final PullResult pullResult = new PullResult(PullStatus.FOUND,2, 0, 1, msgFoundList); + final PullResult pullResult = new PullResult(PullStatus.FOUND, 2, 0, 1, msgFoundList); when(defaultMQPullConsumer.pull(any(MessageQueue.class), anyString(), anyLong(), anyInt())).thenReturn(pullResult); when(defaultMQPullConsumer.minOffset(any(MessageQueue.class))).thenReturn(Long.valueOf(0)); @@ -73,8 +72,9 @@ public static void init() throws MQClientException, RemotingException, MQBrokerE Field producerField = ConsumeMessageCommand.class.getDeclaredField("defaultMQPullConsumer"); producerField.setAccessible(true); - producerField.set(consumeMessageCommand,defaultMQPullConsumer); + producerField.set(consumeMessageCommand, defaultMQPullConsumer); } + @AfterClass public static void terminate() { } @@ -102,11 +102,55 @@ public void testExecuteByCondition() throws SubCommandException { System.setOut(new PrintStream(bos)); Options options = ServerUtil.buildCommandlineOptions(new Options()); - String[] subargs = new String[] {"-t mytopic","-b localhost","-i 0", "-n localhost:9876"}; + String[] subargs = new String[] {"-t mytopic", "-b localhost", "-i 0", "-n localhost:9876"}; CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + consumeMessageCommand.commandName(), subargs, consumeMessageCommand.buildCommandlineOptions(options), new PosixParser()); consumeMessageCommand.execute(commandLine, options, null); System.setOut(out); String s = new String(bos.toByteArray()); Assert.assertTrue(s.contains("Consume ok")); } + + @Test + public void testExecuteDefaultWhenPullMessageByQueueGotException() throws SubCommandException, InterruptedException, RemotingException, MQClientException, MQBrokerException, NoSuchFieldException, IllegalAccessException { + DefaultMQPullConsumer defaultMQPullConsumer = mock(DefaultMQPullConsumer.class); + when(defaultMQPullConsumer.pull(any(MessageQueue.class), anyString(), anyLong(), anyInt())).thenThrow(Exception.class); + Field producerField = ConsumeMessageCommand.class.getDeclaredField("defaultMQPullConsumer"); + producerField.setAccessible(true); + producerField.set(consumeMessageCommand, defaultMQPullConsumer); + + PrintStream out = System.out; + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(bos)); + Options options = ServerUtil.buildCommandlineOptions(new Options()); + String[] subargs = new String[] {"-t topic-not-existu", "-n localhost:9876"}; + CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + consumeMessageCommand.commandName(), + subargs, consumeMessageCommand.buildCommandlineOptions(options), new PosixParser()); + consumeMessageCommand.execute(commandLine, options, null); + + System.setOut(out); + String s = new String(bos.toByteArray()); + Assert.assertTrue(!s.contains("Consume ok")); + } + + @Test + public void testExecuteByConditionWhenPullMessageByQueueGotException() throws IllegalAccessException, InterruptedException, RemotingException, MQClientException, MQBrokerException, NoSuchFieldException, SubCommandException { + DefaultMQPullConsumer defaultMQPullConsumer = mock(DefaultMQPullConsumer.class); + when(defaultMQPullConsumer.pull(any(MessageQueue.class), anyString(), anyLong(), anyInt())).thenThrow(Exception.class); + Field producerField = ConsumeMessageCommand.class.getDeclaredField("defaultMQPullConsumer"); + producerField.setAccessible(true); + producerField.set(consumeMessageCommand, defaultMQPullConsumer); + + PrintStream out = System.out; + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(bos)); + Options options = ServerUtil.buildCommandlineOptions(new Options()); + + String[] subargs = new String[] {"-t mytopic", "-b localhost", "-i 0", "-n localhost:9876"}; + CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + consumeMessageCommand.commandName(), subargs, consumeMessageCommand.buildCommandlineOptions(options), new PosixParser()); + consumeMessageCommand.execute(commandLine, options, null); + + System.setOut(out); + String s = new String(bos.toByteArray()); + Assert.assertTrue(!s.contains("Consume ok")); + } } \ No newline at end of file From 439796f7c4c916734a334b95b1c17de63ffe1620 Mon Sep 17 00:00:00 2001 From: huangyiming Date: Mon, 23 Jul 2018 19:58:10 +0800 Subject: [PATCH 69/73] [ISSUE #375] Replace the message_magic_code and blank_magic_code value --- .../org/apache/rocketmq/common/message/MessageDecoder.java | 2 +- store/src/main/java/org/apache/rocketmq/store/CommitLog.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java index 2d7bcbbfd45..1eb9f4d9948 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java @@ -38,7 +38,7 @@ public class MessageDecoder { public final static int MESSAGE_FLAG_POSTION = 16; public final static int MESSAGE_PHYSIC_OFFSET_POSTION = 28; public final static int MESSAGE_STORE_TIMESTAMP_POSTION = 56; - public final static int MESSAGE_MAGIC_CODE = 0xAABBCCDD ^ 1880681586 + 8; + public final static int MESSAGE_MAGIC_CODE = -626843481; public static final char NAME_VALUE_SEPARATOR = 1; public static final char PROPERTY_SEPARATOR = 2; public static final int BODY_SIZE_POSITION = 4 // 1 TOTALSIZE diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 7f772b243b4..d3caf8d84a8 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -44,10 +44,10 @@ */ public class CommitLog { // Message's MAGIC CODE daa320a7 - public final static int MESSAGE_MAGIC_CODE = 0xAABBCCDD ^ 1880681586 + 8; + public final static int MESSAGE_MAGIC_CODE = -626843481; private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); // End of file empty MAGIC CODE cbd43194 - private final static int BLANK_MAGIC_CODE = 0xBBCCDDEE ^ 1880681586 + 8; + private final static int BLANK_MAGIC_CODE = -875286124; private final MappedFileQueue mappedFileQueue; private final DefaultMessageStore defaultMessageStore; private final FlushCommitLogService flushCommitLogService; From ba91ec8a7ed83cb312cd0d5251e6ddc2373e117f Mon Sep 17 00:00:00 2001 From: XiaoZYang Date: Mon, 23 Jul 2018 23:09:26 +0800 Subject: [PATCH 70/73] FIX admin subcommand consumeMessage can pull message with timestamp greater then now (#380) --- .../command/message/ConsumeMessageCommand.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java index aa98ee62870..6bf91840aeb 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java @@ -142,7 +142,7 @@ public void execute(final CommandLine commandLine, final Options options, RPCHoo if (commandLine.hasOption('c')) { messageCount = Long.parseLong(commandLine.getOptionValue('c').trim()); if (messageCount <= 0) { - System.out.print("please input a positive messageNumber!"); + System.out.print("Please input a positive messageNumber!"); return; } } @@ -161,20 +161,33 @@ public void execute(final CommandLine commandLine, final Options options, RPCHoo } if (commandLine.hasOption('o')) { if (consumeType != ConsumeType.BYQUEUE) { - System.out.print("please set queueId before offset!"); + System.out.print("Please set queueId before offset!"); return; } offset = Long.parseLong(commandLine.getOptionValue('o').trim()); consumeType = ConsumeType.BYOFFSET; } + long now = System.currentTimeMillis(); if (commandLine.hasOption('s')) { String timestampStr = commandLine.getOptionValue('s').trim(); timeValueBegin = timestampFormat(timestampStr); + if (timeValueBegin > now) { + System.out.print("Please set the beginTimestamp before now!"); + return; + } } if (commandLine.hasOption('e')) { String timestampStr = commandLine.getOptionValue('e').trim(); timeValueEnd = timestampFormat(timestampStr); + if (timeValueEnd > now) { + System.out.print("Please set the endTimestamp before now!"); + return; + } + if (timeValueBegin > timeValueEnd) { + System.out.print("Please make sure that the beginTimestamp is less than or equal to the endTimestamp"); + return; + } } switch (consumeType) { From 9623f3c31b514f268f2df5ca87f1ed2f63d17106 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Thu, 26 Jul 2018 11:42:22 +0800 Subject: [PATCH 71/73] Change MQVersion current verison to 4.3.0 --- common/src/main/java/org/apache/rocketmq/common/MQVersion.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java index 22a49a1cb55..96a2329cc30 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java +++ b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java @@ -18,7 +18,7 @@ public class MQVersion { - public static final int CURRENT_VERSION = Version.V4_3_0_SNAPSHOT.ordinal(); + public static final int CURRENT_VERSION = Version.V4_3_0.ordinal(); public static String getVersionDesc(int value) { int length = Version.values().length; From 76e3679dba050888aa4c490f1fc1902132134c3f Mon Sep 17 00:00:00 2001 From: dongeforever Date: Thu, 26 Jul 2018 12:00:27 +0800 Subject: [PATCH 72/73] [maven-release-plugin] prepare release rocketmq-all-4.3.0 --- broker/pom.xml | 2 +- client/pom.xml | 2 +- common/pom.xml | 2 +- distribution/pom.xml | 5 ++--- example/pom.xml | 4 ++-- filter/pom.xml | 2 +- logappender/pom.xml | 2 +- logging/pom.xml | 6 ++---- namesrv/pom.xml | 2 +- openmessaging/pom.xml | 2 +- pom.xml | 6 +++--- remoting/pom.xml | 2 +- srvutil/pom.xml | 2 +- store/pom.xml | 2 +- test/pom.xml | 2 +- tools/pom.xml | 2 +- 16 files changed, 21 insertions(+), 24 deletions(-) diff --git a/broker/pom.xml b/broker/pom.xml index e3787c36503..49b16bde843 100644 --- a/broker/pom.xml +++ b/broker/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.3.0-SNAPSHOT + 4.3.0 4.0.0 diff --git a/client/pom.xml b/client/pom.xml index 30b0e18d4a1..da2f73c087e 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.3.0-SNAPSHOT + 4.3.0 4.0.0 diff --git a/common/pom.xml b/common/pom.xml index ce16df4dd2a..2b61bee72d2 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.3.0-SNAPSHOT + 4.3.0 4.0.0 diff --git a/distribution/pom.xml b/distribution/pom.xml index 9293d495b1a..108565ccf06 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -15,13 +15,12 @@ See the License for the specific language governing permissions and limitations under the License. --> - + 4.0.0 org.apache.rocketmq rocketmq-all - 4.3.0-SNAPSHOT + 4.3.0 rocketmq-distribution rocketmq-distribution ${project.version} diff --git a/example/pom.xml b/example/pom.xml index b9d97df3d9a..b2d815e7688 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.3.0-SNAPSHOT + 4.3.0 4.0.0 @@ -51,7 +51,7 @@ org.apache.rocketmq rocketmq-openmessaging - 4.3.0-SNAPSHOT + 4.3.0 diff --git a/filter/pom.xml b/filter/pom.xml index eebfa30af49..1ae8f2e057d 100644 --- a/filter/pom.xml +++ b/filter/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.3.0-SNAPSHOT + 4.3.0 4.0.0 diff --git a/logappender/pom.xml b/logappender/pom.xml index ff5f7e2bced..7b4b18a5eed 100644 --- a/logappender/pom.xml +++ b/logappender/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.3.0-SNAPSHOT + 4.3.0 4.0.0 rocketmq-logappender diff --git a/logging/pom.xml b/logging/pom.xml index baad6ad0c49..ada5143c0e5 100644 --- a/logging/pom.xml +++ b/logging/pom.xml @@ -15,13 +15,11 @@ limitations under the License. --> - + org.apache.rocketmq rocketmq-all - 4.3.0-SNAPSHOT + 4.3.0 4.0.0 diff --git a/namesrv/pom.xml b/namesrv/pom.xml index 959814cc28a..596a1888c95 100644 --- a/namesrv/pom.xml +++ b/namesrv/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.3.0-SNAPSHOT + 4.3.0 4.0.0 diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml index 857a70eeb3e..fed7e1ac96d 100644 --- a/openmessaging/pom.xml +++ b/openmessaging/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.3.0-SNAPSHOT + 4.3.0 4.0.0 diff --git a/pom.xml b/pom.xml index 857dfce65ed..ccf8c3e4fee 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ 2012 org.apache.rocketmq rocketmq-all - 4.3.0-SNAPSHOT + 4.3.0 pom Apache RocketMQ ${project.version} http://rocketmq.apache.org/ @@ -42,7 +42,7 @@ git@github.com:apache/rocketmq.git scm:git:git@github.com:apache/rocketmq.git scm:git:git@github.com:apache/rocketmq.git - HEAD + rocketmq-all-4.3.0 @@ -157,7 +157,7 @@ - + true diff --git a/remoting/pom.xml b/remoting/pom.xml index 55589137943..3ec3af39116 100644 --- a/remoting/pom.xml +++ b/remoting/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.3.0-SNAPSHOT + 4.3.0 4.0.0 diff --git a/srvutil/pom.xml b/srvutil/pom.xml index b22e45f9b5c..41cab8db7e1 100644 --- a/srvutil/pom.xml +++ b/srvutil/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.3.0-SNAPSHOT + 4.3.0 4.0.0 diff --git a/store/pom.xml b/store/pom.xml index 07feb444ae1..b03090a61d3 100644 --- a/store/pom.xml +++ b/store/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.3.0-SNAPSHOT + 4.3.0 4.0.0 diff --git a/test/pom.xml b/test/pom.xml index fc01d2c75e8..6cb57a1d661 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.3.0-SNAPSHOT + 4.3.0 4.0.0 diff --git a/tools/pom.xml b/tools/pom.xml index 97e5e9c5d00..342c8793a2b 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.3.0-SNAPSHOT + 4.3.0 4.0.0 From 795eae6f44587eaaf770571160a604a953fa8716 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Thu, 26 Jul 2018 12:04:15 +0800 Subject: [PATCH 73/73] [maven-release-plugin] prepare for next development iteration --- broker/pom.xml | 2 +- client/pom.xml | 2 +- common/pom.xml | 2 +- distribution/pom.xml | 2 +- example/pom.xml | 4 ++-- filter/pom.xml | 2 +- logappender/pom.xml | 2 +- logging/pom.xml | 2 +- namesrv/pom.xml | 2 +- openmessaging/pom.xml | 2 +- pom.xml | 4 ++-- remoting/pom.xml | 2 +- srvutil/pom.xml | 2 +- store/pom.xml | 2 +- test/pom.xml | 2 +- tools/pom.xml | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/broker/pom.xml b/broker/pom.xml index 49b16bde843..f10ae53730e 100644 --- a/broker/pom.xml +++ b/broker/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.3.0 + 4.4.0-SNAPSHOT 4.0.0 diff --git a/client/pom.xml b/client/pom.xml index da2f73c087e..da6f53a874e 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.3.0 + 4.4.0-SNAPSHOT 4.0.0 diff --git a/common/pom.xml b/common/pom.xml index 2b61bee72d2..62c6d7eb5c2 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.3.0 + 4.4.0-SNAPSHOT 4.0.0 diff --git a/distribution/pom.xml b/distribution/pom.xml index 108565ccf06..c4d3b955c61 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -20,7 +20,7 @@ org.apache.rocketmq rocketmq-all - 4.3.0 + 4.4.0-SNAPSHOT rocketmq-distribution rocketmq-distribution ${project.version} diff --git a/example/pom.xml b/example/pom.xml index b2d815e7688..28dfe922fb1 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.3.0 + 4.4.0-SNAPSHOT 4.0.0 @@ -51,7 +51,7 @@ org.apache.rocketmq rocketmq-openmessaging - 4.3.0 + 4.4.0-SNAPSHOT diff --git a/filter/pom.xml b/filter/pom.xml index 1ae8f2e057d..0134f03d0e5 100644 --- a/filter/pom.xml +++ b/filter/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.3.0 + 4.4.0-SNAPSHOT 4.0.0 diff --git a/logappender/pom.xml b/logappender/pom.xml index 7b4b18a5eed..c250e59e5b6 100644 --- a/logappender/pom.xml +++ b/logappender/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.3.0 + 4.4.0-SNAPSHOT 4.0.0 rocketmq-logappender diff --git a/logging/pom.xml b/logging/pom.xml index ada5143c0e5..a52b9d0ac6c 100644 --- a/logging/pom.xml +++ b/logging/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.3.0 + 4.4.0-SNAPSHOT 4.0.0 diff --git a/namesrv/pom.xml b/namesrv/pom.xml index 596a1888c95..79f0abdc818 100644 --- a/namesrv/pom.xml +++ b/namesrv/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.3.0 + 4.4.0-SNAPSHOT 4.0.0 diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml index fed7e1ac96d..2850b1f2e17 100644 --- a/openmessaging/pom.xml +++ b/openmessaging/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.3.0 + 4.4.0-SNAPSHOT 4.0.0 diff --git a/pom.xml b/pom.xml index ccf8c3e4fee..5785d9cd176 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ 2012 org.apache.rocketmq rocketmq-all - 4.3.0 + 4.4.0-SNAPSHOT pom Apache RocketMQ ${project.version} http://rocketmq.apache.org/ @@ -42,7 +42,7 @@ git@github.com:apache/rocketmq.git scm:git:git@github.com:apache/rocketmq.git scm:git:git@github.com:apache/rocketmq.git - rocketmq-all-4.3.0 + HEAD diff --git a/remoting/pom.xml b/remoting/pom.xml index 3ec3af39116..55d92f37051 100644 --- a/remoting/pom.xml +++ b/remoting/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.3.0 + 4.4.0-SNAPSHOT 4.0.0 diff --git a/srvutil/pom.xml b/srvutil/pom.xml index 41cab8db7e1..73229ac9b1e 100644 --- a/srvutil/pom.xml +++ b/srvutil/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.3.0 + 4.4.0-SNAPSHOT 4.0.0 diff --git a/store/pom.xml b/store/pom.xml index b03090a61d3..59be3739ded 100644 --- a/store/pom.xml +++ b/store/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.3.0 + 4.4.0-SNAPSHOT 4.0.0 diff --git a/test/pom.xml b/test/pom.xml index 6cb57a1d661..08d26b3e29a 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.3.0 + 4.4.0-SNAPSHOT 4.0.0 diff --git a/tools/pom.xml b/tools/pom.xml index 342c8793a2b..dc0e256ed46 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.3.0 + 4.4.0-SNAPSHOT 4.0.0