From 18cea40b4a2b3c08f4088f01e1c0cbd1d9c3b8bc Mon Sep 17 00:00:00 2001 From: "duheng.dh" Date: Thu, 5 Jul 2018 07:43:04 +0800 Subject: [PATCH 1/6] Add support of transactional message --- .../rocketmq/broker/BrokerController.java | 62 ++- .../broker/client/ProducerManager.java | 36 +- .../broker/client/net/Broker2Client.java | 52 +- .../processor/EndTransactionProcessor.java | 221 ++++---- .../processor/SendMessageProcessor.java | 57 +- .../AbstractTransactionCheckListener.java | 109 ++++ .../broker/transaction/OperationResult.java | 51 ++ .../TransactionMsgCheckService.java | 78 +++ .../transaction/TransactionMsgService.java | 77 +++ .../broker/transaction/TransactionRecord.java | 3 +- .../broker/transaction/TransactionStore.java | 2 +- ...faultAbstractTransactionCheckListener.java | 36 ++ .../broker/transaction/queue/GetResult.java | 41 ++ .../queue/QueueTransactionMsgServiceImpl.java | 498 ++++++++++++++++++ .../transaction/queue/TransactionBridge.java | 342 ++++++++++++ .../transaction/queue/TransactionUtil.java | 40 ++ .../broker/util/PositiveAtomicCounter.java | 40 ++ .../rocketmq/broker/util/ServiceProvider.java | 191 +++++++ .../EndTransactionProcessorTest.java | 152 ++++++ .../processor/SendMessageProcessorTest.java | 51 +- ...tAbstractTransactionCheckListenerTest.java | 78 +++ .../QueueTransactionMsgServiceImplTest.java | 247 +++++++++ .../queue/TransactionBridgeTest.java | 189 +++++++ .../util/LogTransactionCheckListener.java | 28 + .../broker/util/ServiceProviderTest.java | 41 ++ .../util/TransactionMsgServiceImpl.java | 67 +++ ...ansaction.AbstractTransactionCheckListener | 1 + ...q.broker.transaction.TransactionMsgService | 1 + .../client/impl/ClientRemotingProcessor.java | 4 + .../client/impl/consumer/PullAPIWrapper.java | 4 + .../impl/producer/DefaultMQProducerImpl.java | 91 ++-- .../client/impl/producer/MQProducerInner.java | 7 +- .../client/producer/DefaultMQProducer.java | 4 +- .../producer/LocalTransactionExecuter.java | 2 +- .../rocketmq/client/producer/MQProducer.java | 3 +- .../producer/TransactionCheckListener.java | 2 +- .../client/producer/TransactionListener.java | 40 ++ .../producer/TransactionMQProducer.java | 50 +- .../apache/rocketmq/common/BrokerConfig.java | 42 ++ .../org/apache/rocketmq/common/MixAll.java | 4 + .../rocketmq/common/message/Message.java | 19 +- .../rocketmq/common/message/MessageConst.java | 3 + .../CheckTransactionStateRequestHeader.java | 9 + .../header/EndTransactionRequestHeader.java | 16 +- .../benchmark/TransactionProducer.java | 87 +-- .../transaction/TransactionExecuterImpl.java | 41 -- ...Impl.java => TransactionListenerImpl.java} | 41 +- .../transaction/TransactionProducer.java | 34 +- .../apache/rocketmq/filter/ParserTest.java | 15 +- .../logging/inner/LoggingBuilderTest.java | 2 +- pom.xml | 5 +- .../rocketmq/store/DefaultMessageStore.java | 2 +- .../rocketmq/store/MessageExtBrokerInner.java | 7 +- .../rocketmq/store/ConsumeQueueTest.java | 2 - .../test/client/rmq/RMQAsyncSendProducer.java | 20 +- .../test/client/rmq/RMQNormalProducer.java | 6 +- .../clientinterface/AbstractMQProducer.java | 4 +- .../test/clientinterface/MQProducer.java | 4 +- .../{SendResult.java => ResultWrapper.java} | 2 +- 59 files changed, 2974 insertions(+), 389 deletions(-) create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionCheckListener.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/transaction/OperationResult.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionMsgCheckService.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionMsgService.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/DefaultAbstractTransactionCheckListener.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/GetResult.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/QueueTransactionMsgServiceImpl.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionBridge.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionUtil.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/util/PositiveAtomicCounter.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/util/ServiceProvider.java create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/processor/EndTransactionProcessorTest.java create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/DefaultAbstractTransactionCheckListenerTest.java create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/QueueTransactionMsgServiceImplTest.java create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionBridgeTest.java create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/util/LogTransactionCheckListener.java create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/util/ServiceProviderTest.java create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/util/TransactionMsgServiceImpl.java create mode 100644 broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionCheckListener create mode 100644 broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.TransactionMsgService create mode 100644 client/src/main/java/org/apache/rocketmq/client/producer/TransactionListener.java delete mode 100644 example/src/main/java/org/apache/rocketmq/example/transaction/TransactionExecuterImpl.java rename example/src/main/java/org/apache/rocketmq/example/transaction/{TransactionCheckListenerImpl.java => TransactionListenerImpl.java} (54%) rename test/src/main/java/org/apache/rocketmq/test/sendresult/{SendResult.java => ResultWrapper.java} (98%) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index ed85a679c05..7b7667ba451 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -61,6 +61,13 @@ import org.apache.rocketmq.broker.slave.SlaveSynchronize; import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager; import org.apache.rocketmq.broker.topic.TopicConfigManager; +import org.apache.rocketmq.broker.transaction.AbstractTransactionCheckListener; +import org.apache.rocketmq.broker.transaction.TransactionMsgCheckService; +import org.apache.rocketmq.broker.transaction.TransactionMsgService; +import org.apache.rocketmq.broker.transaction.queue.DefaultAbstractTransactionCheckListener; +import org.apache.rocketmq.broker.transaction.queue.QueueTransactionMsgServiceImpl; +import org.apache.rocketmq.broker.transaction.queue.TransactionBridge; +import org.apache.rocketmq.broker.util.ServiceProvider; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.Configuration; import org.apache.rocketmq.common.DataVersion; @@ -142,6 +149,9 @@ public class BrokerController { private BrokerFastFailure brokerFastFailure; private Configuration configuration; private FileWatchService fileWatchService; + private TransactionMsgCheckService transactionMsgCheckService; + private TransactionMsgService transactionMsgService; + private AbstractTransactionCheckListener transactionCheckListener; public BrokerController( final BrokerConfig brokerConfig, @@ -432,11 +442,26 @@ private void reloadServerSslContext() { log.warn("FileWatchService created error, can't load the certificate dynamically"); } } + initialTransaction(); } - return result; } + private void initialTransaction() { + this.transactionMsgService = ServiceProvider.loadClass(ServiceProvider.TRANSACTION_SERVICE_ID, TransactionMsgService.class); + if (null == this.transactionMsgService) { + this.transactionMsgService = new QueueTransactionMsgServiceImpl(new TransactionBridge(this, this.getMessageStore())); + log.warn("Load default transaction message hook service: {}", QueueTransactionMsgServiceImpl.class.getSimpleName()); + } + this.transactionCheckListener = ServiceProvider.loadClass(ServiceProvider.TRANSACTION_LISTENER_ID, AbstractTransactionCheckListener.class); + if (null == this.transactionCheckListener) { + this.transactionCheckListener = new DefaultAbstractTransactionCheckListener(); + log.warn("Load default discard message hook service: {}", DefaultAbstractTransactionCheckListener.class.getSimpleName()); + } + this.transactionCheckListener.setBrokerController(this); + this.transactionMsgCheckService = new TransactionMsgCheckService(this); + } + public void registerProcessor() { /** * SendMessageProcessor @@ -700,6 +725,10 @@ public void shutdown() { if (this.fileWatchService != null) { this.fileWatchService.shutdown(); } + + if (this.transactionMsgCheckService != null) { + this.transactionMsgCheckService.shutdown(false); + } } private void unregisterBrokerAll() { @@ -768,6 +797,13 @@ public void run() { if (this.brokerFastFailure != null) { this.brokerFastFailure.start(); } + + if (BrokerRole.SLAVE != messageStoreConfig.getBrokerRole()) { + if (this.transactionMsgCheckService != null) { + log.info("start transaction service!"); + this.transactionMsgCheckService.start(); + } + } } public synchronized void registerIncrementBrokerData(TopicConfig topicConfig, DataVersion dataVersion) { @@ -949,4 +985,28 @@ public void setStoreHost(InetSocketAddress storeHost) { public Configuration getConfiguration() { return this.configuration; } + + public TransactionMsgCheckService getTransactionMsgCheckService() { + return transactionMsgCheckService; + } + + public void setTransactionMsgCheckService(TransactionMsgCheckService transactionMsgCheckService) { + this.transactionMsgCheckService = transactionMsgCheckService; + } + + public TransactionMsgService getTransactionMsgService() { + return transactionMsgService; + } + + public void setTransactionMsgService(TransactionMsgService transactionMsgService) { + this.transactionMsgService = transactionMsgService; + } + + public AbstractTransactionCheckListener getTransactionCheckListener() { + return transactionCheckListener; + } + + public void setTransactionCheckListener(AbstractTransactionCheckListener transactionCheckListener) { + this.transactionCheckListener = transactionCheckListener; + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java index 88f1fdeaf73..11f8fe521be 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java @@ -17,13 +17,18 @@ package org.apache.rocketmq.broker.client; import io.netty.channel.Channel; + +import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; + +import org.apache.rocketmq.broker.util.PositiveAtomicCounter; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; @@ -37,7 +42,7 @@ public class ProducerManager { private final Lock groupChannelLock = new ReentrantLock(); private final HashMap> groupChannelTable = new HashMap>(); - + private PositiveAtomicCounter positiveAtomicCounter = new PositiveAtomicCounter(); public ProducerManager() { } @@ -185,4 +190,33 @@ public void unregisterProducer(final String group, final ClientChannelInfo clien log.error("", e); } } + + public Channel getAvaliableChannel(String groupId) { + HashMap channelClientChannelInfoHashMap = groupChannelTable.get(groupId); + List channelList = new ArrayList(); + if (channelClientChannelInfoHashMap != null) { + for (Channel channel : channelClientChannelInfoHashMap.keySet()) { + channelList.add(channel); + } + int size = channelList.size(); + if (0 == size) { + log.warn("channel list is empty. groupId={}", groupId); + return null; + } + + int index = positiveAtomicCounter.incrementAndGet() % size; + Channel channel = channelList.get(index); + int count = 0; + boolean isOk = channel.isActive() && channel.isWritable(); + while (isOk && count++ < 3) { + index = (++index) % size; + channel = channelList.get(index); + return channel; + } + } else { + log.warn("check transaction failed, channel table is empty. groupId={}", groupId); + return null; + } + return null; + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java b/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java index 2a104457805..ae5a73f1590 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java @@ -17,25 +17,15 @@ package org.apache.rocketmq.broker.client.net; import io.netty.channel.Channel; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.FileRegion; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.ConcurrentMap; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.client.ClientChannelInfo; import org.apache.rocketmq.broker.client.ConsumerGroupInfo; -import org.apache.rocketmq.broker.pagecache.OneMessageTransfer; import org.apache.rocketmq.common.MQVersion; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.message.MessageQueueForC; import org.apache.rocketmq.common.protocol.RequestCode; @@ -47,11 +37,19 @@ import org.apache.rocketmq.common.protocol.header.GetConsumerStatusRequestHeader; import org.apache.rocketmq.common.protocol.header.NotifyConsumerIdsChangedRequestHeader; import org.apache.rocketmq.common.protocol.header.ResetOffsetRequestHeader; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import org.apache.rocketmq.store.SelectMappedBufferResult; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentMap; public class Broker2Client { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); @@ -62,34 +60,22 @@ public Broker2Client(BrokerController brokerController) { } public void checkProducerTransactionState( + final String group, final Channel channel, final CheckTransactionStateRequestHeader requestHeader, - final SelectMappedBufferResult selectMappedBufferResult) { + final MessageExt messageExt) throws Exception { RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHECK_TRANSACTION_STATE, requestHeader); - request.markOnewayRPC(); - + request.setBody(MessageDecoder.encode(messageExt, false)); try { - FileRegion fileRegion = - new OneMessageTransfer(request.encodeHeader(selectMappedBufferResult.getSize()), - selectMappedBufferResult); - channel.writeAndFlush(fileRegion).addListener(new ChannelFutureListener() { - @Override - public void operationComplete(ChannelFuture future) throws Exception { - selectMappedBufferResult.release(); - if (!future.isSuccess()) { - log.error("invokeProducer failed,", future.cause()); - } - } - }); - } catch (Throwable e) { - log.error("invokeProducer exception", e); - selectMappedBufferResult.release(); + this.brokerController.getRemotingServer().invokeOneway(channel, request, 10); + } catch (Exception e) { + log.error("check transaction failed because invoke producer exception. group={}, msgId={}", group, messageExt.getMsgId(), e.getMessage()); } } public RemotingCommand callClient(final Channel channel, - final RemotingCommand request + final RemotingCommand request ) throws RemotingSendRequestException, RemotingTimeoutException, InterruptedException { return this.brokerController.getRemotingServer().invokeSync(channel, request, 10000); } @@ -119,7 +105,7 @@ public RemotingCommand resetOffset(String topic, String group, long timeStamp, b } public RemotingCommand resetOffset(String topic, String group, long timeStamp, boolean isForce, - boolean isC) { + boolean isC) { final RemotingCommand response = RemotingCommand.createResponseCommand(null); TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java index 6801f759ea5..3982b66427c 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java @@ -18,10 +18,9 @@ import io.netty.channel.ChannelHandlerContext; import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.transaction.OperationResult; import org.apache.rocketmq.common.TopicFilterType; import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; @@ -34,11 +33,16 @@ import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.store.MessageExtBrokerInner; -import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.config.BrokerRole; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +/** + * EndTransaction processor: process commit and rollback message + */ public class EndTransactionProcessor implements NettyRequestProcessor { - private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); + private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); private final BrokerController brokerController; public EndTransactionProcessor(final BrokerController brokerController) { @@ -46,16 +50,22 @@ public EndTransactionProcessor(final BrokerController brokerController) { } @Override - public RemotingCommand processRequest(ChannelHandlerContext ctx, - RemotingCommand request) throws RemotingCommandException { + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws + RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); final EndTransactionRequestHeader requestHeader = - (EndTransactionRequestHeader) request.decodeCommandCustomHeader(EndTransactionRequestHeader.class); + (EndTransactionRequestHeader)request.decodeCommandCustomHeader(EndTransactionRequestHeader.class); + LOGGER.info("Transaction request:{}", requestHeader); + if (BrokerRole.SLAVE == brokerController.getMessageStoreConfig().getBrokerRole()) { + response.setCode(ResponseCode.SLAVE_NOT_AVAILABLE); + LOGGER.warn("Message store is slave mode, so end transaction is forbidden. "); + return response; + } if (requestHeader.getFromTransactionCheck()) { switch (requestHeader.getCommitOrRollback()) { case MessageSysFlag.TRANSACTION_NOT_TYPE: { - LOGGER.warn("check producer[{}] transaction state, but it's pending status." + LOGGER.warn("Check producer[{}] transaction state, but it's pending status." + "RequestHeader: {} Remark: {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.toString(), @@ -64,7 +74,7 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx, } case MessageSysFlag.TRANSACTION_COMMIT_TYPE: { - LOGGER.warn("check producer[{}] transaction state, the producer commit the message." + LOGGER.warn("Check producer[{}] transaction state, the producer commit the message." + "RequestHeader: {} Remark: {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.toString(), @@ -74,7 +84,7 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx, } case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: { - LOGGER.warn("check producer[{}] transaction state, the producer rollback the message." + LOGGER.warn("Check producer[{}] transaction state, the producer rollback the message." + "RequestHeader: {} Remark: {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.toString(), @@ -87,7 +97,7 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx, } else { switch (requestHeader.getCommitOrRollback()) { case MessageSysFlag.TRANSACTION_NOT_TYPE: { - LOGGER.warn("the producer[{}] end transaction in sending message, and it's pending status." + LOGGER.warn("The producer[{}] end transaction in sending message, and it's pending status." + "RequestHeader: {} Remark: {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.toString(), @@ -100,7 +110,7 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx, } case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: { - LOGGER.warn("the producer[{}] end transaction in sending message, rollback the message." + LOGGER.warn("The producer[{}] end transaction in sending message, rollback the message." + "RequestHeader: {} Remark: {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.toString(), @@ -111,122 +121,145 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx, return null; } } + OperationResult result = new OperationResult(); + if (MessageSysFlag.TRANSACTION_COMMIT_TYPE == requestHeader.getCommitOrRollback()) { + result = this.brokerController.getTransactionMsgService().commitMessage(requestHeader); + if (result.getResponseCode() == ResponseCode.SUCCESS) { + RemotingCommand res = checkPrepareMessage(result.getPrepareMessage(), requestHeader); + if (res.getCode() == ResponseCode.SUCCESS) { + MessageExtBrokerInner msgInner = endMessageTransaction(result.getPrepareMessage()); + msgInner.setSysFlag(MessageSysFlag.resetTransactionValue(msgInner.getSysFlag(), requestHeader.getCommitOrRollback())); + msgInner.setQueueOffset(requestHeader.getTranStateTableOffset()); + msgInner.setPreparedTransactionOffset(requestHeader.getCommitLogOffset()); + msgInner.setStoreTimestamp(result.getPrepareMessage().getStoreTimestamp()); + RemotingCommand sendResult = sendFinalMessage(msgInner); + if (sendResult.getCode() == ResponseCode.SUCCESS) { + this.brokerController.getTransactionMsgService().deletePrepareMessage(result.getPrepareMessage()); + } + return sendResult; + } + return res; + } + } else if (MessageSysFlag.TRANSACTION_ROLLBACK_TYPE == requestHeader.getCommitOrRollback()) { + result = this.brokerController.getTransactionMsgService().rollbackMessage(requestHeader); + if (result.getResponseCode() == ResponseCode.SUCCESS) { + RemotingCommand res = checkPrepareMessage(result.getPrepareMessage(), requestHeader); + if (res.getCode() == ResponseCode.SUCCESS) { + this.brokerController.getTransactionMsgService().deletePrepareMessage(result.getPrepareMessage()); + } + return res; + } + } + response.setCode(result.getResponseCode()); + response.setRemark(result.getResponseRemark()); + return response; + } + + @Override + public boolean rejectRequest() { + return false; + } - final MessageExt msgExt = this.brokerController.getMessageStore().lookMessageByOffset(requestHeader.getCommitLogOffset()); + private RemotingCommand checkPrepareMessage(MessageExt msgExt, EndTransactionRequestHeader requestHeader) { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); if (msgExt != null) { final String pgroupRead = msgExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP); if (!pgroupRead.equals(requestHeader.getProducerGroup())) { response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("the producer group wrong"); + response.setRemark("The producer group wrong"); return response; } if (msgExt.getQueueOffset() != requestHeader.getTranStateTableOffset()) { response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("the transaction state table offset wrong"); + response.setRemark("The transaction state table offset wrong"); return response; } if (msgExt.getCommitLogOffset() != requestHeader.getCommitLogOffset()) { response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("the commit log offset wrong"); - return response; - } - - MessageExtBrokerInner msgInner = this.endMessageTransaction(msgExt); - msgInner.setSysFlag(MessageSysFlag.resetTransactionValue(msgInner.getSysFlag(), requestHeader.getCommitOrRollback())); - - msgInner.setQueueOffset(requestHeader.getTranStateTableOffset()); - msgInner.setPreparedTransactionOffset(requestHeader.getCommitLogOffset()); - msgInner.setStoreTimestamp(msgExt.getStoreTimestamp()); - if (MessageSysFlag.TRANSACTION_ROLLBACK_TYPE == requestHeader.getCommitOrRollback()) { - msgInner.setBody(null); - } - - final MessageStore messageStore = this.brokerController.getMessageStore(); - final PutMessageResult putMessageResult = messageStore.putMessage(msgInner); - if (putMessageResult != null) { - switch (putMessageResult.getPutMessageStatus()) { - // Success - case PUT_OK: - case FLUSH_DISK_TIMEOUT: - case FLUSH_SLAVE_TIMEOUT: - case SLAVE_NOT_AVAILABLE: - response.setCode(ResponseCode.SUCCESS); - response.setRemark(null); - break; - // Failed - case CREATE_MAPEDFILE_FAILED: - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("create mapped file failed."); - break; - case MESSAGE_ILLEGAL: - case PROPERTIES_SIZE_EXCEEDED: - response.setCode(ResponseCode.MESSAGE_ILLEGAL); - response.setRemark("the message is illegal, maybe msg body or properties length not matched. msg body length limit 128k, msg properties length limit 32k."); - break; - case SERVICE_NOT_AVAILABLE: - response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE); - response.setRemark("service not available now."); - break; - case OS_PAGECACHE_BUSY: - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("OS page cache busy, please try another machine"); - break; - case UNKNOWN_ERROR: - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("UNKNOWN_ERROR"); - break; - default: - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("UNKNOWN_ERROR DEFAULT"); - break; - } - + response.setRemark("The commit log offset wrong"); return response; - } else { - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("store putMessage return null"); } } else { response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("find prepared transaction message failed"); + response.setRemark("Find prepared transaction message failed"); return response; } - + response.setCode(ResponseCode.SUCCESS); return response; } - @Override - public boolean rejectRequest() { - return false; - } - private MessageExtBrokerInner endMessageTransaction(MessageExt msgExt) { MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + msgInner.setTopic(msgExt.getUserProperty(MessageConst.PROPERTY_REAL_TOPIC)); + msgInner.setQueueId(Integer.parseInt(msgExt.getUserProperty(MessageConst.PROPERTY_REAL_QUEUE_ID))); msgInner.setBody(msgExt.getBody()); msgInner.setFlag(msgExt.getFlag()); - MessageAccessor.setProperties(msgInner, msgExt.getProperties()); - + msgInner.setBornTimestamp(msgExt.getBornTimestamp()); + msgInner.setBornHost(msgExt.getBornHost()); + msgInner.setStoreHost(msgExt.getStoreHost()); + msgInner.setReconsumeTimes(msgExt.getReconsumeTimes()); + msgInner.setWaitStoreMsgOK(false); + msgInner.setTransactionId(msgExt.getUserProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX)); + msgInner.setSysFlag(msgExt.getSysFlag()); TopicFilterType topicFilterType = (msgInner.getSysFlag() & MessageSysFlag.MULTI_TAGS_FLAG) == MessageSysFlag.MULTI_TAGS_FLAG ? TopicFilterType.MULTI_TAG : TopicFilterType.SINGLE_TAG; long tagsCodeValue = MessageExtBrokerInner.tagsString2tagsCode(topicFilterType, msgInner.getTags()); msgInner.setTagsCode(tagsCodeValue); + MessageAccessor.setProperties(msgInner, msgExt.getProperties()); msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties())); - - msgInner.setSysFlag(msgExt.getSysFlag()); - msgInner.setBornTimestamp(msgExt.getBornTimestamp()); - msgInner.setBornHost(msgExt.getBornHost()); - msgInner.setStoreHost(msgExt.getStoreHost()); - msgInner.setReconsumeTimes(msgExt.getReconsumeTimes()); - - msgInner.setWaitStoreMsgOK(false); - MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_DELAY_TIME_LEVEL); - - msgInner.setTopic(msgExt.getTopic()); - msgInner.setQueueId(msgExt.getQueueId()); - + MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_REAL_TOPIC); + MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_REAL_QUEUE_ID); return msgInner; } + + private RemotingCommand sendFinalMessage(MessageExtBrokerInner msgInner) { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner); + if (putMessageResult != null) { + switch (putMessageResult.getPutMessageStatus()) { + // Success + case PUT_OK: + case FLUSH_DISK_TIMEOUT: + case FLUSH_SLAVE_TIMEOUT: + case SLAVE_NOT_AVAILABLE: + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + break; + // Failed + case CREATE_MAPEDFILE_FAILED: + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("Create mapped file failed."); + break; + case MESSAGE_ILLEGAL: + case PROPERTIES_SIZE_EXCEEDED: + response.setCode(ResponseCode.MESSAGE_ILLEGAL); + response.setRemark("The message is illegal, maybe msg body or properties length not matched. msg body length limit 128k, msg properties length limit 32k."); + break; + case SERVICE_NOT_AVAILABLE: + response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE); + response.setRemark("Service not available now."); + break; + case OS_PAGECACHE_BUSY: + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("OS page cache busy, please try another machine"); + break; + case UNKNOWN_ERROR: + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UNKNOWN_ERROR"); + break; + default: + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UNKNOWN_ERROR DEFAULT"); + break; + } + return response; + } else { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("store putMessage return null"); + } + return response; + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java index 227a23e6b49..64bbe05338a 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java @@ -17,8 +17,6 @@ package org.apache.rocketmq.broker.processor; import io.netty.channel.ChannelHandlerContext; -import java.net.SocketAddress; -import java.util.List; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.mqtrace.ConsumeMessageContext; import org.apache.rocketmq.broker.mqtrace.ConsumeMessageHook; @@ -51,6 +49,10 @@ import org.apache.rocketmq.store.config.StorePathConfigHelper; import org.apache.rocketmq.store.stats.BrokerStatsManager; +import java.net.SocketAddress; +import java.util.List; +import java.util.Map; + public class SendMessageProcessor extends AbstractSendMessageProcessor implements NettyRequestProcessor { private List consumeMessageHookList; @@ -61,7 +63,7 @@ public SendMessageProcessor(final BrokerController brokerController) { @Override public RemotingCommand processRequest(ChannelHandlerContext ctx, - RemotingCommand request) throws RemotingCommandException { + RemotingCommand request) throws RemotingCommandException { SendMessageContext mqtraceContext; switch (request.getCode()) { case RequestCode.CONSUMER_SEND_MSG_BACK: @@ -97,7 +99,7 @@ private RemotingCommand consumerSendMsgBack(final ChannelHandlerContext ctx, fin throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); final ConsumerSendMsgBackRequestHeader requestHeader = - (ConsumerSendMsgBackRequestHeader) request.decodeCommandCustomHeader(ConsumerSendMsgBackRequestHeader.class); + (ConsumerSendMsgBackRequestHeader)request.decodeCommandCustomHeader(ConsumerSendMsgBackRequestHeader.class); if (this.hasConsumeMessageHook() && !UtilAll.isBlank(requestHeader.getOriginMsgId())) { @@ -247,8 +249,8 @@ private RemotingCommand consumerSendMsgBack(final ChannelHandlerContext ctx, fin } private boolean handleRetryAndDLQ(SendMessageRequestHeader requestHeader, RemotingCommand response, - RemotingCommand request, - MessageExt msg, TopicConfig topicConfig) { + RemotingCommand request, + MessageExt msg, TopicConfig topicConfig) { String newTopic = requestHeader.getTopic(); if (null != newTopic && newTopic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { String groupName = newTopic.substring(MixAll.RETRY_GROUP_TOPIC_PREFIX.length()); @@ -291,12 +293,12 @@ private boolean handleRetryAndDLQ(SendMessageRequestHeader requestHeader, Remoti } private RemotingCommand sendMessage(final ChannelHandlerContext ctx, - final RemotingCommand request, - final SendMessageContext sendMessageContext, - final SendMessageRequestHeader requestHeader) throws RemotingCommandException { + final RemotingCommand request, + final SendMessageContext sendMessageContext, + final SendMessageRequestHeader requestHeader) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); - final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader) response.readCustomHeader(); + final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader)response.readCustomHeader(); response.setOpaque(request.getOpaque()); @@ -343,27 +345,30 @@ private RemotingCommand sendMessage(final ChannelHandlerContext ctx, msgInner.setBornHost(ctx.channel().remoteAddress()); msgInner.setStoreHost(this.getStoreHost()); msgInner.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes()); - - if (this.brokerController.getBrokerConfig().isRejectTransactionMessage()) { - String traFlag = msgInner.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED); - if (traFlag != null) { + PutMessageResult putMessageResult = null; + Map oriProps = MessageDecoder.string2messageProperties(requestHeader.getProperties()); + String traFlag = oriProps.get(MessageConst.PROPERTY_TRANSACTION_PREPARED); + if (traFlag != null && Boolean.parseBoolean(traFlag)) { + if (this.brokerController.getBrokerConfig().isRejectTransactionMessage()) { response.setCode(ResponseCode.NO_PERMISSION); response.setRemark( - "the broker[" + this.brokerController.getBrokerConfig().getBrokerIP1() + "] sending transaction message is forbidden"); + "the broker[" + this.brokerController.getBrokerConfig().getBrokerIP1() + + "] sending transaction message is forbidden"); return response; } + putMessageResult = this.brokerController.getTransactionMsgService().prepareMessage(msgInner); + } else { + putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner); } - PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner); - return handlePutMessageResult(putMessageResult, response, request, msgInner, responseHeader, sendMessageContext, ctx, queueIdInt); } private RemotingCommand handlePutMessageResult(PutMessageResult putMessageResult, RemotingCommand response, - RemotingCommand request, MessageExt msg, - SendMessageResponseHeader responseHeader, SendMessageContext sendMessageContext, ChannelHandlerContext ctx, - int queueIdInt) { + RemotingCommand request, MessageExt msg, + SendMessageResponseHeader responseHeader, SendMessageContext sendMessageContext, ChannelHandlerContext ctx, + int queueIdInt) { if (putMessageResult == null) { response.setCode(ResponseCode.SYSTEM_ERROR); response.setRemark("store putMessage return null"); @@ -443,7 +448,7 @@ private RemotingCommand handlePutMessageResult(PutMessageResult putMessageResult int commercialBaseCount = brokerController.getBrokerConfig().getCommercialBaseCount(); int wroteSize = putMessageResult.getAppendMessageResult().getWroteBytes(); - int incValue = (int) Math.ceil(wroteSize / BrokerStatsManager.SIZE_PER_COUNT) * commercialBaseCount; + int incValue = (int)Math.ceil(wroteSize / BrokerStatsManager.SIZE_PER_COUNT) * commercialBaseCount; sendMessageContext.setCommercialSendStats(BrokerStatsManager.StatsType.SEND_SUCCESS); sendMessageContext.setCommercialSendTimes(incValue); @@ -454,7 +459,7 @@ private RemotingCommand handlePutMessageResult(PutMessageResult putMessageResult } else { if (hasSendMessageHook()) { int wroteSize = request.getBody().length; - int incValue = (int) Math.ceil(wroteSize / BrokerStatsManager.SIZE_PER_COUNT); + int incValue = (int)Math.ceil(wroteSize / BrokerStatsManager.SIZE_PER_COUNT); sendMessageContext.setCommercialSendStats(BrokerStatsManager.StatsType.SEND_FAILURE); sendMessageContext.setCommercialSendTimes(incValue); @@ -466,12 +471,12 @@ private RemotingCommand handlePutMessageResult(PutMessageResult putMessageResult } private RemotingCommand sendBatchMessage(final ChannelHandlerContext ctx, - final RemotingCommand request, - final SendMessageContext sendMessageContext, - final SendMessageRequestHeader requestHeader) throws RemotingCommandException { + final RemotingCommand request, + final SendMessageContext sendMessageContext, + final SendMessageRequestHeader requestHeader) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); - final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader) response.readCustomHeader(); + final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader)response.readCustomHeader(); response.setOpaque(request.getOpaque()); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionCheckListener.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionCheckListener.java new file mode 100644 index 00000000000..2a3b3d71bf8 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionCheckListener.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.transaction; + +import io.netty.channel.Channel; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +public abstract class AbstractTransactionCheckListener { + private static final Logger log = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); + + private BrokerController brokerController; + + private static ExecutorService executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, new ArrayBlockingQueue(2000), new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + Thread thread = new Thread(r); + thread.setName("Transaction-msg-check-thread"); + return thread; + } + }); + + public AbstractTransactionCheckListener() {} + + public AbstractTransactionCheckListener(BrokerController brokerController) { + this.brokerController = brokerController; + } + + public void sendCheckMessage(MessageExt msgExt) throws Exception { + CheckTransactionStateRequestHeader checkTransactionStateRequestHeader = new CheckTransactionStateRequestHeader(); + checkTransactionStateRequestHeader.setCommitLogOffset(msgExt.getCommitLogOffset()); + checkTransactionStateRequestHeader.setOffsetMsgId(msgExt.getMsgId()); + checkTransactionStateRequestHeader.setMsgId(msgExt.getUserProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX)); + checkTransactionStateRequestHeader.setTransactionId(checkTransactionStateRequestHeader.getMsgId()); + checkTransactionStateRequestHeader.setTranStateTableOffset(msgExt.getQueueOffset()); + msgExt.setTopic(msgExt.getUserProperty(MessageConst.PROPERTY_REAL_TOPIC)); + msgExt.setQueueId(Integer.parseInt(msgExt.getUserProperty(MessageConst.PROPERTY_REAL_QUEUE_ID))); + msgExt.setStoreSize(0); + String groupId = msgExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP); + Channel channel = brokerController.getProducerManager().getAvaliableChannel(groupId); + if (channel != null) { + brokerController.getBroker2Client().checkProducerTransactionState(groupId, channel, checkTransactionStateRequestHeader, msgExt); + } else { + log.warn("Check transaction failed, channel is null. groupId={}", groupId); + } + } + + public void resolveHalfMsg(final MessageExt msgExt) { + executorService.execute(new Runnable() { + @Override + public void run() { + try { + sendCheckMessage(msgExt); + } catch (Exception e) { + log.error("Send check message error!", e); + } + } + }); + } + + public BrokerController getBrokerController() { + return brokerController; + } + + public void shutDown() { + executorService.shutdown(); + } + + /** + * Inject brokerController for this listener + * + * @param brokerController + */ + public void setBrokerController(BrokerController brokerController) { + this.brokerController = brokerController; + } + + /** + * In order to avoid check back unlimited, we will discard this message after checked too many times + * + * @param msgExt Message to be discarded. + */ + public abstract void resolveDiscardMsg(MessageExt msgExt); +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/OperationResult.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/OperationResult.java new file mode 100644 index 00000000000..6c35f6adeb5 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/OperationResult.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.transaction; + +import org.apache.rocketmq.common.message.MessageExt; + +public class OperationResult { + private MessageExt prepareMessage; + + private int responseCode; + + private String responseRemark; + + public MessageExt getPrepareMessage() { + return prepareMessage; + } + + public void setPrepareMessage(MessageExt prepareMessage) { + this.prepareMessage = prepareMessage; + } + + public int getResponseCode() { + return responseCode; + } + + public void setResponseCode(int responseCode) { + this.responseCode = responseCode; + } + + public String getResponseRemark() { + return responseRemark; + } + + public void setResponseRemark(String responseRemark) { + this.responseRemark = responseRemark; + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionMsgCheckService.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionMsgCheckService.java new file mode 100644 index 00000000000..7f34eb95e5e --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionMsgCheckService.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.transaction; + +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.ServiceThread; +import org.apache.rocketmq.common.constant.LoggerName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.atomic.AtomicBoolean; + +public class TransactionMsgCheckService extends ServiceThread { + private static final Logger log = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); + + private BrokerController brokerController; + + private final AtomicBoolean started = new AtomicBoolean(false); + + public TransactionMsgCheckService(BrokerController brokerController) { + this.brokerController = brokerController; + } + + @Override + public void start() { + if (started.compareAndSet(false, true)) { + super.start(); + this.brokerController.getTransactionMsgService().open(); + } + } + + @Override + public void shutdown(boolean interrupt) { + if (started.compareAndSet(true, false)) { + super.shutdown(interrupt); + this.brokerController.getTransactionMsgService().close(); + this.brokerController.getTransactionCheckListener().shutDown(); + } + } + + @Override + public String getServiceName() { + return TransactionMsgCheckService.class.getSimpleName(); + } + + @Override + public void run() { + log.info("Start transaction service thread!"); + long timeout = brokerController.getBrokerConfig().getTransactionTimeOut(); + int checkMax = brokerController.getBrokerConfig().getTransactionCheckMax(); + long checkInterval = brokerController.getBrokerConfig().getTransactionCheckInterval(); + log.info("Check parameter: transactionCheckMax: {}, transactionTimeOut: {} transactionCheckInterval: {}", checkMax, timeout, checkInterval); + while (!this.isStopped()) { + try { + Thread.sleep(checkInterval); + this.brokerController.getTransactionMsgService().check(timeout, checkMax, this.brokerController.getTransactionCheckListener()); + } catch (Exception e) { + log.error("", e); + } + } + log.info("End transaction service thread!"); + } + +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionMsgService.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionMsgService.java new file mode 100644 index 00000000000..77462a6b60a --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionMsgService.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.transaction; + +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.PutMessageResult; + +public interface TransactionMsgService { + + /** + * Process prepare message, in common, we should put this message to storage service. + * + * @param messageInner Prepare(Half) message. + * @return Prepare message storage result. + */ + PutMessageResult prepareMessage(MessageExtBrokerInner messageInner); + + /** + * Delete prepare message when this message has been committed or rolled back. + * + * @param messageExt + */ + boolean deletePrepareMessage(MessageExt messageExt); + + /** + * Invoked to process commit prepare message. + * + * @param requestHeader Commit message request header. + * @return Operate result contains prepare message and relative error code. + */ + OperationResult commitMessage(EndTransactionRequestHeader requestHeader); + + /** + * Invoked to roll back prepare message. + * + * @param requestHeader Prepare message request header. + * @return Operate result contains prepare message and relative error code. + */ + OperationResult rollbackMessage(EndTransactionRequestHeader requestHeader); + + /** + * Traverse uncommitted/unroll back half message and send check back request to producer to obtain transaction status. + * + * @param transactionTimeout The minimum time of the transactional message to be checked firstly, one message only exceed this time interval that can be checked. + * @param transactionCheckMax The maximum number of times the message was checked, if exceed this value, this message will be discarded. + * @param listener When the message is considered to be checked or discarded, the relative method of this class will be invoked. + */ + void check(long transactionTimeout, int transactionCheckMax, AbstractTransactionCheckListener listener); + + /** + * Open transaction service. + * + * @return If open success, return true. + */ + boolean open(); + + /** + * Close transaction service. + */ + void close(); +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionRecord.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionRecord.java index 5f81ebb3303..cebde4a4f87 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionRecord.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionRecord.java @@ -14,9 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.rocketmq.broker.transaction; - +@Deprecated public class TransactionRecord { // Commit Log Offset private long offset; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionStore.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionStore.java index 3decc01f5eb..4dcdd772e62 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionStore.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionStore.java @@ -18,7 +18,7 @@ package org.apache.rocketmq.broker.transaction; import java.util.List; - +@Deprecated public interface TransactionStore { boolean open(); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/DefaultAbstractTransactionCheckListener.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/DefaultAbstractTransactionCheckListener.java new file mode 100644 index 00000000000..a3bc1a0b800 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/DefaultAbstractTransactionCheckListener.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.transaction.queue; + +import org.apache.rocketmq.broker.transaction.AbstractTransactionCheckListener; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageExt; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DefaultAbstractTransactionCheckListener extends AbstractTransactionCheckListener { + private static final Logger log = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); + + public DefaultAbstractTransactionCheckListener() { + super(); + } + + @Override + public void resolveDiscardMsg(MessageExt msgExt) { + log.error("MsgExt:{} has been checked too many times, so discard it", msgExt); + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/GetResult.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/GetResult.java new file mode 100644 index 00000000000..a78970e83ad --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/GetResult.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.transaction.queue; + +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.common.message.MessageExt; + +public class GetResult { + private MessageExt msg; + private PullResult pullResult; + + public MessageExt getMsg() { + return msg; + } + + public void setMsg(MessageExt msg) { + this.msg = msg; + } + + public PullResult getPullResult() { + return pullResult; + } + + public void setPullResult(PullResult pullResult) { + this.pullResult = pullResult; + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/QueueTransactionMsgServiceImpl.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/QueueTransactionMsgServiceImpl.java new file mode 100644 index 00000000000..ac7155eab9a --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/QueueTransactionMsgServiceImpl.java @@ -0,0 +1,498 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.transaction.queue; + +import org.apache.rocketmq.broker.transaction.AbstractTransactionCheckListener; +import org.apache.rocketmq.broker.transaction.OperationResult; +import org.apache.rocketmq.broker.transaction.TransactionMsgService; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.PullStatus; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.PutMessageStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +public class QueueTransactionMsgServiceImpl implements TransactionMsgService { + private static final Logger log = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); + + private TransactionBridge transactionBridge; + + private static final int TRY_PULL_MSG_NUMBER = 1; + + private final int queueTime = 60000; + + private static final int MAX_RETRY_COUNT_WHEN_HALF_NULL = 1; + + public QueueTransactionMsgServiceImpl(TransactionBridge transactionBridge) { + this.transactionBridge = transactionBridge; + } + + private ConcurrentHashMap opQueueMap = new ConcurrentHashMap<>(); + + @Override + public PutMessageResult prepareMessage(MessageExtBrokerInner messageInner) { + return transactionBridge.putHalfMessage(messageInner); + } + + private boolean isNeedDiscard(MessageExt msgExt, int transactionCheckMax) { + String checkTimes = msgExt.getProperty(MessageConst.PROPERTY_TRANSACTION_CHECK_TIMES); + int checkTime = 1; + if (null != checkTimes) { + checkTime = getInt(checkTimes); + if (checkTime >= transactionCheckMax) { + return true; + } else { + checkTime++; + } + } + msgExt.putUserProperty(MessageConst.PROPERTY_TRANSACTION_CHECK_TIMES, String.valueOf(checkTime)); + return false; + } + + private boolean isNeedSkip(MessageExt msgExt) { + long valueOfCurrentMinusBorn = System.currentTimeMillis() - msgExt.getBornTimestamp(); + if (valueOfCurrentMinusBorn + > transactionBridge.getBrokerController().getMessageStoreConfig().getFileReservedTime() + * 3600L * 1000) { + log.info("Half message status too log ,so skip it.messageId {},bornTime {}", + msgExt.getMsgId(), msgExt.getBornTimestamp()); + return true; + } + return false; + } + + private boolean putBackHalfMsgQueue(MessageExt msgExt, long offset) { + PutMessageResult putMessageResult = putBackToHalfQueueReturnResult(msgExt); + if (putMessageResult != null + && putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK) { + msgExt.setQueueOffset( + putMessageResult.getAppendMessageResult().getLogicsOffset()); + msgExt.setCommitLogOffset( + putMessageResult.getAppendMessageResult().getWroteOffset()); + msgExt.setMsgId(putMessageResult.getAppendMessageResult().getMsgId()); + log.info( + "Send check message, the offset={} restored in queueOffset={} " + + "commitLogOffset={} " + + "newMsgId={} realMsgId={} topic={}", + offset, msgExt.getQueueOffset(), msgExt.getCommitLogOffset(), msgExt.getMsgId(), + msgExt.getUserProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX), + msgExt.getTopic()); + return true; + } else { + log.error( + "PutBackToHalfQueueReturnResult write failed, topic: {}, queueId: {}, " + + "msgId: {}", + msgExt.getTopic(), msgExt.getQueueId(), msgExt.getMsgId()); + return false; + } + } + + @Override + public void check(long transactionTimeout, int transactionCheckMax, AbstractTransactionCheckListener listener) { + try { + String topic = MixAll.RMQ_SYS_TRANS_HALF_TOPIC; + Set msgQueues = transactionBridge.fetchMessageQueues(topic); + if (msgQueues == null || msgQueues.size() == 0) { + log.warn("The queue of topic is empty :" + topic); + return; + } + log.info("Check topic={}, queues={}", topic, msgQueues); + for (MessageQueue messageQueue : msgQueues) { + long startTime = System.currentTimeMillis(); + MessageQueue opQueue = getOpQueue(messageQueue); + long halfOffset = transactionBridge.fetchConsumeOffset(messageQueue); + long opOffset = transactionBridge.fetchConsumeOffset(opQueue); + log.info("Before check, the queue={} msgOffset={} opOffset={}", messageQueue, halfOffset, opOffset); + if (halfOffset < 0 || opOffset < 0) { + log.error("MessageQueue: {} illegal offset read: {}, op offset: {},skip this queue", messageQueue, + halfOffset, opOffset); + continue; + } + + List doneOpOffset = new ArrayList<>(); + HashMap removeMap = new HashMap<>(); + PullResult pullResult = fillOpRemoveMap(removeMap, opQueue, opOffset, halfOffset, doneOpOffset); + if (null == pullResult) { + log.error("The queue={} check msgOffset={} with opOffset={} failed, pullResult is null", + messageQueue, halfOffset, opOffset); + continue; + } + // single thread + int getMessageNullCount = 1; + long newOffset = halfOffset; + long i = halfOffset; + while (true) { + if (System.currentTimeMillis() - startTime > queueTime) { + log.info("Queue={} process time reach max={}", messageQueue, queueTime); + break; + } + if (removeMap.containsKey(i)) { + log.info("Half offset {} has been committed/rolled back", i); + removeMap.remove(i); + } else { + GetResult getResult = getHalfMsg(messageQueue, i); + MessageExt msgExt = getResult.getMsg(); + if (msgExt == null) { + if (getMessageNullCount++ > MAX_RETRY_COUNT_WHEN_HALF_NULL) { + break; + } + if (getResult.getPullResult().getPullStatus() == PullStatus.NO_NEW_MSG) { + log.info("No new msg, the miss offset={} in={}, continue check={}, pull result={}", i, + messageQueue, getMessageNullCount, getResult.getPullResult()); + break; + } else { + log.info("Illegal offset, the miss offset={} in={}, continue check={}, pull result={}", + i, messageQueue, getMessageNullCount, getResult.getPullResult()); + i = getResult.getPullResult().getNextBeginOffset(); + newOffset = i; + continue; + } + } + + if (isNeedDiscard(msgExt, transactionCheckMax) || isNeedSkip(msgExt)) { + listener.resolveDiscardMsg(msgExt); + newOffset = i + 1; + i++; + continue; + } + if (msgExt.getStoreTimestamp() >= startTime) { + log.info("Fresh stored. the miss offset={}, check it later, store={}", i, + new Date(msgExt.getStoreTimestamp())); + break; + } + + long valueOfCurrentMinusBorn = System.currentTimeMillis() - msgExt.getBornTimestamp(); + long checkImmunityTime = transactionTimeout; + String checkImmunityTimeStr = msgExt.getUserProperty(MessageConst.PROPERTY_CHECK_IMMUNITY_TIME_IN_SECONDS); + if (null != checkImmunityTimeStr) { + checkImmunityTime = getImmunityTime(checkImmunityTimeStr, transactionTimeout); + if (valueOfCurrentMinusBorn < checkImmunityTime) { + if (checkPrepareQueueOffset(removeMap, doneOpOffset, msgExt, checkImmunityTime)) { + newOffset = i + 1; + i++; + continue; + } + } + } else { + if ((0 <= valueOfCurrentMinusBorn) && (valueOfCurrentMinusBorn < checkImmunityTime)) { + log.info("New arrived, the miss offset={}, check it later checkImmunity={}, born={}", i, + checkImmunityTime, new Date(msgExt.getBornTimestamp())); + break; + } + } + List opMsg = pullResult.getMsgFoundList(); + boolean isNeedCheck = (opMsg == null && valueOfCurrentMinusBorn > checkImmunityTime) + || (opMsg != null && (opMsg.get(opMsg.size() - 1).getBornTimestamp() - startTime > transactionTimeout)) + || (valueOfCurrentMinusBorn <= -1); + + if (isNeedCheck) { + if (!putBackHalfMsgQueue(msgExt, i)) { + continue; + } + listener.resolveHalfMsg(msgExt); + } else { + pullResult = fillOpRemoveMap(removeMap, opQueue, pullResult.getNextBeginOffset(), halfOffset, doneOpOffset); + log.info("The miss offset:{} in messageQueue:{} need to get more opMsg, result is:{}", i, + messageQueue, pullResult); + continue; + } + } + newOffset = i + 1; + i++; + } + if (newOffset != halfOffset) { + transactionBridge.updateConsumeOffset(messageQueue, newOffset); + } + long newOpOffset = calculateOpOffset(doneOpOffset, opOffset); + if (newOpOffset != opOffset) { + transactionBridge.updateConsumeOffset(opQueue, newOpOffset); + } + } + } catch (Exception e) { + e.printStackTrace(); + log.error("Check error", e); + } + + } + + private long getImmunityTime(String checkImmunityTimeStr, long transactionTimeout) { + long checkImmunityTime; + + checkImmunityTime = getLong(checkImmunityTimeStr); + if (-1 == checkImmunityTime) { + checkImmunityTime = transactionTimeout; + } else { + checkImmunityTime *= 1000; + } + return checkImmunityTime; + } + + /** + * Read op message, parse op message, and fill removeMap + * + * @param removeMap Half message to be remove, key:halfOffset, value: opOffset. + * @param opQueue Op message queue. + * @param pullOffsetOfOp The begin offset of op message queue. + * @param miniOffset The current minimum offset of half message queue. + * @param doneOpOffset Stored op messages that have been processed. + * @return Op message result. + */ + private PullResult fillOpRemoveMap(HashMap removeMap, + MessageQueue opQueue, long pullOffsetOfOp, long miniOffset, List doneOpOffset) { + PullResult pullResult = pullOpMsg(opQueue, pullOffsetOfOp, 32); + if (null == pullResult) { + return null; + } + if (pullResult.getPullStatus() == PullStatus.OFFSET_ILLEGAL + || pullResult.getPullStatus() == PullStatus.NO_MATCHED_MSG) { + log.warn("The miss op offset={} in queue={} is illegal, pullResult={}", pullOffsetOfOp, opQueue, + pullResult); + transactionBridge.updateConsumeOffset(opQueue, pullResult.getNextBeginOffset()); + return pullResult; + } else if (pullResult.getPullStatus() == PullStatus.NO_NEW_MSG) { + log.warn("The miss op offset={} in queue={} is NO_NEW_MSG, pullResult={}", pullOffsetOfOp, opQueue, + pullResult); + return pullResult; + } + List opMsg = pullResult.getMsgFoundList(); + if (opMsg == null) { + log.warn("The miss op offset={} in queue={} is empty, pullResult={}", pullOffsetOfOp, opQueue, pullResult); + return pullResult; + } + for (MessageExt opMessageExt : opMsg) { + Long queueOffset = getLong(new String(opMessageExt.getBody(), TransactionUtil.charset)); + log.info("Topic: {} tags: {}, OpOffset: {}, HalfOffset: {}", opMessageExt.getTopic(), + opMessageExt.getTags(), opMessageExt.getQueueOffset(), queueOffset); + if (TransactionUtil.REMOVETAG.equals(opMessageExt.getTags())) { + if (queueOffset < miniOffset) { + doneOpOffset.add(opMessageExt.getQueueOffset()); + } else { + removeMap.put(queueOffset, opMessageExt.getQueueOffset()); + } + } else { + log.error("Found a illegal tag in opMessageExt= {} ", opMessageExt); + } + } + if (log.isDebugEnabled()) { + log.debug("Remove map: {}" + removeMap); + log.debug("Done op list: {}" + doneOpOffset); + } + return pullResult; + } + + /** + * If return true, skip this msg + * + * @param removeMap Op message map to determine whether a half message was responded by producer. + * @param doneOpOffset Op Message which has been checked. + * @param msgExt Half message + * @param checkImmunityTime User defined time to avoid being detected early. + * @return Return true if put success, otherwise return false. + */ + private boolean checkPrepareQueueOffset(HashMap removeMap, List doneOpOffset, MessageExt msgExt, long checkImmunityTime) { + if (System.currentTimeMillis() - msgExt.getBornTimestamp() < checkImmunityTime) { + String prepareQueueOffsetStr = msgExt.getUserProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET); + if (null == prepareQueueOffsetStr) { + return putImmunityMsgBackToHalfQueue(msgExt); + } else { + long prepareQueueOffset = getLong(prepareQueueOffsetStr); + if (-1 == prepareQueueOffset) { + return false; + } else { + if (removeMap.containsKey(prepareQueueOffset)) { + long tmpOpOffset = removeMap.remove(prepareQueueOffset); + doneOpOffset.add(tmpOpOffset); + return true; + } else { + return putImmunityMsgBackToHalfQueue(msgExt); + } + } + + } + + } else { + return true; + } + } + + /** + * Write messageExt to Half topic again + * + * @param messageExt Message will be write back to queue + * @return Put result can used to determine the specific results of storage. + */ + private PutMessageResult putBackToHalfQueueReturnResult(MessageExt messageExt) { + PutMessageResult putMessageResult = null; + try { + MessageExtBrokerInner msgInner = transactionBridge.renewHalfMessageInner(messageExt); + putMessageResult = transactionBridge.putMessageReturnResult(msgInner); + } catch (Exception e) { + e.printStackTrace(); + } + return putMessageResult; + } + + private boolean putImmunityMsgBackToHalfQueue(MessageExt messageExt) { + MessageExtBrokerInner msgInner = transactionBridge.renewImmunityHalfMessageInner(messageExt); + return transactionBridge.putMessage(msgInner); + } + + /** + * Read half message from Half Topic + * + * @param mq Target message queue, in this method, it means the half message queue. + * @param offset Offset in the message queue. + * @param nums Pull message number. + * @return Messages pulled from half message queue. + */ + private PullResult pullHalfMsg(MessageQueue mq, long offset, int nums) { + return transactionBridge.getHalfMessage(mq.getQueueId(), offset, nums); + } + + /** + * Read op message from Op Topic + * + * @param mq Target Message Queue + * @param offset Offset in the message queue + * @param nums Pull message number + * @return Messages pulled from operate message queue. + */ + private PullResult pullOpMsg(MessageQueue mq, long offset, int nums) { + return transactionBridge.getOpMessage(mq.getQueueId(), offset, nums); + } + + private Long getLong(String s) { + long v = -1; + try { + v = Long.valueOf(s); + } catch (Exception e) { + log.error("GetLong error", e); + } + return v; + + } + + private Integer getInt(String s) { + int v = -1; + try { + v = Integer.valueOf(s); + } catch (Exception e) { + log.error("GetInt error", e); + } + return v; + + } + + private long calculateOpOffset(List doneOffset, long oldOffset) { + Collections.sort(doneOffset); + long newOffset = oldOffset; + for (int i = 0; i < doneOffset.size(); i++) { + if (doneOffset.get(i) == newOffset) { + newOffset++; + } else { + break; + } + } + return newOffset; + + } + + private MessageQueue getOpQueue(MessageQueue messageQueue) { + MessageQueue opQueue = opQueueMap.get(messageQueue); + if (opQueue == null) { + opQueue = new MessageQueue(TransactionUtil.buildOpTopic(), messageQueue.getBrokerName(), + messageQueue.getQueueId()); + opQueueMap.put(messageQueue, opQueue); + } + return opQueue; + + } + + private GetResult getHalfMsg(MessageQueue messageQueue, long offset) { + GetResult getResult = new GetResult(); + + PullResult result = pullHalfMsg(messageQueue, offset, TRY_PULL_MSG_NUMBER); + getResult.setPullResult(result); + List messageExts = result.getMsgFoundList(); + if (messageExts == null) { + return getResult; + } + getResult.setMsg(messageExts.get(0)); + return getResult; + } + + private OperationResult getHalfMessageByOffset(long commitLogOffset) { + OperationResult response = new OperationResult(); + MessageExt messageExt = this.transactionBridge.lookMessageByOffset(commitLogOffset); + if (messageExt != null) { + response.setPrepareMessage(messageExt); + response.setResponseCode(ResponseCode.SUCCESS); + } else { + response.setResponseCode(ResponseCode.SYSTEM_ERROR); + response.setResponseRemark("Find prepared transaction message failed"); + } + return response; + } + + @Override + public boolean deletePrepareMessage(MessageExt msgExt) { + if (this.transactionBridge.putOpMessage(msgExt, TransactionUtil.REMOVETAG)) { + log.info("Transaction op message write successfully. messageId={}, queueId={} msgExt:{}", msgExt.getMsgId(), msgExt.getQueueId(), msgExt); + return true; + } else { + log.error("Transaction op message write failed. messageId is {}, queueId is {}", msgExt.getMsgId(), msgExt.getQueueId()); + return false; + } + } + + @Override + public OperationResult commitMessage(EndTransactionRequestHeader requestHeader) { + return getHalfMessageByOffset(requestHeader.getCommitLogOffset()); + } + + @Override + public OperationResult rollbackMessage(EndTransactionRequestHeader requestHeader) { + return getHalfMessageByOffset(requestHeader.getCommitLogOffset()); + } + + @Override + public boolean open() { + return true; + } + + @Override + public void close() { + + } + +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionBridge.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionBridge.java new file mode 100644 index 00000000000..8330d32469f --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionBridge.java @@ -0,0 +1,342 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.transaction.queue; + +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.PullStatus; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.constant.PermName; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageClientIDSetter; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.remoting.common.RemotingHelper; +import org.apache.rocketmq.store.GetMessageResult; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.PutMessageStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +public class TransactionBridge { + private static Logger logger = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); + + private final ConcurrentHashMap opQueueMap = new ConcurrentHashMap<>(); + private final BrokerController brokerController; + private final MessageStore store; + private final SocketAddress storeHost; + + public TransactionBridge(BrokerController brokerController, MessageStore store) { + try { + this.brokerController = brokerController; + this.store = store; + this.storeHost = + new InetSocketAddress(brokerController.getBrokerConfig().getBrokerIP1(), + brokerController.getNettyServerConfig().getListenPort()); + } catch (Exception e) { + logger.error("Init TransactionBridge error", e); + throw new RuntimeException(e); + } + + } + + public long fetchConsumeOffset(MessageQueue mq) { + long offset = brokerController.getConsumerOffsetManager().queryOffset(TransactionUtil.buildConsumerGroup(), + mq.getTopic(), mq.getQueueId()); + if (offset == -1) { + offset = store.getMinOffsetInQueue(mq.getTopic(), mq.getQueueId()); + } + return offset; + } + + public Set fetchMessageQueues(String topic) { + Set mqSet = new HashSet<>(); + TopicConfig topicConfig = selectTopicConfig(topic); + if (topicConfig != null && topicConfig.getReadQueueNums() > 0) { + for (int i = 0; i < topicConfig.getReadQueueNums(); i++) { + MessageQueue mq = new MessageQueue(); + mq.setTopic(topic); + mq.setBrokerName(brokerController.getBrokerConfig().getBrokerName()); + mq.setQueueId(i); + mqSet.add(mq); + } + } + return mqSet; + } + + public void updateConsumeOffset(MessageQueue mq, long offset) { + this.brokerController.getConsumerOffsetManager().commitOffset( + RemotingHelper.parseSocketAddressAddr(this.storeHost), TransactionUtil.buildConsumerGroup(), mq.getTopic(), + mq.getQueueId(), offset); + } + + public PullResult getHalfMessage(int queueId, long offset, int nums) { + String group = TransactionUtil.buildConsumerGroup(); + String topic = TransactionUtil.buildHalfTopic(); + SubscriptionData sub = new SubscriptionData(topic, "*"); + return getMessage(group, topic, queueId, offset, nums, sub); + } + + public PullResult getOpMessage(int queueId, long offset, int nums) { + String group = TransactionUtil.buildConsumerGroup(); + String topic = TransactionUtil.buildOpTopic(); + SubscriptionData sub = new SubscriptionData(topic, "*"); + return getMessage(group, topic, queueId, offset, nums, sub); + } + + private PullResult getMessage(String group, String topic, int queueId, long offset, int nums, SubscriptionData sub) { + GetMessageResult getMessageResult = store.getMessage(group, topic, queueId, offset, nums, null); + + if (getMessageResult != null) { + PullStatus pullStatus = PullStatus.NO_NEW_MSG; + List foundList = null; + switch (getMessageResult.getStatus()) { + case FOUND: + pullStatus = PullStatus.FOUND; + foundList = decodeMsgList(getMessageResult); + this.brokerController.getBrokerStatsManager().incGroupGetNums(group, topic, + getMessageResult.getMessageCount()); + this.brokerController.getBrokerStatsManager().incGroupGetSize(group, topic, + getMessageResult.getBufferTotalSize()); + this.brokerController.getBrokerStatsManager().incBrokerGetNums(getMessageResult.getMessageCount()); + this.brokerController.getBrokerStatsManager().recordDiskFallBehindTime(group, topic, queueId, + this.brokerController.getMessageStore().now() - foundList.get(foundList.size() - 1) + .getStoreTimestamp()); + break; + case NO_MATCHED_MESSAGE: + pullStatus = PullStatus.NO_MATCHED_MSG; + logger.warn("No matched message. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}", + getMessageResult.getStatus(), topic, group, offset); + break; + case NO_MESSAGE_IN_QUEUE: + pullStatus = PullStatus.NO_NEW_MSG; + logger.warn("No new message. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}", + getMessageResult.getStatus(), topic, group, offset); + break; + case MESSAGE_WAS_REMOVING: + case NO_MATCHED_LOGIC_QUEUE: + case OFFSET_FOUND_NULL: + case OFFSET_OVERFLOW_BADLY: + case OFFSET_OVERFLOW_ONE: + case OFFSET_TOO_SMALL: + pullStatus = PullStatus.OFFSET_ILLEGAL; + logger.warn("Offset illegal. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}", + getMessageResult.getStatus(), topic, group, offset); + break; + default: + assert false; + break; + } + + return new PullResult(pullStatus, getMessageResult.getNextBeginOffset(), getMessageResult.getMinOffset(), + getMessageResult.getMaxOffset(), foundList); + + } else { + logger.error("Get message from store return null. topic={}, groupId={}, requestOffset={}", topic, group, + offset); + return null; + } + } + + private List decodeMsgList(GetMessageResult getMessageResult) { + List foundList = new ArrayList<>(); + try { + List messageBufferList = getMessageResult.getMessageBufferList(); + for (ByteBuffer bb : messageBufferList) { + MessageExt msgExt = MessageDecoder.decode(bb); + foundList.add(msgExt); + } + + } finally { + getMessageResult.release(); + } + + return foundList; + } + + public PutMessageResult putHalfMessage(MessageExtBrokerInner messageInner) { + return store.putMessage(parseHalfMessageInner(messageInner)); + } + + private MessageExtBrokerInner parseHalfMessageInner(MessageExtBrokerInner msgInner) { + MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_REAL_TOPIC, msgInner.getTopic()); + MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_REAL_QUEUE_ID, + String.valueOf(msgInner.getQueueId())); + msgInner.setSysFlag( + MessageSysFlag.resetTransactionValue(msgInner.getSysFlag(), MessageSysFlag.TRANSACTION_NOT_TYPE)); + msgInner.setTopic(TransactionUtil.buildHalfTopic()); + msgInner.setQueueId(0); + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); + return msgInner; + } + + public boolean putOpMessage(MessageExt messageExt, String opType) { + MessageQueue messageQueue = new MessageQueue(messageExt.getTopic(), + this.brokerController.getBrokerConfig().getBrokerName(), messageExt.getQueueId()); + if (TransactionUtil.REMOVETAG.equals(opType)) { + return addRemoveTagInTransactionOp(messageExt, messageQueue); + } + return true; + } + + public PutMessageResult putMessageReturnResult(MessageExtBrokerInner messageInner) { + logger.debug("[BUG-TO-FIX] Thread:{} msgID:{}", Thread.currentThread().getName(), messageInner.getMsgId()); + return store.putMessage(messageInner); + } + + public boolean putMessage(MessageExtBrokerInner messageInner) { + PutMessageResult putMessageResult = store.putMessage(messageInner); + if (putMessageResult != null + && putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK) { + return true; + } else { + logger.error("Put message failed, topic: {}, queueId: {}, msgId: {}", + messageInner.getTopic(), messageInner.getQueueId(), messageInner.getMsgId()); + return false; + } + } + + + public MessageExtBrokerInner renewImmunityHalfMessageInner(MessageExt msgExt) { + MessageExtBrokerInner msgInner = renewHalfMessageInner(msgExt); + String queueOffsetFromPrepare = msgExt.getUserProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET); + if (null != queueOffsetFromPrepare) { + MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET, + String.valueOf(queueOffsetFromPrepare)); + } else { + MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET, + String.valueOf(msgExt.getQueueOffset())); + } + + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); + + return msgInner; + } + + public MessageExtBrokerInner renewHalfMessageInner(MessageExt msgExt) { + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + msgInner.setTopic(msgExt.getTopic()); + msgInner.setBody(msgExt.getBody()); + msgInner.setQueueId(msgExt.getQueueId()); + msgInner.setMsgId(msgExt.getMsgId()); + msgInner.setSysFlag(msgExt.getSysFlag()); + msgInner.setTags(msgExt.getTags()); + msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(msgInner.getTags())); + MessageAccessor.setProperties(msgInner, msgExt.getProperties()); + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties())); + msgInner.setBornTimestamp(msgExt.getBornTimestamp()); + msgInner.setBornHost(msgExt.getBornHost()); + msgInner.setStoreHost(msgExt.getStoreHost()); + msgInner.setWaitStoreMsgOK(false); + return msgInner; + } + + private MessageExtBrokerInner makeOpMessageInner(Message message, MessageQueue messageQueue) { + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + msgInner.setTopic(message.getTopic()); + msgInner.setBody(message.getBody()); + msgInner.setQueueId(messageQueue.getQueueId()); + msgInner.setTags(message.getTags()); + msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(msgInner.getTags())); + msgInner.setSysFlag(0); + MessageAccessor.setProperties(msgInner, message.getProperties()); + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(message.getProperties())); + msgInner.setBornTimestamp(System.currentTimeMillis()); + msgInner.setBornHost(this.storeHost); + msgInner.setStoreHost(this.storeHost); + msgInner.setWaitStoreMsgOK(false); + MessageClientIDSetter.setUniqID(msgInner); + return msgInner; + } + + private TopicConfig selectTopicConfig(String topic) { + TopicConfig topicConfig = brokerController.getTopicConfigManager().selectTopicConfig(topic); + if (topicConfig == null) { + topicConfig = this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod( + topic, 1, PermName.PERM_WRITE | PermName.PERM_READ, 0); + } + return topicConfig; + } + + /** + * Use this function while transaction msg is committed or rollback write a flag 'd' to operation queue for the msg's offset + * + * @param messageExt Op message + * @param messageQueue Op message queue + * @return This method will always return true. + */ + private boolean addRemoveTagInTransactionOp(MessageExt messageExt, MessageQueue messageQueue) { + Message message = new Message(TransactionUtil.buildOpTopic(), TransactionUtil.REMOVETAG, + String.valueOf(messageExt.getQueueOffset()).getBytes(TransactionUtil.charset)); + writeOp(message, messageQueue); + return true; + } + + private void writeOp(Message message, MessageQueue mq) { + MessageQueue opQueue; + if (opQueueMap.containsKey(mq)) { + opQueue = opQueueMap.get(mq); + } else { + opQueue = getOpQueueByHalf(mq); + MessageQueue oldQueue = opQueueMap.putIfAbsent(mq, opQueue); + if (oldQueue != null) { + opQueue = oldQueue; + } + } + if (opQueue == null) { + opQueue = new MessageQueue(TransactionUtil.buildOpTopic(), mq.getBrokerName(), mq.getQueueId()); + } + if (logger.isDebugEnabled()) { + logger.debug(opQueue + "---------Op msg " + message.getTags() + "---" + new String(message.getBody(), + TransactionUtil.charset)); + } + putMessage(makeOpMessageInner(message, opQueue)); + } + + private MessageQueue getOpQueueByHalf(MessageQueue halfMQ) { + MessageQueue opQueue = new MessageQueue(); + opQueue.setTopic(TransactionUtil.buildOpTopic()); + opQueue.setBrokerName(halfMQ.getBrokerName()); + opQueue.setQueueId(halfMQ.getQueueId()); + return opQueue; + } + + public MessageExt lookMessageByOffset(final long commitLogOffset) { + return this.store.lookMessageByOffset(commitLogOffset); + } + + public BrokerController getBrokerController() { + return brokerController; + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionUtil.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionUtil.java new file mode 100644 index 00000000000..f0ea8c79455 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionUtil.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.transaction.queue; + +import org.apache.rocketmq.common.MixAll; + +import java.nio.charset.Charset; + +public class TransactionUtil { + public static final String REMOVETAG = "d"; + public static Charset charset = Charset.forName("utf-8"); + + + public static String buildOpTopic() { + return MixAll.RMQ_SYS_TRANS_OP_HALF_TOPIC; + } + + public static String buildHalfTopic() { + return MixAll.RMQ_SYS_TRANS_HALF_TOPIC; + } + + public static String buildConsumerGroup() { + return MixAll.CID_SYS_RMQ_TRANS; + } + +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/util/PositiveAtomicCounter.java b/broker/src/main/java/org/apache/rocketmq/broker/util/PositiveAtomicCounter.java new file mode 100644 index 00000000000..8d92f43085e --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/util/PositiveAtomicCounter.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.util; + +import java.util.concurrent.atomic.AtomicInteger; + +public class PositiveAtomicCounter { + private static final int MASK = 0x7FFFFFFF; + private final AtomicInteger atom; + + + public PositiveAtomicCounter() { + atom = new AtomicInteger(0); + } + + + public final int incrementAndGet() { + final int rt = atom.incrementAndGet(); + return rt & MASK; + } + + + public int intValue() { + return atom.intValue(); + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/util/ServiceProvider.java b/broker/src/main/java/org/apache/rocketmq/broker/util/ServiceProvider.java new file mode 100644 index 00000000000..74e3ddd9be5 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/util/ServiceProvider.java @@ -0,0 +1,191 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to + * You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.apache.rocketmq.broker.util; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; + +public class ServiceProvider { + + private final static Logger LOG = LoggerFactory + .getLogger(ServiceProvider.class); + /** + * A reference to the classloader that loaded this class. It's more efficient to compute it once and cache it here. + */ + private static ClassLoader thisClassLoader; + + /** + * JDK1.3+ 'Service Provider' specification. + */ + public static final String TRANSACTION_SERVICE_ID = "META-INF/service/org.apache.rocketmq.broker.transaction.TransactionMsgService"; + + public static final String TRANSACTION_LISTENER_ID = "META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionCheckListener"; + + static { + thisClassLoader = getClassLoader(ServiceProvider.class); + } + + /** + * Returns a string that uniquely identifies the specified object, including its class. + *

+ * The returned string is of form "classname@hashcode", ie is the same as the return value of the Object.toString() method, but works even when the specified object's class has overidden the toString method. + * + * @param o may be null. + * @return a string of form classname@hashcode, or "null" if param o is null. + */ + protected static String objectId(Object o) { + if (o == null) { + return "null"; + } else { + return o.getClass().getName() + "@" + System.identityHashCode(o); + } + } + + protected static ClassLoader getClassLoader(Class clazz) { + try { + return clazz.getClassLoader(); + } catch (SecurityException e) { + LOG.error("Unable to get classloader for class {} due to security restrictions !", + clazz, e.getMessage()); + throw e; + } + } + + protected static ClassLoader getContextClassLoader() { + ClassLoader classLoader = null; + try { + classLoader = Thread.currentThread().getContextClassLoader(); + } catch (SecurityException ex) { + /** + * The getContextClassLoader() method throws SecurityException when the context + * class loader isn't an ancestor of the calling class's class + * loader, or if security permissions are restricted. + */ + } + return classLoader; + } + + protected static InputStream getResourceAsStream(ClassLoader loader, String name) { + if (loader != null) { + return loader.getResourceAsStream(name); + } else { + return ClassLoader.getSystemResourceAsStream(name); + } + } + + public static List load(String name, Class clazz) { + LOG.info("Looking for a resource file of name [{}] ...", name); + List services = new ArrayList(); + try { + ArrayList names = new ArrayList(); + final InputStream is = getResourceAsStream(getContextClassLoader(), name); + if (is != null) { + BufferedReader reader; + try { + reader = new BufferedReader(new InputStreamReader(is, "UTF-8")); + } catch (java.io.UnsupportedEncodingException e) { + reader = new BufferedReader(new InputStreamReader(is)); + } + String serviceName = reader.readLine(); + while (serviceName != null && !"".equals(serviceName)) { + LOG.info( + "Creating an instance as specified by file {} which was present in the path of the context classloader.", + name); + if (!names.contains(serviceName)) { + names.add(serviceName); + } + + services.add((T)initService(getContextClassLoader(), serviceName, clazz)); + + serviceName = reader.readLine(); + } + reader.close(); + } else { + // is == null + LOG.warn("No resource file with name [{}] found.", name); + } + } catch (Exception e) { + LOG.error("Error occured when looking for resource file " + name, e); + } + return services; + } + + public static T loadClass(String name, Class clazz) { + final InputStream is = getResourceAsStream(getContextClassLoader(), name); + BufferedReader reader; + try { + try { + reader = new BufferedReader(new InputStreamReader(is, "UTF-8")); + } catch (java.io.UnsupportedEncodingException e) { + reader = new BufferedReader(new InputStreamReader(is)); + } + String serviceName = reader.readLine(); + reader.close(); + if (serviceName != null && !"".equals(serviceName)) { + return initService(getContextClassLoader(), serviceName, clazz); + } else { + LOG.warn("ServiceName is empty!"); + return null; + } + } catch (Exception e) { + LOG.error("Error occured when looking for resource file " + name, e); + } + return null; + } + + protected static T initService(ClassLoader classLoader, String serviceName, Class clazz) { + Class serviceClazz = null; + try { + if (classLoader != null) { + try { + // Warning: must typecast here & allow exception to be generated/caught & recast properly + serviceClazz = classLoader.loadClass(serviceName); + if (clazz.isAssignableFrom(serviceClazz)) { + LOG.info("Loaded class {} from classloader {}", serviceClazz.getName(), + objectId(classLoader)); + } else { + // This indicates a problem with the ClassLoader tree. An incompatible ClassLoader was used to load the implementation. + LOG.error( + "Class {} loaded from classloader {} does not extend {} as loaded by this classloader.", + new Object[] {serviceClazz.getName(), + objectId(serviceClazz.getClassLoader()), clazz.getName()}); + } + return (T)serviceClazz.newInstance(); + } catch (ClassNotFoundException ex) { + if (classLoader == thisClassLoader) { + // Nothing more to try, onwards. + LOG.warn("Unable to locate any class {} via classloader", serviceName, + objectId(classLoader)); + throw ex; + } + // Ignore exception, continue + } catch (NoClassDefFoundError e) { + if (classLoader == thisClassLoader) { + // Nothing more to try, onwards. + LOG.warn( + "Class {} cannot be loaded via classloader {}.it depends on some other class that cannot be found.", + serviceClazz, objectId(classLoader)); + throw e; + } + // Ignore exception, continue + } + } + } catch (Exception e) { + LOG.error("Unable to init service.", e); + } + return (T)serviceClazz; + } +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/EndTransactionProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/EndTransactionProcessorTest.java new file mode 100644 index 00000000000..0a2827d680a --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/EndTransactionProcessorTest.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.processor; + +import io.netty.channel.ChannelHandlerContext; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.transaction.OperationResult; +import org.apache.rocketmq.broker.transaction.TransactionMsgService; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.netty.NettyServerConfig; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.store.AppendMessageResult; +import org.apache.rocketmq.store.AppendMessageStatus; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class EndTransactionProcessorTest { + + private EndTransactionProcessor endTransactionProcessor; + + @Mock + private ChannelHandlerContext handlerContext; + + @Spy + private BrokerController + brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), + new MessageStoreConfig()); + + @Mock + private MessageStore messageStore; + + @Mock + private TransactionMsgService transactionMsgService; + + @Before + public void init() { + brokerController.setMessageStore(messageStore); + brokerController.setTransactionMsgService(transactionMsgService); + endTransactionProcessor = new EndTransactionProcessor(brokerController); + } + + private OperationResult createResponse(int status){ + OperationResult response = new OperationResult(); + response.setPrepareMessage(createDefaultMessageExt()); + response.setResponseCode(status); + response.setResponseRemark(null); + return response; + } + + @Test + public void testProcessRequest() throws RemotingCommandException { + when(transactionMsgService.commitMessage(any(EndTransactionRequestHeader.class))).thenReturn(createResponse(ResponseCode.SUCCESS)); + when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult + (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + RemotingCommand request = createEndTransactionMsgCommand(MessageSysFlag.TRANSACTION_COMMIT_TYPE, false); + RemotingCommand response = endTransactionProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testProcessRequest_CheckMessage() throws RemotingCommandException { + when(transactionMsgService.commitMessage(any(EndTransactionRequestHeader.class))).thenReturn(createResponse(ResponseCode.SUCCESS)); + when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult + (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + RemotingCommand request = createEndTransactionMsgCommand(MessageSysFlag.TRANSACTION_COMMIT_TYPE, true); + RemotingCommand response = endTransactionProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testProcessRequest_NotType() throws RemotingCommandException { + RemotingCommand request = createEndTransactionMsgCommand(MessageSysFlag.TRANSACTION_NOT_TYPE, true); + RemotingCommand response = endTransactionProcessor.processRequest(handlerContext, request); + assertThat(response).isNull(); + } + + @Test + public void testProcessRequest_RollBack() throws RemotingCommandException { + when(transactionMsgService.rollbackMessage(any(EndTransactionRequestHeader.class))).thenReturn(createResponse(ResponseCode.SUCCESS)); + RemotingCommand request = createEndTransactionMsgCommand(MessageSysFlag.TRANSACTION_ROLLBACK_TYPE, true); + RemotingCommand response = endTransactionProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + private MessageExt createDefaultMessageExt() { + MessageExt messageExt = new MessageExt(); + messageExt.setMsgId("12345678"); + messageExt.setQueueId(0); + messageExt.setCommitLogOffset(123456789L); + messageExt.setQueueOffset(1234); + MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_REAL_QUEUE_ID, "0"); + MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_TRANSACTION_PREPARED, "true"); + MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_PRODUCER_GROUP, "testTransactionGroup"); + return messageExt; + } + + private EndTransactionRequestHeader createEndTransactionRequestHeader(int status, boolean isCheckMsg) { + EndTransactionRequestHeader header = new EndTransactionRequestHeader(); + header.setCommitLogOffset(123456789L); + header.setFromTransactionCheck(isCheckMsg); + header.setCommitOrRollback(status); + header.setMsgId("12345678"); + header.setTransactionId("123"); + header.setProducerGroup("testTransactionGroup"); + header.setTranStateTableOffset(1234L); + return header; + } + + private RemotingCommand createEndTransactionMsgCommand(int status, boolean isCheckMsg) { + EndTransactionRequestHeader header = createEndTransactionRequestHeader(status, isCheckMsg); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.END_TRANSACTION, header); + request.makeCustomHeaderToNet(); + return request; + } +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java index 7828e7a91cd..566da9692b1 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java @@ -21,16 +21,22 @@ import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.List; +import java.util.Map; + import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.mqtrace.SendMessageContext; import org.apache.rocketmq.broker.mqtrace.SendMessageHook; +import org.apache.rocketmq.broker.transaction.TransactionMsgService; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.ConsumerSendMsgBackRequestHeader; import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyServerConfig; @@ -68,6 +74,9 @@ public class SendMessageProcessorTest { @Mock private MessageStore messageStore; + @Mock + private TransactionMsgService transactionMsgService; + private String topic = "FooBar"; private String group = "FooBarGroup"; @@ -177,7 +186,42 @@ public void testProcessRequest_WithMsgBack() throws RemotingCommandException { assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); } - private RemotingCommand createSendMsgCommand(int requestCode) { + @Test + public void testProcessRequest_Transaction() throws RemotingCommandException { + brokerController.setTransactionMsgService(transactionMsgService); + when(brokerController.getTransactionMsgService().prepareMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + RemotingCommand request = createSendTransactionMsgCommand(RequestCode.SEND_MESSAGE); + final RemotingCommand[] response = new RemotingCommand[1]; + doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + response[0] = invocation.getArgument(0); + return null; + } + }).when(handlerContext).writeAndFlush(any(Object.class)); + RemotingCommand responseToReturn = sendMessageProcessor.processRequest(handlerContext, request); + if (responseToReturn != null) { + assertThat(response[0]).isNull(); + response[0] = responseToReturn; + } + assertThat(response[0].getCode()).isEqualTo(ResponseCode.SUCCESS); + + } + private RemotingCommand createSendTransactionMsgCommand(int requestCode) { + SendMessageRequestHeader header = createSendMsgRequestHeader(); + int sysFlag = header.getSysFlag(); + Map oriProps = MessageDecoder.string2messageProperties(header.getProperties()); + oriProps.put(MessageConst.PROPERTY_TRANSACTION_PREPARED, "true"); + header.setProperties(MessageDecoder.messageProperties2String(oriProps)); + sysFlag |= MessageSysFlag.TRANSACTION_PREPARED_TYPE; + header.setSysFlag(sysFlag); + RemotingCommand request = RemotingCommand.createRequestCommand(requestCode, header); + request.setBody(new byte[] {'a'}); + request.makeCustomHeaderToNet(); + return request; + } + + private SendMessageRequestHeader createSendMsgRequestHeader() { SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); requestHeader.setProducerGroup(group); requestHeader.setTopic(topic); @@ -188,6 +232,11 @@ private RemotingCommand createSendMsgCommand(int requestCode) { requestHeader.setBornTimestamp(System.currentTimeMillis()); requestHeader.setFlag(124); requestHeader.setReconsumeTimes(0); + return requestHeader; + } + + private RemotingCommand createSendMsgCommand(int requestCode) { + SendMessageRequestHeader requestHeader = createSendMsgRequestHeader(); RemotingCommand request = RemotingCommand.createRequestCommand(requestCode, requestHeader); request.setBody(new byte[] {'a'}); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/DefaultAbstractTransactionCheckListenerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/DefaultAbstractTransactionCheckListenerTest.java new file mode 100644 index 00000000000..7b33e94f6bf --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/DefaultAbstractTransactionCheckListenerTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.transaction.queue; + +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.netty.NettyServerConfig; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class DefaultAbstractTransactionCheckListenerTest { + + private DefaultAbstractTransactionCheckListener listener; + + @Spy + private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), + new NettyClientConfig(), new MessageStoreConfig()); + + + @Before + public void init() { + listener = new DefaultAbstractTransactionCheckListener(); + listener.setBrokerController(brokerController); + } + + @Test + public void testResolveHalfMsg() { + listener.resolveHalfMsg(createMessageExt()); + } + + @Test + public void testSendCheckMessage() throws Exception{ + MessageExt messageExt = createMessageExt(); + listener.sendCheckMessage(messageExt); + } + + @Test + public void sendCheckMessage(){ + listener.resolveDiscardMsg(createMessageExt()); + } + + private MessageExtBrokerInner createMessageExt() { + MessageExtBrokerInner inner = new MessageExtBrokerInner(); + MessageAccessor.putProperty(inner,MessageConst.PROPERTY_REAL_QUEUE_ID,"1"); + MessageAccessor.putProperty(inner,MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX,"1234255"); + MessageAccessor.putProperty(inner,MessageConst.PROPERTY_REAL_TOPIC,"realTopic"); + inner.setTransactionId(inner.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX)); + inner.setBody("check".getBytes()); + inner.setMsgId("12344567890"); + inner.setQueueId(0); + return inner; + } + +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/QueueTransactionMsgServiceImplTest.java b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/QueueTransactionMsgServiceImplTest.java new file mode 100644 index 00000000000..595fb57b000 --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/QueueTransactionMsgServiceImplTest.java @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.transaction.queue; + +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.transaction.OperationResult; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.PullStatus; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.netty.NettyServerConfig; +import org.apache.rocketmq.store.AppendMessageResult; +import org.apache.rocketmq.store.AppendMessageStatus; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class QueueTransactionMsgServiceImplTest { + + private QueueTransactionMsgServiceImpl queueTransactionMsgService; + + @Mock + private TransactionBridge bridge; + + @Spy + private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), + new NettyClientConfig(), new MessageStoreConfig()); + + @Mock + private DefaultAbstractTransactionCheckListener listener; + + @Before + public void init() { + listener.setBrokerController(brokerController); + queueTransactionMsgService = new QueueTransactionMsgServiceImpl(bridge); + brokerController.getMessageStoreConfig().setFileReservedTime(3); + } + + @Test + public void testPrepareMessage() { + MessageExtBrokerInner inner = createMessageBrokerInner(); + when(bridge.putHalfMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult + (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + PutMessageResult result = queueTransactionMsgService.prepareMessage(inner); + assert result.isOk(); + } + + @Test + public void testCommitMessage() { + when(bridge.lookMessageByOffset(anyLong())).thenReturn(createMessageBrokerInner()); + OperationResult result = queueTransactionMsgService.commitMessage(createEndTransactionRequestHeader(MessageSysFlag.TRANSACTION_COMMIT_TYPE)); + assertThat(result.getResponseCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testRollbackMessage() { + when(bridge.lookMessageByOffset(anyLong())).thenReturn(createMessageBrokerInner()); + OperationResult result = queueTransactionMsgService.commitMessage(createEndTransactionRequestHeader(MessageSysFlag.TRANSACTION_ROLLBACK_TYPE)); + assertThat(result.getResponseCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testCheck_withDiscard() { + when(bridge.fetchMessageQueues(MixAll.RMQ_SYS_TRANS_HALF_TOPIC)).thenReturn(createMessageQueueSet(MixAll.RMQ_SYS_TRANS_HALF_TOPIC)); + when(bridge.getHalfMessage(0, 0, 1)).thenReturn(createDiscardPullResult(MixAll.RMQ_SYS_TRANS_HALF_TOPIC, 5, "hellp", 1)); + when(bridge.getHalfMessage(0, 1, 1)).thenReturn(createPullResult(MixAll.RMQ_SYS_TRANS_HALF_TOPIC, 6, "hellp", 0)); + when(bridge.getOpMessage(anyInt(), anyLong(), anyInt())).thenReturn(createOpPulResult(MixAll.RMQ_SYS_TRANS_OP_HALF_TOPIC, 1, "10", 1)); + long timeOut = this.brokerController.getBrokerConfig().getTransactionTimeOut(); + int checkMax = this.brokerController.getBrokerConfig().getTransactionCheckMax(); + final AtomicInteger checkMessage = new AtomicInteger(0); + doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invocation) { + checkMessage.addAndGet(1); + return null; + } + }).when(listener).resolveDiscardMsg(any(MessageExt.class)); + queueTransactionMsgService.check(timeOut, checkMax, listener); + assertThat(checkMessage.get()).isEqualTo(1); + } + + @Test + public void testCheck_withCheck() { + when(bridge.fetchMessageQueues(MixAll.RMQ_SYS_TRANS_HALF_TOPIC)).thenReturn(createMessageQueueSet(MixAll.RMQ_SYS_TRANS_HALF_TOPIC)); + when(bridge.getHalfMessage(0, 0, 1)).thenReturn(createPullResult(MixAll.RMQ_SYS_TRANS_HALF_TOPIC, 5, "hello", 1)); + when(bridge.getHalfMessage(0, 1, 1)).thenReturn(createPullResult(MixAll.RMQ_SYS_TRANS_HALF_TOPIC, 6, "hellp", 0)); + when(bridge.getOpMessage(anyInt(), anyLong(), anyInt())).thenReturn(createPullResult(MixAll.RMQ_SYS_TRANS_OP_HALF_TOPIC, 1, "5", 0)); + when(bridge.getBrokerController()).thenReturn(this.brokerController); + when(bridge.renewHalfMessageInner(any(MessageExtBrokerInner.class))).thenReturn(createMessageBrokerInner()); + when(bridge.putMessageReturnResult(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult + (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + long timeOut = this.brokerController.getBrokerConfig().getTransactionTimeOut(); + final int checkMax = this.brokerController.getBrokerConfig().getTransactionCheckMax(); + final AtomicInteger checkMessage = new AtomicInteger(0); + doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invocation) { + checkMessage.addAndGet(1); + return checkMessage; + } + }).when(listener).resolveHalfMsg(any(MessageExt.class)); + queueTransactionMsgService.check(timeOut, checkMax, listener); + assertThat(checkMessage.get()).isEqualTo(1); + } + + @Test + public void testDeletePrepareMessage() { + when(bridge.putOpMessage(any(MessageExt.class), anyString())).thenReturn(true); + boolean res = queueTransactionMsgService.deletePrepareMessage(createMessageBrokerInner()); + assertThat(res).isTrue(); + } + + @Test + public void testOpen() { + boolean isOpen = queueTransactionMsgService.open(); + assertThat(isOpen).isTrue(); + } + + private PullResult createDiscardPullResult(String topic, long queueOffset, String body, int size) { + PullResult result = createPullResult(topic, queueOffset, body, size); + List msgs = result.getMsgFoundList(); + for (MessageExt msg : msgs) { + msg.putUserProperty(MessageConst.PROPERTY_TRANSACTION_CHECK_TIMES, "100000"); + } + return result; + } + + private PullResult createPullResult(String topic, long queueOffset, String body, int size) { + PullResult result = null; + if (0 == size) { + result = new PullResult(PullStatus.NO_NEW_MSG, 1, 0, 1, + null); + } else { + result = new PullResult(PullStatus.FOUND, 1, 0, 1, + getMessageList(queueOffset, topic, body, 1)); + return result; + } + return result; + } + + private PullResult createOpPulResult(String topic, long queueOffset, String body, int size) { + PullResult result = createPullResult(topic, queueOffset, body, size); + List msgs = result.getMsgFoundList(); + for (MessageExt msg : msgs) { + msg.setTags(TransactionUtil.REMOVETAG); + } + return result; + } + + private PullResult createImmunityPulResult(String topic, long queueOffset, String body, int size) { + PullResult result = createPullResult(topic, queueOffset, body, size); + List msgs = result.getMsgFoundList(); + for (MessageExt msg : msgs) { + msg.putUserProperty(MessageConst.PROPERTY_CHECK_IMMUNITY_TIME_IN_SECONDS, "0"); + } + return result; + } + + private List getMessageList(long queueOffset, String topic, String body, int size) { + List msgs = new ArrayList<>(); + for (int i = 0; i < size; i++) { + MessageExt messageExt = createMessageBrokerInner(queueOffset, topic, body); + msgs.add(messageExt); + } + return msgs; + } + + private Set createMessageQueueSet(String topic) { + Set messageQueues = new HashSet<>(); + MessageQueue messageQueue = new MessageQueue(topic, "DefaultCluster", 0); + messageQueues.add(messageQueue); + return messageQueues; + } + + private EndTransactionRequestHeader createEndTransactionRequestHeader(int status) { + EndTransactionRequestHeader header = new EndTransactionRequestHeader(); + header.setCommitLogOffset(123456789L); + header.setCommitOrRollback(status); + header.setMsgId("12345678"); + header.setTransactionId("123"); + header.setProducerGroup("testTransactionGroup"); + header.setTranStateTableOffset(1234L); + return header; + } + + private MessageExtBrokerInner createMessageBrokerInner(long queueOffset, String topic, String body) { + MessageExtBrokerInner inner = new MessageExtBrokerInner(); + inner.setBornTimestamp(System.currentTimeMillis() - 80000); + inner.setTransactionId("123456123"); + inner.setTopic(topic); + inner.setQueueOffset(queueOffset); + inner.setBody(body.getBytes()); + inner.setMsgId("123456123"); + inner.setQueueId(0); + inner.setTopic("hello"); + return inner; + } + + private MessageExtBrokerInner createMessageBrokerInner() { + return createMessageBrokerInner(1, "testTopic", "hello world"); + } +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionBridgeTest.java b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionBridgeTest.java new file mode 100644 index 00000000000..940b57ae6a2 --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionBridgeTest.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.transaction.queue; + +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.PullStatus; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.netty.NettyServerConfig; +import org.apache.rocketmq.store.AppendMessageResult; +import org.apache.rocketmq.store.AppendMessageStatus; +import org.apache.rocketmq.store.GetMessageResult; +import org.apache.rocketmq.store.GetMessageStatus; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.MessageFilter; +import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentMatchers; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.Map; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class TransactionBridgeTest { + + private TransactionBridge transactionBridge; + + @Spy + private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), + new NettyClientConfig(), new MessageStoreConfig()); + + @Mock + private MessageStore messageStore; + + @Before + public void init() { + brokerController.setMessageStore(messageStore); + transactionBridge = new TransactionBridge(brokerController, messageStore); + } + + @Test + public void testPutOpMessage() { + boolean isSuccess = transactionBridge.putOpMessage(createMessageBrokerInner(), TransactionUtil.REMOVETAG); + assertThat(isSuccess).isTrue(); + } + + @Test + public void testPutHalfMessage() { + when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult + (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + PutMessageResult result = transactionBridge.putHalfMessage(createMessageBrokerInner()); + assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK); + } + + @Test + public void testFetchMessageQueues() { + Set messageQueues = transactionBridge.fetchMessageQueues(MixAll.RMQ_SYS_TRANS_HALF_TOPIC); + assertThat(messageQueues.size()).isEqualTo(1); + } + + @Test + public void testFetchConsumeOffset() { + MessageQueue mq = new MessageQueue(TransactionUtil.buildOpTopic(), this.brokerController.getBrokerConfig().getBrokerName(), + 0); + long offset = transactionBridge.fetchConsumeOffset(mq); + assertThat(offset).isGreaterThan(-1); + } + + @Test + public void updateConsumeOffset() { + MessageQueue mq = new MessageQueue(TransactionUtil.buildOpTopic(), this.brokerController.getBrokerConfig().getBrokerName(), + 0); + transactionBridge.updateConsumeOffset(mq, 0); + } + + @Test + public void testGetHalfMessage() { + when(messageStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), ArgumentMatchers.nullable(MessageFilter.class))).thenReturn(createGetMessageResult(GetMessageStatus.NO_MESSAGE_IN_QUEUE)); + PullResult result = transactionBridge.getHalfMessage(0, 0, 1); + assertThat(result.getPullStatus()).isEqualTo(PullStatus.NO_NEW_MSG); + } + + @Test + public void testGetOpMessage() { + when(messageStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), ArgumentMatchers.nullable(MessageFilter.class))).thenReturn(createGetMessageResult(GetMessageStatus.NO_MESSAGE_IN_QUEUE)); + PullResult result = transactionBridge.getOpMessage(0, 0, 1); + assertThat(result.getPullStatus()).isEqualTo(PullStatus.NO_NEW_MSG); + } + + @Test + public void testPutMessageReturnResult() { + when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult + (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + PutMessageResult result = transactionBridge.putMessageReturnResult(createMessageBrokerInner()); + assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK); + } + + @Test + public void testPutMessage() { + when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult + (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + Boolean success = transactionBridge.putMessage(createMessageBrokerInner()); + assertThat(success).isEqualTo(true); + } + + @Test + public void testRenewImmunityHalfMessageInner() { + MessageExt messageExt = createMessageBrokerInner(); + final String offset = "123456789"; + MessageExtBrokerInner msgInner = transactionBridge.renewImmunityHalfMessageInner(messageExt); + MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET,offset); + assertThat(msgInner).isNotNull(); + Map properties = msgInner.getProperties(); + assertThat(properties).isNotNull(); + String resOffset = properties.get(MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET); + assertThat(resOffset).isEqualTo(offset); + } + + + @Test + public void testRenewHalfMessageInner() { + MessageExt messageExt = new MessageExt(); + long bornTimeStamp = messageExt.getBornTimestamp(); + MessageExt messageExtRes = transactionBridge.renewHalfMessageInner(messageExt); + assertThat( messageExtRes.getBornTimestamp()).isEqualTo(bornTimeStamp); + } + + @Test + public void testLookMessageByOffset(){ + when(messageStore.lookMessageByOffset(anyLong())).thenReturn(new MessageExt()); + MessageExt messageExt = transactionBridge.lookMessageByOffset(123); + assertThat(messageExt).isNotNull(); + } + + private GetMessageResult createGetMessageResult(GetMessageStatus status) { + GetMessageResult getMessageResult = new GetMessageResult(); + getMessageResult.setStatus(status); + getMessageResult.setMinOffset(100); + getMessageResult.setMaxOffset(1024); + getMessageResult.setNextBeginOffset(516); + return getMessageResult; + } + + private MessageExtBrokerInner createMessageBrokerInner() { + MessageExtBrokerInner inner = new MessageExtBrokerInner(); + inner.setTransactionId("12342123444"); + inner.setBornTimestamp(System.currentTimeMillis()); + inner.setBody("prepare".getBytes()); + inner.setMsgId("123456-123"); + inner.setQueueId(0); + inner.setTopic("hello"); + return inner; + } +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/util/LogTransactionCheckListener.java b/broker/src/test/java/org/apache/rocketmq/broker/util/LogTransactionCheckListener.java new file mode 100644 index 00000000000..d5adb886533 --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/util/LogTransactionCheckListener.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.util; + +import org.apache.rocketmq.broker.transaction.AbstractTransactionCheckListener; +import org.apache.rocketmq.common.message.MessageExt; + +public class LogTransactionCheckListener extends AbstractTransactionCheckListener { + + @Override + public void resolveDiscardMsg(MessageExt msgExt) { + + } +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/util/ServiceProviderTest.java b/broker/src/test/java/org/apache/rocketmq/broker/util/ServiceProviderTest.java new file mode 100644 index 00000000000..c739db5d865 --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/util/ServiceProviderTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.broker.util; + +import org.apache.rocketmq.broker.transaction.AbstractTransactionCheckListener; +import org.apache.rocketmq.broker.transaction.TransactionMsgService; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ServiceProviderTest { + + @Test + public void loadTransactionMsgServiceTest() { + TransactionMsgService transactionService = ServiceProvider.loadClass(ServiceProvider.TRANSACTION_SERVICE_ID, + TransactionMsgService.class); + assertThat(transactionService).isNotNull(); + } + + @Test + public void loadAbstractTransactionListenerTest() { + AbstractTransactionCheckListener listener = ServiceProvider.loadClass(ServiceProvider.TRANSACTION_LISTENER_ID, + AbstractTransactionCheckListener.class); + assertThat(listener).isNotNull(); + } +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/util/TransactionMsgServiceImpl.java b/broker/src/test/java/org/apache/rocketmq/broker/util/TransactionMsgServiceImpl.java new file mode 100644 index 00000000000..76172a277da --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/util/TransactionMsgServiceImpl.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.util; + +import org.apache.rocketmq.broker.transaction.AbstractTransactionCheckListener; +import org.apache.rocketmq.broker.transaction.OperationResult; +import org.apache.rocketmq.broker.transaction.TransactionMsgService; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.PutMessageResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TransactionMsgServiceImpl implements TransactionMsgService { + private static final Logger log = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); + + @Override + public PutMessageResult prepareMessage(MessageExtBrokerInner messageInner) { + return null; + } + + @Override + public boolean deletePrepareMessage(MessageExt messageExt) { + return false; + } + + @Override + public OperationResult commitMessage(EndTransactionRequestHeader requestHeader) { + return null; + } + + @Override + public OperationResult rollbackMessage(EndTransactionRequestHeader requestHeader) { + return null; + } + + @Override + public void check(long transactionTimeout, int transactionCheckMax, AbstractTransactionCheckListener listener) { + log.warn("check check!"); + } + + @Override + public boolean open() { + return true; + } + + @Override + public void close() { + + } +} diff --git a/broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionCheckListener b/broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionCheckListener new file mode 100644 index 00000000000..0a88c133ee8 --- /dev/null +++ b/broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionCheckListener @@ -0,0 +1 @@ +org.apache.rocketmq.broker.util.LogTransactionCheckListener \ No newline at end of file diff --git a/broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.TransactionMsgService b/broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.TransactionMsgService new file mode 100644 index 00000000000..01ee14278fc --- /dev/null +++ b/broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.TransactionMsgService @@ -0,0 +1 @@ +org.apache.rocketmq.broker.util.TransactionMsgServiceImpl \ No newline at end of file diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java b/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java index 9771f145fb5..fe0db96c907 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java @@ -91,6 +91,10 @@ public RemotingCommand checkTransactionState(ChannelHandlerContext ctx, final ByteBuffer byteBuffer = ByteBuffer.wrap(request.getBody()); final MessageExt messageExt = MessageDecoder.decode(byteBuffer); if (messageExt != null) { + String transactionId = messageExt.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX); + if (null != transactionId && !"".equals(transactionId)) { + messageExt.setTransactionId(transactionId); + } final String group = messageExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP); if (group != null) { MQProducerInner producer = this.mqClientFactory.selectProducer(group); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java index 42c324fe2d3..b650e35e02f 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java @@ -96,6 +96,10 @@ public PullResult processPullResult(final MessageQueue mq, final PullResult pull } for (MessageExt msg : msgListFilterAgain) { + String traFlag = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED); + if (traFlag != null && Boolean.parseBoolean(traFlag)) { + msg.setTransactionId(msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX)); + } MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MIN_OFFSET, Long.toString(pullResult.getMinOffset())); MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MAX_OFFSET, diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index 81461f5e9b2..4b5e373fd27 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -16,33 +16,6 @@ */ package org.apache.rocketmq.client.impl.producer; -import java.io.IOException; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Random; -import java.util.Set; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.common.message.Message; -import org.apache.rocketmq.common.message.MessageClientIDSetter; -import org.apache.rocketmq.common.message.MessageExt; -import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.common.message.MessageConst; -import org.apache.rocketmq.common.message.MessageDecoder; -import org.apache.rocketmq.common.message.MessageBatch; -import org.apache.rocketmq.common.message.MessageAccessor; -import org.apache.rocketmq.common.message.MessageType; -import org.apache.rocketmq.common.message.MessageId; import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.Validators; import org.apache.rocketmq.client.common.ClientErrorCode; @@ -58,30 +31,56 @@ import org.apache.rocketmq.client.latency.MQFaultStrategy; import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.client.producer.DefaultMQProducer; -import org.apache.rocketmq.client.producer.LocalTransactionExecuter; import org.apache.rocketmq.client.producer.LocalTransactionState; import org.apache.rocketmq.client.producer.MessageQueueSelector; import org.apache.rocketmq.client.producer.SendCallback; import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.client.producer.SendStatus; -import org.apache.rocketmq.client.producer.TransactionCheckListener; +import org.apache.rocketmq.client.producer.TransactionListener; import org.apache.rocketmq.client.producer.TransactionMQProducer; import org.apache.rocketmq.client.producer.TransactionSendResult; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.ServiceState; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageBatch; +import org.apache.rocketmq.common.message.MessageClientIDSetter; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageId; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.message.MessageType; import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader; import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingConnectException; import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; +import java.io.IOException; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + public class DefaultMQProducerImpl implements MQProducerInner { private final InternalLogger log = ClientLogger.getLog(); private final Random random = new Random(); @@ -116,13 +115,17 @@ public void registerCheckForbiddenHook(CheckForbiddenHook checkForbiddenHook) { public void initTransactionEnv() { TransactionMQProducer producer = (TransactionMQProducer) this.defaultMQProducer; - this.checkRequestQueue = new LinkedBlockingQueue(producer.getCheckRequestHoldMax()); - this.checkExecutor = new ThreadPoolExecutor( - producer.getCheckThreadPoolMinSize(), - producer.getCheckThreadPoolMaxSize(), - 1000 * 60, - TimeUnit.MILLISECONDS, - this.checkRequestQueue); + if (producer.getExecutorService() != null) { + this.checkExecutor = producer.getExecutorService(); + } else { + this.checkRequestQueue = new LinkedBlockingQueue(2000); + this.checkExecutor = new ThreadPoolExecutor( + 1, + 1, + 1000 * 60, + TimeUnit.MILLISECONDS, + this.checkRequestQueue); + } } public void destroyTransactionEnv() { @@ -239,10 +242,10 @@ public boolean isPublishTopicNeedUpdate(String topic) { } @Override - public TransactionCheckListener checkListener() { + public TransactionListener checkListener() { if (this.defaultMQProducer instanceof TransactionMQProducer) { TransactionMQProducer producer = (TransactionMQProducer) defaultMQProducer; - return producer.getTransactionCheckListener(); + return producer.getTransactionListener(); } return null; @@ -259,12 +262,12 @@ public void checkTransactionState(final String addr, final MessageExt msg, @Override public void run() { - TransactionCheckListener transactionCheckListener = DefaultMQProducerImpl.this.checkListener(); + TransactionListener transactionCheckListener = DefaultMQProducerImpl.this.checkListener(); if (transactionCheckListener != null) { LocalTransactionState localTransactionState = LocalTransactionState.UNKNOW; Throwable exception = null; try { - localTransactionState = transactionCheckListener.checkLocalTransactionState(message); + localTransactionState = transactionCheckListener.checkLocalTransaction(message); } catch (Throwable e) { log.error("Broker call checkTransactionState, but checkLocalTransactionState exception", e); exception = e; @@ -962,7 +965,7 @@ public void sendOneway(Message msg, MessageQueueSelector selector, Object arg) } public TransactionSendResult sendMessageInTransaction(final Message msg, - final LocalTransactionExecuter tranExecuter, final Object arg) + final TransactionListener tranExecuter, final Object arg) throws MQClientException { if (null == tranExecuter) { throw new MQClientException("tranExecutor is null", null); @@ -986,7 +989,11 @@ public TransactionSendResult sendMessageInTransaction(final Message msg, if (sendResult.getTransactionId() != null) { msg.putUserProperty("__transactionId__", sendResult.getTransactionId()); } - localTransactionState = tranExecuter.executeLocalTransactionBranch(msg, arg); + String transactionId = msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX); + if (null != transactionId && !"".equals(transactionId)) { + msg.setTransactionId(transactionId); + } + localTransactionState = tranExecuter.executeLocalTransaction(msg, arg); if (null == localTransactionState) { localTransactionState = LocalTransactionState.UNKNOW; } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/MQProducerInner.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/MQProducerInner.java index dfd485dd909..52ebe1b57d3 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/MQProducerInner.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/MQProducerInner.java @@ -16,17 +16,18 @@ */ package org.apache.rocketmq.client.impl.producer; -import java.util.Set; -import org.apache.rocketmq.client.producer.TransactionCheckListener; +import org.apache.rocketmq.client.producer.TransactionListener; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader; +import java.util.Set; + public interface MQProducerInner { Set getPublishTopicList(); boolean isPublishTopicNeedUpdate(final String topic); - TransactionCheckListener checkListener(); + TransactionListener checkListener(); void checkTransactionState( final String addr, diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java index a2f25dd0f8f..03e394553a1 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java @@ -464,14 +464,12 @@ public void sendOneway(Message msg, MessageQueueSelector selector, Object arg) * This method is to send transactional messages. * * @param msg Transactional message to send. - * @param tranExecuter local transaction executor. * @param arg Argument used along with local transaction executor. * @return Transaction result. * @throws MQClientException if there is any client error. */ @Override - public TransactionSendResult sendMessageInTransaction(Message msg, LocalTransactionExecuter tranExecuter, - final Object arg) + public TransactionSendResult sendMessageInTransaction(Message msg, final Object arg) throws MQClientException { throw new RuntimeException("sendMessageInTransaction not implement, please use TransactionMQProducer class"); } diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/LocalTransactionExecuter.java b/client/src/main/java/org/apache/rocketmq/client/producer/LocalTransactionExecuter.java index 80b5546925f..3e72088bf48 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/LocalTransactionExecuter.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/LocalTransactionExecuter.java @@ -17,7 +17,7 @@ package org.apache.rocketmq.client.producer; import org.apache.rocketmq.common.message.Message; - +@Deprecated public interface LocalTransactionExecuter { LocalTransactionState executeLocalTransactionBranch(final Message msg, final Object arg); } diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java index 14caf6ffac9..0776ee155cf 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java @@ -80,8 +80,7 @@ void send(final Message msg, final MessageQueueSelector selector, final Object a void sendOneway(final Message msg, final MessageQueueSelector selector, final Object arg) throws MQClientException, RemotingException, InterruptedException; - TransactionSendResult sendMessageInTransaction(final Message msg, - final LocalTransactionExecuter tranExecuter, final Object arg) throws MQClientException; + TransactionSendResult sendMessageInTransaction(final Message msg, final Object arg) throws MQClientException; //for batch SendResult send(final Collection msgs) throws MQClientException, RemotingException, MQBrokerException, diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/TransactionCheckListener.java b/client/src/main/java/org/apache/rocketmq/client/producer/TransactionCheckListener.java index 1cf5c4d90a5..420ed621d6f 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/TransactionCheckListener.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/TransactionCheckListener.java @@ -17,7 +17,7 @@ package org.apache.rocketmq.client.producer; import org.apache.rocketmq.common.message.MessageExt; - +@Deprecated public interface TransactionCheckListener { LocalTransactionState checkLocalTransactionState(final MessageExt msg); } diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/TransactionListener.java b/client/src/main/java/org/apache/rocketmq/client/producer/TransactionListener.java new file mode 100644 index 00000000000..c750e53845a --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/producer/TransactionListener.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.producer; + +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageExt; + +public interface TransactionListener { + /** + * When send transactional prepare(half) message succeed, this method will be invoked to execute local transaction. + * + * @param msg Half(prepare) message + * @param arg Custom business parameter + * @return Transaction state + */ + LocalTransactionState executeLocalTransaction(final Message msg, final Object arg); + + /** + * When no response to prepare(half) message. broker will send check message to check the transaction status, and this + * method will be invoked to get local transaction status. + * + * @param msg Check message + * @return Transaction state + */ + LocalTransactionState checkLocalTransaction(final MessageExt msg); +} diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/TransactionMQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/TransactionMQProducer.java index 7b87e8f720b..c4f122c5848 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/TransactionMQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/TransactionMQProducer.java @@ -20,11 +20,12 @@ import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.remoting.RPCHook; +import java.util.concurrent.ExecutorService; + public class TransactionMQProducer extends DefaultMQProducer { - private TransactionCheckListener transactionCheckListener; - private int checkThreadPoolMinSize = 1; - private int checkThreadPoolMaxSize = 1; - private int checkRequestHoldMax = 2000; + private TransactionListener transactionListener; + + private ExecutorService executorService; public TransactionMQProducer() { } @@ -50,44 +51,27 @@ public void shutdown() { } @Override - public TransactionSendResult sendMessageInTransaction(final Message msg, - final LocalTransactionExecuter tranExecuter, final Object arg) throws MQClientException { - if (null == this.transactionCheckListener) { - throw new MQClientException("localTransactionBranchCheckListener is null", null); + public TransactionSendResult sendMessageInTransaction(final Message msg, final Object arg) throws MQClientException { + if (null == this.transactionListener) { + throw new MQClientException("TransactionListener is null", null); } - return this.defaultMQProducerImpl.sendMessageInTransaction(msg, tranExecuter, arg); - } - - public TransactionCheckListener getTransactionCheckListener() { - return transactionCheckListener; - } - - public void setTransactionCheckListener(TransactionCheckListener transactionCheckListener) { - this.transactionCheckListener = transactionCheckListener; - } - - public int getCheckThreadPoolMinSize() { - return checkThreadPoolMinSize; - } - - public void setCheckThreadPoolMinSize(int checkThreadPoolMinSize) { - this.checkThreadPoolMinSize = checkThreadPoolMinSize; + return this.defaultMQProducerImpl.sendMessageInTransaction(msg, transactionListener, arg); } - public int getCheckThreadPoolMaxSize() { - return checkThreadPoolMaxSize; + public TransactionListener getTransactionListener() { + return transactionListener; } - public void setCheckThreadPoolMaxSize(int checkThreadPoolMaxSize) { - this.checkThreadPoolMaxSize = checkThreadPoolMaxSize; + public void setTransactionListener(TransactionListener transactionListener) { + this.transactionListener = transactionListener; } - public int getCheckRequestHoldMax() { - return checkRequestHoldMax; + public ExecutorService getExecutorService() { + return executorService; } - public void setCheckRequestHoldMax(int checkRequestHoldMax) { - this.checkRequestHoldMax = checkRequestHoldMax; + public void setExecutorService(ExecutorService executorService) { + this.executorService = executorService; } } diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index 203431aee1a..300333574f5 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -144,6 +144,24 @@ public class BrokerConfig { */ private int registerNameServerPeriod = 1000 * 30; + /** + * The minimum time of the transactional message to be checked firstly, one message only exceed this time interval that can be checked. + */ + @ImportantField + private long transactionTimeOut = 3 * 1000; + + /** + * The maximum number of times the message was checked, if exceed this value, this message will be discarded. + */ + @ImportantField + private int transactionCheckMax = 5; + + /** + * Transaction message check interval. + */ + @ImportantField + private long transactionCheckInterval = 60 * 1000; + public boolean isTraceOn() { return traceOn; } @@ -633,4 +651,28 @@ public int getRegisterNameServerPeriod() { public void setRegisterNameServerPeriod(int registerNameServerPeriod) { this.registerNameServerPeriod = registerNameServerPeriod; } + + public long getTransactionTimeOut() { + return transactionTimeOut; + } + + public void setTransactionTimeOut(long transactionTimeOut) { + this.transactionTimeOut = transactionTimeOut; + } + + public int getTransactionCheckMax() { + return transactionCheckMax; + } + + public void setTransactionCheckMax(int transactionCheckMax) { + this.transactionCheckMax = transactionCheckMax; + } + + public long getTransactionCheckInterval() { + return transactionCheckInterval; + } + + public void setTransactionCheckInterval(long transactionCheckInterval) { + this.transactionCheckInterval = transactionCheckInterval; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/MixAll.java b/common/src/main/java/org/apache/rocketmq/common/MixAll.java index 49085c3f1e1..835c8c91be1 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MixAll.java +++ b/common/src/main/java/org/apache/rocketmq/common/MixAll.java @@ -89,6 +89,10 @@ public class MixAll { public static final String DEFAULT_TRACE_REGION_ID = "DefaultRegion"; public static final String CONSUME_CONTEXT_TYPE = "ConsumeContextType"; + public static final String RMQ_SYS_TRANS_HALF_TOPIC = "RMQ_SYS_TRANS_HALF_TOPIC"; + public static final String RMQ_SYS_TRANS_OP_HALF_TOPIC = "RMQ_SYS_TRANS_OP_HALF_TOPIC"; + public static final String CID_SYS_RMQ_TRANS = "CID_RMQ_SYS_TRANS"; + public static String getWSAddr() { String wsDomainName = System.getProperty("rocketmq.namesrv.domain", DEFAULT_NAMESRV_ADDR_LOOKUP); String wsDomainSubgroup = System.getProperty("rocketmq.namesrv.domain.subgroup", "nsaddr"); diff --git a/common/src/main/java/org/apache/rocketmq/common/message/Message.java b/common/src/main/java/org/apache/rocketmq/common/message/Message.java index 15ba2142c3e..287be13088f 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/Message.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/Message.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.common.message; import java.io.Serializable; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.Map; @@ -28,6 +29,7 @@ public class Message implements Serializable { private int flag; private Map properties; private byte[] body; + private String transactionId; public Message() { } @@ -191,9 +193,22 @@ public void setBuyerId(String buyerId) { putProperty(MessageConst.PROPERTY_BUYER_ID, buyerId); } + public String getTransactionId() { + return transactionId; + } + + public void setTransactionId(String transactionId) { + this.transactionId = transactionId; + } + @Override public String toString() { - return "Message [topic=" + topic + ", flag=" + flag + ", properties=" + properties + ", body=" - + (body != null ? body.length : 0) + "]"; + return "Message{" + + "topic='" + topic + '\'' + + ", flag=" + flag + + ", properties=" + properties + + ", body=" + Arrays.toString(body) + + ", transactionId='" + transactionId + '\'' + + '}'; } } diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java index 1edbbec70c6..bf5c807c7f3 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java @@ -41,6 +41,9 @@ public class MessageConst { public static final String PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX = "UNIQ_KEY"; public static final String PROPERTY_MAX_RECONSUME_TIMES = "MAX_RECONSUME_TIMES"; public static final String PROPERTY_CONSUME_START_TIMESTAMP = "CONSUME_START_TIME"; + public static final String PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET = "TRAN_PREPARED_QUEUE_OFFSET"; + public static final String PROPERTY_TRANSACTION_CHECK_TIMES = "TRANSACTION_CHECK_TIMES"; + public static final String PROPERTY_CHECK_IMMUNITY_TIME_IN_SECONDS = "CHECK_IMMUNITY_TIME_IN_SECONDS"; public static final String KEY_SEPARATOR = " "; diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/CheckTransactionStateRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/CheckTransactionStateRequestHeader.java index 76c7b42791f..6cba71c7e91 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/CheckTransactionStateRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/CheckTransactionStateRequestHeader.java @@ -31,6 +31,7 @@ public class CheckTransactionStateRequestHeader implements CommandCustomHeader { private Long commitLogOffset; private String msgId; private String transactionId; + private String offsetMsgId; @Override public void checkFields() throws RemotingCommandException { @@ -67,4 +68,12 @@ public String getTransactionId() { public void setTransactionId(String transactionId) { this.transactionId = transactionId; } + + public String getOffsetMsgId() { + return offsetMsgId; + } + + public void setOffsetMsgId(String offsetMsgId) { + this.offsetMsgId = offsetMsgId; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/EndTransactionRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/EndTransactionRequestHeader.java index 67785e23cfb..87661c320ad 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/EndTransactionRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/EndTransactionRequestHeader.java @@ -15,9 +15,6 @@ * limitations under the License. */ -/** - * $Id: EndTransactionRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $ - */ package org.apache.rocketmq.common.protocol.header; import org.apache.rocketmq.common.sysflag.MessageSysFlag; @@ -121,9 +118,14 @@ public void setTransactionId(String transactionId) { @Override public String toString() { - return "EndTransactionRequestHeader [producerGroup=" + producerGroup + ", tranStateTableOffset=" - + tranStateTableOffset + ", commitLogOffset=" + commitLogOffset + ", commitOrRollback=" - + commitOrRollback + ", fromTransactionCheck=" + fromTransactionCheck + ", msgId=" + msgId - + "]"; + return "EndTransactionRequestHeader{" + + "producerGroup='" + producerGroup + '\'' + + ", tranStateTableOffset=" + tranStateTableOffset + + ", commitLogOffset=" + commitLogOffset + + ", commitOrRollback=" + commitOrRollback + + ", fromTransactionCheck=" + fromTransactionCheck + + ", msgId='" + msgId + '\'' + + ", transactionId='" + transactionId + '\'' + + '}'; } } diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java index d9fafdd08e8..9cb05c2b185 100644 --- a/example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java +++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java @@ -17,23 +17,23 @@ package org.apache.rocketmq.example.benchmark; -import java.io.UnsupportedEncodingException; -import java.util.LinkedList; -import java.util.Timer; -import java.util.TimerTask; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicLong; import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.producer.LocalTransactionExecuter; import org.apache.rocketmq.client.producer.LocalTransactionState; import org.apache.rocketmq.client.producer.SendResult; -import org.apache.rocketmq.client.producer.TransactionCheckListener; +import org.apache.rocketmq.client.producer.TransactionListener; import org.apache.rocketmq.client.producer.TransactionMQProducer; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.remoting.common.RemotingHelper; +import java.io.UnsupportedEncodingException; +import java.util.LinkedList; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; + public class TransactionProducer { private static int threadCount; private static int messageSize; @@ -73,8 +73,8 @@ private void printStats() { Long[] end = snapshotList.getLast(); final long sendTps = - (long) (((end[3] - begin[3]) / (double) (end[0] - begin[0])) * 1000L); - final double averageRT = (end[5] - begin[5]) / (double) (end[3] - begin[3]); + (long)(((end[3] - begin[3]) / (double)(end[0] - begin[0])) * 1000L); + final double averageRT = (end[5] - begin[5]) / (double)(end[3] - begin[3]); System.out.printf( "Send TPS: %d Max RT: %d Average RT: %7.3f Send Failed: %d Response Failed: %d transaction checkCount: %d %n", @@ -92,16 +92,14 @@ public void run() { } }, 10000, 10000); - final TransactionCheckListener transactionCheckListener = - new TransactionCheckListenerBImpl(ischeckffalse, statsBenchmark); + final TransactionListener transactionListener = + new TransactionListenerImpl(ischeckffalse, ischeck, statsBenchmark); final TransactionMQProducer producer = new TransactionMQProducer("benchmark_transaction_producer"); producer.setInstanceName(Long.toString(System.currentTimeMillis())); - producer.setTransactionCheckListener(transactionCheckListener); + producer.setTransactionListener(transactionListener); producer.setDefaultTopicQueueNums(1000); producer.start(); - final TransactionExecuterBImpl tranExecuter = new TransactionExecuterBImpl(ischeck); - for (int i = 0; i < threadCount; i++) { sendThreadPool.execute(new Runnable() { @Override @@ -111,7 +109,7 @@ public void run() { // Thread.sleep(1000); final long beginTimestamp = System.currentTimeMillis(); SendResult sendResult = - producer.sendMessageInTransaction(msg, tranExecuter, null); + producer.sendMessageInTransaction(msg, null); if (sendResult != null) { statsBenchmark.getSendRequestSuccessCount().incrementAndGet(); statsBenchmark.getReceiveResponseSuccessCount().incrementAndGet(); @@ -124,8 +122,7 @@ public void run() { boolean updated = statsBenchmark.getSendMessageMaxRT().compareAndSet(prevMaxRT, currentRT); - if (updated) - break; + if (updated) { break; } prevMaxRT = statsBenchmark.getSendMessageMaxRT().get(); } @@ -153,35 +150,37 @@ private static Message buildMessage(final int messageSize) throws UnsupportedEnc } } -class TransactionExecuterBImpl implements LocalTransactionExecuter { - - private boolean ischeck; - - public TransactionExecuterBImpl(boolean ischeck) { - this.ischeck = ischeck; - } - - @Override - public LocalTransactionState executeLocalTransactionBranch(final Message msg, final Object arg) { - if (ischeck) { - return LocalTransactionState.UNKNOW; - } - return LocalTransactionState.COMMIT_MESSAGE; - } -} - -class TransactionCheckListenerBImpl implements TransactionCheckListener { +//class TransactionExecuterBImpl implements LocalTransactionExecuter { +// +// private boolean ischeckLocal; +// +// public TransactionExecuterBImpl(boolean ischeck) { +// this.ischeck = ischeck; +// } +// +// @Override +// public LocalTransactionState executeLocalTransactionBranch(final Message msg, final Object arg) { +// if (ischeck) { +// return LocalTransactionState.UNKNOW; +// } +// return LocalTransactionState.COMMIT_MESSAGE; +// } +//} + +class TransactionListenerImpl implements TransactionListener { private boolean ischeckffalse; private StatsBenchmarkTProducer statsBenchmarkTProducer; + private boolean ischeckLocal; - public TransactionCheckListenerBImpl(boolean ischeckffalse, - StatsBenchmarkTProducer statsBenchmarkTProducer) { + public TransactionListenerImpl(boolean ischeckffalse, boolean isCheckLocal, + StatsBenchmarkTProducer statsBenchmarkTProducer) { this.ischeckffalse = ischeckffalse; + this.ischeckLocal = isCheckLocal; this.statsBenchmarkTProducer = statsBenchmarkTProducer; } @Override - public LocalTransactionState checkLocalTransactionState(MessageExt msg) { + public LocalTransactionState checkLocalTransaction(MessageExt msg) { statsBenchmarkTProducer.getCheckRequestSuccessCount().incrementAndGet(); if (ischeckffalse) { @@ -190,6 +189,14 @@ public LocalTransactionState checkLocalTransactionState(MessageExt msg) { return LocalTransactionState.COMMIT_MESSAGE; } + + @Override + public LocalTransactionState executeLocalTransaction(final Message msg, final Object arg) { + if (ischeckLocal) { + return LocalTransactionState.UNKNOW; + } + return LocalTransactionState.COMMIT_MESSAGE; + } } class StatsBenchmarkTProducer { diff --git a/example/src/main/java/org/apache/rocketmq/example/transaction/TransactionExecuterImpl.java b/example/src/main/java/org/apache/rocketmq/example/transaction/TransactionExecuterImpl.java deleted file mode 100644 index e7d193ee85b..00000000000 --- a/example/src/main/java/org/apache/rocketmq/example/transaction/TransactionExecuterImpl.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.example.transaction; - -import java.util.concurrent.atomic.AtomicInteger; -import org.apache.rocketmq.client.producer.LocalTransactionExecuter; -import org.apache.rocketmq.client.producer.LocalTransactionState; -import org.apache.rocketmq.common.message.Message; - -public class TransactionExecuterImpl implements LocalTransactionExecuter { - private AtomicInteger transactionIndex = new AtomicInteger(1); - - @Override - public LocalTransactionState executeLocalTransactionBranch(final Message msg, final Object arg) { - int value = transactionIndex.getAndIncrement(); - - if (value == 0) { - throw new RuntimeException("Could not find db"); - } else if ((value % 5) == 0) { - return LocalTransactionState.ROLLBACK_MESSAGE; - } else if ((value % 4) == 0) { - return LocalTransactionState.COMMIT_MESSAGE; - } - - return LocalTransactionState.UNKNOW; - } -} diff --git a/example/src/main/java/org/apache/rocketmq/example/transaction/TransactionCheckListenerImpl.java b/example/src/main/java/org/apache/rocketmq/example/transaction/TransactionListenerImpl.java similarity index 54% rename from example/src/main/java/org/apache/rocketmq/example/transaction/TransactionCheckListenerImpl.java rename to example/src/main/java/org/apache/rocketmq/example/transaction/TransactionListenerImpl.java index acb755ecfd7..ce471d2ce92 100644 --- a/example/src/main/java/org/apache/rocketmq/example/transaction/TransactionCheckListenerImpl.java +++ b/example/src/main/java/org/apache/rocketmq/example/transaction/TransactionListenerImpl.java @@ -16,27 +16,40 @@ */ package org.apache.rocketmq.example.transaction; -import java.util.concurrent.atomic.AtomicInteger; import org.apache.rocketmq.client.producer.LocalTransactionState; -import org.apache.rocketmq.client.producer.TransactionCheckListener; +import org.apache.rocketmq.client.producer.TransactionListener; +import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageExt; -public class TransactionCheckListenerImpl implements TransactionCheckListener { +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +public class TransactionListenerImpl implements TransactionListener { private AtomicInteger transactionIndex = new AtomicInteger(0); - @Override - public LocalTransactionState checkLocalTransactionState(MessageExt msg) { - System.out.printf("server checking TrMsg %s%n", msg); + private ConcurrentHashMap localTrans = new ConcurrentHashMap<>(); + @Override + public LocalTransactionState executeLocalTransaction(Message msg, Object arg) { int value = transactionIndex.getAndIncrement(); - if ((value % 6) == 0) { - throw new RuntimeException("Could not find db"); - } else if ((value % 5) == 0) { - return LocalTransactionState.ROLLBACK_MESSAGE; - } else if ((value % 4) == 0) { - return LocalTransactionState.COMMIT_MESSAGE; - } - + int status = value % 3; + localTrans.put(msg.getTransactionId(), status); return LocalTransactionState.UNKNOW; } + + @Override + public LocalTransactionState checkLocalTransaction(MessageExt msg) { + Integer status = localTrans.get(msg.getTransactionId()); + if (null != status) { + switch (status) { + case 0: + return LocalTransactionState.UNKNOW; + case 1: + return LocalTransactionState.COMMIT_MESSAGE; + case 2: + return LocalTransactionState.ROLLBACK_MESSAGE; + } + } + return LocalTransactionState.COMMIT_MESSAGE; + } } diff --git a/example/src/main/java/org/apache/rocketmq/example/transaction/TransactionProducer.java b/example/src/main/java/org/apache/rocketmq/example/transaction/TransactionProducer.java index edfad2485f6..75c805b9698 100644 --- a/example/src/main/java/org/apache/rocketmq/example/transaction/TransactionProducer.java +++ b/example/src/main/java/org/apache/rocketmq/example/transaction/TransactionProducer.java @@ -16,32 +16,44 @@ */ package org.apache.rocketmq.example.transaction; -import java.io.UnsupportedEncodingException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.producer.SendResult; -import org.apache.rocketmq.client.producer.TransactionCheckListener; +import org.apache.rocketmq.client.producer.TransactionListener; import org.apache.rocketmq.client.producer.TransactionMQProducer; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.remoting.common.RemotingHelper; +import java.io.UnsupportedEncodingException; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + public class TransactionProducer { public static void main(String[] args) throws MQClientException, InterruptedException { - TransactionCheckListener transactionCheckListener = new TransactionCheckListenerImpl(); + TransactionListener transactionListener = new TransactionListenerImpl(); TransactionMQProducer producer = new TransactionMQProducer("please_rename_unique_group_name"); - producer.setCheckThreadPoolMinSize(2); - producer.setCheckThreadPoolMaxSize(2); - producer.setCheckRequestHoldMax(2000); - producer.setTransactionCheckListener(transactionCheckListener); + ExecutorService executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, new ArrayBlockingQueue(2000), new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + Thread thread = new Thread(r); + thread.setName("client-transaction-msg-check-thread"); + return thread; + } + }); + + producer.setExecutorService(executorService); + producer.setTransactionListener(transactionListener); producer.start(); String[] tags = new String[] {"TagA", "TagB", "TagC", "TagD", "TagE"}; - TransactionExecuterImpl tranExecuter = new TransactionExecuterImpl(); - for (int i = 0; i < 100; i++) { + for (int i = 0; i < 10; i++) { try { Message msg = - new Message("TopicTest", tags[i % tags.length], "KEY" + i, + new Message("TopicTest1234", tags[i % tags.length], "KEY" + i, ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET)); - SendResult sendResult = producer.sendMessageInTransaction(msg, tranExecuter, null); + SendResult sendResult = producer.sendMessageInTransaction(msg, null); System.out.printf("%s%n", sendResult); Thread.sleep(10); diff --git a/filter/src/test/java/org/apache/rocketmq/filter/ParserTest.java b/filter/src/test/java/org/apache/rocketmq/filter/ParserTest.java index 36ef2714fcc..115632faa0a 100644 --- a/filter/src/test/java/org/apache/rocketmq/filter/ParserTest.java +++ b/filter/src/test/java/org/apache/rocketmq/filter/ParserTest.java @@ -84,14 +84,17 @@ public void testParse_decimalOverFlow() { @Test public void testParse_floatOverFlow() { try { - String str = "1"; - for (int i = 0; i < 2048; i++) { - str += "111111111111111111111111111111111111111111111111111"; + StringBuffer sb = new StringBuffer(210000); + sb.append("1"); + for (int i = 0; i < 2048; i ++) { + sb.append("111111111111111111111111111111111111111111111111111"); } - str += "."; - for (int i = 0; i < 2048; i++) { - str += "111111111111111111111111111111111111111111111111111"; + sb.append("."); + for (int i = 0; i < 2048; i ++) { + sb.append("111111111111111111111111111111111111111111111111111"); } + String str = sb.toString(); + SelectorParser.parse("a > " + str); diff --git a/logging/src/test/java/org/apache/rocketmq/logging/inner/LoggingBuilderTest.java b/logging/src/test/java/org/apache/rocketmq/logging/inner/LoggingBuilderTest.java index 977e553b70b..6c816a6815c 100644 --- a/logging/src/test/java/org/apache/rocketmq/logging/inner/LoggingBuilderTest.java +++ b/logging/src/test/java/org/apache/rocketmq/logging/inner/LoggingBuilderTest.java @@ -88,7 +88,7 @@ public void testRollingFileAppender() throws InterruptedException { Assert.assertTrue(cc >= 2); } - @Test + //@Test public void testDailyRollingFileAppender() throws InterruptedException { String rollingFile = loggingDir + "/daily-rolling--222.log"; Appender rollingFileAppender = LoggingBuilder.newAppenderBuilder().withAsync(false, 1024) diff --git a/pom.xml b/pom.xml index 6fe4f73f1b9..857dfce65ed 100644 --- a/pom.xml +++ b/pom.xml @@ -157,7 +157,7 @@ - + true @@ -254,6 +254,7 @@ bin/README.md .github/* src/test/resources/certs/* + src/test/resources/META-INF/service/* */target/** */*.iml @@ -315,6 +316,7 @@ maven-surefire-plugin 2.19.1 + 1 1 true @@ -333,6 +335,7 @@ maven-failsafe-plugin 2.19.1 + 1 **/NormalMsgDelayIT.java diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index 4fc7412b1fd..1ade7c2838e 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -247,7 +247,7 @@ public void shutdown() { try { - Thread.sleep(1000 * 3); + Thread.sleep(1000); } catch (InterruptedException e) { log.error("shutdown Exception, ", e); } diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageExtBrokerInner.java b/store/src/main/java/org/apache/rocketmq/store/MessageExtBrokerInner.java index c7879af0645..e5f087b697d 100644 --- a/store/src/main/java/org/apache/rocketmq/store/MessageExtBrokerInner.java +++ b/store/src/main/java/org/apache/rocketmq/store/MessageExtBrokerInner.java @@ -25,12 +25,15 @@ public class MessageExtBrokerInner extends MessageExt { private long tagsCode; public static long tagsString2tagsCode(final TopicFilterType filter, final String tags) { - if (null == tags || tags.length() == 0) - return 0; + if (null == tags || tags.length() == 0) { return 0; } return tags.hashCode(); } + public static long tagsString2tagsCode(final String tags) { + return tagsString2tagsCode(null, tags); + } + public String getPropertiesString() { return propertiesString; } diff --git a/store/src/test/java/org/apache/rocketmq/store/ConsumeQueueTest.java b/store/src/test/java/org/apache/rocketmq/store/ConsumeQueueTest.java index b7d38f8c78f..7e01b8513c9 100644 --- a/store/src/test/java/org/apache/rocketmq/store/ConsumeQueueTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/ConsumeQueueTest.java @@ -212,8 +212,6 @@ public void dispatch(DispatchRequest request) { try { try { putMsg(master); - // wait build consume queue - Thread.sleep(1000); } catch (Exception e) { e.printStackTrace(); assertThat(Boolean.FALSE).isTrue(); diff --git a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQAsyncSendProducer.java b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQAsyncSendProducer.java index 2f8af68b31e..9907cac8008 100644 --- a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQAsyncSendProducer.java +++ b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQAsyncSendProducer.java @@ -26,10 +26,11 @@ import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.client.producer.MessageQueueSelector; import org.apache.rocketmq.client.producer.SendCallback; +import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.test.clientinterface.AbstractMQProducer; -import org.apache.rocketmq.test.sendresult.SendResult; +import org.apache.rocketmq.test.sendresult.ResultWrapper; import org.apache.rocketmq.test.util.RandomUtil; import org.apache.rocketmq.test.util.TestUtil; @@ -39,20 +40,19 @@ public class RMQAsyncSendProducer extends AbstractMQProducer { private String nsAddr = null; private DefaultMQProducer producer = null; private SendCallback sendCallback = null; - private List successSendResult = Collections - .synchronizedList(new ArrayList()); - private AtomicInteger exceptionMsgCount = new AtomicInteger( - 0); + private List successSendResult = Collections.synchronizedList(new ArrayList()); + private AtomicInteger exceptionMsgCount = new AtomicInteger(0); private int msgSize = 0; public RMQAsyncSendProducer(String nsAddr, String topic) { super(topic); this.nsAddr = nsAddr; sendCallback = new SendCallback() { - public void onSuccess(org.apache.rocketmq.client.producer.SendResult sendResult) { + @Override + public void onSuccess(SendResult sendResult) { successSendResult.add(sendResult); } - + @Override public void onException(Throwable throwable) { exceptionMsgCount.getAndIncrement(); } @@ -66,7 +66,7 @@ public int getSuccessMsgCount() { return successSendResult.size(); } - public List getSuccessSendResult() { + public List getSuccessSendResult() { return successSendResult; } @@ -94,10 +94,12 @@ private void start() { } } - public SendResult send(Object msg, Object arg) { + @Override + public ResultWrapper send(Object msg, Object arg) { return null; } + @Override public void shutdown() { producer.shutdown(); } diff --git a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQNormalProducer.java b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQNormalProducer.java index 42949339e33..6bbec6cb948 100644 --- a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQNormalProducer.java +++ b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQNormalProducer.java @@ -26,7 +26,7 @@ import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.test.clientinterface.AbstractMQProducer; -import org.apache.rocketmq.test.sendresult.SendResult; +import org.apache.rocketmq.test.sendresult.ResultWrapper; public class RMQNormalProducer extends AbstractMQProducer { private static Logger logger = Logger.getLogger(RMQNormalProducer.class); @@ -91,7 +91,7 @@ public void start() { } } - public SendResult send(Object msg, Object orderKey) { + public ResultWrapper send(Object msg, Object orderKey) { org.apache.rocketmq.client.producer.SendResult metaqResult = null; Message message = (Message) msg; try { @@ -132,7 +132,7 @@ public void send(List msgs, MessageQueue mq) { } } - public SendResult sendMQ(Message msg, MessageQueue mq) { + public ResultWrapper sendMQ(Message msg, MessageQueue mq) { org.apache.rocketmq.client.producer.SendResult metaqResult = null; try { long start = System.currentTimeMillis(); diff --git a/test/src/main/java/org/apache/rocketmq/test/clientinterface/AbstractMQProducer.java b/test/src/main/java/org/apache/rocketmq/test/clientinterface/AbstractMQProducer.java index 8201e6313c3..df6abfc662b 100644 --- a/test/src/main/java/org/apache/rocketmq/test/clientinterface/AbstractMQProducer.java +++ b/test/src/main/java/org/apache/rocketmq/test/clientinterface/AbstractMQProducer.java @@ -21,14 +21,14 @@ import java.util.List; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.test.client.rmq.RMQNormalProducer; -import org.apache.rocketmq.test.sendresult.SendResult; +import org.apache.rocketmq.test.sendresult.ResultWrapper; import org.apache.rocketmq.test.util.RandomUtil; import org.apache.rocketmq.test.util.TestUtil; public abstract class AbstractMQProducer extends MQCollector implements MQProducer { protected String topic = null; - protected SendResult sendResult = new SendResult(); + protected ResultWrapper sendResult = new ResultWrapper(); protected boolean startSuccess = false; protected String producerGroupName = null; protected String producerInstanceName = null; diff --git a/test/src/main/java/org/apache/rocketmq/test/clientinterface/MQProducer.java b/test/src/main/java/org/apache/rocketmq/test/clientinterface/MQProducer.java index 795457d1992..e9ed0d3fdfc 100644 --- a/test/src/main/java/org/apache/rocketmq/test/clientinterface/MQProducer.java +++ b/test/src/main/java/org/apache/rocketmq/test/clientinterface/MQProducer.java @@ -17,10 +17,10 @@ package org.apache.rocketmq.test.clientinterface; -import org.apache.rocketmq.test.sendresult.SendResult; +import org.apache.rocketmq.test.sendresult.ResultWrapper; public interface MQProducer { - SendResult send(Object msg, Object arg); + ResultWrapper send(Object msg, Object arg); void setDebug(); diff --git a/test/src/main/java/org/apache/rocketmq/test/sendresult/SendResult.java b/test/src/main/java/org/apache/rocketmq/test/sendresult/ResultWrapper.java similarity index 98% rename from test/src/main/java/org/apache/rocketmq/test/sendresult/SendResult.java rename to test/src/main/java/org/apache/rocketmq/test/sendresult/ResultWrapper.java index d53ee7da25c..9fe31463e44 100644 --- a/test/src/main/java/org/apache/rocketmq/test/sendresult/SendResult.java +++ b/test/src/main/java/org/apache/rocketmq/test/sendresult/ResultWrapper.java @@ -17,7 +17,7 @@ package org.apache.rocketmq.test.sendresult; -public class SendResult { +public class ResultWrapper { private boolean sendResult = false; private String msgId = null; private Exception sendException = null; From 6ec0c7f2ea4cf5c0c304a8f80327ac510e0c2e39 Mon Sep 17 00:00:00 2001 From: duhengforever Date: Tue, 10 Jul 2018 01:26:02 +0800 Subject: [PATCH 2/6] Rename class name to make it seems more standardized --- .../rocketmq/broker/BrokerController.java | 116 +++++++++--------- .../apache/rocketmq/broker/BrokerStartup.java | 11 +- .../broker/client/ProducerManager.java | 4 +- .../broker/client/net/Broker2Client.java | 2 +- .../processor/EndTransactionProcessor.java | 14 +-- .../processor/SendMessageProcessor.java | 2 +- ...actTransactionalMessageCheckListener.java} | 18 +-- ... => TransactionalMessageCheckService.java} | 20 +-- ....java => TransactionalMessageService.java} | 4 +- ...ultTransactionalMessageCheckListener.java} | 12 +- ...e.java => TransactionalMessageBridge.java} | 52 ++++---- ...a => TransactionalMessageServiceImpl.java} | 34 +++-- ...til.java => TransactionalMessageUtil.java} | 2 +- .../rocketmq/broker/util/ServiceProvider.java | 4 +- .../EndTransactionProcessorTest.java | 6 +- .../processor/SendMessageProcessorTest.java | 18 +-- ...ransactionalMessageCheckListenerTest.java} | 6 +- ...va => TransactionalMessageBridgeTest.java} | 12 +- ... TransactionalMessageServiceImplTest.java} | 14 ++- ...LogTransactionalMessageCheckListener.java} | 4 +- .../broker/util/ServiceProviderTest.java | 12 +- ...a => TransactionalMessageServiceImpl.java} | 14 +-- ...ansaction.AbstractTransactionCheckListener | 1 - ....AbstractTransactionalMessageCheckListener | 1 + ...q.broker.transaction.TransactionMsgService | 1 - ...er.transaction.TransactionalMessageService | 1 + .../producer/LocalTransactionExecuter.java | 23 ---- .../producer/TransactionCheckListener.java | 23 ---- .../benchmark/TransactionProducer.java | 40 ++---- 29 files changed, 204 insertions(+), 267 deletions(-) rename broker/src/main/java/org/apache/rocketmq/broker/transaction/{AbstractTransactionCheckListener.java => AbstractTransactionalMessageCheckListener.java} (86%) rename broker/src/main/java/org/apache/rocketmq/broker/transaction/{TransactionMsgCheckService.java => TransactionalMessageCheckService.java} (73%) rename broker/src/main/java/org/apache/rocketmq/broker/transaction/{TransactionMsgService.java => TransactionalMessageService.java} (96%) rename broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/{DefaultAbstractTransactionCheckListener.java => DefaultTransactionalMessageCheckListener.java} (74%) rename broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/{TransactionBridge.java => TransactionalMessageBridge.java} (89%) rename broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/{QueueTransactionMsgServiceImpl.java => TransactionalMessageServiceImpl.java} (95%) rename broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/{TransactionUtil.java => TransactionalMessageUtil.java} (97%) rename broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/{DefaultAbstractTransactionCheckListenerTest.java => DefaultTransactionalMessageCheckListenerTest.java} (93%) rename broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/{TransactionBridgeTest.java => TransactionalMessageBridgeTest.java} (93%) rename broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/{QueueTransactionMsgServiceImplTest.java => TransactionalMessageServiceImplTest.java} (95%) rename broker/src/test/java/org/apache/rocketmq/broker/util/{LogTransactionCheckListener.java => LogTransactionalMessageCheckListener.java} (89%) rename broker/src/test/java/org/apache/rocketmq/broker/util/{TransactionMsgServiceImpl.java => TransactionalMessageServiceImpl.java} (80%) delete mode 100644 broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionCheckListener create mode 100644 broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener delete mode 100644 broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.TransactionMsgService create mode 100644 broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.TransactionalMessageService delete mode 100644 client/src/main/java/org/apache/rocketmq/client/producer/LocalTransactionExecuter.java delete mode 100644 client/src/main/java/org/apache/rocketmq/client/producer/TransactionCheckListener.java diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 7b7667ba451..af2781f9c64 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -16,21 +16,6 @@ */ package org.apache.rocketmq.broker; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; import org.apache.rocketmq.broker.client.ClientHousekeepingService; import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener; import org.apache.rocketmq.broker.client.ConsumerManager; @@ -61,12 +46,12 @@ import org.apache.rocketmq.broker.slave.SlaveSynchronize; import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager; import org.apache.rocketmq.broker.topic.TopicConfigManager; -import org.apache.rocketmq.broker.transaction.AbstractTransactionCheckListener; -import org.apache.rocketmq.broker.transaction.TransactionMsgCheckService; -import org.apache.rocketmq.broker.transaction.TransactionMsgService; -import org.apache.rocketmq.broker.transaction.queue.DefaultAbstractTransactionCheckListener; -import org.apache.rocketmq.broker.transaction.queue.QueueTransactionMsgServiceImpl; -import org.apache.rocketmq.broker.transaction.queue.TransactionBridge; +import org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener; +import org.apache.rocketmq.broker.transaction.TransactionalMessageCheckService; +import org.apache.rocketmq.broker.transaction.TransactionalMessageService; +import org.apache.rocketmq.broker.transaction.queue.DefaultTransactionalMessageCheckListener; +import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageBridge; +import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageServiceImpl; import org.apache.rocketmq.broker.util.ServiceProvider; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.Configuration; @@ -76,12 +61,12 @@ import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.PermName; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.namesrv.RegisterBrokerResult; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.common.stats.MomentStatsItem; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.RemotingServer; import org.apache.rocketmq.remoting.common.TlsMode; @@ -100,6 +85,22 @@ import org.apache.rocketmq.store.stats.BrokerStats; import org.apache.rocketmq.store.stats.BrokerStatsManager; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + public class BrokerController { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final InternalLogger LOG_PROTECTION = InternalLoggerFactory.getLogger(LoggerName.PROTECTION_LOGGER_NAME); @@ -149,9 +150,9 @@ public class BrokerController { private BrokerFastFailure brokerFastFailure; private Configuration configuration; private FileWatchService fileWatchService; - private TransactionMsgCheckService transactionMsgCheckService; - private TransactionMsgService transactionMsgService; - private AbstractTransactionCheckListener transactionCheckListener; + private TransactionalMessageCheckService transactionMsgCheckService; + private TransactionalMessageService transactionalMessageService; + private AbstractTransactionalMessageCheckListener transactionalMessageCheckListener; public BrokerController( final BrokerConfig brokerConfig, @@ -225,7 +226,7 @@ public boolean initialize() throws CloneNotSupportedException { this.messageStore = new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig); - this.brokerStats = new BrokerStats((DefaultMessageStore) this.messageStore); + this.brokerStats = new BrokerStats((DefaultMessageStore)this.messageStore); //load plugin MessageStorePluginContext context = new MessageStorePluginContext(messageStoreConfig, brokerStatsManager, messageArrivingListener, brokerConfig); this.messageStore = MessageStoreFactory.build(context, this.messageStore); @@ -240,7 +241,7 @@ public boolean initialize() throws CloneNotSupportedException { if (result) { this.remotingServer = new NettyRemotingServer(this.nettyServerConfig, this.clientHousekeepingService); - NettyServerConfig fastConfig = (NettyServerConfig) this.nettyServerConfig.clone(); + NettyServerConfig fastConfig = (NettyServerConfig)this.nettyServerConfig.clone(); fastConfig.setListenPort(nettyServerConfig.getListenPort() - 2); this.fastRemotingServer = new NettyRemotingServer(fastConfig, this.clientHousekeepingService); this.sendMessageExecutor = new BrokerFixedThreadPoolExecutor( @@ -415,6 +416,7 @@ public void run() { }, new FileWatchService.Listener() { boolean certChanged, keyChanged = false; + @Override public void onChanged(String path) { if (path.equals(TlsSystemConfig.tlsServerTrustCertPath)) { @@ -433,9 +435,10 @@ public void onChanged(String path) { reloadServerSslContext(); } } + private void reloadServerSslContext() { - ((NettyRemotingServer) remotingServer).loadSslContext(); - ((NettyRemotingServer) fastRemotingServer).loadSslContext(); + ((NettyRemotingServer)remotingServer).loadSslContext(); + ((NettyRemotingServer)fastRemotingServer).loadSslContext(); } }); } catch (Exception e) { @@ -448,18 +451,18 @@ private void reloadServerSslContext() { } private void initialTransaction() { - this.transactionMsgService = ServiceProvider.loadClass(ServiceProvider.TRANSACTION_SERVICE_ID, TransactionMsgService.class); - if (null == this.transactionMsgService) { - this.transactionMsgService = new QueueTransactionMsgServiceImpl(new TransactionBridge(this, this.getMessageStore())); - log.warn("Load default transaction message hook service: {}", QueueTransactionMsgServiceImpl.class.getSimpleName()); + this.transactionalMessageService = ServiceProvider.loadClass(ServiceProvider.TRANSACTION_SERVICE_ID, TransactionalMessageService.class); + if (null == this.transactionalMessageService) { + this.transactionalMessageService = new TransactionalMessageServiceImpl(new TransactionalMessageBridge(this, this.getMessageStore())); + log.warn("Load default transaction message hook service: {}", TransactionalMessageServiceImpl.class.getSimpleName()); } - this.transactionCheckListener = ServiceProvider.loadClass(ServiceProvider.TRANSACTION_LISTENER_ID, AbstractTransactionCheckListener.class); - if (null == this.transactionCheckListener) { - this.transactionCheckListener = new DefaultAbstractTransactionCheckListener(); - log.warn("Load default discard message hook service: {}", DefaultAbstractTransactionCheckListener.class.getSimpleName()); + this.transactionalMessageCheckListener = ServiceProvider.loadClass(ServiceProvider.TRANSACTION_LISTENER_ID, AbstractTransactionalMessageCheckListener.class); + if (null == this.transactionalMessageCheckListener) { + this.transactionalMessageCheckListener = new DefaultTransactionalMessageCheckListener(); + log.warn("Load default discard message hook service: {}", DefaultTransactionalMessageCheckListener.class.getSimpleName()); } - this.transactionCheckListener.setBrokerController(this); - this.transactionMsgCheckService = new TransactionMsgCheckService(this); + this.transactionalMessageCheckListener.setBrokerController(this); + this.transactionMsgCheckService = new TransactionalMessageCheckService(this); } public void registerProcessor() { @@ -564,8 +567,7 @@ public long headSlowTimeMills(BlockingQueue q) { slowTimeMills = rt == null ? 0 : this.messageStore.now() - rt.getCreateTimestamp(); } - if (slowTimeMills < 0) - slowTimeMills = 0; + if (slowTimeMills < 0) { slowTimeMills = 0; } return slowTimeMills; } @@ -849,7 +851,7 @@ public synchronized void registerBrokerAll(final boolean checkOrderConfig, boole } private void doRegisterBrokerAll(boolean checkOrderConfig, boolean oneway, - TopicConfigSerializeWrapper topicConfigWrapper) { + TopicConfigSerializeWrapper topicConfigWrapper) { List registerBrokerResultList = this.brokerOuterAPI.registerBrokerAll( this.brokerConfig.getBrokerClusterName(), this.getBrokerAddr(), @@ -879,10 +881,10 @@ private void doRegisterBrokerAll(boolean checkOrderConfig, boolean oneway, } private boolean needRegister(final String clusterName, - final String brokerAddr, - final String brokerName, - final long brokerId, - final int timeoutMills) { + final String brokerAddr, + final String brokerName, + final long brokerId, + final int timeoutMills) { TopicConfigSerializeWrapper topicConfigWrapper = this.getTopicConfigManager().buildTopicConfigSerializeWrapper(); List changeList = brokerOuterAPI.needRegister(clusterName, brokerAddr, brokerName, brokerId, topicConfigWrapper, timeoutMills); @@ -986,27 +988,27 @@ public Configuration getConfiguration() { return this.configuration; } - public TransactionMsgCheckService getTransactionMsgCheckService() { + public TransactionalMessageCheckService getTransactionMsgCheckService() { return transactionMsgCheckService; } - public void setTransactionMsgCheckService(TransactionMsgCheckService transactionMsgCheckService) { + public void setTransactionMsgCheckService(TransactionalMessageCheckService transactionMsgCheckService) { this.transactionMsgCheckService = transactionMsgCheckService; } - public TransactionMsgService getTransactionMsgService() { - return transactionMsgService; + public TransactionalMessageService getTransactionalMessageService() { + return transactionalMessageService; } - public void setTransactionMsgService(TransactionMsgService transactionMsgService) { - this.transactionMsgService = transactionMsgService; + public void setTransactionalMessageService(TransactionalMessageService transactionalMessageService) { + this.transactionalMessageService = transactionalMessageService; } - public AbstractTransactionCheckListener getTransactionCheckListener() { - return transactionCheckListener; + public AbstractTransactionalMessageCheckListener getTransactionalMessageCheckListener() { + return transactionalMessageCheckListener; } - public void setTransactionCheckListener(AbstractTransactionCheckListener transactionCheckListener) { - this.transactionCheckListener = transactionCheckListener; + public void setTransactionalMessageCheckListener(AbstractTransactionalMessageCheckListener transactionalMessageCheckListener) { + this.transactionalMessageCheckListener = transactionalMessageCheckListener; } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java index 1fc1b3b8474..4b986c0d249 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java @@ -18,11 +18,6 @@ import ch.qos.logback.classic.LoggerContext; import ch.qos.logback.classic.joran.JoranConfigurator; -import java.io.BufferedInputStream; -import java.io.FileInputStream; -import java.io.InputStream; -import java.util.Properties; -import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; @@ -45,6 +40,12 @@ import org.apache.rocketmq.store.config.MessageStoreConfig; import org.slf4j.LoggerFactory; +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; + import static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_ENABLE; public class BrokerStartup { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java index 11f8fe521be..5dedf7029fe 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java @@ -200,7 +200,7 @@ public Channel getAvaliableChannel(String groupId) { } int size = channelList.size(); if (0 == size) { - log.warn("channel list is empty. groupId={}", groupId); + log.warn("Channel list is empty. groupId={}", groupId); return null; } @@ -214,7 +214,7 @@ public Channel getAvaliableChannel(String groupId) { return channel; } } else { - log.warn("check transaction failed, channel table is empty. groupId={}", groupId); + log.warn("Check transaction failed, channel table is empty. groupId={}", groupId); return null; } return null; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java b/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java index ae5a73f1590..4c409f2254a 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java @@ -70,7 +70,7 @@ public void checkProducerTransactionState( try { this.brokerController.getRemotingServer().invokeOneway(channel, request, 10); } catch (Exception e) { - log.error("check transaction failed because invoke producer exception. group={}, msgId={}", group, messageExt.getMsgId(), e.getMessage()); + log.error("Check transaction failed because invoke producer exception. group={}, msgId={}", group, messageExt.getMsgId(), e.getMessage()); } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java index 3982b66427c..c9e85ed1a50 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java @@ -28,6 +28,8 @@ import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; @@ -35,14 +37,12 @@ import org.apache.rocketmq.store.MessageExtBrokerInner; import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.config.BrokerRole; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * EndTransaction processor: process commit and rollback message */ public class EndTransactionProcessor implements NettyRequestProcessor { - private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); + private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); private final BrokerController brokerController; public EndTransactionProcessor(final BrokerController brokerController) { @@ -123,7 +123,7 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand } OperationResult result = new OperationResult(); if (MessageSysFlag.TRANSACTION_COMMIT_TYPE == requestHeader.getCommitOrRollback()) { - result = this.brokerController.getTransactionMsgService().commitMessage(requestHeader); + result = this.brokerController.getTransactionalMessageService().commitMessage(requestHeader); if (result.getResponseCode() == ResponseCode.SUCCESS) { RemotingCommand res = checkPrepareMessage(result.getPrepareMessage(), requestHeader); if (res.getCode() == ResponseCode.SUCCESS) { @@ -134,18 +134,18 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand msgInner.setStoreTimestamp(result.getPrepareMessage().getStoreTimestamp()); RemotingCommand sendResult = sendFinalMessage(msgInner); if (sendResult.getCode() == ResponseCode.SUCCESS) { - this.brokerController.getTransactionMsgService().deletePrepareMessage(result.getPrepareMessage()); + this.brokerController.getTransactionalMessageService().deletePrepareMessage(result.getPrepareMessage()); } return sendResult; } return res; } } else if (MessageSysFlag.TRANSACTION_ROLLBACK_TYPE == requestHeader.getCommitOrRollback()) { - result = this.brokerController.getTransactionMsgService().rollbackMessage(requestHeader); + result = this.brokerController.getTransactionalMessageService().rollbackMessage(requestHeader); if (result.getResponseCode() == ResponseCode.SUCCESS) { RemotingCommand res = checkPrepareMessage(result.getPrepareMessage(), requestHeader); if (res.getCode() == ResponseCode.SUCCESS) { - this.brokerController.getTransactionMsgService().deletePrepareMessage(result.getPrepareMessage()); + this.brokerController.getTransactionalMessageService().deletePrepareMessage(result.getPrepareMessage()); } return res; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java index 64bbe05338a..b7e7a6187b7 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java @@ -356,7 +356,7 @@ private RemotingCommand sendMessage(final ChannelHandlerContext ctx, + "] sending transaction message is forbidden"); return response; } - putMessageResult = this.brokerController.getTransactionMsgService().prepareMessage(msgInner); + putMessageResult = this.brokerController.getTransactionalMessageService().prepareMessage(msgInner); } else { putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionCheckListener.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java similarity index 86% rename from broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionCheckListener.java rename to broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java index 2a3b3d71bf8..b9b48dac4d0 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionCheckListener.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java @@ -22,8 +22,8 @@ import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ExecutorService; @@ -31,8 +31,8 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -public abstract class AbstractTransactionCheckListener { - private static final Logger log = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); +public abstract class AbstractTransactionalMessageCheckListener { + private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); private BrokerController brokerController; @@ -45,9 +45,9 @@ public Thread newThread(Runnable r) { } }); - public AbstractTransactionCheckListener() {} + public AbstractTransactionalMessageCheckListener() {} - public AbstractTransactionCheckListener(BrokerController brokerController) { + public AbstractTransactionalMessageCheckListener(BrokerController brokerController) { this.brokerController = brokerController; } @@ -66,7 +66,7 @@ public void sendCheckMessage(MessageExt msgExt) throws Exception { if (channel != null) { brokerController.getBroker2Client().checkProducerTransactionState(groupId, channel, checkTransactionStateRequestHeader, msgExt); } else { - log.warn("Check transaction failed, channel is null. groupId={}", groupId); + LOGGER.warn("Check transaction failed, channel is null. groupId={}", groupId); } } @@ -77,7 +77,7 @@ public void run() { try { sendCheckMessage(msgExt); } catch (Exception e) { - log.error("Send check message error!", e); + LOGGER.error("Send check message error!", e); } } }); @@ -101,7 +101,7 @@ public void setBrokerController(BrokerController brokerController) { } /** - * In order to avoid check back unlimited, we will discard this message after checked too many times + * In order to avoid check back unlimited, we will discard the message that have been checked more than a certain number of times. * * @param msgExt Message to be discarded. */ diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionMsgCheckService.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java similarity index 73% rename from broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionMsgCheckService.java rename to broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java index 7f34eb95e5e..87c10310f32 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionMsgCheckService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java @@ -19,19 +19,19 @@ import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.ServiceThread; import org.apache.rocketmq.common.constant.LoggerName; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import java.util.concurrent.atomic.AtomicBoolean; -public class TransactionMsgCheckService extends ServiceThread { - private static final Logger log = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); +public class TransactionalMessageCheckService extends ServiceThread { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); private BrokerController brokerController; private final AtomicBoolean started = new AtomicBoolean(false); - public TransactionMsgCheckService(BrokerController brokerController) { + public TransactionalMessageCheckService(BrokerController brokerController) { this.brokerController = brokerController; } @@ -39,7 +39,7 @@ public TransactionMsgCheckService(BrokerController brokerController) { public void start() { if (started.compareAndSet(false, true)) { super.start(); - this.brokerController.getTransactionMsgService().open(); + this.brokerController.getTransactionalMessageService().open(); } } @@ -47,14 +47,14 @@ public void start() { public void shutdown(boolean interrupt) { if (started.compareAndSet(true, false)) { super.shutdown(interrupt); - this.brokerController.getTransactionMsgService().close(); - this.brokerController.getTransactionCheckListener().shutDown(); + this.brokerController.getTransactionalMessageService().close(); + this.brokerController.getTransactionalMessageCheckListener().shutDown(); } } @Override public String getServiceName() { - return TransactionMsgCheckService.class.getSimpleName(); + return TransactionalMessageCheckService.class.getSimpleName(); } @Override @@ -67,7 +67,7 @@ public void run() { while (!this.isStopped()) { try { Thread.sleep(checkInterval); - this.brokerController.getTransactionMsgService().check(timeout, checkMax, this.brokerController.getTransactionCheckListener()); + this.brokerController.getTransactionalMessageService().check(timeout, checkMax, this.brokerController.getTransactionalMessageCheckListener()); } catch (Exception e) { log.error("", e); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionMsgService.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageService.java similarity index 96% rename from broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionMsgService.java rename to broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageService.java index 77462a6b60a..7e1401a1f7b 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionMsgService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageService.java @@ -21,7 +21,7 @@ import org.apache.rocketmq.store.MessageExtBrokerInner; import org.apache.rocketmq.store.PutMessageResult; -public interface TransactionMsgService { +public interface TransactionalMessageService { /** * Process prepare message, in common, we should put this message to storage service. @@ -61,7 +61,7 @@ public interface TransactionMsgService { * @param transactionCheckMax The maximum number of times the message was checked, if exceed this value, this message will be discarded. * @param listener When the message is considered to be checked or discarded, the relative method of this class will be invoked. */ - void check(long transactionTimeout, int transactionCheckMax, AbstractTransactionCheckListener listener); + void check(long transactionTimeout, int transactionCheckMax, AbstractTransactionalMessageCheckListener listener); /** * Open transaction service. diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/DefaultAbstractTransactionCheckListener.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/DefaultTransactionalMessageCheckListener.java similarity index 74% rename from broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/DefaultAbstractTransactionCheckListener.java rename to broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/DefaultTransactionalMessageCheckListener.java index a3bc1a0b800..529bfe4f349 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/DefaultAbstractTransactionCheckListener.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/DefaultTransactionalMessageCheckListener.java @@ -16,16 +16,16 @@ */ package org.apache.rocketmq.broker.transaction.queue; -import org.apache.rocketmq.broker.transaction.AbstractTransactionCheckListener; +import org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.message.MessageExt; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; -public class DefaultAbstractTransactionCheckListener extends AbstractTransactionCheckListener { - private static final Logger log = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); +public class DefaultTransactionalMessageCheckListener extends AbstractTransactionalMessageCheckListener { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); - public DefaultAbstractTransactionCheckListener() { + public DefaultTransactionalMessageCheckListener() { super(); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionBridge.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java similarity index 89% rename from broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionBridge.java rename to broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java index 8330d32469f..71fdd206784 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionBridge.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java @@ -31,14 +31,14 @@ import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.logging.InnerLoggerFactory; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.store.GetMessageResult; import org.apache.rocketmq.store.MessageExtBrokerInner; import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageStatus; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.net.InetSocketAddress; import java.net.SocketAddress; @@ -49,15 +49,15 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -public class TransactionBridge { - private static Logger logger = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); +public class TransactionalMessageBridge { + private static final InternalLogger LOGGER = InnerLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); private final ConcurrentHashMap opQueueMap = new ConcurrentHashMap<>(); private final BrokerController brokerController; private final MessageStore store; private final SocketAddress storeHost; - public TransactionBridge(BrokerController brokerController, MessageStore store) { + public TransactionalMessageBridge(BrokerController brokerController, MessageStore store) { try { this.brokerController = brokerController; this.store = store; @@ -65,14 +65,14 @@ public TransactionBridge(BrokerController brokerController, MessageStore store) new InetSocketAddress(brokerController.getBrokerConfig().getBrokerIP1(), brokerController.getNettyServerConfig().getListenPort()); } catch (Exception e) { - logger.error("Init TransactionBridge error", e); + LOGGER.error("Init TransactionBridge error", e); throw new RuntimeException(e); } } public long fetchConsumeOffset(MessageQueue mq) { - long offset = brokerController.getConsumerOffsetManager().queryOffset(TransactionUtil.buildConsumerGroup(), + long offset = brokerController.getConsumerOffsetManager().queryOffset(TransactionalMessageUtil.buildConsumerGroup(), mq.getTopic(), mq.getQueueId()); if (offset == -1) { offset = store.getMinOffsetInQueue(mq.getTopic(), mq.getQueueId()); @@ -97,20 +97,20 @@ public Set fetchMessageQueues(String topic) { public void updateConsumeOffset(MessageQueue mq, long offset) { this.brokerController.getConsumerOffsetManager().commitOffset( - RemotingHelper.parseSocketAddressAddr(this.storeHost), TransactionUtil.buildConsumerGroup(), mq.getTopic(), + RemotingHelper.parseSocketAddressAddr(this.storeHost), TransactionalMessageUtil.buildConsumerGroup(), mq.getTopic(), mq.getQueueId(), offset); } public PullResult getHalfMessage(int queueId, long offset, int nums) { - String group = TransactionUtil.buildConsumerGroup(); - String topic = TransactionUtil.buildHalfTopic(); + String group = TransactionalMessageUtil.buildConsumerGroup(); + String topic = TransactionalMessageUtil.buildHalfTopic(); SubscriptionData sub = new SubscriptionData(topic, "*"); return getMessage(group, topic, queueId, offset, nums, sub); } public PullResult getOpMessage(int queueId, long offset, int nums) { - String group = TransactionUtil.buildConsumerGroup(); - String topic = TransactionUtil.buildOpTopic(); + String group = TransactionalMessageUtil.buildConsumerGroup(); + String topic = TransactionalMessageUtil.buildOpTopic(); SubscriptionData sub = new SubscriptionData(topic, "*"); return getMessage(group, topic, queueId, offset, nums, sub); } @@ -136,12 +136,12 @@ private PullResult getMessage(String group, String topic, int queueId, long offs break; case NO_MATCHED_MESSAGE: pullStatus = PullStatus.NO_MATCHED_MSG; - logger.warn("No matched message. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}", + LOGGER.warn("No matched message. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}", getMessageResult.getStatus(), topic, group, offset); break; case NO_MESSAGE_IN_QUEUE: pullStatus = PullStatus.NO_NEW_MSG; - logger.warn("No new message. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}", + LOGGER.warn("No new message. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}", getMessageResult.getStatus(), topic, group, offset); break; case MESSAGE_WAS_REMOVING: @@ -151,7 +151,7 @@ private PullResult getMessage(String group, String topic, int queueId, long offs case OFFSET_OVERFLOW_ONE: case OFFSET_TOO_SMALL: pullStatus = PullStatus.OFFSET_ILLEGAL; - logger.warn("Offset illegal. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}", + LOGGER.warn("Offset illegal. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}", getMessageResult.getStatus(), topic, group, offset); break; default: @@ -163,7 +163,7 @@ private PullResult getMessage(String group, String topic, int queueId, long offs getMessageResult.getMaxOffset(), foundList); } else { - logger.error("Get message from store return null. topic={}, groupId={}, requestOffset={}", topic, group, + LOGGER.error("Get message from store return null. topic={}, groupId={}, requestOffset={}", topic, group, offset); return null; } @@ -195,7 +195,7 @@ private MessageExtBrokerInner parseHalfMessageInner(MessageExtBrokerInner msgInn String.valueOf(msgInner.getQueueId())); msgInner.setSysFlag( MessageSysFlag.resetTransactionValue(msgInner.getSysFlag(), MessageSysFlag.TRANSACTION_NOT_TYPE)); - msgInner.setTopic(TransactionUtil.buildHalfTopic()); + msgInner.setTopic(TransactionalMessageUtil.buildHalfTopic()); msgInner.setQueueId(0); msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); return msgInner; @@ -204,14 +204,14 @@ private MessageExtBrokerInner parseHalfMessageInner(MessageExtBrokerInner msgInn public boolean putOpMessage(MessageExt messageExt, String opType) { MessageQueue messageQueue = new MessageQueue(messageExt.getTopic(), this.brokerController.getBrokerConfig().getBrokerName(), messageExt.getQueueId()); - if (TransactionUtil.REMOVETAG.equals(opType)) { + if (TransactionalMessageUtil.REMOVETAG.equals(opType)) { return addRemoveTagInTransactionOp(messageExt, messageQueue); } return true; } public PutMessageResult putMessageReturnResult(MessageExtBrokerInner messageInner) { - logger.debug("[BUG-TO-FIX] Thread:{} msgID:{}", Thread.currentThread().getName(), messageInner.getMsgId()); + LOGGER.debug("[BUG-TO-FIX] Thread:{} msgID:{}", Thread.currentThread().getName(), messageInner.getMsgId()); return store.putMessage(messageInner); } @@ -221,7 +221,7 @@ public boolean putMessage(MessageExtBrokerInner messageInner) { && putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK) { return true; } else { - logger.error("Put message failed, topic: {}, queueId: {}, msgId: {}", + LOGGER.error("Put message failed, topic: {}, queueId: {}, msgId: {}", messageInner.getTopic(), messageInner.getQueueId(), messageInner.getMsgId()); return false; } @@ -297,8 +297,8 @@ private TopicConfig selectTopicConfig(String topic) { * @return This method will always return true. */ private boolean addRemoveTagInTransactionOp(MessageExt messageExt, MessageQueue messageQueue) { - Message message = new Message(TransactionUtil.buildOpTopic(), TransactionUtil.REMOVETAG, - String.valueOf(messageExt.getQueueOffset()).getBytes(TransactionUtil.charset)); + Message message = new Message(TransactionalMessageUtil.buildOpTopic(), TransactionalMessageUtil.REMOVETAG, + String.valueOf(messageExt.getQueueOffset()).getBytes(TransactionalMessageUtil.charset)); writeOp(message, messageQueue); return true; } @@ -315,18 +315,14 @@ private void writeOp(Message message, MessageQueue mq) { } } if (opQueue == null) { - opQueue = new MessageQueue(TransactionUtil.buildOpTopic(), mq.getBrokerName(), mq.getQueueId()); - } - if (logger.isDebugEnabled()) { - logger.debug(opQueue + "---------Op msg " + message.getTags() + "---" + new String(message.getBody(), - TransactionUtil.charset)); + opQueue = new MessageQueue(TransactionalMessageUtil.buildOpTopic(), mq.getBrokerName(), mq.getQueueId()); } putMessage(makeOpMessageInner(message, opQueue)); } private MessageQueue getOpQueueByHalf(MessageQueue halfMQ) { MessageQueue opQueue = new MessageQueue(); - opQueue.setTopic(TransactionUtil.buildOpTopic()); + opQueue.setTopic(TransactionalMessageUtil.buildOpTopic()); opQueue.setBrokerName(halfMQ.getBrokerName()); opQueue.setQueueId(halfMQ.getQueueId()); return opQueue; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/QueueTransactionMsgServiceImpl.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java similarity index 95% rename from broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/QueueTransactionMsgServiceImpl.java rename to broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java index ac7155eab9a..ee1cab28eab 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/QueueTransactionMsgServiceImpl.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java @@ -16,9 +16,9 @@ */ package org.apache.rocketmq.broker.transaction.queue; -import org.apache.rocketmq.broker.transaction.AbstractTransactionCheckListener; +import org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener; import org.apache.rocketmq.broker.transaction.OperationResult; -import org.apache.rocketmq.broker.transaction.TransactionMsgService; +import org.apache.rocketmq.broker.transaction.TransactionalMessageService; import org.apache.rocketmq.client.consumer.PullResult; import org.apache.rocketmq.client.consumer.PullStatus; import org.apache.rocketmq.common.MixAll; @@ -28,11 +28,11 @@ import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.store.MessageExtBrokerInner; import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageStatus; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Collections; @@ -42,10 +42,10 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -public class QueueTransactionMsgServiceImpl implements TransactionMsgService { - private static final Logger log = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); +public class TransactionalMessageServiceImpl implements TransactionalMessageService { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); - private TransactionBridge transactionBridge; + private TransactionalMessageBridge transactionBridge; private static final int TRY_PULL_MSG_NUMBER = 1; @@ -53,7 +53,7 @@ public class QueueTransactionMsgServiceImpl implements TransactionMsgService { private static final int MAX_RETRY_COUNT_WHEN_HALF_NULL = 1; - public QueueTransactionMsgServiceImpl(TransactionBridge transactionBridge) { + public TransactionalMessageServiceImpl(TransactionalMessageBridge transactionBridge) { this.transactionBridge = transactionBridge; } @@ -84,7 +84,7 @@ private boolean isNeedSkip(MessageExt msgExt) { if (valueOfCurrentMinusBorn > transactionBridge.getBrokerController().getMessageStoreConfig().getFileReservedTime() * 3600L * 1000) { - log.info("Half message status too log ,so skip it.messageId {},bornTime {}", + log.info("Half message exceed file reserved time ,so skip it.messageId {},bornTime {}", msgExt.getMsgId(), msgExt.getBornTimestamp()); return true; } @@ -118,7 +118,7 @@ private boolean putBackHalfMsgQueue(MessageExt msgExt, long offset) { } @Override - public void check(long transactionTimeout, int transactionCheckMax, AbstractTransactionCheckListener listener) { + public void check(long transactionTimeout, int transactionCheckMax, AbstractTransactionalMessageCheckListener listener) { try { String topic = MixAll.RMQ_SYS_TRANS_HALF_TOPIC; Set msgQueues = transactionBridge.fetchMessageQueues(topic); @@ -290,10 +290,10 @@ private PullResult fillOpRemoveMap(HashMap removeMap, return pullResult; } for (MessageExt opMessageExt : opMsg) { - Long queueOffset = getLong(new String(opMessageExt.getBody(), TransactionUtil.charset)); + Long queueOffset = getLong(new String(opMessageExt.getBody(), TransactionalMessageUtil.charset)); log.info("Topic: {} tags: {}, OpOffset: {}, HalfOffset: {}", opMessageExt.getTopic(), opMessageExt.getTags(), opMessageExt.getQueueOffset(), queueOffset); - if (TransactionUtil.REMOVETAG.equals(opMessageExt.getTags())) { + if (TransactionalMessageUtil.REMOVETAG.equals(opMessageExt.getTags())) { if (queueOffset < miniOffset) { doneOpOffset.add(opMessageExt.getQueueOffset()); } else { @@ -303,10 +303,8 @@ private PullResult fillOpRemoveMap(HashMap removeMap, log.error("Found a illegal tag in opMessageExt= {} ", opMessageExt); } } - if (log.isDebugEnabled()) { - log.debug("Remove map: {}" + removeMap); - log.debug("Done op list: {}" + doneOpOffset); - } + log.debug("Remove map: {}", removeMap); + log.debug("Done op list: {}", doneOpOffset); return pullResult; } @@ -430,7 +428,7 @@ private long calculateOpOffset(List doneOffset, long oldOffset) { private MessageQueue getOpQueue(MessageQueue messageQueue) { MessageQueue opQueue = opQueueMap.get(messageQueue); if (opQueue == null) { - opQueue = new MessageQueue(TransactionUtil.buildOpTopic(), messageQueue.getBrokerName(), + opQueue = new MessageQueue(TransactionalMessageUtil.buildOpTopic(), messageQueue.getBrokerName(), messageQueue.getQueueId()); opQueueMap.put(messageQueue, opQueue); } @@ -466,7 +464,7 @@ private OperationResult getHalfMessageByOffset(long commitLogOffset) { @Override public boolean deletePrepareMessage(MessageExt msgExt) { - if (this.transactionBridge.putOpMessage(msgExt, TransactionUtil.REMOVETAG)) { + if (this.transactionBridge.putOpMessage(msgExt, TransactionalMessageUtil.REMOVETAG)) { log.info("Transaction op message write successfully. messageId={}, queueId={} msgExt:{}", msgExt.getMsgId(), msgExt.getQueueId(), msgExt); return true; } else { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionUtil.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageUtil.java similarity index 97% rename from broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionUtil.java rename to broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageUtil.java index f0ea8c79455..a64dd76017e 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionUtil.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageUtil.java @@ -20,7 +20,7 @@ import java.nio.charset.Charset; -public class TransactionUtil { +public class TransactionalMessageUtil { public static final String REMOVETAG = "d"; public static Charset charset = Charset.forName("utf-8"); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/util/ServiceProvider.java b/broker/src/main/java/org/apache/rocketmq/broker/util/ServiceProvider.java index 74e3ddd9be5..59be7a7e26c 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/util/ServiceProvider.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/util/ServiceProvider.java @@ -30,9 +30,9 @@ public class ServiceProvider { /** * JDK1.3+ 'Service Provider' specification. */ - public static final String TRANSACTION_SERVICE_ID = "META-INF/service/org.apache.rocketmq.broker.transaction.TransactionMsgService"; + public static final String TRANSACTION_SERVICE_ID = "META-INF/service/org.apache.rocketmq.broker.transaction.TransactionalMessageService"; - public static final String TRANSACTION_LISTENER_ID = "META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionCheckListener"; + public static final String TRANSACTION_LISTENER_ID = "META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener"; static { thisClassLoader = getClassLoader(ServiceProvider.class); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/EndTransactionProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/EndTransactionProcessorTest.java index 0a2827d680a..019cc6a316c 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/EndTransactionProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/EndTransactionProcessorTest.java @@ -19,7 +19,7 @@ import io.netty.channel.ChannelHandlerContext; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.transaction.OperationResult; -import org.apache.rocketmq.broker.transaction.TransactionMsgService; +import org.apache.rocketmq.broker.transaction.TransactionalMessageService; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; @@ -67,12 +67,12 @@ public class EndTransactionProcessorTest { private MessageStore messageStore; @Mock - private TransactionMsgService transactionMsgService; + private TransactionalMessageService transactionMsgService; @Before public void init() { brokerController.setMessageStore(messageStore); - brokerController.setTransactionMsgService(transactionMsgService); + brokerController.setTransactionalMessageService(transactionMsgService); endTransactionProcessor = new EndTransactionProcessor(brokerController); } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java index 566da9692b1..a4da9518f08 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java @@ -18,15 +18,10 @@ import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.mqtrace.SendMessageContext; import org.apache.rocketmq.broker.mqtrace.SendMessageHook; -import org.apache.rocketmq.broker.transaction.TransactionMsgService; +import org.apache.rocketmq.broker.transaction.TransactionalMessageService; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.message.MessageConst; @@ -57,6 +52,11 @@ import org.mockito.junit.MockitoJUnitRunner; import org.mockito.stubbing.Answer; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; @@ -75,7 +75,7 @@ public class SendMessageProcessorTest { private MessageStore messageStore; @Mock - private TransactionMsgService transactionMsgService; + private TransactionalMessageService transactionMsgService; private String topic = "FooBar"; private String group = "FooBarGroup"; @@ -188,8 +188,8 @@ public void testProcessRequest_WithMsgBack() throws RemotingCommandException { @Test public void testProcessRequest_Transaction() throws RemotingCommandException { - brokerController.setTransactionMsgService(transactionMsgService); - when(brokerController.getTransactionMsgService().prepareMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + brokerController.setTransactionalMessageService(transactionMsgService); + when(brokerController.getTransactionalMessageService().prepareMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); RemotingCommand request = createSendTransactionMsgCommand(RequestCode.SEND_MESSAGE); final RemotingCommand[] response = new RemotingCommand[1]; doAnswer(new Answer() { diff --git a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/DefaultAbstractTransactionCheckListenerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/DefaultTransactionalMessageCheckListenerTest.java similarity index 93% rename from broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/DefaultAbstractTransactionCheckListenerTest.java rename to broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/DefaultTransactionalMessageCheckListenerTest.java index 7b33e94f6bf..17bf00b4310 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/DefaultAbstractTransactionCheckListenerTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/DefaultTransactionalMessageCheckListenerTest.java @@ -32,9 +32,9 @@ import org.mockito.junit.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) -public class DefaultAbstractTransactionCheckListenerTest { +public class DefaultTransactionalMessageCheckListenerTest { - private DefaultAbstractTransactionCheckListener listener; + private DefaultTransactionalMessageCheckListener listener; @Spy private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), @@ -43,7 +43,7 @@ public class DefaultAbstractTransactionCheckListenerTest { @Before public void init() { - listener = new DefaultAbstractTransactionCheckListener(); + listener = new DefaultTransactionalMessageCheckListener(); listener.setBrokerController(brokerController); } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionBridgeTest.java b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java similarity index 93% rename from broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionBridgeTest.java rename to broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java index 940b57ae6a2..b1c669c9933 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionBridgeTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java @@ -56,9 +56,9 @@ import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) -public class TransactionBridgeTest { +public class TransactionalMessageBridgeTest { - private TransactionBridge transactionBridge; + private TransactionalMessageBridge transactionBridge; @Spy private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), @@ -70,12 +70,12 @@ public class TransactionBridgeTest { @Before public void init() { brokerController.setMessageStore(messageStore); - transactionBridge = new TransactionBridge(brokerController, messageStore); + transactionBridge = new TransactionalMessageBridge(brokerController, messageStore); } @Test public void testPutOpMessage() { - boolean isSuccess = transactionBridge.putOpMessage(createMessageBrokerInner(), TransactionUtil.REMOVETAG); + boolean isSuccess = transactionBridge.putOpMessage(createMessageBrokerInner(), TransactionalMessageUtil.REMOVETAG); assertThat(isSuccess).isTrue(); } @@ -95,7 +95,7 @@ public void testFetchMessageQueues() { @Test public void testFetchConsumeOffset() { - MessageQueue mq = new MessageQueue(TransactionUtil.buildOpTopic(), this.brokerController.getBrokerConfig().getBrokerName(), + MessageQueue mq = new MessageQueue(TransactionalMessageUtil.buildOpTopic(), this.brokerController.getBrokerConfig().getBrokerName(), 0); long offset = transactionBridge.fetchConsumeOffset(mq); assertThat(offset).isGreaterThan(-1); @@ -103,7 +103,7 @@ public void testFetchConsumeOffset() { @Test public void updateConsumeOffset() { - MessageQueue mq = new MessageQueue(TransactionUtil.buildOpTopic(), this.brokerController.getBrokerConfig().getBrokerName(), + MessageQueue mq = new MessageQueue(TransactionalMessageUtil.buildOpTopic(), this.brokerController.getBrokerConfig().getBrokerName(), 0); transactionBridge.updateConsumeOffset(mq, 0); } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/QueueTransactionMsgServiceImplTest.java b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImplTest.java similarity index 95% rename from broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/QueueTransactionMsgServiceImplTest.java rename to broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImplTest.java index 595fb57b000..47eccbe026e 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/QueueTransactionMsgServiceImplTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImplTest.java @@ -17,7 +17,9 @@ package org.apache.rocketmq.broker.transaction.queue; import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener; import org.apache.rocketmq.broker.transaction.OperationResult; +import org.apache.rocketmq.broker.transaction.TransactionalMessageService; import org.apache.rocketmq.client.consumer.PullResult; import org.apache.rocketmq.client.consumer.PullStatus; import org.apache.rocketmq.common.BrokerConfig; @@ -60,24 +62,24 @@ import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) -public class QueueTransactionMsgServiceImplTest { +public class TransactionalMessageServiceImplTest { - private QueueTransactionMsgServiceImpl queueTransactionMsgService; + private TransactionalMessageService queueTransactionMsgService; @Mock - private TransactionBridge bridge; + private TransactionalMessageBridge bridge; @Spy private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig()); @Mock - private DefaultAbstractTransactionCheckListener listener; + private AbstractTransactionalMessageCheckListener listener; @Before public void init() { listener.setBrokerController(brokerController); - queueTransactionMsgService = new QueueTransactionMsgServiceImpl(bridge); + queueTransactionMsgService = new TransactionalMessageServiceImpl(bridge); brokerController.getMessageStoreConfig().setFileReservedTime(3); } @@ -187,7 +189,7 @@ private PullResult createOpPulResult(String topic, long queueOffset, String body PullResult result = createPullResult(topic, queueOffset, body, size); List msgs = result.getMsgFoundList(); for (MessageExt msg : msgs) { - msg.setTags(TransactionUtil.REMOVETAG); + msg.setTags(TransactionalMessageUtil.REMOVETAG); } return result; } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/util/LogTransactionCheckListener.java b/broker/src/test/java/org/apache/rocketmq/broker/util/LogTransactionalMessageCheckListener.java similarity index 89% rename from broker/src/test/java/org/apache/rocketmq/broker/util/LogTransactionCheckListener.java rename to broker/src/test/java/org/apache/rocketmq/broker/util/LogTransactionalMessageCheckListener.java index d5adb886533..3a2098eabd1 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/util/LogTransactionCheckListener.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/util/LogTransactionalMessageCheckListener.java @@ -16,10 +16,10 @@ */ package org.apache.rocketmq.broker.util; -import org.apache.rocketmq.broker.transaction.AbstractTransactionCheckListener; +import org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener; import org.apache.rocketmq.common.message.MessageExt; -public class LogTransactionCheckListener extends AbstractTransactionCheckListener { +public class LogTransactionalMessageCheckListener extends AbstractTransactionalMessageCheckListener { @Override public void resolveDiscardMsg(MessageExt msgExt) { diff --git a/broker/src/test/java/org/apache/rocketmq/broker/util/ServiceProviderTest.java b/broker/src/test/java/org/apache/rocketmq/broker/util/ServiceProviderTest.java index c739db5d865..22228a6e0ef 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/util/ServiceProviderTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/util/ServiceProviderTest.java @@ -17,8 +17,8 @@ package org.apache.rocketmq.broker.util; -import org.apache.rocketmq.broker.transaction.AbstractTransactionCheckListener; -import org.apache.rocketmq.broker.transaction.TransactionMsgService; +import org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener; +import org.apache.rocketmq.broker.transaction.TransactionalMessageService; import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -27,15 +27,15 @@ public class ServiceProviderTest { @Test public void loadTransactionMsgServiceTest() { - TransactionMsgService transactionService = ServiceProvider.loadClass(ServiceProvider.TRANSACTION_SERVICE_ID, - TransactionMsgService.class); + TransactionalMessageService transactionService = ServiceProvider.loadClass(ServiceProvider.TRANSACTION_SERVICE_ID, + TransactionalMessageService.class); assertThat(transactionService).isNotNull(); } @Test public void loadAbstractTransactionListenerTest() { - AbstractTransactionCheckListener listener = ServiceProvider.loadClass(ServiceProvider.TRANSACTION_LISTENER_ID, - AbstractTransactionCheckListener.class); + AbstractTransactionalMessageCheckListener listener = ServiceProvider.loadClass(ServiceProvider.TRANSACTION_LISTENER_ID, + AbstractTransactionalMessageCheckListener.class); assertThat(listener).isNotNull(); } } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/util/TransactionMsgServiceImpl.java b/broker/src/test/java/org/apache/rocketmq/broker/util/TransactionalMessageServiceImpl.java similarity index 80% rename from broker/src/test/java/org/apache/rocketmq/broker/util/TransactionMsgServiceImpl.java rename to broker/src/test/java/org/apache/rocketmq/broker/util/TransactionalMessageServiceImpl.java index 76172a277da..3fedf7722ee 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/util/TransactionMsgServiceImpl.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/util/TransactionalMessageServiceImpl.java @@ -16,19 +16,19 @@ */ package org.apache.rocketmq.broker.util; -import org.apache.rocketmq.broker.transaction.AbstractTransactionCheckListener; +import org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener; import org.apache.rocketmq.broker.transaction.OperationResult; -import org.apache.rocketmq.broker.transaction.TransactionMsgService; +import org.apache.rocketmq.broker.transaction.TransactionalMessageService; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.store.MessageExtBrokerInner; import org.apache.rocketmq.store.PutMessageResult; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -public class TransactionMsgServiceImpl implements TransactionMsgService { - private static final Logger log = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); +public class TransactionalMessageServiceImpl implements TransactionalMessageService { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); @Override public PutMessageResult prepareMessage(MessageExtBrokerInner messageInner) { @@ -51,7 +51,7 @@ public OperationResult rollbackMessage(EndTransactionRequestHeader requestHeader } @Override - public void check(long transactionTimeout, int transactionCheckMax, AbstractTransactionCheckListener listener) { + public void check(long transactionTimeout, int transactionCheckMax, AbstractTransactionalMessageCheckListener listener) { log.warn("check check!"); } diff --git a/broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionCheckListener b/broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionCheckListener deleted file mode 100644 index 0a88c133ee8..00000000000 --- a/broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionCheckListener +++ /dev/null @@ -1 +0,0 @@ -org.apache.rocketmq.broker.util.LogTransactionCheckListener \ No newline at end of file diff --git a/broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener b/broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener new file mode 100644 index 00000000000..455b266b9f1 --- /dev/null +++ b/broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener @@ -0,0 +1 @@ +org.apache.rocketmq.broker.util.LogTransactionalMessageCheckListener \ No newline at end of file diff --git a/broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.TransactionMsgService b/broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.TransactionMsgService deleted file mode 100644 index 01ee14278fc..00000000000 --- a/broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.TransactionMsgService +++ /dev/null @@ -1 +0,0 @@ -org.apache.rocketmq.broker.util.TransactionMsgServiceImpl \ No newline at end of file diff --git a/broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.TransactionalMessageService b/broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.TransactionalMessageService new file mode 100644 index 00000000000..b012e1451c7 --- /dev/null +++ b/broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.TransactionalMessageService @@ -0,0 +1 @@ +org.apache.rocketmq.broker.util.TransactionalMessageServiceImpl \ No newline at end of file diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/LocalTransactionExecuter.java b/client/src/main/java/org/apache/rocketmq/client/producer/LocalTransactionExecuter.java deleted file mode 100644 index 3e72088bf48..00000000000 --- a/client/src/main/java/org/apache/rocketmq/client/producer/LocalTransactionExecuter.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.client.producer; - -import org.apache.rocketmq.common.message.Message; -@Deprecated -public interface LocalTransactionExecuter { - LocalTransactionState executeLocalTransactionBranch(final Message msg, final Object arg); -} diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/TransactionCheckListener.java b/client/src/main/java/org/apache/rocketmq/client/producer/TransactionCheckListener.java deleted file mode 100644 index 420ed621d6f..00000000000 --- a/client/src/main/java/org/apache/rocketmq/client/producer/TransactionCheckListener.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.client.producer; - -import org.apache.rocketmq.common.message.MessageExt; -@Deprecated -public interface TransactionCheckListener { - LocalTransactionState checkLocalTransactionState(final MessageExt msg); -} diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java index 9cb05c2b185..2d8a5fead37 100644 --- a/example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java +++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java @@ -37,14 +37,14 @@ public class TransactionProducer { private static int threadCount; private static int messageSize; - private static boolean ischeck; - private static boolean ischeckffalse; + private static boolean isCheck; + private static boolean isCheckFalse; public static void main(String[] args) throws MQClientException, UnsupportedEncodingException { threadCount = args.length >= 1 ? Integer.parseInt(args[0]) : 32; messageSize = args.length >= 2 ? Integer.parseInt(args[1]) : 1024 * 2; - ischeck = args.length >= 3 && Boolean.parseBoolean(args[2]); - ischeckffalse = args.length >= 4 && Boolean.parseBoolean(args[3]); + isCheck = args.length >= 3 && Boolean.parseBoolean(args[2]); + isCheckFalse = args.length >= 4 && Boolean.parseBoolean(args[3]); final Message msg = buildMessage(messageSize); @@ -93,7 +93,7 @@ public void run() { }, 10000, 10000); final TransactionListener transactionListener = - new TransactionListenerImpl(ischeckffalse, ischeck, statsBenchmark); + new TransactionListenerImpl(isCheckFalse, isCheck, statsBenchmark); final TransactionMQProducer producer = new TransactionMQProducer("benchmark_transaction_producer"); producer.setInstanceName(Long.toString(System.currentTimeMillis())); producer.setTransactionListener(transactionListener); @@ -150,39 +150,23 @@ private static Message buildMessage(final int messageSize) throws UnsupportedEnc } } -//class TransactionExecuterBImpl implements LocalTransactionExecuter { -// -// private boolean ischeckLocal; -// -// public TransactionExecuterBImpl(boolean ischeck) { -// this.ischeck = ischeck; -// } -// -// @Override -// public LocalTransactionState executeLocalTransactionBranch(final Message msg, final Object arg) { -// if (ischeck) { -// return LocalTransactionState.UNKNOW; -// } -// return LocalTransactionState.COMMIT_MESSAGE; -// } -//} class TransactionListenerImpl implements TransactionListener { - private boolean ischeckffalse; + private boolean isCheckFalse; private StatsBenchmarkTProducer statsBenchmarkTProducer; - private boolean ischeckLocal; + private boolean isCheckLocal; - public TransactionListenerImpl(boolean ischeckffalse, boolean isCheckLocal, + public TransactionListenerImpl(boolean isCheckFalse, boolean isCheckLocal, StatsBenchmarkTProducer statsBenchmarkTProducer) { - this.ischeckffalse = ischeckffalse; - this.ischeckLocal = isCheckLocal; + this.isCheckFalse = isCheckFalse; + this.isCheckLocal = isCheckLocal; this.statsBenchmarkTProducer = statsBenchmarkTProducer; } @Override public LocalTransactionState checkLocalTransaction(MessageExt msg) { statsBenchmarkTProducer.getCheckRequestSuccessCount().incrementAndGet(); - if (ischeckffalse) { + if (isCheckFalse) { return LocalTransactionState.ROLLBACK_MESSAGE; } @@ -192,7 +176,7 @@ public LocalTransactionState checkLocalTransaction(MessageExt msg) { @Override public LocalTransactionState executeLocalTransaction(final Message msg, final Object arg) { - if (ischeckLocal) { + if (isCheckLocal) { return LocalTransactionState.UNKNOW; } return LocalTransactionState.COMMIT_MESSAGE; From aeca8a1cc5795c9f68b703180dbade654352adc7 Mon Sep 17 00:00:00 2001 From: duhengforever Date: Thu, 12 Jul 2018 08:44:51 +0800 Subject: [PATCH 3/6] Format code with RMQ style --- .../rocketmq/broker/BrokerController.java | 27 +++++++++-------- ...ractTransactionalMessageCheckListener.java | 6 ++-- .../broker/transaction/TransactionRecord.java | 4 +++ .../broker/transaction/TransactionStore.java | 4 +++ .../TransactionalMessageService.java | 12 +++++--- .../queue/TransactionalMessageBridge.java | 9 +++--- .../TransactionalMessageServiceImpl.java | 30 ++++++++++--------- .../queue/TransactionalMessageUtil.java | 1 - .../apache/rocketmq/common/BrokerConfig.java | 5 ++-- 9 files changed, 58 insertions(+), 40 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index af2781f9c64..8a33aa23f1a 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -226,7 +226,7 @@ public boolean initialize() throws CloneNotSupportedException { this.messageStore = new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig); - this.brokerStats = new BrokerStats((DefaultMessageStore)this.messageStore); + this.brokerStats = new BrokerStats((DefaultMessageStore) this.messageStore); //load plugin MessageStorePluginContext context = new MessageStorePluginContext(messageStoreConfig, brokerStatsManager, messageArrivingListener, brokerConfig); this.messageStore = MessageStoreFactory.build(context, this.messageStore); @@ -241,7 +241,7 @@ public boolean initialize() throws CloneNotSupportedException { if (result) { this.remotingServer = new NettyRemotingServer(this.nettyServerConfig, this.clientHousekeepingService); - NettyServerConfig fastConfig = (NettyServerConfig)this.nettyServerConfig.clone(); + NettyServerConfig fastConfig = (NettyServerConfig) this.nettyServerConfig.clone(); fastConfig.setListenPort(nettyServerConfig.getListenPort() - 2); this.fastRemotingServer = new NettyRemotingServer(fastConfig, this.clientHousekeepingService); this.sendMessageExecutor = new BrokerFixedThreadPoolExecutor( @@ -437,8 +437,8 @@ public void onChanged(String path) { } private void reloadServerSslContext() { - ((NettyRemotingServer)remotingServer).loadSslContext(); - ((NettyRemotingServer)fastRemotingServer).loadSslContext(); + ((NettyRemotingServer) remotingServer).loadSslContext(); + ((NettyRemotingServer) fastRemotingServer).loadSslContext(); } }); } catch (Exception e) { @@ -567,7 +567,9 @@ public long headSlowTimeMills(BlockingQueue q) { slowTimeMills = rt == null ? 0 : this.messageStore.now() - rt.getCreateTimestamp(); } - if (slowTimeMills < 0) { slowTimeMills = 0; } + if (slowTimeMills < 0) { + slowTimeMills = 0; + } return slowTimeMills; } @@ -802,7 +804,7 @@ public void run() { if (BrokerRole.SLAVE != messageStoreConfig.getBrokerRole()) { if (this.transactionMsgCheckService != null) { - log.info("start transaction service!"); + log.info("Start transaction service!"); this.transactionMsgCheckService.start(); } } @@ -851,7 +853,7 @@ public synchronized void registerBrokerAll(final boolean checkOrderConfig, boole } private void doRegisterBrokerAll(boolean checkOrderConfig, boolean oneway, - TopicConfigSerializeWrapper topicConfigWrapper) { + TopicConfigSerializeWrapper topicConfigWrapper) { List registerBrokerResultList = this.brokerOuterAPI.registerBrokerAll( this.brokerConfig.getBrokerClusterName(), this.getBrokerAddr(), @@ -881,10 +883,10 @@ private void doRegisterBrokerAll(boolean checkOrderConfig, boolean oneway, } private boolean needRegister(final String clusterName, - final String brokerAddr, - final String brokerName, - final long brokerId, - final int timeoutMills) { + final String brokerAddr, + final String brokerName, + final long brokerId, + final int timeoutMills) { TopicConfigSerializeWrapper topicConfigWrapper = this.getTopicConfigManager().buildTopicConfigSerializeWrapper(); List changeList = brokerOuterAPI.needRegister(clusterName, brokerAddr, brokerName, brokerId, topicConfigWrapper, timeoutMills); @@ -1008,7 +1010,8 @@ public AbstractTransactionalMessageCheckListener getTransactionalMessageCheckLis return transactionalMessageCheckListener; } - public void setTransactionalMessageCheckListener(AbstractTransactionalMessageCheckListener transactionalMessageCheckListener) { + public void setTransactionalMessageCheckListener( + AbstractTransactionalMessageCheckListener transactionalMessageCheckListener) { this.transactionalMessageCheckListener = transactionalMessageCheckListener; } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java index b9b48dac4d0..659c6af16f0 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java @@ -45,7 +45,8 @@ public Thread newThread(Runnable r) { } }); - public AbstractTransactionalMessageCheckListener() {} + public AbstractTransactionalMessageCheckListener() { + } public AbstractTransactionalMessageCheckListener(BrokerController brokerController) { this.brokerController = brokerController; @@ -101,7 +102,8 @@ public void setBrokerController(BrokerController brokerController) { } /** - * In order to avoid check back unlimited, we will discard the message that have been checked more than a certain number of times. + * In order to avoid check back unlimited, we will discard the message that have been checked more than a certain + * number of times. * * @param msgExt Message to be discarded. */ diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionRecord.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionRecord.java index cebde4a4f87..772f08e52c7 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionRecord.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionRecord.java @@ -15,6 +15,10 @@ * limitations under the License. */ package org.apache.rocketmq.broker.transaction; + +/** + * This class will be removed in the version 4.4.0 and {@link OperationResult} class is recommended. + */ @Deprecated public class TransactionRecord { // Commit Log Offset diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionStore.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionStore.java index 4dcdd772e62..03e02273934 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionStore.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionStore.java @@ -18,6 +18,10 @@ package org.apache.rocketmq.broker.transaction; import java.util.List; + +/** + * This class will be removed in ther version 4.4.0, and {@link TransactionalMessageService} class is recommended. + */ @Deprecated public interface TransactionStore { boolean open(); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageService.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageService.java index 7e1401a1f7b..143909fd972 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageService.java @@ -55,11 +55,15 @@ public interface TransactionalMessageService { OperationResult rollbackMessage(EndTransactionRequestHeader requestHeader); /** - * Traverse uncommitted/unroll back half message and send check back request to producer to obtain transaction status. + * Traverse uncommitted/unroll back half message and send check back request to producer to obtain transaction + * status. * - * @param transactionTimeout The minimum time of the transactional message to be checked firstly, one message only exceed this time interval that can be checked. - * @param transactionCheckMax The maximum number of times the message was checked, if exceed this value, this message will be discarded. - * @param listener When the message is considered to be checked or discarded, the relative method of this class will be invoked. + * @param transactionTimeout The minimum time of the transactional message to be checked firstly, one message only + * exceed this time interval that can be checked. + * @param transactionCheckMax The maximum number of times the message was checked, if exceed this value, this + * message will be discarded. + * @param listener When the message is considered to be checked or discarded, the relative method of this class will + * be invoked. */ void check(long transactionTimeout, int transactionCheckMax, AbstractTransactionalMessageCheckListener listener); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java index 71fdd206784..84a62761bd6 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java @@ -115,7 +115,8 @@ public PullResult getOpMessage(int queueId, long offset, int nums) { return getMessage(group, topic, queueId, offset, nums, sub); } - private PullResult getMessage(String group, String topic, int queueId, long offset, int nums, SubscriptionData sub) { + private PullResult getMessage(String group, String topic, int queueId, long offset, int nums, + SubscriptionData sub) { GetMessageResult getMessageResult = store.getMessage(group, topic, queueId, offset, nums, null); if (getMessageResult != null) { @@ -227,7 +228,6 @@ public boolean putMessage(MessageExtBrokerInner messageInner) { } } - public MessageExtBrokerInner renewImmunityHalfMessageInner(MessageExt msgExt) { MessageExtBrokerInner msgInner = renewHalfMessageInner(msgExt); String queueOffsetFromPrepare = msgExt.getUserProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET); @@ -290,9 +290,10 @@ private TopicConfig selectTopicConfig(String topic) { } /** - * Use this function while transaction msg is committed or rollback write a flag 'd' to operation queue for the msg's offset + * Use this function while transaction msg is committed or rollback write a flag 'd' to operation queue for the + * msg's offset * - * @param messageExt Op message + * @param messageExt Op message * @param messageQueue Op message queue * @return This method will always return true. */ diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java index ee1cab28eab..9ce9a9703f3 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java @@ -118,7 +118,8 @@ private boolean putBackHalfMsgQueue(MessageExt msgExt, long offset) { } @Override - public void check(long transactionTimeout, int transactionCheckMax, AbstractTransactionalMessageCheckListener listener) { + public void check(long transactionTimeout, int transactionCheckMax, + AbstractTransactionalMessageCheckListener listener) { try { String topic = MixAll.RMQ_SYS_TRANS_HALF_TOPIC; Set msgQueues = transactionBridge.fetchMessageQueues(topic); @@ -260,15 +261,15 @@ private long getImmunityTime(String checkImmunityTimeStr, long transactionTimeou /** * Read op message, parse op message, and fill removeMap * - * @param removeMap Half message to be remove, key:halfOffset, value: opOffset. - * @param opQueue Op message queue. + * @param removeMap Half message to be remove, key:halfOffset, value: opOffset. + * @param opQueue Op message queue. * @param pullOffsetOfOp The begin offset of op message queue. - * @param miniOffset The current minimum offset of half message queue. - * @param doneOpOffset Stored op messages that have been processed. + * @param miniOffset The current minimum offset of half message queue. + * @param doneOpOffset Stored op messages that have been processed. * @return Op message result. */ private PullResult fillOpRemoveMap(HashMap removeMap, - MessageQueue opQueue, long pullOffsetOfOp, long miniOffset, List doneOpOffset) { + MessageQueue opQueue, long pullOffsetOfOp, long miniOffset, List doneOpOffset) { PullResult pullResult = pullOpMsg(opQueue, pullOffsetOfOp, 32); if (null == pullResult) { return null; @@ -311,13 +312,14 @@ private PullResult fillOpRemoveMap(HashMap removeMap, /** * If return true, skip this msg * - * @param removeMap Op message map to determine whether a half message was responded by producer. - * @param doneOpOffset Op Message which has been checked. - * @param msgExt Half message + * @param removeMap Op message map to determine whether a half message was responded by producer. + * @param doneOpOffset Op Message which has been checked. + * @param msgExt Half message * @param checkImmunityTime User defined time to avoid being detected early. * @return Return true if put success, otherwise return false. */ - private boolean checkPrepareQueueOffset(HashMap removeMap, List doneOpOffset, MessageExt msgExt, long checkImmunityTime) { + private boolean checkPrepareQueueOffset(HashMap removeMap, List doneOpOffset, MessageExt msgExt, + long checkImmunityTime) { if (System.currentTimeMillis() - msgExt.getBornTimestamp() < checkImmunityTime) { String prepareQueueOffsetStr = msgExt.getUserProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET); if (null == prepareQueueOffsetStr) { @@ -368,9 +370,9 @@ private boolean putImmunityMsgBackToHalfQueue(MessageExt messageExt) { /** * Read half message from Half Topic * - * @param mq Target message queue, in this method, it means the half message queue. + * @param mq Target message queue, in this method, it means the half message queue. * @param offset Offset in the message queue. - * @param nums Pull message number. + * @param nums Pull message number. * @return Messages pulled from half message queue. */ private PullResult pullHalfMsg(MessageQueue mq, long offset, int nums) { @@ -380,9 +382,9 @@ private PullResult pullHalfMsg(MessageQueue mq, long offset, int nums) { /** * Read op message from Op Topic * - * @param mq Target Message Queue + * @param mq Target Message Queue * @param offset Offset in the message queue - * @param nums Pull message number + * @param nums Pull message number * @return Messages pulled from operate message queue. */ private PullResult pullOpMsg(MessageQueue mq, long offset, int nums) { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageUtil.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageUtil.java index a64dd76017e..3042b4c3e56 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageUtil.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageUtil.java @@ -24,7 +24,6 @@ public class TransactionalMessageUtil { public static final String REMOVETAG = "d"; public static Charset charset = Charset.forName("utf-8"); - public static String buildOpTopic() { return MixAll.RMQ_SYS_TRANS_OP_HALF_TOPIC; } diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index 300333574f5..e4486da0d15 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -137,15 +137,14 @@ public class BrokerConfig { private boolean forceRegister = true; /** - * * This configurable item defines interval of topics registration of broker to name server. Allowing values are * between 10, 000 and 60, 000 milliseconds. - * */ private int registerNameServerPeriod = 1000 * 30; /** - * The minimum time of the transactional message to be checked firstly, one message only exceed this time interval that can be checked. + * The minimum time of the transactional message to be checked firstly, one message only exceed this time interval + * that can be checked. */ @ImportantField private long transactionTimeOut = 3 * 1000; From 0ce72f1728948be180b9b52ec1b50b7d47bb4290 Mon Sep 17 00:00:00 2001 From: duhengforever Date: Thu, 12 Jul 2018 22:32:27 +0800 Subject: [PATCH 4/6] Remove some magic numbers and redefine some constants. --- .../apache/rocketmq/broker/client/ProducerManager.java | 3 ++- .../queue/TransactionalMessageServiceImpl.java | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java index 5dedf7029fe..28d103c0bfc 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java @@ -39,6 +39,7 @@ public class ProducerManager { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final long LOCK_TIMEOUT_MILLIS = 3000; private static final long CHANNEL_EXPIRED_TIMEOUT = 1000 * 120; + private static final int GET_AVALIABLE_CHANNEL_RETRY_COUNT = 3; private final Lock groupChannelLock = new ReentrantLock(); private final HashMap> groupChannelTable = new HashMap>(); @@ -208,7 +209,7 @@ public Channel getAvaliableChannel(String groupId) { Channel channel = channelList.get(index); int count = 0; boolean isOk = channel.isActive() && channel.isWritable(); - while (isOk && count++ < 3) { + while (isOk && count++ < GET_AVALIABLE_CHANNEL_RETRY_COUNT) { index = (++index) % size; channel = channelList.get(index); return channel; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java index 9ce9a9703f3..97166c29467 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java @@ -47,9 +47,9 @@ public class TransactionalMessageServiceImpl implements TransactionalMessageServ private TransactionalMessageBridge transactionBridge; - private static final int TRY_PULL_MSG_NUMBER = 1; + private static final int PULL_MSG_RETRY_NUMBER = 1; - private final int queueTime = 60000; + private final int MAX_QUEUE_PROCESS_TIME = 60000; private static final int MAX_RETRY_COUNT_WHEN_HALF_NULL = 1; @@ -153,8 +153,8 @@ public void check(long transactionTimeout, int transactionCheckMax, long newOffset = halfOffset; long i = halfOffset; while (true) { - if (System.currentTimeMillis() - startTime > queueTime) { - log.info("Queue={} process time reach max={}", messageQueue, queueTime); + if (System.currentTimeMillis() - startTime > MAX_QUEUE_PROCESS_TIME) { + log.info("Queue={} process time reach max={}", messageQueue, MAX_QUEUE_PROCESS_TIME); break; } if (removeMap.containsKey(i)) { @@ -441,7 +441,7 @@ private MessageQueue getOpQueue(MessageQueue messageQueue) { private GetResult getHalfMsg(MessageQueue messageQueue, long offset) { GetResult getResult = new GetResult(); - PullResult result = pullHalfMsg(messageQueue, offset, TRY_PULL_MSG_NUMBER); + PullResult result = pullHalfMsg(messageQueue, offset, PULL_MSG_RETRY_NUMBER); getResult.setPullResult(result); List messageExts = result.getMsgFoundList(); if (messageExts == null) { From 6825247361a761401ffc9aed558becfcbd0ddbe7 Mon Sep 17 00:00:00 2001 From: duhengforever Date: Fri, 13 Jul 2018 10:15:42 +0800 Subject: [PATCH 5/6] Rename some constants --- .../transaction/queue/TransactionalMessageServiceImpl.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java index 97166c29467..c2e7f3f4dd5 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java @@ -49,7 +49,7 @@ public class TransactionalMessageServiceImpl implements TransactionalMessageServ private static final int PULL_MSG_RETRY_NUMBER = 1; - private final int MAX_QUEUE_PROCESS_TIME = 60000; + private static final int MAX_PROCESS_TIME_LIMIT = 60000; private static final int MAX_RETRY_COUNT_WHEN_HALF_NULL = 1; @@ -153,8 +153,8 @@ public void check(long transactionTimeout, int transactionCheckMax, long newOffset = halfOffset; long i = halfOffset; while (true) { - if (System.currentTimeMillis() - startTime > MAX_QUEUE_PROCESS_TIME) { - log.info("Queue={} process time reach max={}", messageQueue, MAX_QUEUE_PROCESS_TIME); + if (System.currentTimeMillis() - startTime > MAX_PROCESS_TIME_LIMIT) { + log.info("Queue={} process time reach max={}", messageQueue, MAX_PROCESS_TIME_LIMIT); break; } if (removeMap.containsKey(i)) { From 835e0134ea681b149fd5397ab4b6e0a2d307cd82 Mon Sep 17 00:00:00 2001 From: duhengforever Date: Sat, 14 Jul 2018 11:32:58 +0800 Subject: [PATCH 6/6] Improved transaction service implementation and formatted some identifier --- .../rocketmq/broker/BrokerController.java | 21 ++++----- .../TransactionalMessageCheckService.java | 24 +++++----- .../TransactionalMessageServiceImpl.java | 44 +++++++++---------- 3 files changed, 46 insertions(+), 43 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 8a33aa23f1a..9dbee82e705 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -150,7 +150,7 @@ public class BrokerController { private BrokerFastFailure brokerFastFailure; private Configuration configuration; private FileWatchService fileWatchService; - private TransactionalMessageCheckService transactionMsgCheckService; + private TransactionalMessageCheckService transactionalMessageCheckService; private TransactionalMessageService transactionalMessageService; private AbstractTransactionalMessageCheckListener transactionalMessageCheckListener; @@ -462,7 +462,7 @@ private void initialTransaction() { log.warn("Load default discard message hook service: {}", DefaultTransactionalMessageCheckListener.class.getSimpleName()); } this.transactionalMessageCheckListener.setBrokerController(this); - this.transactionMsgCheckService = new TransactionalMessageCheckService(this); + this.transactionalMessageCheckService = new TransactionalMessageCheckService(this); } public void registerProcessor() { @@ -730,8 +730,8 @@ public void shutdown() { this.fileWatchService.shutdown(); } - if (this.transactionMsgCheckService != null) { - this.transactionMsgCheckService.shutdown(false); + if (this.transactionalMessageCheckService != null) { + this.transactionalMessageCheckService.shutdown(false); } } @@ -803,9 +803,9 @@ public void run() { } if (BrokerRole.SLAVE != messageStoreConfig.getBrokerRole()) { - if (this.transactionMsgCheckService != null) { + if (this.transactionalMessageCheckService != null) { log.info("Start transaction service!"); - this.transactionMsgCheckService.start(); + this.transactionalMessageCheckService.start(); } } } @@ -990,12 +990,13 @@ public Configuration getConfiguration() { return this.configuration; } - public TransactionalMessageCheckService getTransactionMsgCheckService() { - return transactionMsgCheckService; + public TransactionalMessageCheckService getTransactionalMessageCheckService() { + return transactionalMessageCheckService; } - public void setTransactionMsgCheckService(TransactionalMessageCheckService transactionMsgCheckService) { - this.transactionMsgCheckService = transactionMsgCheckService; + public void setTransactionalMessageCheckService( + TransactionalMessageCheckService transactionalMessageCheckService) { + this.transactionalMessageCheckService = transactionalMessageCheckService; } public TransactionalMessageService getTransactionalMessageService() { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java index 87c10310f32..5d515d6f11b 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java @@ -59,20 +59,22 @@ public String getServiceName() { @Override public void run() { - log.info("Start transaction service thread!"); - long timeout = brokerController.getBrokerConfig().getTransactionTimeOut(); - int checkMax = brokerController.getBrokerConfig().getTransactionCheckMax(); + log.info("Start transaction check service thread!"); long checkInterval = brokerController.getBrokerConfig().getTransactionCheckInterval(); - log.info("Check parameter: transactionCheckMax: {}, transactionTimeOut: {} transactionCheckInterval: {}", checkMax, timeout, checkInterval); while (!this.isStopped()) { - try { - Thread.sleep(checkInterval); - this.brokerController.getTransactionalMessageService().check(timeout, checkMax, this.brokerController.getTransactionalMessageCheckListener()); - } catch (Exception e) { - log.error("", e); - } + this.waitForRunning(checkInterval); } - log.info("End transaction service thread!"); + log.info("End transaction check service thread!"); + } + + @Override + protected void onWaitEnd() { + long timeout = brokerController.getBrokerConfig().getTransactionTimeOut(); + int checkMax = brokerController.getBrokerConfig().getTransactionCheckMax(); + long begin = System.currentTimeMillis(); + log.info("Begin to check prepare message, begin time:{}", begin); + this.brokerController.getTransactionalMessageService().check(timeout, checkMax, this.brokerController.getTransactionalMessageCheckListener()); + log.info("End to check prepare message, consumed time:{}", System.currentTimeMillis() - begin); } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java index c2e7f3f4dd5..15e5c84ff2d 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java @@ -45,7 +45,7 @@ public class TransactionalMessageServiceImpl implements TransactionalMessageService { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); - private TransactionalMessageBridge transactionBridge; + private TransactionalMessageBridge transactionalMessageBridge; private static final int PULL_MSG_RETRY_NUMBER = 1; @@ -54,17 +54,17 @@ public class TransactionalMessageServiceImpl implements TransactionalMessageServ private static final int MAX_RETRY_COUNT_WHEN_HALF_NULL = 1; public TransactionalMessageServiceImpl(TransactionalMessageBridge transactionBridge) { - this.transactionBridge = transactionBridge; + this.transactionalMessageBridge = transactionBridge; } private ConcurrentHashMap opQueueMap = new ConcurrentHashMap<>(); @Override public PutMessageResult prepareMessage(MessageExtBrokerInner messageInner) { - return transactionBridge.putHalfMessage(messageInner); + return transactionalMessageBridge.putHalfMessage(messageInner); } - private boolean isNeedDiscard(MessageExt msgExt, int transactionCheckMax) { + private boolean needDiscard(MessageExt msgExt, int transactionCheckMax) { String checkTimes = msgExt.getProperty(MessageConst.PROPERTY_TRANSACTION_CHECK_TIMES); int checkTime = 1; if (null != checkTimes) { @@ -79,10 +79,10 @@ private boolean isNeedDiscard(MessageExt msgExt, int transactionCheckMax) { return false; } - private boolean isNeedSkip(MessageExt msgExt) { + private boolean needSkip(MessageExt msgExt) { long valueOfCurrentMinusBorn = System.currentTimeMillis() - msgExt.getBornTimestamp(); if (valueOfCurrentMinusBorn - > transactionBridge.getBrokerController().getMessageStoreConfig().getFileReservedTime() + > transactionalMessageBridge.getBrokerController().getMessageStoreConfig().getFileReservedTime() * 3600L * 1000) { log.info("Half message exceed file reserved time ,so skip it.messageId {},bornTime {}", msgExt.getMsgId(), msgExt.getBornTimestamp()); @@ -122,7 +122,7 @@ public void check(long transactionTimeout, int transactionCheckMax, AbstractTransactionalMessageCheckListener listener) { try { String topic = MixAll.RMQ_SYS_TRANS_HALF_TOPIC; - Set msgQueues = transactionBridge.fetchMessageQueues(topic); + Set msgQueues = transactionalMessageBridge.fetchMessageQueues(topic); if (msgQueues == null || msgQueues.size() == 0) { log.warn("The queue of topic is empty :" + topic); return; @@ -131,8 +131,8 @@ public void check(long transactionTimeout, int transactionCheckMax, for (MessageQueue messageQueue : msgQueues) { long startTime = System.currentTimeMillis(); MessageQueue opQueue = getOpQueue(messageQueue); - long halfOffset = transactionBridge.fetchConsumeOffset(messageQueue); - long opOffset = transactionBridge.fetchConsumeOffset(opQueue); + long halfOffset = transactionalMessageBridge.fetchConsumeOffset(messageQueue); + long opOffset = transactionalMessageBridge.fetchConsumeOffset(opQueue); log.info("Before check, the queue={} msgOffset={} opOffset={}", messageQueue, halfOffset, opOffset); if (halfOffset < 0 || opOffset < 0) { log.error("MessageQueue: {} illegal offset read: {}, op offset: {},skip this queue", messageQueue, @@ -180,7 +180,7 @@ public void check(long transactionTimeout, int transactionCheckMax, } } - if (isNeedDiscard(msgExt, transactionCheckMax) || isNeedSkip(msgExt)) { + if (needDiscard(msgExt, transactionCheckMax) || needSkip(msgExt)) { listener.resolveDiscardMsg(msgExt); newOffset = i + 1; i++; @@ -232,11 +232,11 @@ public void check(long transactionTimeout, int transactionCheckMax, i++; } if (newOffset != halfOffset) { - transactionBridge.updateConsumeOffset(messageQueue, newOffset); + transactionalMessageBridge.updateConsumeOffset(messageQueue, newOffset); } long newOpOffset = calculateOpOffset(doneOpOffset, opOffset); if (newOpOffset != opOffset) { - transactionBridge.updateConsumeOffset(opQueue, newOpOffset); + transactionalMessageBridge.updateConsumeOffset(opQueue, newOpOffset); } } } catch (Exception e) { @@ -278,7 +278,7 @@ private PullResult fillOpRemoveMap(HashMap removeMap, || pullResult.getPullStatus() == PullStatus.NO_MATCHED_MSG) { log.warn("The miss op offset={} in queue={} is illegal, pullResult={}", pullOffsetOfOp, opQueue, pullResult); - transactionBridge.updateConsumeOffset(opQueue, pullResult.getNextBeginOffset()); + transactionalMessageBridge.updateConsumeOffset(opQueue, pullResult.getNextBeginOffset()); return pullResult; } else if (pullResult.getPullStatus() == PullStatus.NO_NEW_MSG) { log.warn("The miss op offset={} in queue={} is NO_NEW_MSG, pullResult={}", pullOffsetOfOp, opQueue, @@ -354,17 +354,17 @@ private boolean checkPrepareQueueOffset(HashMap removeMap, List