diff --git a/.gitignore b/.gitignore index c4e43cd66fb..264f48d0dd7 100644 --- a/.gitignore +++ b/.gitignore @@ -10,5 +10,6 @@ devenv *.versionsBackup !NOTICE-BIN !LICENSE-BIN -.DS_Store -localbin/ +.DS_Store +localbin +nohup.out diff --git a/NOTICE b/NOTICE index c218b26089b..703c28b27c5 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ Apache RocketMQ -Copyright 2016-2017 The Apache Software Foundation +Copyright 2016-2018 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). \ No newline at end of file diff --git a/README.md b/README.md index 8f670c2f94c..f964f46fc99 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,8 @@ It offers a variety of features: * Various message filter mechanics such as SQL and Tag * Docker images for isolated testing and cloud isolated clusters * Feature-rich administrative dashboard for configuration, metrics and monitoring +* Access control list +* Message trace ---------- @@ -37,8 +39,7 @@ It offers a variety of features: ---------- ## Apache RocketMQ Community -* [RocketMQ Community Incubator Projects](https://github.com/apache/rocketmq-externals) - +* [RocketMQ Community Projects](https://github.com/apache/rocketmq-externals) ---------- ## Contributing diff --git a/acl/pom.xml b/acl/pom.xml new file mode 100644 index 00000000000..4482cd1e7cc --- /dev/null +++ b/acl/pom.xml @@ -0,0 +1,72 @@ + + + 4.0.0 + + org.apache.rocketmq + rocketmq-all + 4.4.1-SNAPSHOT + + rocketmq-acl + rocketmq-acl ${project.version} + + http://maven.apache.org + + UTF-8 + + + + ${project.groupId} + rocketmq-remoting + + + ${project.groupId} + rocketmq-logging + + + ${project.groupId} + rocketmq-common + + + ${project.groupId} + rocketmq-srvutil + + + org.yaml + snakeyaml + + + commons-codec + commons-codec + + + org.apache.commons + commons-lang3 + + + + org.slf4j + slf4j-api + test + + + ch.qos.logback + logback-classic + test + + + ch.qos.logback + logback-core + test + + + + diff --git a/acl/src/main/java/org/apache/rocketmq/acl/AccessResource.java b/acl/src/main/java/org/apache/rocketmq/acl/AccessResource.java new file mode 100644 index 00000000000..e30febc5719 --- /dev/null +++ b/acl/src/main/java/org/apache/rocketmq/acl/AccessResource.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.acl; + +public interface AccessResource { +} diff --git a/acl/src/main/java/org/apache/rocketmq/acl/AccessValidator.java b/acl/src/main/java/org/apache/rocketmq/acl/AccessValidator.java new file mode 100644 index 00000000000..c915cf35d1c --- /dev/null +++ b/acl/src/main/java/org/apache/rocketmq/acl/AccessValidator.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.acl; + +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +public interface AccessValidator { + /** + * Parse to get the AccessResource(user, resource, needed permission) + * + * @param request + * @param remoteAddr + * @return Plain access resource result,include access key,signature and some other access attributes. + */ + AccessResource parse(RemotingCommand request, String remoteAddr); + + /** + * Validate the access resource. + * + * @param accessResource + */ + void validate(AccessResource accessResource); +} diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/AclClientRPCHook.java b/acl/src/main/java/org/apache/rocketmq/acl/common/AclClientRPCHook.java new file mode 100644 index 00000000000..9e5bf1fb5d9 --- /dev/null +++ b/acl/src/main/java/org/apache/rocketmq/acl/common/AclClientRPCHook.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.acl.common; + +import java.lang.reflect.Field; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +import static org.apache.rocketmq.acl.common.SessionCredentials.ACCESS_KEY; +import static org.apache.rocketmq.acl.common.SessionCredentials.SECURITY_TOKEN; +import static org.apache.rocketmq.acl.common.SessionCredentials.SIGNATURE; + +public class AclClientRPCHook implements RPCHook { + private final SessionCredentials sessionCredentials; + protected ConcurrentHashMap, Field[]> fieldCache = + new ConcurrentHashMap, Field[]>(); + + public AclClientRPCHook(SessionCredentials sessionCredentials) { + this.sessionCredentials = sessionCredentials; + } + + @Override + public void doBeforeRequest(String remoteAddr, RemotingCommand request) { + byte[] total = AclUtils.combineRequestContent(request, + parseRequestContent(request, sessionCredentials.getAccessKey(), sessionCredentials.getSecurityToken())); + String signature = AclUtils.calSignature(total, sessionCredentials.getSecretKey()); + request.addExtField(SIGNATURE, signature); + request.addExtField(ACCESS_KEY, sessionCredentials.getAccessKey()); + + // The SecurityToken value is unneccessary,user can choose this one. + if (sessionCredentials.getSecurityToken() != null) { + request.addExtField(SECURITY_TOKEN, sessionCredentials.getSecurityToken()); + } + } + + @Override + public void doAfterResponse(String remoteAddr, RemotingCommand request, RemotingCommand response) { + + } + + protected SortedMap parseRequestContent(RemotingCommand request, String ak, String securityToken) { + CommandCustomHeader header = request.readCustomHeader(); + // Sort property + SortedMap map = new TreeMap(); + map.put(ACCESS_KEY, ak); + if (securityToken != null) { + map.put(SECURITY_TOKEN, securityToken); + } + try { + // Add header properties + if (null != header) { + Field[] fields = fieldCache.get(header.getClass()); + if (null == fields) { + fields = header.getClass().getDeclaredFields(); + for (Field field : fields) { + field.setAccessible(true); + } + Field[] tmp = fieldCache.putIfAbsent(header.getClass(), fields); + if (null != tmp) { + fields = tmp; + } + } + + for (Field field : fields) { + Object value = field.get(header); + if (null != value && !field.isSynthetic()) { + map.put(field.getName(), value.toString()); + } + } + } + return map; + } catch (Exception e) { + throw new RuntimeException("incompatible exception.", e); + } + } + + public SessionCredentials getSessionCredentials() { + return sessionCredentials; + } +} diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/AclException.java b/acl/src/main/java/org/apache/rocketmq/acl/common/AclException.java new file mode 100644 index 00000000000..54579d48a7d --- /dev/null +++ b/acl/src/main/java/org/apache/rocketmq/acl/common/AclException.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.acl.common; + +public class AclException extends RuntimeException { + private static final long serialVersionUID = -7256002576788700354L; + + private String status; + private int code; + + public AclException(String status, int code) { + super(); + this.status = status; + this.code = code; + } + + public AclException(String status, int code, String message) { + super(message); + this.status = status; + this.code = code; + } + + public AclException(String message) { + super(message); + } + + public AclException(String message, Throwable throwable) { + super(message, throwable); + } + + public AclException(String status, int code, String message, Throwable throwable) { + super(message, throwable); + this.status = status; + this.code = code; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public int getCode() { + return code; + } + + public void setCode(int code) { + this.code = code; + } +} diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/AclSigner.java b/acl/src/main/java/org/apache/rocketmq/acl/common/AclSigner.java new file mode 100644 index 00000000000..61e9350663f --- /dev/null +++ b/acl/src/main/java/org/apache/rocketmq/acl/common/AclSigner.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.acl.common; + +import java.nio.charset.Charset; +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import org.apache.commons.codec.binary.Base64; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; + +public class AclSigner { + public static final Charset DEFAULT_CHARSET = Charset.forName("UTF-8"); + public static final SigningAlgorithm DEFAULT_ALGORITHM = SigningAlgorithm.HmacSHA1; + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.ROCKETMQ_AUTHORIZE_LOGGER_NAME); + private static final int CAL_SIGNATURE_FAILED = 10015; + private static final String CAL_SIGNATURE_FAILED_MSG = "[%s:signature-failed] unable to calculate a request signature. error=%s"; + + public static String calSignature(String data, String key) throws AclException { + return calSignature(data, key, DEFAULT_ALGORITHM, DEFAULT_CHARSET); + } + + public static String calSignature(String data, String key, SigningAlgorithm algorithm, + Charset charset) throws AclException { + return signAndBase64Encode(data, key, algorithm, charset); + } + + private static String signAndBase64Encode(String data, String key, SigningAlgorithm algorithm, Charset charset) + throws AclException { + try { + byte[] signature = sign(data.getBytes(charset), key.getBytes(charset), algorithm); + return new String(Base64.encodeBase64(signature), DEFAULT_CHARSET); + } catch (Exception e) { + String message = String.format(CAL_SIGNATURE_FAILED_MSG, CAL_SIGNATURE_FAILED, e.getMessage()); + log.error(message, e); + throw new AclException("CAL_SIGNATURE_FAILED", CAL_SIGNATURE_FAILED, message, e); + } + } + + private static byte[] sign(byte[] data, byte[] key, SigningAlgorithm algorithm) throws AclException { + try { + Mac mac = Mac.getInstance(algorithm.toString()); + mac.init(new SecretKeySpec(key, algorithm.toString())); + return mac.doFinal(data); + } catch (Exception e) { + String message = String.format(CAL_SIGNATURE_FAILED_MSG, CAL_SIGNATURE_FAILED, e.getMessage()); + log.error(message, e); + throw new AclException("CAL_SIGNATURE_FAILED", CAL_SIGNATURE_FAILED, message, e); + } + } + + public static String calSignature(byte[] data, String key) throws AclException { + return calSignature(data, key, DEFAULT_ALGORITHM, DEFAULT_CHARSET); + } + + public static String calSignature(byte[] data, String key, SigningAlgorithm algorithm, + Charset charset) throws AclException { + return signAndBase64Encode(data, key, algorithm, charset); + } + + private static String signAndBase64Encode(byte[] data, String key, SigningAlgorithm algorithm, Charset charset) + throws AclException { + try { + byte[] signature = sign(data, key.getBytes(charset), algorithm); + return new String(Base64.encodeBase64(signature), DEFAULT_CHARSET); + } catch (Exception e) { + String message = String.format(CAL_SIGNATURE_FAILED_MSG, CAL_SIGNATURE_FAILED, e.getMessage()); + log.error(message, e); + throw new AclException("CAL_SIGNATURE_FAILED", CAL_SIGNATURE_FAILED, message, e); + } + } + +} diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java b/acl/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java new file mode 100644 index 00000000000..1a618456f40 --- /dev/null +++ b/acl/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.acl.common; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.Map; +import java.util.SortedMap; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.yaml.snakeyaml.Yaml; + +import static org.apache.rocketmq.acl.common.SessionCredentials.CHARSET; + +public class AclUtils { + + public static byte[] combineRequestContent(RemotingCommand request, SortedMap fieldsMap) { + try { + StringBuilder sb = new StringBuilder(""); + for (Map.Entry entry : fieldsMap.entrySet()) { + if (!SessionCredentials.SIGNATURE.equals(entry.getKey())) { + sb.append(entry.getValue()); + } + } + + return AclUtils.combineBytes(sb.toString().getBytes(CHARSET), request.getBody()); + } catch (Exception e) { + throw new RuntimeException("incompatible exception.", e); + } + } + + public static byte[] combineBytes(byte[] b1, byte[] b2) { + int size = (null != b1 ? b1.length : 0) + (null != b2 ? b2.length : 0); + byte[] total = new byte[size]; + if (null != b1) + System.arraycopy(b1, 0, total, 0, b1.length); + if (null != b2) + System.arraycopy(b2, 0, total, b1.length, b2.length); + return total; + } + + public static String calSignature(byte[] data, String secretKey) { + String signature = AclSigner.calSignature(data, secretKey); + return signature; + } + + public static void verify(String netaddress, int index) { + if (!AclUtils.isScope(netaddress, index)) { + throw new AclException(String.format("netaddress examine scope Exception netaddress is %s", netaddress)); + } + } + + public static String[] getAddreeStrArray(String netaddress, String four) { + String[] fourStrArray = StringUtils.split(four.substring(1, four.length() - 1), ","); + String address = netaddress.substring(0, netaddress.indexOf("{")); + String[] addreeStrArray = new String[fourStrArray.length]; + for (int i = 0; i < fourStrArray.length; i++) { + addreeStrArray[i] = address + fourStrArray[i]; + } + return addreeStrArray; + } + + public static boolean isScope(String num, int index) { + String[] strArray = StringUtils.split(num, "."); + if (strArray.length != 4) { + return false; + } + return isScope(strArray, index); + + } + + public static boolean isScope(String[] num, int index) { + if (num.length <= index) { + + } + for (int i = 0; i < index; i++) { + if (!isScope(num[i])) { + return false; + } + } + return true; + + } + + public static boolean isScope(String num) { + return isScope(Integer.valueOf(num.trim())); + } + + public static boolean isScope(int num) { + return num >= 0 && num <= 255; + } + + public static boolean isAsterisk(String asterisk) { + return asterisk.indexOf('*') > -1; + } + + public static boolean isColon(String colon) { + return colon.indexOf(',') > -1; + } + + public static boolean isMinus(String minus) { + return minus.indexOf('-') > -1; + + } + + public static T getYamlDataObject(String path, Class clazz) { + Yaml ymal = new Yaml(); + FileInputStream fis = null; + try { + fis = new FileInputStream(new File(path)); + return ymal.loadAs(fis, clazz); + } catch (Exception e) { + throw new AclException(String.format("The file for Plain mode was not found , paths %s", path), e); + } finally { + if (fis != null) { + try { + fis.close(); + } catch (IOException e) { + throw new AclException("close transport fileInputStream Exception", e); + } + } + } + } + +} diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/Permission.java b/acl/src/main/java/org/apache/rocketmq/acl/common/Permission.java new file mode 100644 index 00000000000..0acc8e95081 --- /dev/null +++ b/acl/src/main/java/org/apache/rocketmq/acl/common/Permission.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.acl.common; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.acl.plain.PlainAccessResource; +import org.apache.rocketmq.common.protocol.RequestCode; + +public class Permission { + + public static final byte DENY = 1; + public static final byte ANY = 1 << 1; + public static final byte PUB = 1 << 2; + public static final byte SUB = 1 << 3; + + public static final Set ADMIN_CODE = new HashSet(); + + static { + // UPDATE_AND_CREATE_TOPIC + ADMIN_CODE.add(RequestCode.UPDATE_AND_CREATE_TOPIC); + // UPDATE_BROKER_CONFIG + ADMIN_CODE.add(RequestCode.UPDATE_BROKER_CONFIG); + // DELETE_TOPIC_IN_BROKER + ADMIN_CODE.add(RequestCode.DELETE_TOPIC_IN_BROKER); + // UPDATE_AND_CREATE_SUBSCRIPTIONGROUP + ADMIN_CODE.add(RequestCode.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP); + // DELETE_SUBSCRIPTIONGROUP + ADMIN_CODE.add(RequestCode.DELETE_SUBSCRIPTIONGROUP); + } + + public static boolean checkPermission(byte neededPerm, byte ownedPerm) { + if ((ownedPerm & DENY) > 0) { + return false; + } + if ((neededPerm & ANY) > 0) { + return ((ownedPerm & PUB) > 0) || ((ownedPerm & SUB) > 0); + } + return (neededPerm & ownedPerm) > 0; + } + + public static byte parsePermFromString(String permString) { + if (permString == null) { + return Permission.DENY; + } + switch (permString.trim()) { + case "PUB": + return Permission.PUB; + case "SUB": + return Permission.SUB; + case "PUB|SUB": + return Permission.PUB | Permission.SUB; + case "SUB|PUB": + return Permission.PUB | Permission.SUB; + case "DENY": + return Permission.DENY; + default: + return Permission.DENY; + } + } + + public static void parseResourcePerms(PlainAccessResource plainAccessResource, Boolean isTopic, + List resources) { + if (resources == null || resources.isEmpty()) { + return; + } + for (String resource : resources) { + String[] items = StringUtils.split(resource, "="); + if (items.length == 2) { + plainAccessResource.addResourceAndPerm(isTopic ? items[0].trim() : PlainAccessResource.getRetryTopic(items[0].trim()), parsePermFromString(items[1].trim())); + } else { + throw new AclException(String.format("Parse resource permission failed for %s:%s", isTopic ? "topic" : "group", resource)); + } + } + } + + public static boolean needAdminPerm(Integer code) { + return ADMIN_CODE.contains(code); + } +} diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/SessionCredentials.java b/acl/src/main/java/org/apache/rocketmq/acl/common/SessionCredentials.java new file mode 100644 index 00000000000..33a8a34350c --- /dev/null +++ b/acl/src/main/java/org/apache/rocketmq/acl/common/SessionCredentials.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.acl.common; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.Properties; +import org.apache.rocketmq.common.MixAll; + +public class SessionCredentials { + public static final Charset CHARSET = Charset.forName("UTF-8"); + public static final String ACCESS_KEY = "AccessKey"; + public static final String SECRET_KEY = "SecretKey"; + public static final String SIGNATURE = "Signature"; + public static final String SECURITY_TOKEN = "SecurityToken"; + + public static final String KEY_FILE = System.getProperty("rocketmq.client.keyFile", + System.getProperty("user.home") + File.separator + "key"); + + private String accessKey; + private String secretKey; + private String securityToken; + private String signature; + + public SessionCredentials() { + String keyContent = null; + try { + keyContent = MixAll.file2String(KEY_FILE); + } catch (IOException ignore) { + } + if (keyContent != null) { + Properties prop = MixAll.string2Properties(keyContent); + if (prop != null) { + this.updateContent(prop); + } + } + } + + public SessionCredentials(String accessKey, String secretKey) { + this.accessKey = accessKey; + this.secretKey = secretKey; + } + + public SessionCredentials(String accessKey, String secretKey, String securityToken) { + this(accessKey, secretKey); + this.securityToken = securityToken; + } + + public void updateContent(Properties prop) { + { + String value = prop.getProperty(ACCESS_KEY); + if (value != null) { + this.accessKey = value.trim(); + } + } + { + String value = prop.getProperty(SECRET_KEY); + if (value != null) { + this.secretKey = value.trim(); + } + } + { + String value = prop.getProperty(SECURITY_TOKEN); + if (value != null) { + this.securityToken = value.trim(); + } + } + } + + public String getAccessKey() { + return accessKey; + } + + public void setAccessKey(String accessKey) { + this.accessKey = accessKey; + } + + public String getSecretKey() { + return secretKey; + } + + public void setSecretKey(String secretKey) { + this.secretKey = secretKey; + } + + public String getSignature() { + return signature; + } + + public void setSignature(String signature) { + this.signature = signature; + } + + public String getSecurityToken() { + return securityToken; + } + + public void setSecurityToken(final String securityToken) { + this.securityToken = securityToken; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((accessKey == null) ? 0 : accessKey.hashCode()); + result = prime * result + ((secretKey == null) ? 0 : secretKey.hashCode()); + result = prime * result + ((signature == null) ? 0 : signature.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + + SessionCredentials other = (SessionCredentials) obj; + if (accessKey == null) { + if (other.accessKey != null) + return false; + } else if (!accessKey.equals(other.accessKey)) + return false; + + if (secretKey == null) { + if (other.secretKey != null) + return false; + } else if (!secretKey.equals(other.secretKey)) + return false; + + if (signature == null) { + if (other.signature != null) + return false; + } else if (!signature.equals(other.signature)) + return false; + + return true; + } + + @Override + public String toString() { + return "SessionCredentials [accessKey=" + accessKey + ", secretKey=" + secretKey + ", signature=" + + signature + ", SecurityToken=" + securityToken + "]"; + } +} \ No newline at end of file diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/SigningAlgorithm.java b/acl/src/main/java/org/apache/rocketmq/acl/common/SigningAlgorithm.java new file mode 100644 index 00000000000..bfed7b2f233 --- /dev/null +++ b/acl/src/main/java/org/apache/rocketmq/acl/common/SigningAlgorithm.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.acl.common; + +public enum SigningAlgorithm { + HmacSHA1, + HmacSHA256, + HmacMD5; + +} diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessResource.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessResource.java new file mode 100644 index 00000000000..00072e8e2e2 --- /dev/null +++ b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessResource.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.acl.plain; + +import java.util.HashMap; +import java.util.Map; +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.apache.rocketmq.acl.AccessResource; +import org.apache.rocketmq.common.MixAll; + +public class PlainAccessResource implements AccessResource { + + // Identify the user + private String accessKey; + + private String secretKey; + + private String whiteRemoteAddress; + + private boolean admin; + + private byte defaultTopicPerm = 1; + + private byte defaultGroupPerm = 1; + + private Map resourcePermMap; + + private RemoteAddressStrategy remoteAddressStrategy; + + private int requestCode; + + //the content to calculate the content + private byte[] content; + + private String signature; + + private String secretToken; + + private String recognition; + + public PlainAccessResource() { + } + + public static boolean isRetryTopic(String topic) { + return null != topic && topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX); + } + + public static String printStr(String resource, boolean isGroup) { + if (resource == null) { + return null; + } + if (isGroup) { + return String.format("%s:%s", "group", getGroupFromRetryTopic(resource)); + } else { + return String.format("%s:%s", "topic", resource); + } + } + + public static String getGroupFromRetryTopic(String retryTopic) { + if (retryTopic == null) { + return null; + } + return retryTopic.substring(MixAll.RETRY_GROUP_TOPIC_PREFIX.length()); + } + + public static String getRetryTopic(String group) { + if (group == null) { + return null; + } + return MixAll.getRetryTopic(group); + } + + public void addResourceAndPerm(String resource, byte perm) { + if (resource == null) { + return; + } + if (resourcePermMap == null) { + resourcePermMap = new HashMap<>(); + } + resourcePermMap.put(resource, perm); + } + + public String getAccessKey() { + return accessKey; + } + + public void setAccessKey(String accessKey) { + this.accessKey = accessKey; + } + + public String getSecretKey() { + return secretKey; + } + + public void setSecretKey(String secretKey) { + this.secretKey = secretKey; + } + + public String getWhiteRemoteAddress() { + return whiteRemoteAddress; + } + + public void setWhiteRemoteAddress(String whiteRemoteAddress) { + this.whiteRemoteAddress = whiteRemoteAddress; + } + + public boolean isAdmin() { + return admin; + } + + public void setAdmin(boolean admin) { + this.admin = admin; + } + + public byte getDefaultTopicPerm() { + return defaultTopicPerm; + } + + public void setDefaultTopicPerm(byte defaultTopicPerm) { + this.defaultTopicPerm = defaultTopicPerm; + } + + public byte getDefaultGroupPerm() { + return defaultGroupPerm; + } + + public void setDefaultGroupPerm(byte defaultGroupPerm) { + this.defaultGroupPerm = defaultGroupPerm; + } + + public Map getResourcePermMap() { + return resourcePermMap; + } + + public String getRecognition() { + return recognition; + } + + public void setRecognition(String recognition) { + this.recognition = recognition; + } + + public int getRequestCode() { + return requestCode; + } + + public void setRequestCode(int requestCode) { + this.requestCode = requestCode; + } + + public String getSecretToken() { + return secretToken; + } + + public void setSecretToken(String secretToken) { + this.secretToken = secretToken; + } + + public RemoteAddressStrategy getRemoteAddressStrategy() { + return remoteAddressStrategy; + } + + public void setRemoteAddressStrategy(RemoteAddressStrategy remoteAddressStrategy) { + this.remoteAddressStrategy = remoteAddressStrategy; + } + + public String getSignature() { + return signature; + } + + public void setSignature(String signature) { + this.signature = signature; + } + + @Override + public String toString() { + return ToStringBuilder.reflectionToString(this); + } + + public byte[] getContent() { + return content; + } + + public void setContent(byte[] content) { + this.content = content; + } +} diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessValidator.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessValidator.java new file mode 100644 index 00000000000..34bb1b439e5 --- /dev/null +++ b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessValidator.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.acl.plain; + +import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; +import org.apache.rocketmq.acl.AccessResource; +import org.apache.rocketmq.acl.AccessValidator; +import org.apache.rocketmq.acl.common.AclException; +import org.apache.rocketmq.acl.common.AclUtils; +import org.apache.rocketmq.acl.common.Permission; +import org.apache.rocketmq.acl.common.SessionCredentials; +import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupRequestHeader; +import org.apache.rocketmq.common.protocol.header.UnregisterClientRequestHeader; +import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader; +import org.apache.rocketmq.common.protocol.heartbeat.ConsumerData; +import org.apache.rocketmq.common.protocol.heartbeat.HeartbeatData; +import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +import static org.apache.rocketmq.acl.plain.PlainAccessResource.getRetryTopic; + +public class PlainAccessValidator implements AccessValidator { + + private PlainPermissionLoader aclPlugEngine; + + public PlainAccessValidator() { + aclPlugEngine = new PlainPermissionLoader(); + } + + @Override + public AccessResource parse(RemotingCommand request, String remoteAddr) { + PlainAccessResource accessResource = new PlainAccessResource(); + if (remoteAddr != null && remoteAddr.contains(":")) { + accessResource.setWhiteRemoteAddress(remoteAddr.split(":")[0]); + } else { + accessResource.setWhiteRemoteAddress(remoteAddr); + } + accessResource.setRequestCode(request.getCode()); + accessResource.setAccessKey(request.getExtFields().get(SessionCredentials.ACCESS_KEY)); + accessResource.setSignature(request.getExtFields().get(SessionCredentials.SIGNATURE)); + accessResource.setSecretToken(request.getExtFields().get(SessionCredentials.SECURITY_TOKEN)); + + try { + switch (request.getCode()) { + case RequestCode.SEND_MESSAGE: + accessResource.addResourceAndPerm(request.getExtFields().get("topic"), Permission.PUB); + break; + case RequestCode.SEND_MESSAGE_V2: + accessResource.addResourceAndPerm(request.getExtFields().get("b"), Permission.PUB); + break; + case RequestCode.CONSUMER_SEND_MSG_BACK: + accessResource.addResourceAndPerm(request.getExtFields().get("originTopic"), Permission.PUB); + accessResource.addResourceAndPerm(getRetryTopic(request.getExtFields().get("group")), Permission.SUB); + break; + case RequestCode.PULL_MESSAGE: + accessResource.addResourceAndPerm(request.getExtFields().get("topic"), Permission.SUB); + accessResource.addResourceAndPerm(getRetryTopic(request.getExtFields().get("consumerGroup")), Permission.SUB); + break; + case RequestCode.QUERY_MESSAGE: + accessResource.addResourceAndPerm(request.getExtFields().get("topic"), Permission.SUB); + break; + case RequestCode.HEART_BEAT: + HeartbeatData heartbeatData = HeartbeatData.decode(request.getBody(), HeartbeatData.class); + for (ConsumerData data : heartbeatData.getConsumerDataSet()) { + accessResource.addResourceAndPerm(getRetryTopic(data.getGroupName()), Permission.SUB); + for (SubscriptionData subscriptionData : data.getSubscriptionDataSet()) { + accessResource.addResourceAndPerm(subscriptionData.getTopic(), Permission.SUB); + } + } + break; + case RequestCode.UNREGISTER_CLIENT: + final UnregisterClientRequestHeader unregisterClientRequestHeader = + (UnregisterClientRequestHeader) request + .decodeCommandCustomHeader(UnregisterClientRequestHeader.class); + accessResource.addResourceAndPerm(getRetryTopic(unregisterClientRequestHeader.getConsumerGroup()), Permission.SUB); + break; + case RequestCode.GET_CONSUMER_LIST_BY_GROUP: + final GetConsumerListByGroupRequestHeader getConsumerListByGroupRequestHeader = + (GetConsumerListByGroupRequestHeader) request + .decodeCommandCustomHeader(GetConsumerListByGroupRequestHeader.class); + accessResource.addResourceAndPerm(getRetryTopic(getConsumerListByGroupRequestHeader.getConsumerGroup()), Permission.SUB); + break; + case RequestCode.UPDATE_CONSUMER_OFFSET: + final UpdateConsumerOffsetRequestHeader updateConsumerOffsetRequestHeader = + (UpdateConsumerOffsetRequestHeader) request + .decodeCommandCustomHeader(UpdateConsumerOffsetRequestHeader.class); + accessResource.addResourceAndPerm(getRetryTopic(updateConsumerOffsetRequestHeader.getConsumerGroup()), Permission.SUB); + accessResource.addResourceAndPerm(updateConsumerOffsetRequestHeader.getTopic(), Permission.SUB); + break; + default: + break; + + } + } catch (Throwable t) { + throw new AclException(t.getMessage(), t); + } + // Content + SortedMap map = new TreeMap(); + for (Map.Entry entry : request.getExtFields().entrySet()) { + if (!SessionCredentials.SIGNATURE.equals(entry.getKey())) { + map.put(entry.getKey(), entry.getValue()); + } + } + accessResource.setContent(AclUtils.combineRequestContent(request, map)); + return accessResource; + } + + @Override + public void validate(AccessResource accessResource) { + aclPlugEngine.validate((PlainAccessResource) accessResource); + } + +} diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionLoader.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionLoader.java new file mode 100644 index 00000000000..1da7380b6d9 --- /dev/null +++ b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionLoader.java @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.acl.plain; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.apache.rocketmq.acl.common.AclException; +import org.apache.rocketmq.acl.common.AclUtils; +import org.apache.rocketmq.acl.common.Permission; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.srvutil.FileWatchService; + +public class PlainPermissionLoader { + + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); + + private static final String DEFAULT_PLAIN_ACL_FILE = "/conf/plain_acl.yml"; + + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + + private String fileHome = System.getProperty(MixAll.ROCKETMQ_HOME_PROPERTY, + System.getenv(MixAll.ROCKETMQ_HOME_ENV)); + + private String fileName = System.getProperty("rocketmq.acl.plain.file", DEFAULT_PLAIN_ACL_FILE); + + private Map plainAccessResourceMap = new HashMap<>(); + + private List globalWhiteRemoteAddressStrategy = new ArrayList<>(); + + private RemoteAddressStrategyFactory remoteAddressStrategyFactory = new RemoteAddressStrategyFactory(); + + private boolean isWatchStart; + + public PlainPermissionLoader() { + load(); + watch(); + } + + public void load() { + + Map plainAccessResourceMap = new HashMap<>(); + List globalWhiteRemoteAddressStrategy = new ArrayList<>(); + + JSONObject plainAclConfData = AclUtils.getYamlDataObject(fileHome + File.separator + fileName, + JSONObject.class); + + if (plainAclConfData == null || plainAclConfData.isEmpty()) { + throw new AclException(String.format("%s file is not data", fileHome + File.separator + fileName)); + } + log.info("Broker plain acl conf data is : ", plainAclConfData.toString()); + JSONArray globalWhiteRemoteAddressesList = plainAclConfData.getJSONArray("globalWhiteRemoteAddresses"); + if (globalWhiteRemoteAddressesList != null && !globalWhiteRemoteAddressesList.isEmpty()) { + for (int i = 0; i < globalWhiteRemoteAddressesList.size(); i++) { + globalWhiteRemoteAddressStrategy.add(remoteAddressStrategyFactory. + getRemoteAddressStrategy(globalWhiteRemoteAddressesList.getString(i))); + } + } + + JSONArray accounts = plainAclConfData.getJSONArray("accounts"); + if (accounts != null && !accounts.isEmpty()) { + List plainAccessConfigList = accounts.toJavaList(PlainAccessConfig.class); + for (PlainAccessConfig plainAccessConfig : plainAccessConfigList) { + PlainAccessResource plainAccessResource = buildPlainAccessResource(plainAccessConfig); + plainAccessResourceMap.put(plainAccessResource.getAccessKey(),plainAccessResource); + } + } + + this.globalWhiteRemoteAddressStrategy = globalWhiteRemoteAddressStrategy; + this.plainAccessResourceMap = plainAccessResourceMap; + } + + private void watch() { + try { + String watchFilePath = fileHome + fileName; + FileWatchService fileWatchService = new FileWatchService(new String[] {watchFilePath}, new FileWatchService.Listener() { + @Override + public void onChanged(String path) { + log.info("The plain acl yml changed, reload the context"); + load(); + } + }); + fileWatchService.start(); + log.info("Succeed to start AclWatcherService"); + this.isWatchStart = true; + } catch (Exception e) { + log.error("Failed to start AclWatcherService", e); + } + } + + void checkPerm(PlainAccessResource needCheckedAccess, PlainAccessResource ownedAccess) { + if (Permission.needAdminPerm(needCheckedAccess.getRequestCode()) && !ownedAccess.isAdmin()) { + throw new AclException(String.format("Need admin permission for request code=%d, but accessKey=%s is not", needCheckedAccess.getRequestCode(), ownedAccess.getAccessKey())); + } + Map needCheckedPermMap = needCheckedAccess.getResourcePermMap(); + Map ownedPermMap = ownedAccess.getResourcePermMap(); + + if (needCheckedPermMap == null) { + // If the needCheckedPermMap is null,then return + return; + } + + for (Map.Entry needCheckedEntry : needCheckedPermMap.entrySet()) { + String resource = needCheckedEntry.getKey(); + Byte neededPerm = needCheckedEntry.getValue(); + boolean isGroup = PlainAccessResource.isRetryTopic(resource); + + if (!ownedPermMap.containsKey(resource)) { + // Check the default perm + byte ownedPerm = isGroup ? needCheckedAccess.getDefaultGroupPerm() : + needCheckedAccess.getDefaultTopicPerm(); + if (!Permission.checkPermission(neededPerm, ownedPerm)) { + throw new AclException(String.format("No default permission for %s", PlainAccessResource.printStr(resource, isGroup))); + } + continue; + } + if (!Permission.checkPermission(neededPerm, ownedPermMap.get(resource))) { + throw new AclException(String.format("No default permission for %s", PlainAccessResource.printStr(resource, isGroup))); + } + } + } + + void clearPermissionInfo() { + this.plainAccessResourceMap.clear(); + this.globalWhiteRemoteAddressStrategy.clear(); + } + + public PlainAccessResource buildPlainAccessResource(PlainAccessConfig plainAccessConfig) throws AclException { + if (plainAccessConfig.getAccessKey() == null + || plainAccessConfig.getSecretKey() == null + || plainAccessConfig.getAccessKey().length() <= 6 + || plainAccessConfig.getSecretKey().length() <= 6) { + throw new AclException(String.format( + "The accessKey=%s and secretKey=%s cannot be null and length should longer than 6", + plainAccessConfig.getAccessKey(), plainAccessConfig.getSecretKey())); + } + PlainAccessResource plainAccessResource = new PlainAccessResource(); + plainAccessResource.setAccessKey(plainAccessConfig.getAccessKey()); + plainAccessResource.setSecretKey(plainAccessConfig.getSecretKey()); + plainAccessResource.setWhiteRemoteAddress(plainAccessConfig.getWhiteRemoteAddress()); + + plainAccessResource.setAdmin(plainAccessConfig.isAdmin()); + + plainAccessResource.setDefaultGroupPerm(Permission.parsePermFromString(plainAccessConfig.getDefaultGroupPerm())); + plainAccessResource.setDefaultTopicPerm(Permission.parsePermFromString(plainAccessConfig.getDefaultTopicPerm())); + + Permission.parseResourcePerms(plainAccessResource, false, plainAccessConfig.getGroupPerms()); + Permission.parseResourcePerms(plainAccessResource, true, plainAccessConfig.getTopicPerms()); + + plainAccessResource.setRemoteAddressStrategy(remoteAddressStrategyFactory. + getRemoteAddressStrategy(plainAccessResource.getWhiteRemoteAddress())); + + return plainAccessResource; + } + + public void validate(PlainAccessResource plainAccessResource) { + + // Check the global white remote addr + for (RemoteAddressStrategy remoteAddressStrategy : globalWhiteRemoteAddressStrategy) { + if (remoteAddressStrategy.match(plainAccessResource)) { + return; + } + } + + if (plainAccessResource.getAccessKey() == null) { + throw new AclException(String.format("No accessKey is configured")); + } + + if (!plainAccessResourceMap.containsKey(plainAccessResource.getAccessKey())) { + throw new AclException(String.format("No acl config for %s", plainAccessResource.getAccessKey())); + } + + // Check the white addr for accesskey + PlainAccessResource ownedAccess = plainAccessResourceMap.get(plainAccessResource.getAccessKey()); + if (ownedAccess.getRemoteAddressStrategy().match(plainAccessResource)) { + return; + } + + // Check the signature + String signature = AclUtils.calSignature(plainAccessResource.getContent(), ownedAccess.getSecretKey()); + if (!signature.equals(plainAccessResource.getSignature())) { + throw new AclException(String.format("Check signature failed for accessKey=%s", plainAccessResource.getAccessKey())); + } + // Check perm of each resource + + checkPerm(plainAccessResource, ownedAccess); + } + + public boolean isWatchStart() { + return isWatchStart; + } + + static class PlainAccessConfig { + + private String accessKey; + + private String secretKey; + + private String whiteRemoteAddress; + + private boolean admin; + + private String defaultTopicPerm; + + private String defaultGroupPerm; + + private List topicPerms; + + private List groupPerms; + + public String getAccessKey() { + return accessKey; + } + + public void setAccessKey(String accessKey) { + this.accessKey = accessKey; + } + + public String getSecretKey() { + return secretKey; + } + + public void setSecretKey(String secretKey) { + this.secretKey = secretKey; + } + + public String getWhiteRemoteAddress() { + return whiteRemoteAddress; + } + + public void setWhiteRemoteAddress(String whiteRemoteAddress) { + this.whiteRemoteAddress = whiteRemoteAddress; + } + + public boolean isAdmin() { + return admin; + } + + public void setAdmin(boolean admin) { + this.admin = admin; + } + + public String getDefaultTopicPerm() { + return defaultTopicPerm; + } + + public void setDefaultTopicPerm(String defaultTopicPerm) { + this.defaultTopicPerm = defaultTopicPerm; + } + + public String getDefaultGroupPerm() { + return defaultGroupPerm; + } + + public void setDefaultGroupPerm(String defaultGroupPerm) { + this.defaultGroupPerm = defaultGroupPerm; + } + + public List getTopicPerms() { + return topicPerms; + } + + public void setTopicPerms(List topicPerms) { + this.topicPerms = topicPerms; + } + + public List getGroupPerms() { + return groupPerms; + } + + public void setGroupPerms(List groupPerms) { + this.groupPerms = groupPerms; + } + + } + +} diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategy.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategy.java new file mode 100644 index 00000000000..8eab40c954b --- /dev/null +++ b/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategy.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.acl.plain; + +public interface RemoteAddressStrategy { + + boolean match(PlainAccessResource plainAccessResource); +} diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyFactory.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyFactory.java new file mode 100644 index 00000000000..de29e92fd55 --- /dev/null +++ b/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyFactory.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.acl.plain; + +import java.util.HashSet; +import java.util.Set; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.acl.common.AclException; +import org.apache.rocketmq.acl.common.AclUtils; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; + +public class RemoteAddressStrategyFactory { + + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); + + public static final NullRemoteAddressStrategy NULL_NET_ADDRESS_STRATEGY = new NullRemoteAddressStrategy(); + + public static final BlankRemoteAddressStrategy BLANK_NET_ADDRESS_STRATEGY = new BlankRemoteAddressStrategy(); + + public RemoteAddressStrategy getRemoteAddressStrategy(PlainAccessResource plainAccessResource) { + return getRemoteAddressStrategy(plainAccessResource.getWhiteRemoteAddress()); + } + + public RemoteAddressStrategy getRemoteAddressStrategy(String remoteAddr) { + if (StringUtils.isBlank(remoteAddr)) { + return BLANK_NET_ADDRESS_STRATEGY; + } + if ("*".equals(remoteAddr)) { + return NULL_NET_ADDRESS_STRATEGY; + } + if (remoteAddr.endsWith("}")) { + String[] strArray = StringUtils.split(remoteAddr, "."); + String four = strArray[3]; + if (!four.startsWith("{")) { + throw new AclException(String.format("MultipleRemoteAddressStrategy netaddress examine scope Exception netaddress", remoteAddr)); + } + return new MultipleRemoteAddressStrategy(AclUtils.getAddreeStrArray(remoteAddr, four)); + } else if (AclUtils.isColon(remoteAddr)) { + return new MultipleRemoteAddressStrategy(StringUtils.split(remoteAddr, ",")); + } else if (AclUtils.isAsterisk(remoteAddr) || AclUtils.isMinus(remoteAddr)) { + return new RangeRemoteAddressStrategy(remoteAddr); + } + return new OneRemoteAddressStrategy(remoteAddr); + + } + + public static class NullRemoteAddressStrategy implements RemoteAddressStrategy { + @Override + public boolean match(PlainAccessResource plainAccessResource) { + return true; + } + + } + + public static class BlankRemoteAddressStrategy implements RemoteAddressStrategy { + @Override + public boolean match(PlainAccessResource plainAccessResource) { + return false; + } + + } + + public static class MultipleRemoteAddressStrategy implements RemoteAddressStrategy { + + private final Set multipleSet = new HashSet<>(); + + public MultipleRemoteAddressStrategy(String[] strArray) { + for (String netaddress : strArray) { + AclUtils.verify(netaddress, 4); + multipleSet.add(netaddress); + } + } + + @Override + public boolean match(PlainAccessResource plainAccessResource) { + return multipleSet.contains(plainAccessResource.getWhiteRemoteAddress()); + } + + } + + public static class OneRemoteAddressStrategy implements RemoteAddressStrategy { + + private String netaddress; + + public OneRemoteAddressStrategy(String netaddress) { + this.netaddress = netaddress; + AclUtils.verify(netaddress, 4); + } + + @Override + public boolean match(PlainAccessResource plainAccessResource) { + return netaddress.equals(plainAccessResource.getWhiteRemoteAddress()); + } + + } + + public static class RangeRemoteAddressStrategy implements RemoteAddressStrategy { + + private String head; + + private int start; + + private int end; + + private int index; + + public RangeRemoteAddressStrategy(String remoteAddr) { + String[] strArray = StringUtils.split(remoteAddr, "."); + if (analysis(strArray, 2) || analysis(strArray, 3)) { + AclUtils.verify(remoteAddr, index - 1); + StringBuffer sb = new StringBuffer().append(strArray[0].trim()).append(".").append(strArray[1].trim()).append("."); + if (index == 3) { + sb.append(strArray[2].trim()).append("."); + } + this.head = sb.toString(); + } + } + + private boolean analysis(String[] strArray, int index) { + String value = strArray[index].trim(); + this.index = index; + if ("*".equals(value)) { + setValue(0, 255); + } else if (AclUtils.isMinus(value)) { + if (value.indexOf("-") == 0) { + throw new AclException(String.format("RangeRemoteAddressStrategy netaddress examine scope Exception value %s ", value)); + + } + String[] valueArray = StringUtils.split(value, "-"); + this.start = Integer.valueOf(valueArray[0]); + this.end = Integer.valueOf(valueArray[1]); + if (!(AclUtils.isScope(end) && AclUtils.isScope(start) && start <= end)) { + throw new AclException(String.format("RangeRemoteAddressStrategy netaddress examine scope Exception start is %s , end is %s", start, end)); + } + } + return this.end > 0 ? true : false; + } + + private void setValue(int start, int end) { + this.start = start; + this.end = end; + } + + @Override + public boolean match(PlainAccessResource plainAccessResource) { + String netAddress = plainAccessResource.getWhiteRemoteAddress(); + if (netAddress.startsWith(this.head)) { + String value; + if (index == 3) { + value = netAddress.substring(this.head.length()); + } else { + value = netAddress.substring(this.head.length(), netAddress.lastIndexOf('.')); + } + Integer address = Integer.valueOf(value); + if (address >= this.start && address <= this.end) { + return true; + } + } + return false; + } + + } + +} diff --git a/acl/src/test/java/org/apache/rocketmq/acl/common/AclSignerTest.java b/acl/src/test/java/org/apache/rocketmq/acl/common/AclSignerTest.java new file mode 100644 index 00000000000..eec626357fc --- /dev/null +++ b/acl/src/test/java/org/apache/rocketmq/acl/common/AclSignerTest.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.acl.common; + +import org.junit.Test; + +public class AclSignerTest { + + @Test(expected = Exception.class) + public void calSignatureExceptionTest(){ + AclSigner.calSignature(new byte[]{},""); + } + + @Test + public void calSignatureTest(){ + AclSigner.calSignature("RocketMQ","12345678"); + AclSigner.calSignature("RocketMQ".getBytes(),"12345678"); + } + +} diff --git a/acl/src/test/java/org/apache/rocketmq/acl/common/AclUtilsTest.java b/acl/src/test/java/org/apache/rocketmq/acl/common/AclUtilsTest.java new file mode 100644 index 00000000000..72bcda6bb3a --- /dev/null +++ b/acl/src/test/java/org/apache/rocketmq/acl/common/AclUtilsTest.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.acl.common; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.junit.Assert; +import org.junit.Test; + +public class AclUtilsTest { + + @Test + public void getAddreeStrArray() { + String address = "1.1.1.{1,2,3,4}"; + String[] addressArray = AclUtils.getAddreeStrArray(address, "{1,2,3,4}"); + List newAddressList = new ArrayList<>(); + for (String a : addressArray) { + newAddressList.add(a); + } + + List addressList = new ArrayList<>(); + addressList.add("1.1.1.1"); + addressList.add("1.1.1.2"); + addressList.add("1.1.1.3"); + addressList.add("1.1.1.4"); + Assert.assertEquals(newAddressList, addressList); + } + + @Test + public void isScopeStringArray() { + String adderss = "12"; + + for (int i = 0; i < 6; i++) { + boolean isScope = AclUtils.isScope(adderss, 4); + if (i == 3) { + Assert.assertTrue(isScope); + } else { + Assert.assertFalse(isScope); + } + adderss = adderss + ".12"; + } + } + + @Test + public void isScopeArray() { + String[] adderss = StringUtils.split("12.12.12.12", "."); + boolean isScope = AclUtils.isScope(adderss, 4); + Assert.assertTrue(isScope); + isScope = AclUtils.isScope(adderss, 3); + Assert.assertTrue(isScope); + + adderss = StringUtils.split("12.12.1222.1222", "."); + isScope = AclUtils.isScope(adderss, 4); + Assert.assertFalse(isScope); + isScope = AclUtils.isScope(adderss, 3); + Assert.assertFalse(isScope); + + } + + @Test + public void isScopeStringTest() { + for (int i = 0; i < 256; i++) { + boolean isScope = AclUtils.isScope(i + ""); + Assert.assertTrue(isScope); + } + boolean isScope = AclUtils.isScope("-1"); + Assert.assertFalse(isScope); + isScope = AclUtils.isScope("256"); + Assert.assertFalse(isScope); + } + + @Test + public void isScopeTest() { + for (int i = 0; i < 256; i++) { + boolean isScope = AclUtils.isScope(i); + Assert.assertTrue(isScope); + } + boolean isScope = AclUtils.isScope(-1); + Assert.assertFalse(isScope); + isScope = AclUtils.isScope(256); + Assert.assertFalse(isScope); + + } + + @Test + public void isAsteriskTest() { + boolean isAsterisk = AclUtils.isAsterisk("*"); + Assert.assertTrue(isAsterisk); + + isAsterisk = AclUtils.isAsterisk(","); + Assert.assertFalse(isAsterisk); + } + + @Test + public void isColonTest() { + boolean isColon = AclUtils.isColon(","); + Assert.assertTrue(isColon); + + isColon = AclUtils.isColon("-"); + Assert.assertFalse(isColon); + } + + @Test + public void isMinusTest() { + boolean isMinus = AclUtils.isMinus("-"); + Assert.assertTrue(isMinus); + + isMinus = AclUtils.isMinus("*"); + Assert.assertFalse(isMinus); + } + + @SuppressWarnings("unchecked") + @Test + public void getYamlDataObjectTest() { + + Map map = AclUtils.getYamlDataObject("src/test/resources/conf/plain_acl.yml", Map.class); + Assert.assertFalse(map.isEmpty()); + } + + @Test(expected = Exception.class) + public void getYamlDataObjectExceptionTest() { + + AclUtils.getYamlDataObject("plain_acl.yml", Map.class); + } +} diff --git a/acl/src/test/java/org/apache/rocketmq/acl/common/PermissionTest.java b/acl/src/test/java/org/apache/rocketmq/acl/common/PermissionTest.java new file mode 100644 index 00000000000..253b5b241e0 --- /dev/null +++ b/acl/src/test/java/org/apache/rocketmq/acl/common/PermissionTest.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.acl.common; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.rocketmq.acl.plain.PlainAccessResource; +import org.junit.Assert; +import org.junit.Test; + +public class PermissionTest { + + @Test + public void fromStringGetPermissionTest() { + byte perm = Permission.parsePermFromString("PUB"); + Assert.assertEquals(perm, Permission.PUB); + + perm = Permission.parsePermFromString("SUB"); + Assert.assertEquals(perm, Permission.SUB); + + perm = Permission.parsePermFromString("PUB|SUB"); + Assert.assertEquals(perm, Permission.PUB|Permission.SUB); + + perm = Permission.parsePermFromString("SUB|PUB"); + Assert.assertEquals(perm, Permission.PUB|Permission.SUB); + + perm = Permission.parsePermFromString("DENY"); + Assert.assertEquals(perm, Permission.DENY); + + perm = Permission.parsePermFromString("1"); + Assert.assertEquals(perm, Permission.DENY); + + perm = Permission.parsePermFromString(null); + Assert.assertEquals(perm, Permission.DENY); + + } + + @Test + public void checkPermissionTest() { + boolean boo = Permission.checkPermission(Permission.DENY, Permission.DENY); + Assert.assertFalse(boo); + + boo = Permission.checkPermission(Permission.PUB, Permission.PUB); + Assert.assertTrue(boo); + + boo = Permission.checkPermission(Permission.SUB, Permission.SUB); + Assert.assertTrue(boo); + + boo = Permission.checkPermission(Permission.PUB, (byte) (Permission.PUB|Permission.SUB)); + Assert.assertTrue(boo); + + boo = Permission.checkPermission(Permission.SUB, (byte) (Permission.PUB|Permission.SUB)); + Assert.assertTrue(boo); + + boo = Permission.checkPermission(Permission.ANY, (byte) (Permission.PUB|Permission.SUB)); + Assert.assertTrue(boo); + + boo = Permission.checkPermission(Permission.ANY, Permission.SUB); + Assert.assertTrue(boo); + + boo = Permission.checkPermission(Permission.ANY, Permission.PUB); + Assert.assertTrue(boo); + + boo = Permission.checkPermission(Permission.DENY, Permission.ANY); + Assert.assertFalse(boo); + + boo = Permission.checkPermission(Permission.DENY, Permission.PUB); + Assert.assertFalse(boo); + + boo = Permission.checkPermission(Permission.DENY, Permission.SUB); + Assert.assertFalse(boo); + + } + + @Test(expected = AclException.class) + public void setTopicPermTest() { + PlainAccessResource plainAccessResource = new PlainAccessResource(); + Map resourcePermMap = plainAccessResource.getResourcePermMap(); + + Permission.parseResourcePerms(plainAccessResource, false, null); + Assert.assertNull(resourcePermMap); + + List groups = new ArrayList<>(); + Permission.parseResourcePerms(plainAccessResource, false, groups); + Assert.assertNull(resourcePermMap); + + groups.add("groupA=DENY"); + groups.add("groupB=PUB|SUB"); + groups.add("groupC=PUB"); + Permission.parseResourcePerms(plainAccessResource, false, groups); + resourcePermMap = plainAccessResource.getResourcePermMap(); + + byte perm = resourcePermMap.get(PlainAccessResource.getRetryTopic("groupA")); + Assert.assertEquals(perm, Permission.DENY); + + perm = resourcePermMap.get(PlainAccessResource.getRetryTopic("groupB")); + Assert.assertEquals(perm,Permission.PUB|Permission.SUB); + + perm = resourcePermMap.get(PlainAccessResource.getRetryTopic("groupC")); + Assert.assertEquals(perm, Permission.PUB); + + List topics = new ArrayList<>(); + topics.add("topicA=DENY"); + topics.add("topicB=PUB|SUB"); + topics.add("topicC=PUB"); + + Permission.parseResourcePerms(plainAccessResource, true, topics); + + perm = resourcePermMap.get("topicA"); + Assert.assertEquals(perm, Permission.DENY); + + perm = resourcePermMap.get("topicB"); + Assert.assertEquals(perm, Permission.PUB|Permission.SUB); + + perm = resourcePermMap.get("topicC"); + Assert.assertEquals(perm, Permission.PUB); + + List erron = new ArrayList<>(); + erron.add(""); + Permission.parseResourcePerms(plainAccessResource, false, erron); + } + + @Test + public void checkAdminCodeTest() { + Set code = new HashSet<>(); + code.add(17); + code.add(25); + code.add(215); + code.add(200); + code.add(207); + + for (int i = 0; i < 400; i++) { + boolean boo = Permission.needAdminPerm(i); + if (boo) { + Assert.assertTrue(code.contains(i)); + } + } + } + + @Test + public void AclExceptionTest(){ + AclException aclException = new AclException("CAL_SIGNATURE_FAILED",10015); + AclException aclExceptionWithMessage = new AclException("CAL_SIGNATURE_FAILED",10015,"CAL_SIGNATURE_FAILED Exception"); + Assert.assertEquals(aclException.getCode(),10015); + Assert.assertEquals(aclExceptionWithMessage.getStatus(),"CAL_SIGNATURE_FAILED"); + aclException.setCode(10016); + Assert.assertEquals(aclException.getCode(),10016); + aclException.setStatus("netaddress examine scope Exception netaddress"); + Assert.assertEquals(aclException.getStatus(),"netaddress examine scope Exception netaddress"); + } +} diff --git a/acl/src/test/java/org/apache/rocketmq/acl/common/SessionCredentialsTest.java b/acl/src/test/java/org/apache/rocketmq/acl/common/SessionCredentialsTest.java new file mode 100644 index 00000000000..a1a4bde4f87 --- /dev/null +++ b/acl/src/test/java/org/apache/rocketmq/acl/common/SessionCredentialsTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.acl.common; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.Properties; + +public class SessionCredentialsTest { + + @Test + public void equalsTest(){ + SessionCredentials sessionCredentials=new SessionCredentials("RocketMQ","12345678"); + sessionCredentials.setSecurityToken("abcd"); + SessionCredentials other=new SessionCredentials("RocketMQ","12345678","abcd"); + Assert.assertTrue(sessionCredentials.equals(other)); + } + + @Test + public void updateContentTest(){ + SessionCredentials sessionCredentials=new SessionCredentials(); + Properties properties=new Properties(); + properties.setProperty(SessionCredentials.ACCESS_KEY,"RocketMQ"); + properties.setProperty(SessionCredentials.SECRET_KEY,"12345678"); + properties.setProperty(SessionCredentials.SECURITY_TOKEN,"abcd"); + sessionCredentials.updateContent(properties); + } + + @Test + public void SessionCredentialHashCodeTest(){ + SessionCredentials sessionCredentials=new SessionCredentials(); + Properties properties=new Properties(); + properties.setProperty(SessionCredentials.ACCESS_KEY,"RocketMQ"); + properties.setProperty(SessionCredentials.SECRET_KEY,"12345678"); + properties.setProperty(SessionCredentials.SECURITY_TOKEN,"abcd"); + sessionCredentials.updateContent(properties); + Assert.assertEquals(sessionCredentials.hashCode(),353652211); + } + + @Test + public void SessionCredentialEqualsTest(){ + SessionCredentials sessionCredential1 =new SessionCredentials(); + Properties properties1=new Properties(); + properties1.setProperty(SessionCredentials.ACCESS_KEY,"RocketMQ"); + properties1.setProperty(SessionCredentials.SECRET_KEY,"12345678"); + properties1.setProperty(SessionCredentials.SECURITY_TOKEN,"abcd"); + sessionCredential1.updateContent(properties1); + + SessionCredentials sessionCredential2 =new SessionCredentials(); + Properties properties2=new Properties(); + properties2.setProperty(SessionCredentials.ACCESS_KEY,"RocketMQ"); + properties2.setProperty(SessionCredentials.SECRET_KEY,"12345678"); + properties2.setProperty(SessionCredentials.SECURITY_TOKEN,"abcd"); + sessionCredential2.updateContent(properties2); + + Assert.assertTrue(sessionCredential2.equals(sessionCredential1)); + sessionCredential2.setSecretKey("1234567899"); + sessionCredential2.setSignature("1234567899"); + Assert.assertFalse(sessionCredential2.equals(sessionCredential1)); + } + + @Test + public void SessionCredentialToStringTest(){ + SessionCredentials sessionCredential1 =new SessionCredentials(); + Properties properties1=new Properties(); + properties1.setProperty(SessionCredentials.ACCESS_KEY,"RocketMQ"); + properties1.setProperty(SessionCredentials.SECRET_KEY,"12345678"); + properties1.setProperty(SessionCredentials.SECURITY_TOKEN,"abcd"); + sessionCredential1.updateContent(properties1); + + Assert.assertEquals(sessionCredential1.toString(), + "SessionCredentials [accessKey=RocketMQ, secretKey=12345678, signature=null, SecurityToken=abcd]"); + } + + +} diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java new file mode 100644 index 00000000000..16e770206cb --- /dev/null +++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.acl.plain; + +import java.nio.ByteBuffer; +import java.util.HashSet; +import java.util.Set; + +import org.apache.rocketmq.acl.common.AclClientRPCHook; +import org.apache.rocketmq.acl.common.AclException; +import org.apache.rocketmq.acl.common.AclUtils; +import org.apache.rocketmq.acl.common.SessionCredentials; +import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.header.*; +import org.apache.rocketmq.common.protocol.heartbeat.ConsumerData; +import org.apache.rocketmq.common.protocol.heartbeat.HeartbeatData; +import org.apache.rocketmq.common.protocol.heartbeat.ProducerData; +import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class PlainAccessValidatorTest { + + private PlainAccessValidator plainAccessValidator; + private AclClientRPCHook aclClient; + private SessionCredentials sessionCredentials; + @Before + public void init() { + System.setProperty("rocketmq.home.dir", "src/test/resources"); + plainAccessValidator = new PlainAccessValidator(); + sessionCredentials = new SessionCredentials(); + sessionCredentials.setAccessKey("RocketMQ"); + sessionCredentials.setSecretKey("12345678"); + sessionCredentials.setSecurityToken("87654321"); + aclClient = new AclClientRPCHook(sessionCredentials); + } + + @Test + public void contentTest() { + SendMessageRequestHeader messageRequestHeader = new SendMessageRequestHeader(); + messageRequestHeader.setTopic("topicA"); + RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, messageRequestHeader); + aclClient.doBeforeRequest("", remotingCommand); + + ByteBuffer buf = remotingCommand.encodeHeader(); + buf.getInt(); + buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf); + buf.position(0); + PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "127.0.0.1"); + String signature = AclUtils.calSignature(accessResource.getContent(), sessionCredentials.getSecretKey()); + + Assert.assertEquals(accessResource.getSignature(), signature); + + } + + @Test + public void validateTest() { + SendMessageRequestHeader messageRequestHeader = new SendMessageRequestHeader(); + messageRequestHeader.setTopic("topicB"); + RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, messageRequestHeader); + aclClient.doBeforeRequest("", remotingCommand); + + ByteBuffer buf = remotingCommand.encodeHeader(); + buf.getInt(); + buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf); + buf.position(0); + PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "192.168.0.1"); + plainAccessValidator.validate(accessResource); + + } + + @Test + public void validateSendMessageTest() { + SendMessageRequestHeader messageRequestHeader = new SendMessageRequestHeader(); + messageRequestHeader.setTopic("topicB"); + RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, messageRequestHeader); + aclClient.doBeforeRequest("", remotingCommand); + + ByteBuffer buf = remotingCommand.encodeHeader(); + buf.getInt(); + buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf); + buf.position(0); + PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "192.168.0.1"); + plainAccessValidator.validate(accessResource); + } + + @Test + public void validateSendMessageV2Test() { + SendMessageRequestHeader messageRequestHeader = new SendMessageRequestHeader(); + messageRequestHeader.setTopic("topicC"); + RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(messageRequestHeader)); + aclClient.doBeforeRequest("", remotingCommand); + + ByteBuffer buf = remotingCommand.encodeHeader(); + buf.getInt(); + buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf); + buf.position(0); + PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "192.168.0.1:9876"); + plainAccessValidator.validate(accessResource); + } + + @Test + public void validatePullMessageTest() { + PullMessageRequestHeader pullMessageRequestHeader=new PullMessageRequestHeader(); + pullMessageRequestHeader.setTopic("topicC"); + pullMessageRequestHeader.setConsumerGroup("consumerGroupA"); + RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE,pullMessageRequestHeader); + aclClient.doBeforeRequest("", remotingCommand); + ByteBuffer buf = remotingCommand.encodeHeader(); + buf.getInt(); + buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf); + buf.position(0); + PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "192.168.0.1:9876"); + plainAccessValidator.validate(accessResource); + } + + @Test + public void validateConsumeMessageBackTest() { + ConsumerSendMsgBackRequestHeader consumerSendMsgBackRequestHeader=new ConsumerSendMsgBackRequestHeader(); + consumerSendMsgBackRequestHeader.setOriginTopic("topicC"); + consumerSendMsgBackRequestHeader.setGroup("consumerGroupA"); + RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK,consumerSendMsgBackRequestHeader); + aclClient.doBeforeRequest("", remotingCommand); + ByteBuffer buf = remotingCommand.encodeHeader(); + buf.getInt(); + buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf); + buf.position(0); + PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "192.168.0.1:9876"); + plainAccessValidator.validate(accessResource); + } + + @Test + public void validateQueryMessageTest() { + QueryMessageRequestHeader queryMessageRequestHeader=new QueryMessageRequestHeader(); + queryMessageRequestHeader.setTopic("topicC"); + RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.QUERY_MESSAGE,queryMessageRequestHeader); + aclClient.doBeforeRequest("", remotingCommand); + ByteBuffer buf = remotingCommand.encodeHeader(); + buf.getInt(); + buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf); + buf.position(0); + PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "192.168.0.1:9876"); + plainAccessValidator.validate(accessResource); + } + + @Test + public void validateHeartBeatTest() { + HeartbeatData heartbeatData=new HeartbeatData(); + Set producerDataSet=new HashSet<>(); + Set consumerDataSet=new HashSet<>(); + Set subscriptionDataSet=new HashSet<>(); + ProducerData producerData=new ProducerData(); + producerData.setGroupName("producerGroupA"); + ConsumerData consumerData=new ConsumerData(); + consumerData.setGroupName("consumerGroupA"); + SubscriptionData subscriptionData=new SubscriptionData(); + subscriptionData.setTopic("topicC"); + producerDataSet.add(producerData); + consumerDataSet.add(consumerData); + subscriptionDataSet.add(subscriptionData); + consumerData.setSubscriptionDataSet(subscriptionDataSet); + heartbeatData.setProducerDataSet(producerDataSet); + heartbeatData.setConsumerDataSet(consumerDataSet); + RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.HEART_BEAT,null); + remotingCommand.setBody(heartbeatData.encode()); + aclClient.doBeforeRequest("", remotingCommand); + ByteBuffer buf = remotingCommand.encode(); + buf.getInt(); + buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf); + buf.position(0); + PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "192.168.0.1:9876"); + plainAccessValidator.validate(accessResource); + } + + @Test + public void validateUnRegisterClientTest() { + UnregisterClientRequestHeader unregisterClientRequestHeader=new UnregisterClientRequestHeader(); + unregisterClientRequestHeader.setConsumerGroup("consumerGroupA"); + RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.UNREGISTER_CLIENT,unregisterClientRequestHeader); + aclClient.doBeforeRequest("", remotingCommand); + ByteBuffer buf = remotingCommand.encodeHeader(); + buf.getInt(); + buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf); + buf.position(0); + PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "192.168.0.1:9876"); + plainAccessValidator.validate(accessResource); + } + + @Test + public void validateGetConsumerListByGroupTest() { + GetConsumerListByGroupRequestHeader getConsumerListByGroupRequestHeader=new GetConsumerListByGroupRequestHeader(); + getConsumerListByGroupRequestHeader.setConsumerGroup("consumerGroupA"); + RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_LIST_BY_GROUP,getConsumerListByGroupRequestHeader); + aclClient.doBeforeRequest("", remotingCommand); + ByteBuffer buf = remotingCommand.encodeHeader(); + buf.getInt(); + buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf); + buf.position(0); + PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "192.168.0.1:9876"); + plainAccessValidator.validate(accessResource); + } + + @Test + public void validateUpdateConsumerOffSetTest() { + UpdateConsumerOffsetRequestHeader updateConsumerOffsetRequestHeader=new UpdateConsumerOffsetRequestHeader(); + updateConsumerOffsetRequestHeader.setConsumerGroup("consumerGroupA"); + RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.UPDATE_CONSUMER_OFFSET,updateConsumerOffsetRequestHeader); + aclClient.doBeforeRequest("", remotingCommand); + ByteBuffer buf = remotingCommand.encodeHeader(); + buf.getInt(); + buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf); + buf.position(0); + PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "192.168.0.1:9876"); + plainAccessValidator.validate(accessResource); + } + + @Test(expected = AclException.class) + public void validateNullAccessKeyTest() { + SessionCredentials sessionCredentials=new SessionCredentials(); + sessionCredentials.setAccessKey("RocketMQ1"); + sessionCredentials.setSecretKey("1234"); + AclClientRPCHook aclClientRPCHook=new AclClientRPCHook(sessionCredentials); + SendMessageRequestHeader messageRequestHeader = new SendMessageRequestHeader(); + messageRequestHeader.setTopic("topicB"); + RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, messageRequestHeader); + aclClientRPCHook.doBeforeRequest("", remotingCommand); + + ByteBuffer buf = remotingCommand.encodeHeader(); + buf.getInt(); + buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf); + buf.position(0); + PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "192.168.1.1"); + plainAccessValidator.validate(accessResource); + } + + @Test(expected = AclException.class) + public void validateErrorSecretKeyTest() { + SessionCredentials sessionCredentials=new SessionCredentials(); + sessionCredentials.setAccessKey("RocketMQ"); + sessionCredentials.setSecretKey("1234"); + AclClientRPCHook aclClientRPCHook=new AclClientRPCHook(sessionCredentials); + SendMessageRequestHeader messageRequestHeader = new SendMessageRequestHeader(); + messageRequestHeader.setTopic("topicB"); + RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, messageRequestHeader); + aclClientRPCHook.doBeforeRequest("", remotingCommand); + + ByteBuffer buf = remotingCommand.encodeHeader(); + buf.getInt(); + buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf); + buf.position(0); + PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "192.168.1.1"); + plainAccessValidator.validate(accessResource); + } +} diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionLoaderTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionLoaderTest.java new file mode 100644 index 00000000000..ebbc4fd260c --- /dev/null +++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionLoaderTest.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.acl.plain; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.rocketmq.acl.common.AclException; +import org.apache.rocketmq.acl.common.Permission; +import org.apache.rocketmq.acl.plain.PlainPermissionLoader.PlainAccessConfig; +import org.apache.rocketmq.common.UtilAll; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class PlainPermissionLoaderTest { + + PlainPermissionLoader plainPermissionLoader; + PlainAccessResource PUBPlainAccessResource; + PlainAccessResource SUBPlainAccessResource; + PlainAccessResource ANYPlainAccessResource; + PlainAccessResource DENYPlainAccessResource; + PlainAccessResource plainAccessResource = new PlainAccessResource(); + PlainAccessConfig plainAccessConfig = new PlainAccessConfig(); + PlainAccessResource plainAccessResourceTwo = new PlainAccessResource(); + Set adminCode = new HashSet<>(); + + @Before + public void init() throws NoSuchFieldException, SecurityException, IOException { + // UPDATE_AND_CREATE_TOPIC + adminCode.add(17); + // UPDATE_BROKER_CONFIG + adminCode.add(25); + // DELETE_TOPIC_IN_BROKER + adminCode.add(215); + // UPDATE_AND_CREATE_SUBSCRIPTIONGROUP + adminCode.add(200); + // DELETE_SUBSCRIPTIONGROUP + adminCode.add(207); + + PUBPlainAccessResource = clonePlainAccessResource(Permission.PUB); + SUBPlainAccessResource = clonePlainAccessResource(Permission.SUB); + ANYPlainAccessResource = clonePlainAccessResource(Permission.ANY); + DENYPlainAccessResource = clonePlainAccessResource(Permission.DENY); + + System.setProperty("rocketmq.home.dir", "src/test/resources"); + System.setProperty("rocketmq.acl.plain.file", "/conf/plain_acl.yml"); + plainPermissionLoader = new PlainPermissionLoader(); + + } + + public PlainAccessResource clonePlainAccessResource(byte perm) { + PlainAccessResource painAccessResource = new PlainAccessResource(); + painAccessResource.setAccessKey("RocketMQ"); + painAccessResource.setSecretKey("12345678"); + painAccessResource.setWhiteRemoteAddress("127.0." + perm + ".*"); + painAccessResource.setDefaultGroupPerm(perm); + painAccessResource.setDefaultTopicPerm(perm); + painAccessResource.addResourceAndPerm(PlainAccessResource.getRetryTopic("groupA"), Permission.PUB); + painAccessResource.addResourceAndPerm(PlainAccessResource.getRetryTopic("groupB"), Permission.SUB); + painAccessResource.addResourceAndPerm(PlainAccessResource.getRetryTopic("groupC"), Permission.ANY); + painAccessResource.addResourceAndPerm(PlainAccessResource.getRetryTopic("groupD"), Permission.DENY); + + painAccessResource.addResourceAndPerm("topicA", Permission.PUB); + painAccessResource.addResourceAndPerm("topicB", Permission.SUB); + painAccessResource.addResourceAndPerm("topicC", Permission.ANY); + painAccessResource.addResourceAndPerm("topicD", Permission.DENY); + return painAccessResource; + } + + @Test + public void buildPlainAccessResourceTest() { + PlainAccessResource plainAccessResource = null; + PlainAccessConfig plainAccess = new PlainAccessConfig(); + + plainAccess.setAccessKey("RocketMQ"); + plainAccess.setSecretKey("12345678"); + plainAccessResource = plainPermissionLoader.buildPlainAccessResource(plainAccess); + Assert.assertEquals(plainAccessResource.getAccessKey(), "RocketMQ"); + Assert.assertEquals(plainAccessResource.getSecretKey(), "12345678"); + + plainAccess.setWhiteRemoteAddress("127.0.0.1"); + plainAccessResource = plainPermissionLoader.buildPlainAccessResource(plainAccess); + Assert.assertEquals(plainAccessResource.getWhiteRemoteAddress(), "127.0.0.1"); + + plainAccess.setAdmin(true); + plainAccessResource = plainPermissionLoader.buildPlainAccessResource(plainAccess); + Assert.assertEquals(plainAccessResource.isAdmin(), true); + + List groups = new ArrayList(); + groups.add("groupA=DENY"); + groups.add("groupB=PUB|SUB"); + groups.add("groupC=PUB"); + plainAccess.setGroupPerms(groups); + plainAccessResource = plainPermissionLoader.buildPlainAccessResource(plainAccess); + Map resourcePermMap = plainAccessResource.getResourcePermMap(); + Assert.assertEquals(resourcePermMap.size(), 3); + + Assert.assertEquals(resourcePermMap.get(PlainAccessResource.getRetryTopic("groupA")).byteValue(), Permission.DENY); + Assert.assertEquals(resourcePermMap.get(PlainAccessResource.getRetryTopic("groupB")).byteValue(), Permission.PUB|Permission.SUB); + Assert.assertEquals(resourcePermMap.get(PlainAccessResource.getRetryTopic("groupC")).byteValue(), Permission.PUB); + + List topics = new ArrayList(); + topics.add("topicA=DENY"); + topics.add("topicB=PUB|SUB"); + topics.add("topicC=PUB"); + plainAccess.setTopicPerms(topics); + plainAccessResource = plainPermissionLoader.buildPlainAccessResource(plainAccess); + resourcePermMap = plainAccessResource.getResourcePermMap(); + Assert.assertEquals(resourcePermMap.size(), 6); + + Assert.assertEquals(resourcePermMap.get("topicA").byteValue(), Permission.DENY); + Assert.assertEquals(resourcePermMap.get("topicB").byteValue(), Permission.PUB|Permission.SUB); + Assert.assertEquals(resourcePermMap.get("topicC").byteValue(), Permission.PUB); + } + + @Test(expected = AclException.class) + public void checkPermAdmin() { + PlainAccessResource plainAccessResource = new PlainAccessResource(); + plainAccessResource.setRequestCode(17); + plainPermissionLoader.checkPerm(plainAccessResource, PUBPlainAccessResource); + } + + @Test + public void checkPerm() { + + PlainAccessResource plainAccessResource = new PlainAccessResource(); + plainAccessResource.addResourceAndPerm("topicA", Permission.PUB); + plainPermissionLoader.checkPerm(plainAccessResource, PUBPlainAccessResource); + plainAccessResource.addResourceAndPerm("topicB", Permission.SUB); + plainPermissionLoader.checkPerm(plainAccessResource, ANYPlainAccessResource); + + plainAccessResource = new PlainAccessResource(); + plainAccessResource.addResourceAndPerm("topicB", Permission.SUB); + plainPermissionLoader.checkPerm(plainAccessResource, SUBPlainAccessResource); + plainAccessResource.addResourceAndPerm("topicA", Permission.PUB); + plainPermissionLoader.checkPerm(plainAccessResource, ANYPlainAccessResource); + + } + @Test(expected = AclException.class) + public void checkErrorPerm() { + + plainAccessResource = new PlainAccessResource(); + plainAccessResource.addResourceAndPerm("topicF", Permission.SUB); + plainPermissionLoader.checkPerm(plainAccessResource, SUBPlainAccessResource); + } + @Test(expected = AclException.class) + public void accountNullTest() { + plainAccessConfig.setAccessKey(null); + plainPermissionLoader.buildPlainAccessResource(plainAccessConfig); + } + + @Test(expected = AclException.class) + public void accountThanTest() { + plainAccessConfig.setAccessKey("123"); + plainPermissionLoader.buildPlainAccessResource(plainAccessConfig); + } + + @Test(expected = AclException.class) + public void passWordtNullTest() { + plainAccessConfig.setAccessKey(null); + plainPermissionLoader.buildPlainAccessResource(plainAccessConfig); + } + + @Test(expected = AclException.class) + public void passWordThanTest() { + plainAccessConfig.setAccessKey("123"); + plainPermissionLoader.buildPlainAccessResource(plainAccessConfig); + } + + @Test(expected = AclException.class) + public void testPlainAclPlugEngineInit() { + System.setProperty("rocketmq.home.dir", ""); + new PlainPermissionLoader().load(); + } + + @SuppressWarnings("unchecked") + @Test + public void cleanAuthenticationInfoTest() throws IllegalAccessException { + //plainPermissionLoader.addPlainAccessResource(plainAccessResource); + Map> plainAccessResourceMap = (Map>) FieldUtils.readDeclaredField(plainPermissionLoader, "plainAccessResourceMap", true); + Assert.assertFalse(plainAccessResourceMap.isEmpty()); + + plainPermissionLoader.clearPermissionInfo(); + plainAccessResourceMap = (Map>) FieldUtils.readDeclaredField(plainPermissionLoader, "plainAccessResourceMap", true); + Assert.assertTrue(plainAccessResourceMap.isEmpty()); + } + + @Test + public void isWatchStartTest() { + + PlainPermissionLoader plainPermissionLoader = new PlainPermissionLoader(); + Assert.assertTrue(plainPermissionLoader.isWatchStart()); + } + + + @Test + public void testWatch() throws IOException, IllegalAccessException ,InterruptedException{ + System.setProperty("rocketmq.home.dir", "src/test/resources"); + System.setProperty("rocketmq.acl.plain.file", "/conf/plain_acl-test.yml"); + String fileName =System.getProperty("rocketmq.home.dir", "src/test/resources")+System.getProperty("rocketmq.acl.plain.file", "/conf/plain_acl.yml"); + File transport = new File(fileName); + transport.delete(); + transport.createNewFile(); + FileWriter writer = new FileWriter(transport); + writer.write("accounts:\r\n"); + writer.write("- accessKey: watchrocketmq\r\n"); + writer.write(" secretKey: 12345678\r\n"); + writer.write(" whiteRemoteAddress: 127.0.0.1\r\n"); + writer.write(" admin: true\r\n"); + writer.flush(); + writer.close(); + + PlainPermissionLoader plainPermissionLoader = new PlainPermissionLoader(); + Assert.assertTrue(plainPermissionLoader.isWatchStart()); + + { + Map plainAccessResourceMap = (Map) FieldUtils.readDeclaredField(plainPermissionLoader, "plainAccessResourceMap", true); + PlainAccessResource accessResource = plainAccessResourceMap.get("watchrocketmq"); + Assert.assertNotNull(accessResource); + Assert.assertEquals(accessResource.getSecretKey(), "12345678"); + Assert.assertTrue(accessResource.isAdmin()); + + } + + writer = new FileWriter(new File(fileName), true); + writer.write("- accessKey: watchrocketmq1\r\n"); + writer.write(" secretKey: 88888888\r\n"); + writer.write(" whiteRemoteAddress: 127.0.0.1\r\n"); + writer.write(" admin: false\r\n"); + writer.flush(); + writer.close(); + + Thread.sleep(1000); + { + Map plainAccessResourceMap = (Map) FieldUtils.readDeclaredField(plainPermissionLoader, "plainAccessResourceMap", true); + PlainAccessResource accessResource = plainAccessResourceMap.get("watchrocketmq1"); + Assert.assertNotNull(accessResource); + Assert.assertEquals(accessResource.getSecretKey(), "88888888"); + Assert.assertFalse(accessResource.isAdmin()); + + } + transport.delete(); + System.setProperty("rocketmq.home.dir", "src/test/resources"); + System.setProperty("rocketmq.acl.plain.file", "/conf/plain_acl.yml"); + } + + @Test(expected = AclException.class) + public void initializeTest() { + System.setProperty("rocketmq.acl.plain.file", "/conf/plain_acl_null.yml"); + new PlainPermissionLoader(); + + } + +} diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyTest.java new file mode 100644 index 00000000000..53391f41186 --- /dev/null +++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyTest.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.acl.plain; + +import org.apache.rocketmq.acl.common.AclException; +import org.junit.Assert; +import org.junit.Test; + +public class RemoteAddressStrategyTest { + + RemoteAddressStrategyFactory remoteAddressStrategyFactory = new RemoteAddressStrategyFactory(); + + @Test + public void netaddressStrategyFactoryExceptionTest() { + PlainAccessResource plainAccessResource = new PlainAccessResource(); + remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + Assert.assertEquals(remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource).getClass(), + RemoteAddressStrategyFactory.BlankRemoteAddressStrategy.class); + } + + @Test + public void netaddressStrategyFactoryTest() { + PlainAccessResource plainAccessResource = new PlainAccessResource(); + + plainAccessResource.setWhiteRemoteAddress("*"); + RemoteAddressStrategy remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + Assert.assertEquals(remoteAddressStrategy, RemoteAddressStrategyFactory.NULL_NET_ADDRESS_STRATEGY); + + plainAccessResource.setWhiteRemoteAddress("127.0.0.1"); + remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.OneRemoteAddressStrategy.class); + + plainAccessResource.setWhiteRemoteAddress("127.0.0.1,127.0.0.2,127.0.0.3"); + remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.MultipleRemoteAddressStrategy.class); + + plainAccessResource.setWhiteRemoteAddress("127.0.0.{1,2,3}"); + remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.MultipleRemoteAddressStrategy.class); + + plainAccessResource.setWhiteRemoteAddress("127.0.0.1-200"); + remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.RangeRemoteAddressStrategy.class); + + plainAccessResource.setWhiteRemoteAddress("127.0.0.*"); + remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.RangeRemoteAddressStrategy.class); + + plainAccessResource.setWhiteRemoteAddress("127.0.1-20.*"); + remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.RangeRemoteAddressStrategy.class); + + plainAccessResource.setWhiteRemoteAddress(""); + remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.BlankRemoteAddressStrategy.class); + } + + @Test(expected = AclException.class) + public void verifyTest() { + PlainAccessResource plainAccessResource = new PlainAccessResource(); + plainAccessResource.setWhiteRemoteAddress("127.0.0.1"); + remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + plainAccessResource.setWhiteRemoteAddress("256.0.0.1"); + remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + } + + @Test + public void nullNetaddressStrategyTest() { + boolean isMatch = RemoteAddressStrategyFactory.NULL_NET_ADDRESS_STRATEGY.match(new PlainAccessResource()); + Assert.assertTrue(isMatch); + } + + @Test + public void blankNetaddressStrategyTest() { + boolean isMatch = RemoteAddressStrategyFactory.BLANK_NET_ADDRESS_STRATEGY.match(new PlainAccessResource()); + Assert.assertFalse(isMatch); + } + + public void oneNetaddressStrategyTest() { + PlainAccessResource plainAccessResource = new PlainAccessResource(); + plainAccessResource.setWhiteRemoteAddress("127.0.0.1"); + RemoteAddressStrategy remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + plainAccessResource.setWhiteRemoteAddress(""); + boolean match = remoteAddressStrategy.match(plainAccessResource); + Assert.assertFalse(match); + + plainAccessResource.setWhiteRemoteAddress("127.0.0.2"); + match = remoteAddressStrategy.match(plainAccessResource); + Assert.assertFalse(match); + + plainAccessResource.setWhiteRemoteAddress("127.0.0.1"); + match = remoteAddressStrategy.match(plainAccessResource); + Assert.assertTrue(match); + } + + @Test + public void multipleNetaddressStrategyTest() { + PlainAccessResource plainAccessResource = new PlainAccessResource(); + plainAccessResource.setWhiteRemoteAddress("127.0.0.1,127.0.0.2,127.0.0.3"); + RemoteAddressStrategy remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + multipleNetaddressStrategyTest(remoteAddressStrategy); + + plainAccessResource.setWhiteRemoteAddress("127.0.0.{1,2,3}"); + remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + multipleNetaddressStrategyTest(remoteAddressStrategy); + + } + + @Test(expected = AclException.class) + public void multipleNetaddressStrategyExceptionTest() { + PlainAccessResource plainAccessResource = new PlainAccessResource(); + plainAccessResource.setWhiteRemoteAddress("127.0.0.1,2,3}"); + remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + } + + private void multipleNetaddressStrategyTest(RemoteAddressStrategy remoteAddressStrategy) { + PlainAccessResource plainAccessResource = new PlainAccessResource(); + plainAccessResource.setWhiteRemoteAddress("127.0.0.1"); + boolean match = remoteAddressStrategy.match(plainAccessResource); + Assert.assertTrue(match); + + plainAccessResource.setWhiteRemoteAddress("127.0.0.2"); + match = remoteAddressStrategy.match(plainAccessResource); + Assert.assertTrue(match); + + plainAccessResource.setWhiteRemoteAddress("127.0.0.3"); + match = remoteAddressStrategy.match(plainAccessResource); + Assert.assertTrue(match); + + plainAccessResource.setWhiteRemoteAddress("127.0.0.4"); + match = remoteAddressStrategy.match(plainAccessResource); + Assert.assertFalse(match); + + plainAccessResource.setWhiteRemoteAddress("127.0.0.0"); + match = remoteAddressStrategy.match(plainAccessResource); + Assert.assertFalse(match); + + } + + @Test + public void rangeNetaddressStrategyTest() { + String head = "127.0.0."; + PlainAccessResource plainAccessResource = new PlainAccessResource(); + plainAccessResource.setWhiteRemoteAddress("127.0.0.1-200"); + RemoteAddressStrategy remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + rangeNetaddressStrategyTest(remoteAddressStrategy, head, 1, 200, true); + plainAccessResource.setWhiteRemoteAddress("127.0.0.*"); + remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + rangeNetaddressStrategyTest(remoteAddressStrategy, head, 0, 255, true); + + plainAccessResource.setWhiteRemoteAddress("127.0.1-200.*"); + remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + rangeNetaddressStrategyThirdlyTest(remoteAddressStrategy, head, 1, 200); + } + + private void rangeNetaddressStrategyTest(RemoteAddressStrategy remoteAddressStrategy, String head, int start, + int end, + boolean isFalse) { + PlainAccessResource plainAccessResource = new PlainAccessResource(); + for (int i = -10; i < 300; i++) { + plainAccessResource.setWhiteRemoteAddress(head + i); + boolean match = remoteAddressStrategy.match(plainAccessResource); + if (isFalse && i >= start && i <= end) { + Assert.assertTrue(match); + continue; + } + Assert.assertFalse(match); + + } + } + + private void rangeNetaddressStrategyThirdlyTest(RemoteAddressStrategy remoteAddressStrategy, String head, int start, + int end) { + String newHead; + for (int i = -10; i < 300; i++) { + newHead = head + i; + if (i >= start && i <= end) { + rangeNetaddressStrategyTest(remoteAddressStrategy, newHead, 0, 255, false); + } + } + } + + @Test(expected = AclException.class) + public void rangeNetaddressStrategyExceptionStartGreaterEndTest() { + rangeNetaddressStrategyExceptionTest("127.0.0.2-1"); + } + + @Test(expected = AclException.class) + public void rangeNetaddressStrategyExceptionScopeTest() { + rangeNetaddressStrategyExceptionTest("127.0.0.-1-200"); + } + + @Test(expected = AclException.class) + public void rangeNetaddressStrategyExceptionScopeTwoTest() { + rangeNetaddressStrategyExceptionTest("127.0.0.0-256"); + } + + private void rangeNetaddressStrategyExceptionTest(String netaddress) { + PlainAccessResource plainAccessResource = new PlainAccessResource(); + plainAccessResource.setWhiteRemoteAddress(netaddress); + remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + } + +} diff --git a/acl/src/test/resources/conf/plain_acl.yml b/acl/src/test/resources/conf/plain_acl.yml new file mode 100644 index 00000000000..2c24795ff64 --- /dev/null +++ b/acl/src/test/resources/conf/plain_acl.yml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## suggested format + +globalWhiteRemoteAddresses: +- 10.10.103.* +- 192.168.0.* + +accounts: +- accessKey: RocketMQ + secretKey: 12345678 + whiteRemoteAddress: 192.168.0.* + admin: false + defaultTopicPerm: DENY + defaultGroupPerm: SUB + topicPerms: + - topicA=DENY + - topicB=PUB|SUB + - topicC=SUB + groupPerms: + # the group should convert to retry topic + - groupA=DENY + - groupB=SUB + - groupC=SUB + +- accessKey: rocketmq2 + secretKey: 12345678 + whiteRemoteAddress: 192.168.1.* + # if it is admin, it could access all resources + admin: true + diff --git a/acl/src/test/resources/conf/plain_acl_null.yml b/acl/src/test/resources/conf/plain_acl_null.yml new file mode 100644 index 00000000000..bc30380c888 --- /dev/null +++ b/acl/src/test/resources/conf/plain_acl_null.yml @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## suggested format + + diff --git a/acl/src/test/resources/conf/watch/plain_acl_watch.yml b/acl/src/test/resources/conf/watch/plain_acl_watch.yml new file mode 100644 index 00000000000..9d2c3954941 --- /dev/null +++ b/acl/src/test/resources/conf/watch/plain_acl_watch.yml @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## suggested format +accounts: +- accessKey: watchrocketmq + secretKey: 12345678 + whiteRemoteAddress: 127.0.0.1 + admin: true +- accessKey: watchrocketmq1 + secretKey: 88888888 + whiteRemoteAddress: 127.0.0.1 + admin: false diff --git a/acl/src/test/resources/logback-test.xml b/acl/src/test/resources/logback-test.xml new file mode 100644 index 00000000000..e556c649e52 --- /dev/null +++ b/acl/src/test/resources/logback-test.xml @@ -0,0 +1,34 @@ + + + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{5} - %msg%n + UTF-8 + + + + + + + + + + + diff --git a/broker/pom.xml b/broker/pom.xml index f10ae53730e..01390fd3c53 100644 --- a/broker/pom.xml +++ b/broker/pom.xml @@ -1,25 +1,19 @@ - + org.apache.rocketmq rocketmq-all - 4.4.0-SNAPSHOT + 4.4.1-SNAPSHOT 4.0.0 @@ -52,6 +46,10 @@ ${project.groupId} rocketmq-filter + + ${project.groupId} + rocketmq-acl + ch.qos.logback logback-classic diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 7a5a96e9ea8..427f8611890 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -32,6 +32,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.acl.AccessValidator; import org.apache.rocketmq.broker.client.ClientHousekeepingService; import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener; import org.apache.rocketmq.broker.client.ConsumerManager; @@ -93,6 +94,7 @@ import org.apache.rocketmq.remoting.netty.NettyServerConfig; import org.apache.rocketmq.remoting.netty.RequestTask; import org.apache.rocketmq.remoting.netty.TlsSystemConfig; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.srvutil.FileWatchService; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.MessageArrivingListener; @@ -161,6 +163,7 @@ public class BrokerController { private AbstractTransactionalMessageCheckListener transactionalMessageCheckListener; private Future slaveSyncFuture; + public BrokerController( final BrokerConfig brokerConfig, final NettyServerConfig nettyServerConfig, @@ -464,6 +467,8 @@ private void reloadServerSslContext() { } } initialTransaction(); + initialAcl(); + initialRpcHooks(); } return result; } @@ -483,6 +488,47 @@ private void initialTransaction() { this.transactionalMessageCheckService = new TransactionalMessageCheckService(this); } + private void initialAcl() { + if (!this.brokerConfig.isAclEnable()) { + log.info("The broker dose not enable acl"); + return; + } + + List accessValidators = ServiceProvider.load(ServiceProvider.ACL_VALIDATOR_ID, AccessValidator.class); + if (accessValidators == null || accessValidators.isEmpty()) { + log.info("The broker dose not load the AccessValidator"); + return; + } + + for (AccessValidator accessValidator: accessValidators) { + final AccessValidator validator = accessValidator; + this.registerServerRPCHook(new RPCHook() { + + @Override + public void doBeforeRequest(String remoteAddr, RemotingCommand request) { + //Do not catch the exception + validator.validate(validator.parse(request, remoteAddr)); + } + + @Override + public void doAfterResponse(String remoteAddr, RemotingCommand request, RemotingCommand response) { + } + }); + } + } + + + private void initialRpcHooks() { + + List rpcHooks = ServiceProvider.load(ServiceProvider.RPC_HOOK_ID, RPCHook.class); + if (rpcHooks == null || rpcHooks.isEmpty()) { + return; + } + for (RPCHook rpcHook: rpcHooks) { + this.registerServerRPCHook(rpcHook); + } + } + public void registerProcessor() { /** * SendMessageProcessor @@ -988,6 +1034,7 @@ public void registerConsumeMessageHook(final ConsumeMessageHook hook) { public void registerServerRPCHook(RPCHook rpcHook) { getRemotingServer().registerRPCHook(rpcHook); + this.fastRemotingServer.registerRPCHook(rpcHook); } public RemotingServer getRemotingServer() { @@ -1048,8 +1095,10 @@ public void setTransactionalMessageCheckListener( this.transactionalMessageCheckListener = transactionalMessageCheckListener; } + public BlockingQueue getEndTransactionThreadPoolQueue() { return endTransactionThreadPoolQueue; + } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java index 59c7895eb72..ed353da4bb9 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java @@ -124,6 +124,16 @@ public TopicConfigManager(BrokerController brokerController) { topicConfig.setWriteQueueNums(1); this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); } + { + if (this.brokerController.getBrokerConfig().isTraceTopicEnable()) { + String topic = this.brokerController.getBrokerConfig().getMsgTraceTopicName(); + TopicConfig topicConfig = new TopicConfig(topic); + this.systemTopicList.add(topic); + topicConfig.setReadQueueNums(1); + topicConfig.setWriteQueueNums(1); + this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); + } + } } public boolean isSystemTopic(final String topic) { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java index 15e5c84ff2d..1c227af1502 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java @@ -198,7 +198,7 @@ public void check(long transactionTimeout, int transactionCheckMax, if (null != checkImmunityTimeStr) { checkImmunityTime = getImmunityTime(checkImmunityTimeStr, transactionTimeout); if (valueOfCurrentMinusBorn < checkImmunityTime) { - if (checkPrepareQueueOffset(removeMap, doneOpOffset, msgExt, checkImmunityTime)) { + if (checkPrepareQueueOffset(removeMap, doneOpOffset, msgExt)) { newOffset = i + 1; i++; continue; @@ -315,33 +315,26 @@ private PullResult fillOpRemoveMap(HashMap removeMap, * @param removeMap Op message map to determine whether a half message was responded by producer. * @param doneOpOffset Op Message which has been checked. * @param msgExt Half message - * @param checkImmunityTime User defined time to avoid being detected early. * @return Return true if put success, otherwise return false. */ - private boolean checkPrepareQueueOffset(HashMap removeMap, List doneOpOffset, MessageExt msgExt, - long checkImmunityTime) { - if (System.currentTimeMillis() - msgExt.getBornTimestamp() < checkImmunityTime) { - String prepareQueueOffsetStr = msgExt.getUserProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET); - if (null == prepareQueueOffsetStr) { - return putImmunityMsgBackToHalfQueue(msgExt); + private boolean checkPrepareQueueOffset(HashMap removeMap, List doneOpOffset, + MessageExt msgExt) { + String prepareQueueOffsetStr = msgExt.getUserProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET); + if (null == prepareQueueOffsetStr) { + return putImmunityMsgBackToHalfQueue(msgExt); + } else { + long prepareQueueOffset = getLong(prepareQueueOffsetStr); + if (-1 == prepareQueueOffset) { + return false; } else { - long prepareQueueOffset = getLong(prepareQueueOffsetStr); - if (-1 == prepareQueueOffset) { - return false; + if (removeMap.containsKey(prepareQueueOffset)) { + long tmpOpOffset = removeMap.remove(prepareQueueOffset); + doneOpOffset.add(tmpOpOffset); + return true; } else { - if (removeMap.containsKey(prepareQueueOffset)) { - long tmpOpOffset = removeMap.remove(prepareQueueOffset); - doneOpOffset.add(tmpOpOffset); - return true; - } else { - return putImmunityMsgBackToHalfQueue(msgExt); - } + return putImmunityMsgBackToHalfQueue(msgExt); } - } - - } else { - return true; } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/util/ServiceProvider.java b/broker/src/main/java/org/apache/rocketmq/broker/util/ServiceProvider.java index 8b9b63e4dcb..e679660104d 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/util/ServiceProvider.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/util/ServiceProvider.java @@ -34,6 +34,14 @@ public class ServiceProvider { public static final String TRANSACTION_LISTENER_ID = "META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener"; + + public static final String RPC_HOOK_ID = "META-INF/service/org.apache.rocketmq.remoting.RPCHook"; + + + public static final String ACL_VALIDATOR_ID = "META-INF/service/org.apache.rocketmq.acl.AccessValidator"; + + + static { thisClassLoader = getClassLoader(ServiceProvider.class); } diff --git a/broker/src/main/resources/META-INF/service/org.apache.rocketmq.acl.AccessValidator b/broker/src/main/resources/META-INF/service/org.apache.rocketmq.acl.AccessValidator new file mode 100644 index 00000000000..1abc92e0162 --- /dev/null +++ b/broker/src/main/resources/META-INF/service/org.apache.rocketmq.acl.AccessValidator @@ -0,0 +1 @@ +org.apache.rocketmq.acl.plain.PlainAccessValidator \ No newline at end of file diff --git a/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java index 56abf084a7c..dae1335540b 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java @@ -24,6 +24,7 @@ import org.apache.rocketmq.remoting.netty.NettyServerConfig; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.junit.After; +import org.junit.Ignore; import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; diff --git a/broker/src/test/java/org/apache/rocketmq/broker/pagecache/ManyMessageTransferTest.java b/broker/src/test/java/org/apache/rocketmq/broker/pagecache/ManyMessageTransferTest.java new file mode 100644 index 00000000000..508635c044c --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/pagecache/ManyMessageTransferTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.broker.pagecache; + +import java.nio.ByteBuffer; +import org.apache.rocketmq.store.GetMessageResult; +import org.junit.Assert; +import org.junit.Test; + +public class ManyMessageTransferTest { + + @Test + public void ManyMessageTransferBuilderTest(){ + ByteBuffer byteBuffer = ByteBuffer.allocate(20); + byteBuffer.putInt(20); + GetMessageResult getMessageResult = new GetMessageResult(); + ManyMessageTransfer manyMessageTransfer = new ManyMessageTransfer(byteBuffer,getMessageResult); + } + + @Test + public void ManyMessageTransferPosTest(){ + ByteBuffer byteBuffer = ByteBuffer.allocate(20); + byteBuffer.putInt(20); + GetMessageResult getMessageResult = new GetMessageResult(); + ManyMessageTransfer manyMessageTransfer = new ManyMessageTransfer(byteBuffer,getMessageResult); + Assert.assertEquals(manyMessageTransfer.position(),4); + } + + @Test + public void ManyMessageTransferCountTest(){ + ByteBuffer byteBuffer = ByteBuffer.allocate(20); + byteBuffer.putInt(20); + GetMessageResult getMessageResult = new GetMessageResult(); + ManyMessageTransfer manyMessageTransfer = new ManyMessageTransfer(byteBuffer,getMessageResult); + + Assert.assertEquals(manyMessageTransfer.count(),20); + + } + + @Test + public void ManyMessageTransferCloseTest(){ + ByteBuffer byteBuffer = ByteBuffer.allocate(20); + byteBuffer.putInt(20); + GetMessageResult getMessageResult = new GetMessageResult(); + ManyMessageTransfer manyMessageTransfer = new ManyMessageTransfer(byteBuffer,getMessageResult); + manyMessageTransfer.close(); + manyMessageTransfer.deallocate(); + } +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/pagecache/OneMessageTransferTest.java b/broker/src/test/java/org/apache/rocketmq/broker/pagecache/OneMessageTransferTest.java new file mode 100644 index 00000000000..2cd4bdc1657 --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/pagecache/OneMessageTransferTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.broker.pagecache; + +import java.nio.ByteBuffer; +import org.apache.rocketmq.store.MappedFile; +import org.apache.rocketmq.store.SelectMappedBufferResult; +import org.junit.Assert; +import org.junit.Test; + +public class OneMessageTransferTest { + + @Test + public void OneMessageTransferTest(){ + ByteBuffer byteBuffer = ByteBuffer.allocate(20); + byteBuffer.putInt(20); + SelectMappedBufferResult selectMappedBufferResult = new SelectMappedBufferResult(0,byteBuffer,20,new MappedFile()); + OneMessageTransfer manyMessageTransfer = new OneMessageTransfer(byteBuffer,selectMappedBufferResult); + } + + @Test + public void OneMessageTransferCountTest(){ + ByteBuffer byteBuffer = ByteBuffer.allocate(20); + byteBuffer.putInt(20); + SelectMappedBufferResult selectMappedBufferResult = new SelectMappedBufferResult(0,byteBuffer,20,new MappedFile()); + OneMessageTransfer manyMessageTransfer = new OneMessageTransfer(byteBuffer,selectMappedBufferResult); + Assert.assertEquals(manyMessageTransfer.count(),40); + } + + @Test + public void OneMessageTransferPosTest(){ + ByteBuffer byteBuffer = ByteBuffer.allocate(20); + byteBuffer.putInt(20); + SelectMappedBufferResult selectMappedBufferResult = new SelectMappedBufferResult(0,byteBuffer,20,new MappedFile()); + OneMessageTransfer manyMessageTransfer = new OneMessageTransfer(byteBuffer,selectMappedBufferResult); + Assert.assertEquals(manyMessageTransfer.position(),8); + } +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/util/ServiceProviderTest.java b/broker/src/test/java/org/apache/rocketmq/broker/util/ServiceProviderTest.java index 22228a6e0ef..a3a35c8832d 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/util/ServiceProviderTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/util/ServiceProviderTest.java @@ -17,12 +17,15 @@ package org.apache.rocketmq.broker.util; +import org.apache.rocketmq.acl.AccessValidator; import org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener; import org.apache.rocketmq.broker.transaction.TransactionalMessageService; import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; +import java.util.List; + public class ServiceProviderTest { @Test @@ -38,4 +41,10 @@ public void loadAbstractTransactionListenerTest() { AbstractTransactionalMessageCheckListener.class); assertThat(listener).isNotNull(); } + + @Test + public void loadAccessValidatorTest() { + List accessValidators = ServiceProvider.load(ServiceProvider.ACL_VALIDATOR_ID, AccessValidator.class); + assertThat(accessValidators).isNotNull(); + } } diff --git a/broker/src/test/resources/META-INF/service/org.apache.rocketmq.acl.AccessValidator b/broker/src/test/resources/META-INF/service/org.apache.rocketmq.acl.AccessValidator new file mode 100644 index 00000000000..1abc92e0162 --- /dev/null +++ b/broker/src/test/resources/META-INF/service/org.apache.rocketmq.acl.AccessValidator @@ -0,0 +1 @@ +org.apache.rocketmq.acl.plain.PlainAccessValidator \ No newline at end of file diff --git a/client/pom.xml b/client/pom.xml index da6f53a874e..1b8abfc7666 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.4.0-SNAPSHOT + 4.4.1-SNAPSHOT 4.0.0 diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java index cd706703078..6befbf3b5e9 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java @@ -257,6 +257,18 @@ public PullResult pull(MessageQueue mq, String subExpression, long offset, int m return this.defaultMQPullConsumerImpl.pull(mq, subExpression, offset, maxNums, timeout); } + @Override + public PullResult pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return this.defaultMQPullConsumerImpl.pull(mq, messageSelector, offset, maxNums); + } + + @Override + public PullResult pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums, long timeout) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return this.defaultMQPullConsumerImpl.pull(mq, messageSelector, offset, maxNums, timeout); + } + @Override public void pull(MessageQueue mq, String subExpression, long offset, int maxNums, PullCallback pullCallback) throws MQClientException, RemotingException, InterruptedException { @@ -270,6 +282,20 @@ public void pull(MessageQueue mq, String subExpression, long offset, int maxNums this.defaultMQPullConsumerImpl.pull(mq, subExpression, offset, maxNums, pullCallback, timeout); } + @Override + public void pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums, + PullCallback pullCallback) + throws MQClientException, RemotingException, InterruptedException { + this.defaultMQPullConsumerImpl.pull(mq, messageSelector, offset, maxNums, pullCallback); + } + + @Override + public void pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums, + PullCallback pullCallback, long timeout) + throws MQClientException, RemotingException, InterruptedException { + this.defaultMQPullConsumerImpl.pull(mq, messageSelector, offset, maxNums, pullCallback, timeout); + } + @Override public PullResult pullBlockIfNotFound(MessageQueue mq, String subExpression, long offset, int maxNums) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java index d51030a1592..6cd2ad18a5a 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java @@ -29,6 +29,10 @@ import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl; +import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.client.trace.AsyncTraceDispatcher; +import org.apache.rocketmq.client.trace.TraceDispatcher; +import org.apache.rocketmq.client.trace.hook.ConsumeMessageTraceHookImpl; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; @@ -36,6 +40,7 @@ import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.exception.RemotingException; @@ -56,6 +61,8 @@ */ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsumer { + private final InternalLogger log = ClientLogger.getLog(); + /** * Internal implementation. Most of the functions herein are delegated to it. */ @@ -246,6 +253,11 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume */ private long consumeTimeout = 15; + /** + * Interface of asynchronous transfer data + */ + private TraceDispatcher traceDispatcher = null; + /** * Default constructor. */ @@ -258,7 +270,7 @@ public DefaultMQPushConsumer() { * * @param consumerGroup Consume queue. * @param rpcHook RPC hook to execute before each remoting command. - * @param allocateMessageQueueStrategy message queue allocating algorithm. + * @param allocateMessageQueueStrategy Message queue allocating algorithm. */ public DefaultMQPushConsumer(final String consumerGroup, RPCHook rpcHook, AllocateMessageQueueStrategy allocateMessageQueueStrategy) { @@ -267,6 +279,33 @@ public DefaultMQPushConsumer(final String consumerGroup, RPCHook rpcHook, defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(this, rpcHook); } + /** + * Constructor specifying consumer group, RPC hook, message queue allocating algorithm, enabled msg trace flag and customized trace topic name. + * + * @param consumerGroup Consume queue. + * @param rpcHook RPC hook to execute before each remoting command. + * @param allocateMessageQueueStrategy message queue allocating algorithm. + * @param enableMsgTrace Switch flag instance for message trace. + * @param customizedTraceTopic The name value of message trace topic.If you don't config,you can use the default trace topic name. + */ + public DefaultMQPushConsumer(final String consumerGroup, RPCHook rpcHook, + AllocateMessageQueueStrategy allocateMessageQueueStrategy, boolean enableMsgTrace, final String customizedTraceTopic) { + this.consumerGroup = consumerGroup; + this.allocateMessageQueueStrategy = allocateMessageQueueStrategy; + defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(this, rpcHook); + if (enableMsgTrace) { + try { + AsyncTraceDispatcher dispatcher = new AsyncTraceDispatcher(customizedTraceTopic, rpcHook); + dispatcher.setHostConsumer(this.getDefaultMQPushConsumerImpl()); + traceDispatcher = dispatcher; + this.getDefaultMQPushConsumerImpl().registerConsumeMessageHook( + new ConsumeMessageTraceHookImpl(traceDispatcher)); + } catch (Throwable e) { + log.error("system mqtrace hook init failed ,maybe can't send msg trace data"); + } + } + } + /** * Constructor specifying RPC hook. * @@ -276,6 +315,28 @@ public DefaultMQPushConsumer(RPCHook rpcHook) { this(MixAll.DEFAULT_CONSUMER_GROUP, rpcHook, new AllocateMessageQueueAveragely()); } + + /** + * Constructor specifying consumer group and enabled msg trace flag. + * + * @param consumerGroup Consumer group. + * @param enableMsgTrace Switch flag instance for message trace. + */ + public DefaultMQPushConsumer(final String consumerGroup, boolean enableMsgTrace) { + this(consumerGroup, null, new AllocateMessageQueueAveragely(), enableMsgTrace, null); + } + + /** + * Constructor specifying consumer group, enabled msg trace flag and customized trace topic name. + * + * @param consumerGroup Consumer group. + * @param enableMsgTrace Switch flag instance for message trace. + * @param customizedTraceTopic The name value of message trace topic.If you don't config,you can use the default trace topic name. + */ + public DefaultMQPushConsumer(final String consumerGroup, boolean enableMsgTrace, final String customizedTraceTopic) { + this(consumerGroup, null, new AllocateMessageQueueAveragely(), enableMsgTrace, customizedTraceTopic); + } + /** * Constructor specifying consumer group. * @@ -518,6 +579,13 @@ public Set fetchSubscribeMessageQueues(String topic) throws MQClie @Override public void start() throws MQClientException { this.defaultMQPushConsumerImpl.start(); + if (null != traceDispatcher) { + try { + traceDispatcher.start(this.getNamesrvAddr()); + } catch (MQClientException e) { + log.warn("trace dispatcher start failed ", e); + } + } } /** @@ -526,6 +594,9 @@ public void start() throws MQClientException { @Override public void shutdown() { this.defaultMQPushConsumerImpl.shutdown(); + if (null != traceDispatcher) { + traceDispatcher.shutdown(); + } } @Override @@ -694,4 +765,8 @@ public long getConsumeTimeout() { public void setConsumeTimeout(final long consumeTimeout) { this.consumeTimeout = consumeTimeout; } + + public TraceDispatcher getTraceDispatcher() { + return traceDispatcher; + } } diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumer.java index 33002c98371..28b807c2ed8 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumer.java @@ -66,6 +66,39 @@ PullResult pull(final MessageQueue mq, final String subExpression, final long of final int maxNums, final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException; + /** + * Pulling the messages, not blocking + *

+ * support other message selection, such as {@link org.apache.rocketmq.common.filter.ExpressionType#SQL92} + *

+ * + * @param mq from which message queue + * @param selector message selector({@link MessageSelector}), can be null. + * @param offset from where to pull + * @param maxNums max pulling numbers + * @return The resulting {@code PullRequest} + */ + PullResult pull(final MessageQueue mq, final MessageSelector selector, final long offset, + final int maxNums) throws MQClientException, RemotingException, MQBrokerException, + InterruptedException; + + /** + * Pulling the messages in the specified timeout + *

+ * support other message selection, such as {@link org.apache.rocketmq.common.filter.ExpressionType#SQL92} + *

+ * + * @param mq from which message queue + * @param selector message selector({@link MessageSelector}), can be null. + * @param offset from where to pull + * @param maxNums max pulling numbers + * @param timeout Pulling the messages in the specified timeout + * @return The resulting {@code PullRequest} + */ + PullResult pull(final MessageQueue mq, final MessageSelector selector, final long offset, + final int maxNums, final long timeout) throws MQClientException, RemotingException, MQBrokerException, + InterruptedException; + /** * Pulling the messages in a async. way */ @@ -80,6 +113,20 @@ void pull(final MessageQueue mq, final String subExpression, final long offset, final PullCallback pullCallback, long timeout) throws MQClientException, RemotingException, InterruptedException; + /** + * Pulling the messages in a async. way. Support message selection + */ + void pull(final MessageQueue mq, final MessageSelector selector, final long offset, final int maxNums, + final PullCallback pullCallback) throws MQClientException, RemotingException, + InterruptedException; + + /** + * Pulling the messages in a async. way. Support message selection + */ + void pull(final MessageQueue mq, final MessageSelector selector, final long offset, final int maxNums, + final PullCallback pullCallback, long timeout) throws MQClientException, RemotingException, + InterruptedException; + /** * Pulling the messages,if no message arrival,blocking some time * diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java index 420d89b2fd0..39c43d592d7 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java @@ -26,6 +26,7 @@ import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.Validators; import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; +import org.apache.rocketmq.client.consumer.MessageSelector; import org.apache.rocketmq.client.consumer.PullCallback; import org.apache.rocketmq.client.consumer.PullResult; import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; @@ -46,6 +47,7 @@ import org.apache.rocketmq.common.ServiceState; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.filter.ExpressionType; import org.apache.rocketmq.common.filter.FilterAPI; import org.apache.rocketmq.common.help.FAQUrl; import org.apache.rocketmq.logging.InternalLogger; @@ -158,17 +160,58 @@ public PullResult pull(MessageQueue mq, String subExpression, long offset, int m public PullResult pull(MessageQueue mq, String subExpression, long offset, int maxNums, long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - return this.pullSyncImpl(mq, subExpression, offset, maxNums, false, timeout); + SubscriptionData subscriptionData = getSubscriptionData(mq, subExpression); + return this.pullSyncImpl(mq, subscriptionData, offset, maxNums, false, timeout); } - private PullResult pullSyncImpl(MessageQueue mq, String subExpression, long offset, int maxNums, boolean block, + public PullResult pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return pull(mq, messageSelector, offset, maxNums, this.defaultMQPullConsumer.getConsumerPullTimeoutMillis()); + } + + public PullResult pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums, long timeout) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + SubscriptionData subscriptionData = getSubscriptionData(mq, messageSelector); + return this.pullSyncImpl(mq, subscriptionData, offset, maxNums, false, timeout); + } + + private SubscriptionData getSubscriptionData(MessageQueue mq, String subExpression) + throws MQClientException { + + if (null == mq) { + throw new MQClientException("mq is null", null); + } + + try { + return FilterAPI.buildSubscriptionData(this.defaultMQPullConsumer.getConsumerGroup(), + mq.getTopic(), subExpression); + } catch (Exception e) { + throw new MQClientException("parse subscription error", e); + } + } + + private SubscriptionData getSubscriptionData(MessageQueue mq, MessageSelector messageSelector) + throws MQClientException { + + if (null == mq) { + throw new MQClientException("mq is null", null); + } + + try { + return FilterAPI.build(mq.getTopic(), + messageSelector.getExpression(), messageSelector.getExpressionType()); + } catch (Exception e) { + throw new MQClientException("parse subscription error", e); + } + } + + private PullResult pullSyncImpl(MessageQueue mq, SubscriptionData subscriptionData, long offset, int maxNums, boolean block, long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { this.makeSureStateOK(); if (null == mq) { throw new MQClientException("mq is null", null); - } if (offset < 0) { @@ -183,20 +226,14 @@ private PullResult pullSyncImpl(MessageQueue mq, String subExpression, long offs int sysFlag = PullSysFlag.buildSysFlag(false, block, true, false); - SubscriptionData subscriptionData; - try { - subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPullConsumer.getConsumerGroup(), - mq.getTopic(), subExpression); - } catch (Exception e) { - throw new MQClientException("parse subscription error", e); - } - long timeoutMillis = block ? this.defaultMQPullConsumer.getConsumerTimeoutMillisWhenSuspend() : timeout; + boolean isTagType = ExpressionType.isTagType(subscriptionData.getExpressionType()); PullResult pullResult = this.pullAPIWrapper.pullKernelImpl( mq, subscriptionData.getSubString(), - 0L, + subscriptionData.getExpressionType(), + isTagType ? 0L : subscriptionData.getSubVersion(), offset, maxNums, sysFlag, @@ -369,12 +406,27 @@ public void pull(MessageQueue mq, String subExpression, long offset, int maxNums public void pull(MessageQueue mq, String subExpression, long offset, int maxNums, PullCallback pullCallback, long timeout) throws MQClientException, RemotingException, InterruptedException { - this.pullAsyncImpl(mq, subExpression, offset, maxNums, pullCallback, false, timeout); + SubscriptionData subscriptionData = getSubscriptionData(mq, subExpression); + this.pullAsyncImpl(mq, subscriptionData, offset, maxNums, pullCallback, false, timeout); + } + + public void pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums, + PullCallback pullCallback) + throws MQClientException, RemotingException, InterruptedException { + pull(mq, messageSelector, offset, maxNums, pullCallback, this.defaultMQPullConsumer.getConsumerPullTimeoutMillis()); + } + + public void pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums, + PullCallback pullCallback, + long timeout) + throws MQClientException, RemotingException, InterruptedException { + SubscriptionData subscriptionData = getSubscriptionData(mq, messageSelector); + this.pullAsyncImpl(mq, subscriptionData, offset, maxNums, pullCallback, false, timeout); } private void pullAsyncImpl( final MessageQueue mq, - final String subExpression, + final SubscriptionData subscriptionData, final long offset, final int maxNums, final PullCallback pullCallback, @@ -403,20 +455,14 @@ private void pullAsyncImpl( try { int sysFlag = PullSysFlag.buildSysFlag(false, block, true, false); - final SubscriptionData subscriptionData; - try { - subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPullConsumer.getConsumerGroup(), - mq.getTopic(), subExpression); - } catch (Exception e) { - throw new MQClientException("parse subscription error", e); - } - long timeoutMillis = block ? this.defaultMQPullConsumer.getConsumerTimeoutMillisWhenSuspend() : timeout; + boolean isTagType = ExpressionType.isTagType(subscriptionData.getExpressionType()); this.pullAPIWrapper.pullKernelImpl( mq, subscriptionData.getSubString(), - 0L, + subscriptionData.getExpressionType(), + isTagType ? 0L : subscriptionData.getSubVersion(), offset, maxNums, sysFlag, @@ -444,7 +490,8 @@ public void onException(Throwable e) { public PullResult pullBlockIfNotFound(MessageQueue mq, String subExpression, long offset, int maxNums) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - return this.pullSyncImpl(mq, subExpression, offset, maxNums, true, this.getDefaultMQPullConsumer().getConsumerPullTimeoutMillis()); + SubscriptionData subscriptionData = getSubscriptionData(mq, subExpression); + return this.pullSyncImpl(mq, subscriptionData, offset, maxNums, true, this.getDefaultMQPullConsumer().getConsumerPullTimeoutMillis()); } public DefaultMQPullConsumer getDefaultMQPullConsumer() { @@ -454,7 +501,8 @@ public DefaultMQPullConsumer getDefaultMQPullConsumer() { public void pullBlockIfNotFound(MessageQueue mq, String subExpression, long offset, int maxNums, PullCallback pullCallback) throws MQClientException, RemotingException, InterruptedException { - this.pullAsyncImpl(mq, subExpression, offset, maxNums, pullCallback, true, + SubscriptionData subscriptionData = getSubscriptionData(mq, subExpression); + this.pullAsyncImpl(mq, subscriptionData, offset, maxNums, pullCallback, true, this.getDefaultMQPullConsumer().getConsumerPullTimeoutMillis()); } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java index b650e35e02f..1d2d24fa3b9 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java @@ -209,34 +209,6 @@ public PullResult pullKernelImpl( throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); } - public PullResult pullKernelImpl( - final MessageQueue mq, - final String subExpression, - final long subVersion, - final long offset, - final int maxNums, - final int sysFlag, - final long commitOffset, - final long brokerSuspendMaxTimeMillis, - final long timeoutMillis, - final CommunicationMode communicationMode, - final PullCallback pullCallback - ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - return pullKernelImpl( - mq, - subExpression, - ExpressionType.TAG, - subVersion, offset, - maxNums, - sysFlag, - commitOffset, - brokerSuspendMaxTimeMillis, - timeoutMillis, - communicationMode, - pullCallback - ); - } - public long recalculatePullFromWhichNode(final MessageQueue mq) { if (this.isConnectBrokerByUser()) { return this.defaultBrokerId; diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java index 4b055e0fc2f..88f06ea4ca6 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java @@ -1046,6 +1046,19 @@ public int findBrokerVersion(String brokerName, String brokerAddr) { if (this.brokerVersionTable.get(brokerName).containsKey(brokerAddr)) { return this.brokerVersionTable.get(brokerName).get(brokerAddr); } + } else { + HeartbeatData heartbeatData = prepareHeartbeatData(); + try { + int version = this.mQClientAPIImpl.sendHearbeat(brokerAddr, heartbeatData, 3000); + return version; + } catch (Exception e) { + if (this.isBrokerInNameServer(brokerAddr)) { + log.info("send heart beat to broker[{} {}] failed", brokerName, brokerAddr); + } else { + log.info("send heart beat to broker[{} {}] exception, because the broker not up, forget it", brokerName, + brokerAddr); + } + } } return 0; } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index 7ace9d5b07d..90f4f787635 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -30,8 +30,10 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.Validators; import org.apache.rocketmq.client.common.ClientErrorCode; @@ -101,6 +103,10 @@ public class DefaultMQProducerImpl implements MQProducerInner { private MQFaultStrategy mqFaultStrategy = new MQFaultStrategy(); + private final BlockingQueue asyncSenderThreadPoolQueue; + private final ExecutorService defaultAsyncSenderExecutor; + private ExecutorService asyncSenderExecutor; + public DefaultMQProducerImpl(final DefaultMQProducer defaultMQProducer) { this(defaultMQProducer, null); } @@ -108,6 +114,22 @@ public DefaultMQProducerImpl(final DefaultMQProducer defaultMQProducer) { public DefaultMQProducerImpl(final DefaultMQProducer defaultMQProducer, RPCHook rpcHook) { this.defaultMQProducer = defaultMQProducer; this.rpcHook = rpcHook; + + this.asyncSenderThreadPoolQueue = new LinkedBlockingQueue(50000); + this.defaultAsyncSenderExecutor = new ThreadPoolExecutor( + Runtime.getRuntime().availableProcessors(), + Runtime.getRuntime().availableProcessors(), + 1000 * 60, + TimeUnit.MILLISECONDS, + this.asyncSenderThreadPoolQueue, + new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, "AsyncSenderExecutor_" + this.threadIndex.incrementAndGet()); + } + }); } public void registerCheckForbiddenHook(CheckForbiddenHook checkForbiddenHook) { @@ -456,7 +478,7 @@ public void send(Message msg, public void send(final Message msg, final SendCallback sendCallback, final long timeout) throws MQClientException, RemotingException, InterruptedException { final long beginStartTime = System.currentTimeMillis(); - ExecutorService executor = this.getCallbackExecutor(); + ExecutorService executor = this.getAsyncSenderExecutor(); try { executor.submit(new Runnable() { @Override @@ -957,7 +979,7 @@ public void send(Message msg, MessageQueue mq, SendCallback sendCallback) public void send(final Message msg, final MessageQueue mq, final SendCallback sendCallback, final long timeout) throws MQClientException, RemotingException, InterruptedException { final long beginStartTime = System.currentTimeMillis(); - ExecutorService executor = this.getCallbackExecutor(); + ExecutorService executor = this.getAsyncSenderExecutor(); try { executor.submit(new Runnable() { @Override @@ -1079,7 +1101,7 @@ public void send(Message msg, MessageQueueSelector selector, Object arg, SendCal public void send(final Message msg, final MessageQueueSelector selector, final Object arg, final SendCallback sendCallback, final long timeout) throws MQClientException, RemotingException, InterruptedException { final long beginStartTime = System.currentTimeMillis(); - ExecutorService executor = this.getCallbackExecutor(); + ExecutorService executor = this.getAsyncSenderExecutor(); try { executor.submit(new Runnable() { @Override @@ -1243,9 +1265,13 @@ public void endTransaction( public void setCallbackExecutor(final ExecutorService callbackExecutor) { this.mQClientFactory.getMQClientAPIImpl().getRemotingClient().setCallbackExecutor(callbackExecutor); } - public ExecutorService getCallbackExecutor() { - return this.mQClientFactory.getMQClientAPIImpl().getRemotingClient().getCallbackExecutor(); + public ExecutorService getAsyncSenderExecutor() { + return null == asyncSenderExecutor ? defaultAsyncSenderExecutor : asyncSenderExecutor; + } + + public void setAsyncSenderExecutor(ExecutorService asyncSenderExecutor) { + this.asyncSenderExecutor = asyncSenderExecutor; } public SendResult send(Message msg, diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java index 9732d0eb844..2339142616a 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java @@ -25,6 +25,10 @@ import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; +import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.client.trace.AsyncTraceDispatcher; +import org.apache.rocketmq.client.trace.TraceDispatcher; +import org.apache.rocketmq.client.trace.hook.SendMessageTraceHookImpl; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageBatch; @@ -33,6 +37,7 @@ import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageId; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.netty.NettyRemotingClient; @@ -56,6 +61,8 @@ */ public class DefaultMQProducer extends ClientConfig implements MQProducer { + private final InternalLogger log = ClientLogger.getLog(); + /** * Wrapping internal implementations for virtually all methods presented in this class. */ @@ -119,6 +126,11 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { */ private int maxMessageSize = 1024 * 1024 * 4; // 4M + /** + * Interface of asynchronous transfer data + */ + private TraceDispatcher traceDispatcher = null; + /** * Default constructor. */ @@ -137,6 +149,31 @@ public DefaultMQProducer(final String producerGroup, RPCHook rpcHook) { defaultMQProducerImpl = new DefaultMQProducerImpl(this, rpcHook); } + /** + * Constructor specifying producer group, RPC hook, enabled msgTrace flag and customized trace topic name. + * + * @param producerGroup Producer group, see the name-sake field. + * @param rpcHook RPC hook to execute per each remoting command execution. + * @param enableMsgTrace Switch flag instance for message trace. + * @param customizedTraceTopic The name value of message trace topic.If you don't config,you can use the default trace topic name. + */ + public DefaultMQProducer(final String producerGroup, RPCHook rpcHook, boolean enableMsgTrace,final String customizedTraceTopic) { + this.producerGroup = producerGroup; + defaultMQProducerImpl = new DefaultMQProducerImpl(this, rpcHook); + //if client open the message trace feature + if (enableMsgTrace) { + try { + AsyncTraceDispatcher dispatcher = new AsyncTraceDispatcher(customizedTraceTopic, rpcHook); + dispatcher.setHostProducer(this.getDefaultMQProducerImpl()); + traceDispatcher = dispatcher; + this.getDefaultMQProducerImpl().registerSendMessageHook( + new SendMessageTraceHookImpl(traceDispatcher)); + } catch (Throwable e) { + log.error("system mqtrace hook init failed ,maybe can't send msg trace data"); + } + } + } + /** * Constructor specifying producer group. * @@ -147,8 +184,30 @@ public DefaultMQProducer(final String producerGroup) { } /** - * Constructor specifying the RPC hook. + * Constructor specifying producer group and enabled msg trace flag. + * + * @param producerGroup Producer group, see the name-sake field. + * @param enableMsgTrace Switch flag instance for message trace. + */ + public DefaultMQProducer(final String producerGroup, boolean enableMsgTrace) { + this(producerGroup, null, enableMsgTrace, null); + } + + + /** + * Constructor specifying producer group, enabled msgTrace flag and customized trace topic name. * + * @param producerGroup Producer group, see the name-sake field. + * @param enableMsgTrace Switch flag instance for message trace. + * @param customizedTraceTopic The name value of message trace topic.If you don't config,you can use the default trace topic name. + */ + public DefaultMQProducer(final String producerGroup, boolean enableMsgTrace, final String customizedTraceTopic) { + this(producerGroup, null, enableMsgTrace, customizedTraceTopic); + } + + /** + * Constructor specifying the RPC hook. + * * @param rpcHook RPC hook to execute per each remoting command execution. */ public DefaultMQProducer(RPCHook rpcHook) { @@ -170,6 +229,13 @@ public DefaultMQProducer(RPCHook rpcHook) { @Override public void start() throws MQClientException { this.defaultMQProducerImpl.start(); + if (null != traceDispatcher) { + try { + traceDispatcher.start(this.getNamesrvAddr()); + } catch (MQClientException e) { + log.warn("trace dispatcher start failed ", e); + } + } } /** @@ -178,6 +244,9 @@ public void start() throws MQClientException { @Override public void shutdown() { this.defaultMQProducerImpl.shutdown(); + if (null != traceDispatcher) { + traceDispatcher.shutdown(); + } } /** @@ -655,6 +724,16 @@ public void setCallbackExecutor(final ExecutorService callbackExecutor) { this.defaultMQProducerImpl.setCallbackExecutor(callbackExecutor); } + /** + * Sets an Executor to be used for executing asynchronous send. If the Executor is not set, {@link + * DefaultMQProducerImpl#defaultAsyncSenderExecutor} will be used. + * + * @param asyncSenderExecutor the instance of Executor + */ + public void setAsyncSenderExecutor(final ExecutorService asyncSenderExecutor) { + this.defaultMQProducerImpl.setAsyncSenderExecutor(asyncSenderExecutor); + } + private MessageBatch batch(Collection msgs) throws MQClientException { MessageBatch msgBatch; try { @@ -777,4 +856,9 @@ public int getRetryTimesWhenSendAsyncFailed() { public void setRetryTimesWhenSendAsyncFailed(final int retryTimesWhenSendAsyncFailed) { this.retryTimesWhenSendAsyncFailed = retryTimesWhenSendAsyncFailed; } + + public TraceDispatcher getTraceDispatcher() { + return traceDispatcher; + } + } diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java b/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java new file mode 100644 index 00000000000..87a795e4b4e --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java @@ -0,0 +1,386 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.trace; + +import org.apache.rocketmq.client.common.ThreadLocalIndex; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl; +import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; +import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; +import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.client.producer.MessageQueueSelector; +import org.apache.rocketmq.client.producer.SendCallback; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.ThreadFactoryImpl; +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.remoting.RPCHook; + +import java.io.IOException; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.UUID; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.Set; +import java.util.HashSet; + + +import static org.apache.rocketmq.client.trace.TraceConstants.TRACE_INSTANCE_NAME; + +public class AsyncTraceDispatcher implements TraceDispatcher { + + private final static InternalLogger log = ClientLogger.getLog(); + private final int queueSize; + private final int batchSize; + private final int maxMsgSize; + private final DefaultMQProducer traceProducer; + private final ThreadPoolExecutor traceExecuter; + // The last discard number of log + private AtomicLong discardCount; + private Thread worker; + private ArrayBlockingQueue traceContextQueue; + private ArrayBlockingQueue appenderQueue; + private volatile Thread shutDownHook; + private volatile boolean stopped = false; + private DefaultMQProducerImpl hostProducer; + private DefaultMQPushConsumerImpl hostConsumer; + private volatile ThreadLocalIndex sendWhichQueue = new ThreadLocalIndex(); + private String dispatcherId = UUID.randomUUID().toString(); + private String traceTopicName; + private AtomicBoolean isStarted = new AtomicBoolean(false); + + + public AsyncTraceDispatcher(String traceTopicName, RPCHook rpcHook) throws MQClientException { + // queueSize is greater than or equal to the n power of 2 of value + this.queueSize = 2048; + this.batchSize = 100; + this.maxMsgSize = 128000; + this.discardCount = new AtomicLong(0L); + this.traceContextQueue = new ArrayBlockingQueue(1024); + this.appenderQueue = new ArrayBlockingQueue(queueSize); + if (!UtilAll.isBlank(traceTopicName)) { + this.traceTopicName = traceTopicName; + } else { + this.traceTopicName = MixAll.RMQ_SYS_TRACE_TOPIC; + } + this.traceExecuter = new ThreadPoolExecutor(// + 10, // + 20, // + 1000 * 60, // + TimeUnit.MILLISECONDS, // + this.appenderQueue, // + new ThreadFactoryImpl("MQTraceSendThread_")); + traceProducer = getAndCreateTraceProducer(rpcHook); + } + + public String getTraceTopicName() { + return traceTopicName; + } + + public void setTraceTopicName(String traceTopicName) { + this.traceTopicName = traceTopicName; + } + + public DefaultMQProducer getTraceProducer() { + return traceProducer; + } + + public DefaultMQProducerImpl getHostProducer() { + return hostProducer; + } + + public void setHostProducer(DefaultMQProducerImpl hostProducer) { + this.hostProducer = hostProducer; + } + + public DefaultMQPushConsumerImpl getHostConsumer() { + return hostConsumer; + } + + public void setHostConsumer(DefaultMQPushConsumerImpl hostConsumer) { + this.hostConsumer = hostConsumer; + } + + public void start(String nameSrvAddr) throws MQClientException { + if (isStarted.compareAndSet(false, true)) { + traceProducer.setNamesrvAddr(nameSrvAddr); + traceProducer.setInstanceName(TRACE_INSTANCE_NAME + "_" + nameSrvAddr); + traceProducer.start(); + } + this.worker = new Thread(new AsyncRunnable(), "MQ-AsyncTraceDispatcher-Thread-" + dispatcherId); + this.worker.setDaemon(true); + this.worker.start(); + this.registerShutDownHook(); + } + + private DefaultMQProducer getAndCreateTraceProducer(RPCHook rpcHook) { + DefaultMQProducer traceProducerInstance = this.traceProducer; + if (traceProducerInstance == null) { + traceProducerInstance = new DefaultMQProducer(rpcHook); + traceProducerInstance.setProducerGroup(TraceConstants.GROUP_NAME); + traceProducerInstance.setSendMsgTimeout(5000); + traceProducerInstance.setVipChannelEnabled(false); + // The max size of message is 128K + traceProducerInstance.setMaxMessageSize(maxMsgSize - 10 * 1000); + } + return traceProducerInstance; + } + + @Override + public boolean append(final Object ctx) { + boolean result = traceContextQueue.offer((TraceContext) ctx); + if (!result) { + log.info("buffer full" + discardCount.incrementAndGet() + " ,context is " + ctx); + } + return result; + } + + @Override + public void flush() throws IOException { + // The maximum waiting time for refresh,avoid being written all the time, resulting in failure to return. + long end = System.currentTimeMillis() + 500; + while (traceContextQueue.size() > 0 || appenderQueue.size() > 0 && System.currentTimeMillis() <= end) { + try { + Thread.sleep(1); + } catch (InterruptedException e) { + break; + } + } + log.info("------end trace send " + traceContextQueue.size() + " " + appenderQueue.size()); + } + + @Override + public void shutdown() { + this.stopped = true; + this.traceExecuter.shutdown(); + if (isStarted.get()) { + traceProducer.shutdown(); + } + this.removeShutdownHook(); + } + + public void registerShutDownHook() { + if (shutDownHook == null) { + shutDownHook = new Thread(new Runnable() { + private volatile boolean hasShutdown = false; + + @Override + public void run() { + synchronized (this) { + if (!this.hasShutdown) { + try { + flush(); + } catch (IOException e) { + log.error("system MQTrace hook shutdown failed ,maybe loss some trace data"); + } + } + } + } + }, "ShutdownHookMQTrace"); + Runtime.getRuntime().addShutdownHook(shutDownHook); + } + } + + public void removeShutdownHook() { + if (shutDownHook != null) { + Runtime.getRuntime().removeShutdownHook(shutDownHook); + } + } + + class AsyncRunnable implements Runnable { + private boolean stopped; + + @Override + public void run() { + while (!stopped) { + List contexts = new ArrayList(batchSize); + for (int i = 0; i < batchSize; i++) { + TraceContext context = null; + try { + //get trace data element from blocking Queue — traceContextQueue + context = traceContextQueue.poll(5, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + } + if (context != null) { + contexts.add(context); + } else { + break; + } + } + if (contexts.size() > 0) { + AsyncAppenderRequest request = new AsyncAppenderRequest(contexts); + traceExecuter.submit(request); + } else if (AsyncTraceDispatcher.this.stopped) { + this.stopped = true; + } + } + + } + } + + class AsyncAppenderRequest implements Runnable { + List contextList; + + public AsyncAppenderRequest(final List contextList) { + if (contextList != null) { + this.contextList = contextList; + } else { + this.contextList = new ArrayList(1); + } + } + + @Override + public void run() { + sendTraceData(contextList); + } + + public void sendTraceData(List contextList) { + Map> transBeanMap = new HashMap>(); + for (TraceContext context : contextList) { + if (context.getTraceBeans().isEmpty()) { + continue; + } + // Topic value corresponding to original message entity content + String topic = context.getTraceBeans().get(0).getTopic(); + // Use original message entity's topic as key + String key = topic; + List transBeanList = transBeanMap.get(key); + if (transBeanList == null) { + transBeanList = new ArrayList(); + transBeanMap.put(key, transBeanList); + } + TraceTransferBean traceData = TraceDataEncoder.encoderFromContextBean(context); + transBeanList.add(traceData); + } + for (Map.Entry> entry : transBeanMap.entrySet()) { + flushData(entry.getValue()); + } + } + + /** + * Batch sending data actually + */ + private void flushData(List transBeanList) { + if (transBeanList.size() == 0) { + return; + } + // Temporary buffer + StringBuilder buffer = new StringBuilder(1024); + int count = 0; + Set keySet = new HashSet(); + + for (TraceTransferBean bean : transBeanList) { + // Keyset of message trace includes msgId of or original message + keySet.addAll(bean.getTransKey()); + buffer.append(bean.getTransData()); + count++; + // Ensure that the size of the package should not exceed the upper limit. + if (buffer.length() >= traceProducer.getMaxMessageSize()) { + sendTraceDataByMQ(keySet, buffer.toString()); + // Clear temporary buffer after finishing + buffer.delete(0, buffer.length()); + keySet.clear(); + count = 0; + } + } + if (count > 0) { + sendTraceDataByMQ(keySet, buffer.toString()); + } + transBeanList.clear(); + } + + /** + * Send message trace data + * + * @param keySet the keyset in this batch(including msgId in original message not offsetMsgId) + * @param data the message trace data in this batch + */ + private void sendTraceDataByMQ(Set keySet, final String data) { + String topic = traceTopicName; + final Message message = new Message(topic, data.getBytes()); + + // Keyset of message trace includes msgId of or original message + message.setKeys(keySet); + try { + Set traceBrokerSet = tryGetMessageQueueBrokerSet(traceProducer.getDefaultMQProducerImpl(), topic); + SendCallback callback = new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + } + + @Override + public void onException(Throwable e) { + log.info("send trace data ,the traceData is " + data); + } + }; + if (traceBrokerSet.isEmpty()) { + // No cross set + traceProducer.send(message, callback, 5000); + } else { + traceProducer.send(message, new MessageQueueSelector() { + @Override + public MessageQueue select(List mqs, Message msg, Object arg) { + Set brokerSet = (Set) arg; + List filterMqs = new ArrayList(); + for (MessageQueue queue : mqs) { + if (brokerSet.contains(queue.getBrokerName())) { + filterMqs.add(queue); + } + } + int index = sendWhichQueue.getAndIncrement(); + int pos = Math.abs(index) % filterMqs.size(); + if (pos < 0) { + pos = 0; + } + return filterMqs.get(pos); + } + }, traceBrokerSet, callback); + } + + } catch (Exception e) { + log.info("send trace data,the traceData is" + data); + } + } + + private Set tryGetMessageQueueBrokerSet(DefaultMQProducerImpl producer, String topic) { + Set brokerSet = new HashSet(); + TopicPublishInfo topicPublishInfo = producer.getTopicPublishInfoTable().get(topic); + if (null == topicPublishInfo || !topicPublishInfo.ok()) { + producer.getTopicPublishInfoTable().putIfAbsent(topic, new TopicPublishInfo()); + producer.getmQClientFactory().updateTopicRouteInfoFromNameServer(topic); + topicPublishInfo = producer.getTopicPublishInfoTable().get(topic); + } + if (topicPublishInfo.isHaveTopicRouterInfo() || topicPublishInfo.ok()) { + for (MessageQueue queue : topicPublishInfo.getMessageQueueList()) { + brokerSet.add(queue.getBrokerName()); + } + } + return brokerSet; + } + } + +} diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceBean.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceBean.java new file mode 100644 index 00000000000..f93aa38b829 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceBean.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.trace; + +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.message.MessageType; + +public class TraceBean { + private static final String LOCAL_ADDRESS = UtilAll.ipToIPv4Str(UtilAll.getIP()); + private String topic = ""; + private String msgId = ""; + private String offsetMsgId = ""; + private String tags = ""; + private String keys = ""; + private String storeHost = LOCAL_ADDRESS; + private String clientHost = LOCAL_ADDRESS; + private long storeTime; + private int retryTimes; + private int bodyLength; + private MessageType msgType; + + + public MessageType getMsgType() { + return msgType; + } + + + public void setMsgType(final MessageType msgType) { + this.msgType = msgType; + } + + + public String getOffsetMsgId() { + return offsetMsgId; + } + + + public void setOffsetMsgId(final String offsetMsgId) { + this.offsetMsgId = offsetMsgId; + } + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getMsgId() { + return msgId; + } + + + public void setMsgId(String msgId) { + this.msgId = msgId; + } + + + public String getTags() { + return tags; + } + + + public void setTags(String tags) { + this.tags = tags; + } + + + public String getKeys() { + return keys; + } + + + public void setKeys(String keys) { + this.keys = keys; + } + + + public String getStoreHost() { + return storeHost; + } + + + public void setStoreHost(String storeHost) { + this.storeHost = storeHost; + } + + + public String getClientHost() { + return clientHost; + } + + + public void setClientHost(String clientHost) { + this.clientHost = clientHost; + } + + + public long getStoreTime() { + return storeTime; + } + + + public void setStoreTime(long storeTime) { + this.storeTime = storeTime; + } + + + public int getRetryTimes() { + return retryTimes; + } + + + public void setRetryTimes(int retryTimes) { + this.retryTimes = retryTimes; + } + + + public int getBodyLength() { + return bodyLength; + } + + + public void setBodyLength(int bodyLength) { + this.bodyLength = bodyLength; + } +} diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceConstants.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceConstants.java new file mode 100644 index 00000000000..b9fd8778eab --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceConstants.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.trace; + +public class TraceConstants { + + public static final String GROUP_NAME = "_INNER_TRACE_PRODUCER"; + public static final char CONTENT_SPLITOR = (char) 1; + public static final char FIELD_SPLITOR = (char) 2; + public static final String TRACE_INSTANCE_NAME = "PID_CLIENT_INNER_TRACE_PRODUCER"; +} diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceContext.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceContext.java new file mode 100644 index 00000000000..f61ba888cb3 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceContext.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.trace; + +import org.apache.rocketmq.common.message.MessageClientIDSetter; + +import java.util.List; + +/** + * The context of Trace + */ +public class TraceContext implements Comparable { + + private TraceType traceType; + private long timeStamp = System.currentTimeMillis(); + private String regionId = ""; + private String regionName = ""; + private String groupName = ""; + private int costTime = 0; + private boolean isSuccess = true; + private String requestId = MessageClientIDSetter.createUniqID(); + private int contextCode = 0; + private List traceBeans; + + public int getContextCode() { + return contextCode; + } + + public void setContextCode(final int contextCode) { + this.contextCode = contextCode; + } + + public List getTraceBeans() { + return traceBeans; + } + + public void setTraceBeans(List traceBeans) { + this.traceBeans = traceBeans; + } + + public String getRegionId() { + return regionId; + } + + public void setRegionId(String regionId) { + this.regionId = regionId; + } + + public TraceType getTraceType() { + return traceType; + } + + public void setTraceType(TraceType traceType) { + this.traceType = traceType; + } + + public long getTimeStamp() { + return timeStamp; + } + + public void setTimeStamp(long timeStamp) { + this.timeStamp = timeStamp; + } + + public String getGroupName() { + return groupName; + } + + public void setGroupName(String groupName) { + this.groupName = groupName; + } + + public int getCostTime() { + return costTime; + } + + public void setCostTime(int costTime) { + this.costTime = costTime; + } + + public boolean isSuccess() { + return isSuccess; + } + + public void setSuccess(boolean success) { + isSuccess = success; + } + + public String getRequestId() { + return requestId; + } + + public void setRequestId(String requestId) { + this.requestId = requestId; + } + + public String getRegionName() { + return regionName; + } + + public void setRegionName(String regionName) { + this.regionName = regionName; + } + + @Override + public int compareTo(TraceContext o) { + return (int) (this.timeStamp - o.getTimeStamp()); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(1024); + sb.append(traceType).append("_").append(groupName) + .append("_").append(regionId).append("_").append(isSuccess).append("_"); + if (traceBeans != null && traceBeans.size() > 0) { + for (TraceBean bean : traceBeans) { + sb.append(bean.getMsgId() + "_" + bean.getTopic() + "_"); + } + } + return "TraceContext{" + sb.toString() + '}'; + } +} diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java new file mode 100644 index 00000000000..5a1afaf3617 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.trace; + +import org.apache.rocketmq.common.message.MessageType; + +import java.util.ArrayList; +import java.util.List; + +/** + * Encode/decode for Trace Data + */ +public class TraceDataEncoder { + + /** + * Resolving traceContext list From trace data String + * + * @param traceData + * @return + */ + public static List decoderFromTraceDataString(String traceData) { + List resList = new ArrayList(); + if (traceData == null || traceData.length() <= 0) { + return resList; + } + String[] contextList = traceData.split(String.valueOf(TraceConstants.FIELD_SPLITOR)); + for (String context : contextList) { + String[] line = context.split(String.valueOf(TraceConstants.CONTENT_SPLITOR)); + if (line[0].equals(TraceType.Pub.name())) { + TraceContext pubContext = new TraceContext(); + pubContext.setTraceType(TraceType.Pub); + pubContext.setTimeStamp(Long.parseLong(line[1])); + pubContext.setRegionId(line[2]); + pubContext.setGroupName(line[3]); + TraceBean bean = new TraceBean(); + bean.setTopic(line[4]); + bean.setMsgId(line[5]); + bean.setTags(line[6]); + bean.setKeys(line[7]); + bean.setStoreHost(line[8]); + bean.setBodyLength(Integer.parseInt(line[9])); + pubContext.setCostTime(Integer.parseInt(line[10])); + bean.setMsgType(MessageType.values()[Integer.parseInt(line[11])]); + + if (line.length == 13) { + pubContext.setSuccess(Boolean.parseBoolean(line[12])); + } else if (line.length == 14) { + bean.setOffsetMsgId(line[12]); + pubContext.setSuccess(Boolean.parseBoolean(line[13])); + } + pubContext.setTraceBeans(new ArrayList(1)); + pubContext.getTraceBeans().add(bean); + resList.add(pubContext); + } else if (line[0].equals(TraceType.SubBefore.name())) { + TraceContext subBeforeContext = new TraceContext(); + subBeforeContext.setTraceType(TraceType.SubBefore); + subBeforeContext.setTimeStamp(Long.parseLong(line[1])); + subBeforeContext.setRegionId(line[2]); + subBeforeContext.setGroupName(line[3]); + subBeforeContext.setRequestId(line[4]); + TraceBean bean = new TraceBean(); + bean.setMsgId(line[5]); + bean.setRetryTimes(Integer.parseInt(line[6])); + bean.setKeys(line[7]); + subBeforeContext.setTraceBeans(new ArrayList(1)); + subBeforeContext.getTraceBeans().add(bean); + resList.add(subBeforeContext); + } else if (line[0].equals(TraceType.SubAfter.name())) { + TraceContext subAfterContext = new TraceContext(); + subAfterContext.setTraceType(TraceType.SubAfter); + subAfterContext.setRequestId(line[1]); + TraceBean bean = new TraceBean(); + bean.setMsgId(line[2]); + bean.setKeys(line[5]); + subAfterContext.setTraceBeans(new ArrayList(1)); + subAfterContext.getTraceBeans().add(bean); + subAfterContext.setCostTime(Integer.parseInt(line[3])); + subAfterContext.setSuccess(Boolean.parseBoolean(line[4])); + if (line.length >= 7) { + // add the context type + subAfterContext.setContextCode(Integer.parseInt(line[6])); + } + resList.add(subAfterContext); + } + } + return resList; + } + + /** + * Encoding the trace context into data strings and keyset sets + * + * @param ctx + * @return + */ + public static TraceTransferBean encoderFromContextBean(TraceContext ctx) { + if (ctx == null) { + return null; + } + //build message trace of the transfering entity content bean + TraceTransferBean transferBean = new TraceTransferBean(); + StringBuilder sb = new StringBuilder(256); + switch (ctx.getTraceType()) { + case Pub: { + TraceBean bean = ctx.getTraceBeans().get(0); + //append the content of context and traceBean to transferBean's TransData + sb.append(ctx.getTraceType()).append(TraceConstants.CONTENT_SPLITOR)// + .append(ctx.getTimeStamp()).append(TraceConstants.CONTENT_SPLITOR)// + .append(ctx.getRegionId()).append(TraceConstants.CONTENT_SPLITOR)// + .append(ctx.getGroupName()).append(TraceConstants.CONTENT_SPLITOR)// + .append(bean.getTopic()).append(TraceConstants.CONTENT_SPLITOR)// + .append(bean.getMsgId()).append(TraceConstants.CONTENT_SPLITOR)// + .append(bean.getTags()).append(TraceConstants.CONTENT_SPLITOR)// + .append(bean.getKeys()).append(TraceConstants.CONTENT_SPLITOR)// + .append(bean.getStoreHost()).append(TraceConstants.CONTENT_SPLITOR)// + .append(bean.getBodyLength()).append(TraceConstants.CONTENT_SPLITOR)// + .append(ctx.getCostTime()).append(TraceConstants.CONTENT_SPLITOR)// + .append(bean.getMsgType().ordinal()).append(TraceConstants.CONTENT_SPLITOR)// + .append(bean.getOffsetMsgId()).append(TraceConstants.CONTENT_SPLITOR)// + .append(ctx.isSuccess()).append(TraceConstants.FIELD_SPLITOR); + } + break; + case SubBefore: { + for (TraceBean bean : ctx.getTraceBeans()) { + sb.append(ctx.getTraceType()).append(TraceConstants.CONTENT_SPLITOR)// + .append(ctx.getTimeStamp()).append(TraceConstants.CONTENT_SPLITOR)// + .append(ctx.getRegionId()).append(TraceConstants.CONTENT_SPLITOR)// + .append(ctx.getGroupName()).append(TraceConstants.CONTENT_SPLITOR)// + .append(ctx.getRequestId()).append(TraceConstants.CONTENT_SPLITOR)// + .append(bean.getMsgId()).append(TraceConstants.CONTENT_SPLITOR)// + .append(bean.getRetryTimes()).append(TraceConstants.CONTENT_SPLITOR)// + .append(bean.getKeys()).append(TraceConstants.FIELD_SPLITOR);// + } + } + break; + case SubAfter: { + for (TraceBean bean : ctx.getTraceBeans()) { + sb.append(ctx.getTraceType()).append(TraceConstants.CONTENT_SPLITOR)// + .append(ctx.getRequestId()).append(TraceConstants.CONTENT_SPLITOR)// + .append(bean.getMsgId()).append(TraceConstants.CONTENT_SPLITOR)// + .append(ctx.getCostTime()).append(TraceConstants.CONTENT_SPLITOR)// + .append(ctx.isSuccess()).append(TraceConstants.CONTENT_SPLITOR)// + .append(bean.getKeys()).append(TraceConstants.CONTENT_SPLITOR)// + .append(ctx.getContextCode()).append(TraceConstants.FIELD_SPLITOR); + } + } + break; + default: + } + transferBean.setTransData(sb.toString()); + for (TraceBean bean : ctx.getTraceBeans()) { + + transferBean.getTransKey().add(bean.getMsgId()); + if (bean.getKeys() != null && bean.getKeys().length() > 0) { + transferBean.getTransKey().add(bean.getKeys()); + } + } + return transferBean; + } +} diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceDispatcher.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceDispatcher.java new file mode 100644 index 00000000000..275e6a32207 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceDispatcher.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.trace; + +import org.apache.rocketmq.client.exception.MQClientException; +import java.io.IOException; + +/** + * Interface of asynchronous transfer data + */ +public interface TraceDispatcher { + + /** + * Initialize asynchronous transfer data module + */ + void start(String nameSrvAddr) throws MQClientException; + + /** + * Append the transfering data + * @param ctx data infomation + * @return + */ + boolean append(Object ctx); + + /** + * Write flush action + * + * @throws IOException + */ + void flush() throws IOException; + + /** + * Close the trace Hook + */ + void shutdown(); +} diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceDispatcherType.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceDispatcherType.java new file mode 100644 index 00000000000..f09c9b8db4b --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceDispatcherType.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.trace; + +public enum TraceDispatcherType { + PRODUCER, + CONSUMER +} diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceTransferBean.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceTransferBean.java new file mode 100644 index 00000000000..052ca365213 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceTransferBean.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.trace; + +import java.util.HashSet; +import java.util.Set; + +/** + * Trace transfering bean + */ +public class TraceTransferBean { + private String transData; + private Set transKey = new HashSet(); + + public String getTransData() { + return transData; + } + + public void setTransData(String transData) { + this.transData = transData; + } + + public Set getTransKey() { + return transKey; + } + + public void setTransKey(Set transKey) { + this.transKey = transKey; + } +} diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceType.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceType.java new file mode 100644 index 00000000000..79b19c17e4e --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceType.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.trace; + +public enum TraceType { + Pub, + SubBefore, + SubAfter, +} diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/hook/ConsumeMessageTraceHookImpl.java b/client/src/main/java/org/apache/rocketmq/client/trace/hook/ConsumeMessageTraceHookImpl.java new file mode 100644 index 00000000000..38ec8b97a3f --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/trace/hook/ConsumeMessageTraceHookImpl.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.trace.hook; + +import org.apache.rocketmq.client.consumer.listener.ConsumeReturnType; +import org.apache.rocketmq.client.hook.ConsumeMessageContext; +import org.apache.rocketmq.client.hook.ConsumeMessageHook; +import org.apache.rocketmq.client.trace.TraceContext; +import org.apache.rocketmq.client.trace.TraceDispatcher; +import org.apache.rocketmq.client.trace.TraceBean; +import org.apache.rocketmq.client.trace.TraceType; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; + +import java.util.ArrayList; +import java.util.List; + +public class ConsumeMessageTraceHookImpl implements ConsumeMessageHook { + + private TraceDispatcher localDispatcher; + + public ConsumeMessageTraceHookImpl(TraceDispatcher localDispatcher) { + this.localDispatcher = localDispatcher; + } + + @Override + public String hookName() { + return "ConsumeMessageTraceHook"; + } + + @Override + public void consumeMessageBefore(ConsumeMessageContext context) { + if (context == null || context.getMsgList() == null || context.getMsgList().isEmpty()) { + return; + } + TraceContext traceContext = new TraceContext(); + context.setMqTraceContext(traceContext); + traceContext.setTraceType(TraceType.SubBefore);// + traceContext.setGroupName(context.getConsumerGroup());// + List beans = new ArrayList(); + for (MessageExt msg : context.getMsgList()) { + if (msg == null) { + continue; + } + String regionId = msg.getProperty(MessageConst.PROPERTY_MSG_REGION); + String traceOn = msg.getProperty(MessageConst.PROPERTY_TRACE_SWITCH); + + if (traceOn != null && traceOn.equals("false")) { + // If trace switch is false ,skip it + continue; + } + TraceBean traceBean = new TraceBean(); + traceBean.setTopic(msg.getTopic());// + traceBean.setMsgId(msg.getMsgId());// + traceBean.setTags(msg.getTags());// + traceBean.setKeys(msg.getKeys());// + traceBean.setStoreTime(msg.getStoreTimestamp());// + traceBean.setBodyLength(msg.getStoreSize());// + traceBean.setRetryTimes(msg.getReconsumeTimes());// + traceContext.setRegionId(regionId);// + beans.add(traceBean); + } + if (beans.size() > 0) { + traceContext.setTraceBeans(beans); + traceContext.setTimeStamp(System.currentTimeMillis()); + localDispatcher.append(traceContext); + } + } + + @Override + public void consumeMessageAfter(ConsumeMessageContext context) { + if (context == null || context.getMsgList() == null || context.getMsgList().isEmpty()) { + return; + } + TraceContext subBeforeContext = (TraceContext) context.getMqTraceContext(); + + if (subBeforeContext.getTraceBeans() == null || subBeforeContext.getTraceBeans().size() < 1) { + // If subbefore bean is null ,skip it + return; + } + TraceContext subAfterContext = new TraceContext(); + subAfterContext.setTraceType(TraceType.SubAfter);// + subAfterContext.setRegionId(subBeforeContext.getRegionId());// + subAfterContext.setGroupName(subBeforeContext.getGroupName());// + subAfterContext.setRequestId(subBeforeContext.getRequestId());// + subAfterContext.setSuccess(context.isSuccess());// + + // Caculate the cost time for processing messages + int costTime = (int) ((System.currentTimeMillis() - subBeforeContext.getTimeStamp()) / context.getMsgList().size()); + subAfterContext.setCostTime(costTime);// + subAfterContext.setTraceBeans(subBeforeContext.getTraceBeans()); + String contextType = context.getProps().get(MixAll.CONSUME_CONTEXT_TYPE); + if (contextType != null) { + subAfterContext.setContextCode(ConsumeReturnType.valueOf(contextType).ordinal()); + } + localDispatcher.append(subAfterContext); + } +} diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageTraceHookImpl.java b/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageTraceHookImpl.java new file mode 100644 index 00000000000..20396c6ddcc --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageTraceHookImpl.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.trace.hook; + +import org.apache.rocketmq.client.hook.SendMessageContext; +import org.apache.rocketmq.client.hook.SendMessageHook; +import org.apache.rocketmq.client.producer.SendStatus; +import org.apache.rocketmq.client.trace.AsyncTraceDispatcher; +import org.apache.rocketmq.client.trace.TraceContext; +import org.apache.rocketmq.client.trace.TraceDispatcher; +import org.apache.rocketmq.client.trace.TraceBean; +import org.apache.rocketmq.client.trace.TraceType; +import java.util.ArrayList; + +public class SendMessageTraceHookImpl implements SendMessageHook { + + private TraceDispatcher localDispatcher; + + public SendMessageTraceHookImpl(TraceDispatcher localDispatcher) { + this.localDispatcher = localDispatcher; + } + + @Override + public String hookName() { + return "SendMessageTraceHook"; + } + + @Override + public void sendMessageBefore(SendMessageContext context) { + //if it is message trace data,then it doesn't recorded + if (context == null || context.getMessage().getTopic().startsWith(((AsyncTraceDispatcher) localDispatcher).getTraceTopicName())) { + return; + } + //build the context content of TuxeTraceContext + TraceContext tuxeContext = new TraceContext(); + tuxeContext.setTraceBeans(new ArrayList(1)); + context.setMqTraceContext(tuxeContext); + tuxeContext.setTraceType(TraceType.Pub); + tuxeContext.setGroupName(context.getProducerGroup()); + //build the data bean object of message trace + TraceBean traceBean = new TraceBean(); + traceBean.setTopic(context.getMessage().getTopic()); + traceBean.setTags(context.getMessage().getTags()); + traceBean.setKeys(context.getMessage().getKeys()); + traceBean.setStoreHost(context.getBrokerAddr()); + traceBean.setBodyLength(context.getMessage().getBody().length); + traceBean.setMsgType(context.getMsgType()); + tuxeContext.getTraceBeans().add(traceBean); + } + + @Override + public void sendMessageAfter(SendMessageContext context) { + //if it is message trace data,then it doesn't recorded + if (context == null || context.getMessage().getTopic().startsWith(((AsyncTraceDispatcher) localDispatcher).getTraceTopicName()) + || context.getMqTraceContext() == null) { + return; + } + if (context.getSendResult() == null) { + return; + } + + if (context.getSendResult().getRegionId() == null + || !context.getSendResult().isTraceOn()) { + // if switch is false,skip it + return; + } + + TraceContext tuxeContext = (TraceContext) context.getMqTraceContext(); + TraceBean traceBean = tuxeContext.getTraceBeans().get(0); + int costTime = (int) ((System.currentTimeMillis() - tuxeContext.getTimeStamp()) / tuxeContext.getTraceBeans().size()); + tuxeContext.setCostTime(costTime); + if (context.getSendResult().getSendStatus().equals(SendStatus.SEND_OK)) { + tuxeContext.setSuccess(true); + } else { + tuxeContext.setSuccess(false); + } + tuxeContext.setRegionId(context.getSendResult().getRegionId()); + traceBean.setMsgId(context.getSendResult().getMsgId()); + traceBean.setOffsetMsgId(context.getSendResult().getOffsetMsgId()); + traceBean.setStoreTime(tuxeContext.getTimeStamp() + costTime / 2); + localDispatcher.append(tuxeContext); + } +} diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java index c225afd6842..9540755fe33 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java @@ -167,10 +167,7 @@ public void testSendMessageSync_WithBodyCompressed() throws RemotingException, I @Test public void testSendMessageAsync_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { - ExecutorService callbackExecutor = Executors.newSingleThreadExecutor(); final CountDownLatch countDownLatch = new CountDownLatch(1); - when(mQClientAPIImpl.getRemotingClient()).thenReturn((nettyRemotingClient)); - when(nettyRemotingClient.getCallbackExecutor()).thenReturn(callbackExecutor); producer.send(message, new SendCallback() { @Override public void onSuccess(SendResult sendResult) { @@ -186,15 +183,11 @@ public void onException(Throwable e) { } }); countDownLatch.await(3000L, TimeUnit.MILLISECONDS); - callbackExecutor.shutdown(); } @Test public void testSendMessageAsync() throws RemotingException, MQClientException, InterruptedException { final AtomicInteger cc = new AtomicInteger(0); final CountDownLatch countDownLatch = new CountDownLatch(6); - ExecutorService callbackExecutor = Executors.newSingleThreadExecutor(); - when(mQClientAPIImpl.getRemotingClient()).thenReturn((nettyRemotingClient)); - when(nettyRemotingClient.getCallbackExecutor()).thenReturn(callbackExecutor); SendCallback sendCallback = new SendCallback() { @Override @@ -226,16 +219,13 @@ public MessageQueue select(List mqs, Message msg, Object arg) { producer.send(message,messageQueueSelector,null,sendCallback,1000); countDownLatch.await(3000L, TimeUnit.MILLISECONDS); - callbackExecutor.shutdown(); assertThat(cc.get()).isEqualTo(6); } @Test public void testSendMessageAsync_BodyCompressed() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { - ExecutorService callbackExecutor = Executors.newSingleThreadExecutor(); + final CountDownLatch countDownLatch = new CountDownLatch(1); - when(mQClientAPIImpl.getRemotingClient()).thenReturn((nettyRemotingClient)); - when(nettyRemotingClient.getCallbackExecutor()).thenReturn(callbackExecutor); producer.send(bigMessage, new SendCallback() { @Override public void onSuccess(SendResult sendResult) { @@ -251,7 +241,6 @@ public void onException(Throwable e) { } }); countDownLatch.await(3000L, TimeUnit.MILLISECONDS); - callbackExecutor.shutdown(); } @Test diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithTraceTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithTraceTest.java new file mode 100644 index 00000000000..b45ad028185 --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithTraceTest.java @@ -0,0 +1,314 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.trace; + +import java.io.ByteArrayOutputStream; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.consumer.PullCallback; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.PullStatus; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.impl.CommunicationMode; +import org.apache.rocketmq.client.impl.FindBrokerResult; +import org.apache.rocketmq.client.impl.MQClientAPIImpl; +import org.apache.rocketmq.client.impl.consumer.ConsumeMessageConcurrentlyService; +import org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl; +import org.apache.rocketmq.client.impl.consumer.ProcessQueue; +import org.apache.rocketmq.client.impl.consumer.PullAPIWrapper; +import org.apache.rocketmq.client.impl.consumer.PullMessageService; +import org.apache.rocketmq.client.impl.consumer.PullRequest; +import org.apache.rocketmq.client.impl.consumer.PullResultExt; +import org.apache.rocketmq.client.impl.consumer.RebalancePushImpl; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.producer.SendStatus; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageClientExt; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; + +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.QueueData; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class DefaultMQConsumerWithTraceTest { + private String consumerGroup; + private String consumerGroupNormal; + private String producerGroupTraceTemp = MixAll.RMQ_SYS_TRACE_TOPIC + System.currentTimeMillis(); + + private String topic = "FooBar"; + private String brokerName = "BrokerA"; + private MQClientInstance mQClientFactory; + + @Mock + private MQClientAPIImpl mQClientAPIImpl; + private PullAPIWrapper pullAPIWrapper; + private RebalancePushImpl rebalancePushImpl; + private DefaultMQPushConsumer pushConsumer; + private DefaultMQPushConsumer normalPushConsumer; + private DefaultMQPushConsumer customTraceTopicpushConsumer; + + + private AsyncTraceDispatcher asyncTraceDispatcher; + private MQClientInstance mQClientTraceFactory; + @Mock + private MQClientAPIImpl mQClientTraceAPIImpl; + private DefaultMQProducer traceProducer; + private String customerTraceTopic = "rmq_trace_topic_12345"; + + @Before + public void init() throws Exception { + consumerGroup = "FooBarGroup" + System.currentTimeMillis(); + pushConsumer = new DefaultMQPushConsumer(consumerGroup,true,""); + consumerGroupNormal = "FooBarGroup" + System.currentTimeMillis(); + normalPushConsumer = new DefaultMQPushConsumer(consumerGroupNormal,false,""); + customTraceTopicpushConsumer = new DefaultMQPushConsumer(consumerGroup,true,customerTraceTopic); + pushConsumer.setNamesrvAddr("127.0.0.1:9876"); + pushConsumer.setPullInterval(60 * 1000); + + asyncTraceDispatcher = (AsyncTraceDispatcher)pushConsumer.getTraceDispatcher(); + traceProducer = asyncTraceDispatcher.getTraceProducer(); + + + pushConsumer.registerMessageListener(new MessageListenerConcurrently() { + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + return null; + } + }); + + DefaultMQPushConsumerImpl pushConsumerImpl = pushConsumer.getDefaultMQPushConsumerImpl(); + rebalancePushImpl = spy(new RebalancePushImpl(pushConsumer.getDefaultMQPushConsumerImpl())); + Field field = DefaultMQPushConsumerImpl.class.getDeclaredField("rebalanceImpl"); + field.setAccessible(true); + field.set(pushConsumerImpl, rebalancePushImpl); + pushConsumer.subscribe(topic, "*"); + pushConsumer.start(); + + mQClientFactory = spy(pushConsumerImpl.getmQClientFactory()); + mQClientTraceFactory = spy(pushConsumerImpl.getmQClientFactory()); + + field = DefaultMQPushConsumerImpl.class.getDeclaredField("mQClientFactory"); + field.setAccessible(true); + field.set(pushConsumerImpl, mQClientFactory); + + field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl"); + field.setAccessible(true); + field.set(mQClientFactory, mQClientAPIImpl); + + Field fieldTrace = DefaultMQProducerImpl.class.getDeclaredField("mQClientFactory"); + fieldTrace.setAccessible(true); + fieldTrace.set(traceProducer.getDefaultMQProducerImpl(), mQClientTraceFactory); + + fieldTrace = MQClientInstance.class.getDeclaredField("mQClientAPIImpl"); + fieldTrace.setAccessible(true); + fieldTrace.set(mQClientTraceFactory, mQClientTraceAPIImpl); + + pullAPIWrapper = spy(new PullAPIWrapper(mQClientFactory, consumerGroup, false)); + field = DefaultMQPushConsumerImpl.class.getDeclaredField("pullAPIWrapper"); + field.setAccessible(true); + field.set(pushConsumerImpl, pullAPIWrapper); + + pushConsumer.getDefaultMQPushConsumerImpl().getRebalanceImpl().setmQClientFactory(mQClientFactory); + mQClientFactory.registerConsumer(consumerGroup, pushConsumerImpl); + + when(mQClientFactory.getMQClientAPIImpl().pullMessage(anyString(), any(PullMessageRequestHeader.class), + anyLong(), any(CommunicationMode.class), nullable(PullCallback.class))) + .thenAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock mock) throws Throwable { + PullMessageRequestHeader requestHeader = mock.getArgument(1); + MessageClientExt messageClientExt = new MessageClientExt(); + messageClientExt.setTopic(topic); + messageClientExt.setQueueId(0); + messageClientExt.setMsgId("123"); + messageClientExt.setBody(new byte[] {'a'}); + messageClientExt.setOffsetMsgId("234"); + messageClientExt.setBornHost(new InetSocketAddress(8080)); + messageClientExt.setStoreHost(new InetSocketAddress(8080)); + PullResult pullResult = createPullResult(requestHeader, PullStatus.FOUND, Collections.singletonList(messageClientExt)); + ((PullCallback) mock.getArgument(4)).onSuccess(pullResult); + return pullResult; + } + }); + + doReturn(new FindBrokerResult("127.0.0.1:10911", false)).when(mQClientFactory).findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean()); + Set messageQueueSet = new HashSet(); + messageQueueSet.add(createPullRequest().getMessageQueue()); + pushConsumer.getDefaultMQPushConsumerImpl().updateTopicSubscribeInfo(topic, messageQueueSet); + } + + @After + public void terminate() { + pushConsumer.shutdown(); + } + + @Test + public void testPullMessage_WithTrace_Success() throws InterruptedException, RemotingException, MQBrokerException, MQClientException { + traceProducer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTraceTemp, traceProducer.getDefaultMQProducerImpl()); + //when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + //when(mQClientTraceAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTraceTopicRoute()); + + + final CountDownLatch countDownLatch = new CountDownLatch(1); + final MessageExt[] messageExts = new MessageExt[1]; + pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService(new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() { + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + messageExts[0] = msgs.get(0); + countDownLatch.countDown(); + return null; + } + })); + + PullMessageService pullMessageService = mQClientFactory.getPullMessageService(); + pullMessageService.executePullRequestImmediately(createPullRequest()); + countDownLatch.await(3000L, TimeUnit.MILLISECONDS); + assertThat(messageExts[0].getTopic()).isEqualTo(topic); + assertThat(messageExts[0].getBody()).isEqualTo(new byte[] {'a'}); + } + + private PullRequest createPullRequest() { + PullRequest pullRequest = new PullRequest(); + pullRequest.setConsumerGroup(consumerGroup); + pullRequest.setNextOffset(1024); + + MessageQueue messageQueue = new MessageQueue(); + messageQueue.setBrokerName(brokerName); + messageQueue.setQueueId(0); + messageQueue.setTopic(topic); + pullRequest.setMessageQueue(messageQueue); + ProcessQueue processQueue = new ProcessQueue(); + processQueue.setLocked(true); + processQueue.setLastLockTimestamp(System.currentTimeMillis()); + pullRequest.setProcessQueue(processQueue); + + return pullRequest; + } + + private PullResultExt createPullResult(PullMessageRequestHeader requestHeader, PullStatus pullStatus, + List messageExtList) throws Exception { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + for (MessageExt messageExt : messageExtList) { + outputStream.write(MessageDecoder.encode(messageExt, false)); + } + return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, outputStream.toByteArray()); + } + + public static TopicRouteData createTopicRoute() { + TopicRouteData topicRouteData = new TopicRouteData(); + + topicRouteData.setFilterServerTable(new HashMap>()); + List brokerDataList = new ArrayList(); + BrokerData brokerData = new BrokerData(); + brokerData.setBrokerName("BrokerA"); + brokerData.setCluster("DefaultCluster"); + HashMap brokerAddrs = new HashMap(); + brokerAddrs.put(0L, "127.0.0.1:10911"); + brokerData.setBrokerAddrs(brokerAddrs); + brokerDataList.add(brokerData); + topicRouteData.setBrokerDatas(brokerDataList); + + List queueDataList = new ArrayList(); + QueueData queueData = new QueueData(); + queueData.setBrokerName("BrokerA"); + queueData.setPerm(6); + queueData.setReadQueueNums(3); + queueData.setWriteQueueNums(4); + queueData.setTopicSynFlag(0); + queueDataList.add(queueData); + topicRouteData.setQueueDatas(queueDataList); + return topicRouteData; + } + + private SendResult createSendResult(SendStatus sendStatus) { + SendResult sendResult = new SendResult(); + sendResult.setMsgId("123"); + sendResult.setOffsetMsgId("123"); + sendResult.setQueueOffset(456); + sendResult.setSendStatus(sendStatus); + sendResult.setRegionId("HZ"); + return sendResult; + } + + public static TopicRouteData createTraceTopicRoute() { + TopicRouteData topicRouteData = new TopicRouteData(); + + topicRouteData.setFilterServerTable(new HashMap>()); + List brokerDataList = new ArrayList(); + BrokerData brokerData = new BrokerData(); + brokerData.setBrokerName("broker-trace"); + brokerData.setCluster("DefaultCluster"); + HashMap brokerAddrs = new HashMap(); + brokerAddrs.put(0L, "127.0.0.1:10912"); + brokerData.setBrokerAddrs(brokerAddrs); + brokerDataList.add(brokerData); + topicRouteData.setBrokerDatas(brokerDataList); + + List queueDataList = new ArrayList(); + QueueData queueData = new QueueData(); + queueData.setBrokerName("broker-trace"); + queueData.setPerm(6); + queueData.setReadQueueNums(1); + queueData.setWriteQueueNums(1); + queueData.setTopicSynFlag(1); + queueDataList.add(queueData); + topicRouteData.setQueueDatas(queueDataList); + return topicRouteData; + } +} diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java new file mode 100644 index 00000000000..903be01cddb --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.trace; + +import org.apache.rocketmq.client.ClientConfig; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.hook.SendMessageContext; +import org.apache.rocketmq.client.impl.CommunicationMode; +import org.apache.rocketmq.client.impl.MQClientAPIImpl; +import org.apache.rocketmq.client.impl.MQClientManager; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; +import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.client.producer.SendCallback; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.producer.SendStatus; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.QueueData; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class DefaultMQProducerWithTraceTest { + + @Spy + private MQClientInstance mQClientFactory = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + @Mock + private MQClientAPIImpl mQClientAPIImpl; + + private AsyncTraceDispatcher asyncTraceDispatcher; + + private DefaultMQProducer producer; + private DefaultMQProducer customTraceTopicproducer; + private DefaultMQProducer traceProducer; + private DefaultMQProducer normalProducer; + + private Message message; + private String topic = "FooBar"; + private String producerGroupPrefix = "FooBar_PID"; + private String producerGroupTemp = producerGroupPrefix + System.currentTimeMillis(); + private String producerGroupTraceTemp = MixAll.RMQ_SYS_TRACE_TOPIC + System.currentTimeMillis(); + private String customerTraceTopic = "rmq_trace_topic_12345"; + + @Before + public void init() throws Exception { + + customTraceTopicproducer = new DefaultMQProducer(producerGroupTemp, false, customerTraceTopic); + normalProducer = new DefaultMQProducer(producerGroupTemp, false, ""); + producer = new DefaultMQProducer(producerGroupTemp, true, ""); + producer.setNamesrvAddr("127.0.0.1:9876"); + normalProducer.setNamesrvAddr("127.0.0.1:9877"); + customTraceTopicproducer.setNamesrvAddr("127.0.0.1:9878"); + message = new Message(topic, new byte[]{'a', 'b', 'c'}); + asyncTraceDispatcher = (AsyncTraceDispatcher) producer.getTraceDispatcher(); + asyncTraceDispatcher.setTraceTopicName(customerTraceTopic); + asyncTraceDispatcher.getHostProducer(); + asyncTraceDispatcher.getHostConsumer(); + traceProducer = asyncTraceDispatcher.getTraceProducer(); + + producer.start(); + + Field field = DefaultMQProducerImpl.class.getDeclaredField("mQClientFactory"); + field.setAccessible(true); + field.set(producer.getDefaultMQProducerImpl(), mQClientFactory); + + Field fieldTrace = DefaultMQProducerImpl.class.getDeclaredField("mQClientFactory"); + fieldTrace.setAccessible(true); + fieldTrace.set(traceProducer.getDefaultMQProducerImpl(), mQClientFactory); + + field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl"); + field.setAccessible(true); + field.set(mQClientFactory, mQClientAPIImpl); + + + producer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTemp, producer.getDefaultMQProducerImpl()); + + when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), + nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))).thenCallRealMethod(); + when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), + nullable(SendCallback.class), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) + .thenReturn(createSendResult(SendStatus.SEND_OK)); + + } + + @Test + public void testSendMessageSync_WithTrace_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { + traceProducer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTraceTemp, traceProducer.getDefaultMQProducerImpl()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + final CountDownLatch countDownLatch = new CountDownLatch(1); + try { + producer.send(message); + } catch (MQClientException e) { + } + countDownLatch.await(3000L, TimeUnit.MILLISECONDS); + + } + + @Test + public void testSendMessageSync_WithTrace_NoBrokerSet_Exception() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + final CountDownLatch countDownLatch = new CountDownLatch(1); + try { + producer.send(message); + } catch (MQClientException e) { + } + countDownLatch.await(3000L, TimeUnit.MILLISECONDS); + + } + + @After + public void terminate() { + producer.shutdown(); + } + + public static TopicRouteData createTopicRoute() { + TopicRouteData topicRouteData = new TopicRouteData(); + + topicRouteData.setFilterServerTable(new HashMap>()); + List brokerDataList = new ArrayList(); + BrokerData brokerData = new BrokerData(); + brokerData.setBrokerName("BrokerA"); + brokerData.setCluster("DefaultCluster"); + HashMap brokerAddrs = new HashMap(); + brokerAddrs.put(0L, "127.0.0.1:10911"); + brokerData.setBrokerAddrs(brokerAddrs); + brokerDataList.add(brokerData); + topicRouteData.setBrokerDatas(brokerDataList); + + List queueDataList = new ArrayList(); + QueueData queueData = new QueueData(); + queueData.setBrokerName("BrokerA"); + queueData.setPerm(6); + queueData.setReadQueueNums(3); + queueData.setWriteQueueNums(4); + queueData.setTopicSynFlag(0); + queueDataList.add(queueData); + topicRouteData.setQueueDatas(queueDataList); + return topicRouteData; + } + + private SendResult createSendResult(SendStatus sendStatus) { + SendResult sendResult = new SendResult(); + sendResult.setMsgId("123"); + sendResult.setOffsetMsgId("123"); + sendResult.setQueueOffset(456); + sendResult.setSendStatus(sendStatus); + sendResult.setRegionId("HZ"); + return sendResult; + } + + public static TopicRouteData createTraceTopicRoute() { + TopicRouteData topicRouteData = new TopicRouteData(); + + topicRouteData.setFilterServerTable(new HashMap>()); + List brokerDataList = new ArrayList(); + BrokerData brokerData = new BrokerData(); + brokerData.setBrokerName("broker-trace"); + brokerData.setCluster("DefaultCluster"); + HashMap brokerAddrs = new HashMap(); + brokerAddrs.put(0L, "127.0.0.1:10912"); + brokerData.setBrokerAddrs(brokerAddrs); + brokerDataList.add(brokerData); + topicRouteData.setBrokerDatas(brokerDataList); + + List queueDataList = new ArrayList(); + QueueData queueData = new QueueData(); + queueData.setBrokerName("broker-trace"); + queueData.setPerm(6); + queueData.setReadQueueNums(1); + queueData.setWriteQueueNums(1); + queueData.setTopicSynFlag(1); + queueDataList.add(queueData); + topicRouteData.setQueueDatas(queueDataList); + return topicRouteData; + } +} diff --git a/common/pom.xml b/common/pom.xml index 62c6d7eb5c2..75381b27354 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.4.0-SNAPSHOT + 4.4.1-SNAPSHOT 4.0.0 diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index f81af216515..1c3f37d00d6 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -51,7 +51,10 @@ public class BrokerConfig { @ImportantField private boolean autoCreateSubscriptionGroup = true; private String messageStorePlugIn = ""; - + @ImportantField + private String msgTraceTopicName = MixAll.RMQ_SYS_TRACE_TOPIC; + @ImportantField + private boolean traceTopicEnable = false; /** * thread numbers for send message thread pool, since spin lock will be used by default since 4.0.x, the default * value is 1. @@ -171,6 +174,22 @@ public class BrokerConfig { @ImportantField private long transactionCheckInterval = 60 * 1000; + /** + * Acl feature switch + */ + @ImportantField + private boolean aclEnable = false; + + public static String localHostName() { + try { + return InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + log.error("Failed to obtain the host name", e); + } + + return "DEFAULT_BROKER"; + } + public boolean isTraceOn() { return traceOn; } @@ -235,16 +254,6 @@ public void setSlaveReadEnable(final boolean slaveReadEnable) { this.slaveReadEnable = slaveReadEnable; } - public static String localHostName() { - try { - return InetAddress.getLocalHost().getHostName(); - } catch (UnknownHostException e) { - log.error("Failed to obtain the host name", e); - } - - return "DEFAULT_BROKER"; - } - public int getRegisterBrokerTimeoutMills() { return registerBrokerTimeoutMills; } @@ -732,4 +741,28 @@ public long getWaitTimeMillsInTransactionQueue() { public void setWaitTimeMillsInTransactionQueue(long waitTimeMillsInTransactionQueue) { this.waitTimeMillsInTransactionQueue = waitTimeMillsInTransactionQueue; } + + public String getMsgTraceTopicName() { + return msgTraceTopicName; + } + + public void setMsgTraceTopicName(String msgTraceTopicName) { + this.msgTraceTopicName = msgTraceTopicName; + } + + public boolean isTraceTopicEnable() { + return traceTopicEnable; + } + + public void setTraceTopicEnable(boolean traceTopicEnable) { + this.traceTopicEnable = traceTopicEnable; + } + + public boolean isAclEnable() { + return aclEnable; + } + + public void setAclEnable(boolean aclEnable) { + this.aclEnable = aclEnable; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java index 6c2712d6116..e67b6e7d134 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java +++ b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java @@ -18,7 +18,7 @@ public class MQVersion { - public static final int CURRENT_VERSION = Version.V4_3_1.ordinal(); + public static final int CURRENT_VERSION = Version.V4_4_0.ordinal(); public static String getVersionDesc(int value) { int length = Version.values().length; diff --git a/common/src/main/java/org/apache/rocketmq/common/MixAll.java b/common/src/main/java/org/apache/rocketmq/common/MixAll.java index 20d186764eb..d39e63386f6 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MixAll.java +++ b/common/src/main/java/org/apache/rocketmq/common/MixAll.java @@ -90,6 +90,7 @@ public class MixAll { public static final String CONSUME_CONTEXT_TYPE = "ConsumeContextType"; public static final String RMQ_SYS_TRANS_HALF_TOPIC = "RMQ_SYS_TRANS_HALF_TOPIC"; + public static final String RMQ_SYS_TRACE_TOPIC = "RMQ_SYS_TRACE_TOPIC"; public static final String RMQ_SYS_TRANS_OP_HALF_TOPIC = "RMQ_SYS_TRANS_OP_HALF_TOPIC"; public static final String CID_SYS_RMQ_TRANS = "CID_RMQ_SYS_TRANS"; diff --git a/common/src/main/java/org/apache/rocketmq/common/UtilAll.java b/common/src/main/java/org/apache/rocketmq/common/UtilAll.java index a846755d8db..dee6ca29114 100644 --- a/common/src/main/java/org/apache/rocketmq/common/UtilAll.java +++ b/common/src/main/java/org/apache/rocketmq/common/UtilAll.java @@ -60,6 +60,18 @@ public static int getPid() { } } + public static void sleep(long sleepMs) { + if (sleepMs < 0) { + return; + } + try { + Thread.sleep(sleepMs); + } catch (Throwable ignored) { + + } + + } + public static String currentStackTrace() { StringBuilder sb = new StringBuilder(); StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace(); diff --git a/common/src/main/java/org/apache/rocketmq/common/constant/DBMsgConstants.java b/common/src/main/java/org/apache/rocketmq/common/constant/DBMsgConstants.java index 8273aaa788b..8d2b3449765 100644 --- a/common/src/main/java/org/apache/rocketmq/common/constant/DBMsgConstants.java +++ b/common/src/main/java/org/apache/rocketmq/common/constant/DBMsgConstants.java @@ -18,5 +18,5 @@ package org.apache.rocketmq.common.constant; public class DBMsgConstants { - public static final int MAX_BODY_SIZE = 64 * 1024 * 1204; //64KB + public static final int MAX_BODY_SIZE = 64 * 1024 * 1024; //64KB } diff --git a/common/src/test/java/org/apache/rocketmq/common/BrokerConfigTest.java b/common/src/test/java/org/apache/rocketmq/common/BrokerConfigTest.java index 0fb9b3afb12..aa8bcfa7252 100644 --- a/common/src/test/java/org/apache/rocketmq/common/BrokerConfigTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/BrokerConfigTest.java @@ -27,4 +27,21 @@ public void testConsumerFallBehindThresholdOverflow() { long expect = 1024L * 1024 * 1024 * 16; assertThat(new BrokerConfig().getConsumerFallbehindThreshold()).isEqualTo(expect); } + + @Test + public void testBrokerConfigAttribute() { + BrokerConfig brokerConfig = new BrokerConfig(); + brokerConfig.setNamesrvAddr("127.0.0.1:9876"); + brokerConfig.setAutoCreateTopicEnable(false); + brokerConfig.setBrokerName("broker-a"); + brokerConfig.setBrokerId(0); + brokerConfig.setBrokerClusterName("DefaultCluster"); + brokerConfig.setMsgTraceTopicName("RMQ_SYS_TRACE_TOPIC4"); + assertThat(brokerConfig.getBrokerClusterName()).isEqualTo("DefaultCluster"); + assertThat(brokerConfig.getNamesrvAddr()).isEqualTo("127.0.0.1:9876"); + assertThat(brokerConfig.getMsgTraceTopicName()).isEqualTo("RMQ_SYS_TRACE_TOPIC4"); + assertThat(brokerConfig.getBrokerId()).isEqualTo(0); + assertThat(brokerConfig.getBrokerName()).isEqualTo("broker-a"); + assertThat(brokerConfig.isAutoCreateTopicEnable()).isEqualTo(false); + } } \ No newline at end of file diff --git a/common/src/test/java/org/apache/rocketmq/common/MixAllTest.java b/common/src/test/java/org/apache/rocketmq/common/MixAllTest.java index 0d2dec6fa5c..8d86544be69 100644 --- a/common/src/test/java/org/apache/rocketmq/common/MixAllTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/MixAllTest.java @@ -17,14 +17,13 @@ package org.apache.rocketmq.common; -import org.junit.Test; - import java.io.File; import java.io.IOException; import java.io.PrintWriter; import java.net.InetAddress; import java.util.List; import java.util.concurrent.atomic.AtomicLong; +import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; diff --git a/distribution/bin/runbroker.sh b/distribution/bin/runbroker.sh index 3bd00bb3014..df962b4a275 100644 --- a/distribution/bin/runbroker.sh +++ b/distribution/bin/runbroker.sh @@ -37,7 +37,7 @@ export CLASSPATH=.:${BASE_DIR}/conf:${CLASSPATH} # JVM Configuration #=========================================================================================== JAVA_OPT="${JAVA_OPT} -server -Xms8g -Xmx8g -Xmn4g" -JAVA_OPT="${JAVA_OPT} -XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:SoftRefLRUPolicyMSPerMB=0 -XX:SurvivorRatio=8" +JAVA_OPT="${JAVA_OPT} -XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:SoftRefLRUPolicyMSPerMB=0" JAVA_OPT="${JAVA_OPT} -verbose:gc -Xloggc:/dev/shm/mq_gc_%p.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintAdaptiveSizePolicy" JAVA_OPT="${JAVA_OPT} -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m" JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow" diff --git a/distribution/conf/2m-noslave/broker-trace.properties b/distribution/conf/2m-noslave/broker-trace.properties new file mode 100644 index 00000000000..9dd57a73def --- /dev/null +++ b/distribution/conf/2m-noslave/broker-trace.properties @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +brokerClusterName=DefaultCluster +brokerName=broker-trace +brokerId=0 +deleteWhen=04 +fileReservedTime=48 +brokerRole=ASYNC_MASTER +flushDiskType=ASYNC_FLUSH diff --git a/distribution/conf/plain_acl.yml b/distribution/conf/plain_acl.yml new file mode 100644 index 00000000000..413a7120f28 --- /dev/null +++ b/distribution/conf/plain_acl.yml @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +globalWhiteRemoteAddresses: + +accounts: +- accessKey: RocketMQ + secretKey: 12345678 + whiteRemoteAddress: + admin: false + defaultTopicPerm: DENY + defaultGroupPerm: SUB + topicPerms: + - topicA=DENY + - topicB=PUB|SUB + - topicC=SUB + groupPerms: + # the group should convert to retry topic + - groupA=DENY + - groupB=PUB|SUB + - groupC=SUB + +- accessKey: rocketmq2 + secretKey: 12345678 + whiteRemoteAddress: 192.168.1.* + # if it is admin, it could access all resources + admin: true + diff --git a/distribution/conf/tools.yml b/distribution/conf/tools.yml new file mode 100644 index 00000000000..a4a9ad1b5ae --- /dev/null +++ b/distribution/conf/tools.yml @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +accessKey: rocketmq +secretKey: 12345678 + diff --git a/distribution/pom.xml b/distribution/pom.xml index c4d3b955c61..22966197516 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -20,7 +20,7 @@ org.apache.rocketmq rocketmq-all - 4.4.0-SNAPSHOT + 4.4.1-SNAPSHOT rocketmq-distribution rocketmq-distribution ${project.version} diff --git a/docs/cn/RocketMQ_Example.md b/docs/cn/RocketMQ_Example.md new file mode 100644 index 00000000000..539b3c234d4 --- /dev/null +++ b/docs/cn/RocketMQ_Example.md @@ -0,0 +1,956 @@ +1 基本样例 +-------- + +在基本样例中我们提供如下的功能场景： + +* 使用RocketMQ发送三种类型的消息：同步消息，异步消息和单向消息。其中前两种消息是可靠的，因为会有发送是否成功的应答。 +* 使用RocketMQ来消费接收到的消息。 + +### 1.1 加入依赖： + +`maven:` +``` + + org.apache.rocketmq + rocketmq-client + 4.3.0 + +``` +`gradle` +``` +compile 'org.apache.rocketmq:rocketmq-client:4.3.0' +``` +### 1.2 消息发送 + +#### 1、Producer端发送同步消息 + +这种可靠性同步地发送方式使用的比较广泛，比如：重要的消息通知，短信通知。 +```java +public class SyncProducer { + public static void main(String[] args) throws Exception { + // 实例化消息生产者Producer + DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name"); + // 设置NameServer的地址 + producer.setNamesrvAddr("localhost:9876"); + // 启动Producer实例 + producer.start(); + for (int i = 0; i < 100; i++) { + // 创建消息，并指定Topic，Tag和消息体 + Message msg = new Message("TopicTest" /* Topic */, + "TagA" /* Tag */, + ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET) /* Message body */ + ); + // 发送消息到一个Broker + SendResult sendResult = producer.send(msg); + // 通过sendResult返回消息是否成功送达 + System.out.printf("%s%n", sendResult); + } + // 如果不再发送消息，关闭Producer实例。 + producer.shutdown(); + } +} +``` +#### 2、发送异步消息 + +异步消息通常用在对响应时间敏感的业务场景，即发送端不能容忍长时间地等待Broker的响应。 + +```java +public class AsyncProducer { + public static void main(String[] args) throws Exception { + // 实例化消息生产者Producer + DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name"); + // 设置NameServer的地址 + producer.setNamesrvAddr("localhost:9876"); + // 启动Producer实例 + producer.start(); + producer.setRetryTimesWhenSendAsyncFailed(0); + for (int i = 0; i < 100; i++) { + final int index = i; + // 创建消息，并指定Topic，Tag和消息体 + Message msg = new Message("TopicTest", + "TagA", + "OrderID188", + "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET)); + // SendCallback接收异步返回结果的回调 + producer.send(msg, new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + System.out.printf("%-10d OK %s %n", index, + sendResult.getMsgId()); + } + @Override + public void onException(Throwable e) { + System.out.printf("%-10d Exception %s %n", index, e); + e.printStackTrace(); + } + }); + } + // 如果不再发送消息，关闭Producer实例。 + producer.shutdown(); + } +} +``` + +#### 3、单向发送消息 + +这种方式主要用在不特别关心发送结果的场景，例如日志发送。 + +```java +public class OnewayProducer { + public static void main(String[] args) throws Exception{ + // 实例化消息生产者Producer + DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name"); + // 设置NameServer的地址 + producer.setNamesrvAddr("localhost:9876"); + // 启动Producer实例 + producer.start(); + for (int i = 0; i < 100; i++) { + // 创建消息，并指定Topic，Tag和消息体 + Message msg = new Message("TopicTest" /* Topic */, + "TagA" /* Tag */, + ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET) /* Message body */ + ); + // 发送单向消息，没有任何返回结果 + producer.sendOneway(msg); + + } + // 如果不再发送消息，关闭Producer实例。 + producer.shutdown(); + } +} +``` + +### 1.3 消费消息 + +```java +public class Consumer { + + public static void main(String[] args) throws InterruptedException, MQClientException { + + // 实例化消费者 + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name"); + + // 设置NameServer的地址 + consumer.setNamesrvAddr("localhost:9876"); + + // 订阅一个或者多个Topic，以及Tag来过滤需要消费的消息 + consumer.subscribe("TopicTest", "*"); + // 注册回调实现类来处理从broker拉取回来的消息 + consumer.registerMessageListener(new MessageListenerConcurrently() { + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, ConsumeConcurrentlyContext context) { + System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs); + // 标记该消息已经被成功消费 + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + // 启动消费者实例 + consumer.start(); + System.out.printf("Consumer Started.%n"); + } +} +``` + +2 顺序消息样例 +---------- + +消息有序指的是可以按照消息的发送顺序来消费(FIFO)。RocketMQ可以严格的保证消息有序，可以分为分区有序或者全局有序。 + +顺序消费的原理解析，在默认的情况下消息发送会采取Round Robin轮询方式把消息发送到不同的queue(分区队列)；而消费消息的时候从多个queue上拉取消息，这种情况发送和消费是不能保证顺序。但是如果控制发送的顺序消息只依次发送到同一个queue中，消费的时候只从这个queue上依次拉取，则就保证了顺序。当发送和消费参与的queue只有一个，则是全局有序；如果多个queue参与，则为分区有序，即相对每个queue，消息都是有序的。 + +下面用订单进行分区有序的示例。一个订单的顺序流程是：创建、付款、推送、完成。订单号相同的消息会被先后发送到同一个队列中，消费时，同一个OrderId获取到的肯定是同一个队列。 + +### 2.1 顺序消息生产 + +```java +package org.apache.rocketmq.example.order2; + +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.client.producer.MessageQueueSelector; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageQueue; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +/** +* Producer，发送顺序消息 +*/ +public class Producer { + + public static void main(String[] args) throws Exception { + DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name"); + + producer.setNamesrvAddr("127.0.0.1:9876"); + + producer.start(); + + String[] tags = new String[]{"TagA", "TagC", "TagD"}; + + // 订单列表 + List orderList = new Producer().buildOrders(); + + Date date = new Date(); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + String dateStr = sdf.format(date); + for (int i = 0; i < 10; i++) { + // 加个时间前缀 + String body = dateStr + " Hello RocketMQ " + orderList.get(i); + Message msg = new Message("TopicTest", tags[i % tags.length], "KEY" + i, body.getBytes()); + + SendResult sendResult = producer.send(msg, new MessageQueueSelector() { + @Override + public MessageQueue select(List mqs, Message msg, Object arg) { + Long id = (Long) arg; //根据订单id选择发送queue + long index = id % mqs.size(); + return mqs.get((int) index); + } + }, orderList.get(i).getOrderId());//订单id + + System.out.println(String.format("SendResult status:%s, queueId:%d, body:%s", + sendResult.getSendStatus(), + sendResult.getMessageQueue().getQueueId(), + body)); + } + + producer.shutdown(); + } + + /** + * 订单的步骤 + */ + private static class OrderStep { + private long orderId; + private String desc; + + public long getOrderId() { + return orderId; + } + + public void setOrderId(long orderId) { + this.orderId = orderId; + } + + public String getDesc() { + return desc; + } + + public void setDesc(String desc) { + this.desc = desc; + } + + @Override + public String toString() { + return "OrderStep{" + + "orderId=" + orderId + + ", desc='" + desc + '\'' + + '}'; + } + } + + /** + * 生成模拟订单数据 + */ + private List buildOrders() { + List orderList = new ArrayList(); + + OrderStep orderDemo = new OrderStep(); + orderDemo.setOrderId(15103111039L); + orderDemo.setDesc("创建"); + orderList.add(orderDemo); + + orderDemo = new OrderStep(); + orderDemo.setOrderId(15103111065L); + orderDemo.setDesc("创建"); + orderList.add(orderDemo); + + orderDemo = new OrderStep(); + orderDemo.setOrderId(15103111039L); + orderDemo.setDesc("付款"); + orderList.add(orderDemo); + + orderDemo = new OrderStep(); + orderDemo.setOrderId(15103117235L); + orderDemo.setDesc("创建"); + orderList.add(orderDemo); + + orderDemo = new OrderStep(); + orderDemo.setOrderId(15103111065L); + orderDemo.setDesc("付款"); + orderList.add(orderDemo); + + orderDemo = new OrderStep(); + orderDemo.setOrderId(15103117235L); + orderDemo.setDesc("付款"); + orderList.add(orderDemo); + + orderDemo = new OrderStep(); + orderDemo.setOrderId(15103111065L); + orderDemo.setDesc("完成"); + orderList.add(orderDemo); + + orderDemo = new OrderStep(); + orderDemo.setOrderId(15103111039L); + orderDemo.setDesc("推送"); + orderList.add(orderDemo); + + orderDemo = new OrderStep(); + orderDemo.setOrderId(15103117235L); + orderDemo.setDesc("完成"); + orderList.add(orderDemo); + + orderDemo = new OrderStep(); + orderDemo.setOrderId(15103111039L); + orderDemo.setDesc("完成"); + orderList.add(orderDemo); + + return orderList; + } +} +``` + +### 2.2 顺序消费消息 + +```java +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import org.apache.rocketmq.common.message.MessageExt; +import java.util.List; + +package org.apache.rocketmq.example.order2; + +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext; +import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus; +import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly; +import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.message.MessageExt; + +import java.util.List; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +/** +* 顺序消息消费，带事务方式（应用可控制Offset什么时候提交） +*/ +public class ConsumerInOrder { + + public static void main(String[] args) throws Exception { + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name_3"); + consumer.setNamesrvAddr("127.0.0.1:9876"); + /** + * 设置Consumer第一次启动是从队列头部开始消费还是队列尾部开始消费
+ * 如果非第一次启动，那么按照上次消费的位置继续消费 + */ + consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET); + + consumer.subscribe("TopicTest", "TagA || TagC || TagD"); + + consumer.registerMessageListener(new MessageListenerOrderly() { + + Random random = new Random(); + + @Override + public ConsumeOrderlyStatus consumeMessage(List msgs, ConsumeOrderlyContext context) { + context.setAutoCommit(true); + for (MessageExt msg : msgs) { + // 可以看到每个queue有唯一的consume线程来消费, 订单对每个queue(分区)有序 + System.out.println("consumeThread=" + Thread.currentThread().getName() + "queueId=" + msg.getQueueId() + ", content:" + new String(msg.getBody())); + } + + try { + //模拟业务逻辑处理中... + TimeUnit.SECONDS.sleep(random.nextInt(10)); + } catch (Exception e) { + e.printStackTrace(); + } + return ConsumeOrderlyStatus.SUCCESS; + } + }); + + consumer.start(); + + System.out.println("Consumer Started."); + } +} +``` + +3 延时消息样例 +---------- + +### 3.1 启动消费者等待传入订阅消息 + +```java + +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import org.apache.rocketmq.common.message.MessageExt; +import java.util.List; + +public class ScheduledMessageConsumer { + public static void main(String[] args) throws Exception { + // 实例化消费者 + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("ExampleConsumer"); + // 订阅Topics + consumer.subscribe("TestTopic", "*"); + // 注册消息监听者 + consumer.registerMessageListener(new MessageListenerConcurrently() { + @Override + public ConsumeConcurrentlyStatus consumeMessage(List messages, ConsumeConcurrentlyContext context) { + for (MessageExt message : messages) { + // Print approximate delay time period + System.out.println("Receive message[msgId=" + message.getMsgId() + "] " + (System.currentTimeMillis() - message.getStoreTimestamp()) + "ms later"); + } + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + // 启动消费者 + consumer.start(); + } +} + +``` + +### 3.2 发送延时消息 + +```java + +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.common.message.Message; + +public class ScheduledMessageProducer { + public static void main(String[] args) throws Exception { + // 实例化一个生产者来产生延时消息 + DefaultMQProducer producer = new DefaultMQProducer("ExampleProducerGroup"); + // 启动生产者 + producer.start(); + int totalMessagesToSend = 100; + for (int i = 0; i < totalMessagesToSend; i++) { + Message message = new Message("TestTopic", ("Hello scheduled message " + i).getBytes()); + // 设置延时等级3,这个消息将在10s之后发送(现在只支持固定的几个时间,详看delayTimeLevel) + message.setDelayTimeLevel(3); + // 发送消息 + producer.send(message); + } + // 关闭生产者 + producer.shutdown(); + } +} +``` + +### 3.3 验证 + +您将会看到消息的消费比存储时间晚10秒。 + +### 3.4 延时消息的使用场景 +1. 比如电商里，提交了一个订单就可以发送一个延时消息，1h后去检查这个订单的状态，如果还是未付款就取消订单释放库存。 + +### 3.5 延时消息的使用限制 + +```java +// org/apache/rocketmq/store/config/MessageStoreConfig.java + +private String messageDelayLevel = "1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h"; +``` +现在RocketMq并不支持任意时间的延时，需要设置几个固定的延时等级，从1s到2h分别对应着等级1到18 +消息消费失败会进入延时消息队列，消息发送时间与设置的延时等级和重试次数有关，详见代码`SendMessageProcessor.java` + + +4 批量消息样例 +---------- + +批量发送消息能显著提高传递小消息的性能。限制是这些批量消息应该有相同的topic，相同的waitStoreMsgOK，而且不能是延时消息。此外，这一批消息的总大小不应超过4MB。 + +### 4.1 发送批量消息 + +如果您每次只发送不超过4MB的消息，则很容易使用批处理，样例如下： + +```java +String topic = "BatchTest"; +List messages = new ArrayList<>(); +messages.add(new Message(topic, "TagA", "OrderID001", "Hello world 0".getBytes())); +messages.add(new Message(topic, "TagA", "OrderID002", "Hello world 1".getBytes())); +messages.add(new Message(topic, "TagA", "OrderID003", "Hello world 2".getBytes())); +try { + producer.send(messages); +} catch (Exception e) { + e.printStackTrace(); + //处理error +} + +``` + +### 4.2 消息列表分割 + +复杂度只有当你发送大批量时才会增长，你可能不确定它是否超过了大小限制（4MB）。这时候你最好把你的消息列表分割一下： + +```java + +public class ListSplitter implements Iterator> { + private final int SIZE_LIMIT = 1024 * 1024 * 4; + private final List messages; + private int currIndex; + public ListSplitter(List messages) { + this.messages = messages; + } + @Override public boolean hasNext() { + return currIndex < messages.size(); + } + @Override public List next() { + int nextIndex = currIndex; + int totalSize = 0; + for (; nextIndex < messages.size(); nextIndex++) { + Message message = messages.get(nextIndex); + int tmpSize = message.getTopic().length() + message.getBody().length; + Map properties = message.getProperties(); + for (Map.Entry entry : properties.entrySet()) { + tmpSize += entry.getKey().length() + entry.getValue().length(); + } + tmpSize = tmpSize + 20; // 增加日志的开销20字节 + if (tmpSize > SIZE_LIMIT) { + //单个消息超过了最大的限制 + //忽略,否则会阻塞分裂的进程 + if (nextIndex - currIndex == 0) { + //假如下一个子列表没有元素,则添加这个子列表然后退出循环,否则只是退出循环 + nextIndex++; + } + break; + } + if (tmpSize + totalSize > SIZE_LIMIT) { + break; + } else { + totalSize += tmpSize; + } + + } + List subList = messages.subList(currIndex, nextIndex); + currIndex = nextIndex; + return subList; + } +} +//把大的消息分裂成若干个小的消息 +ListSplitter splitter = new ListSplitter(messages); +while (splitter.hasNext()) { + try { + List listItem = splitter.next(); + producer.send(listItem); + } catch (Exception e) { + e.printStackTrace(); + //处理error + } +} +``` + +5 过滤消息样例 +---------- + +在大多数情况下，TAG是一个简单而有用的设计，其可以来选择您想要的消息。例如： + +```java +DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("CID_EXAMPLE"); +consumer.subscribe("TOPIC", "TAGA || TAGB || TAGC"); +``` + +消费者将接收包含TAGA或TAGB或TAGC的消息。但是限制是一个消息只能有一个标签，这对于复杂的场景可能不起作用。在这种情况下，可以使用SQL表达式筛选消息。SQL特性可以通过发送消息时的属性来进行计算。在RocketMQ定义的语法下，可以实现一些简单的逻辑。下面是一个例子： +``` +------------ +| message | +|----------| a > 5 AND b = 'abc' +| a = 10 | --------------------> Gotten +| b = 'abc'| +| c = true | +------------ +------------ +| message | +|----------| a > 5 AND b = 'abc' +| a = 1 | --------------------> Missed +| b = 'abc'| +| c = true | +------------ +``` +### 5.1 基本语法 + +RocketMQ只定义了一些基本语法来支持这个特性。你也可以很容易地扩展它。 + +- 数值比较，比如：**>，>=，<，<=，BETWEEN，=；** +- 字符比较，比如：**=，<>，IN；** +- **IS NULL** 或者 **IS NOT NULL；** +- 逻辑符号 **AND，OR，NOT；** + +常量支持类型为： + +- 数值，比如：**123，3.1415；** +- 字符，比如：**'abc'，必须用单引号包裹起来；** +- **NULL**，特殊的常量 +- 布尔值，**TRUE** 或 **FALSE** + +只有使用push模式的消费者才能用使用SQL92标准的sql语句，接口如下： +``` +public void subscribe(finalString topic, final MessageSelector messageSelector) +``` + +### 5.2 使用样例 + +#### 1、生产者样例 + +发送消息时，你能通过`putUserProperty`来设置消息的属性 + +```java +DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name"); +producer.start(); +Message msg = new Message("TopicTest", + tag, + ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET) +); +// 设置一些属性 +msg.putUserProperty("a", String.valueOf(i)); +SendResult sendResult = producer.send(msg); + +producer.shutdown(); +``` + +#### 2、消费者样例 + +用MessageSelector.bySql来使用sql筛选消息 + +```java +DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name_4"); +// 只有订阅的消息有这个属性a, a >=0 and a <= 3 +consumer.subscribe("TopicTest", MessageSelector.bySql("a between 0 and 3"); +consumer.registerMessageListener(new MessageListenerConcurrently() { + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, ConsumeConcurrentlyContext context) { + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } +}); +consumer.start(); + +``` + +6 消息事务样例 +---------- + +事务消息共有三种状态，提交状态、回滚状态、中间状态： + +1. TransactionStatus.CommitTransaction: 提交事务，它允许消费者消费此消息。 +2. TransactionStatus.RollbackTransaction: 回滚事务，它代表该消息将被删除，不允许被消费。 +3. TransactionStatus.Unknown: 中间状态，它代表需要检查消息队列来确定状态。 + +### 6.1 发送事务消息样例 + +#### 1、创建事务性生产者 + +使用 `TransactionMQProducer`类创建生产者，并指定唯一的 `ProducerGroup`，就可以设置自定义线程池来处理这些检查请求。执行本地事务后、需要根据执行结果对消息队列进行回复。回传的事务状态在请参考前一节。 + +```java +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import org.apache.rocketmq.common.message.MessageExt; +import java.util.List; +public class TransactionProducer { + public static void main(String[] args) throws MQClientException, InterruptedException { + TransactionListener transactionListener = new TransactionListenerImpl(); + TransactionMQProducer producer = new TransactionMQProducer("please_rename_unique_group_name"); + ExecutorService executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, new ArrayBlockingQueue(2000), new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + Thread thread = new Thread(r); + thread.setName("client-transaction-msg-check-thread"); + return thread; + } + }); + producer.setExecutorService(executorService); + producer.setTransactionListener(transactionListener); + producer.start(); + String[] tags = new String[] {"TagA", "TagB", "TagC", "TagD", "TagE"}; + for (int i = 0; i < 10; i++) { + try { + Message msg = + new Message("TopicTest1234", tags[i % tags.length], "KEY" + i, + ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET)); + SendResult sendResult = producer.sendMessageInTransaction(msg, null); + System.out.printf("%s%n", sendResult); + Thread.sleep(10); + } catch (MQClientException | UnsupportedEncodingException e) { + e.printStackTrace(); + } + } + for (int i = 0; i < 100000; i++) { + Thread.sleep(1000); + } + producer.shutdown(); + } +} + +``` +#### 2、实现事务的监听接口 + +当发送半消息成功时，我们使用 `executeLocalTransaction` 方法来执行本地事务。它返回前一节中提到的三个事务状态之一。`checkLocalTranscation` 方法用于检查本地事务状态，并回应消息队列的检查请求。它也是返回前一节中提到的三个事务状态之一。 + +```java +public class TransactionListenerImpl implements TransactionListener { + private AtomicInteger transactionIndex = new AtomicInteger(0); + private ConcurrentHashMap localTrans = new ConcurrentHashMap<>(); + @Override + public LocalTransactionState executeLocalTransaction(Message msg, Object arg) { + int value = transactionIndex.getAndIncrement(); + int status = value % 3; + localTrans.put(msg.getTransactionId(), status); + return LocalTransactionState.UNKNOW; + } + @Override + public LocalTransactionState checkLocalTransaction(MessageExt msg) { + Integer status = localTrans.get(msg.getTransactionId()); + if (null != status) { + switch (status) { + case 0: + return LocalTransactionState.UNKNOW; + case 1: + return LocalTransactionState.COMMIT_MESSAGE; + case 2: + return LocalTransactionState.ROLLBACK_MESSAGE; + } + } + return LocalTransactionState.COMMIT_MESSAGE; + } +} + +``` + +### 6.2 事务消息使用上的限制 + +1. 事务消息不支持延时消息和批量消息。 +2. 为了避免单个消息被检查太多次而导致半队列消息累积，我们默认将单个消息的检查次数限制为 15 次，但是用户可以通过 Broker 配置文件的 `transactionCheckMax`参数来修改此限制。如果已经检查某条消息超过 N 次的话（ N = `transactionCheckMax` ） 则 Broker 将丢弃此消息，并在默认情况下同时打印错误日志。用户可以通过重写 `AbstractTransactionCheckListener` 类来修改这个行为。 +3. 事务消息将在 Broker 配置文件中的参数 transactionMsgTimeout 这样的特定时间长度之后被检查。当发送事务消息时，用户还可以通过设置用户属性 CHECK_IMMUNITY_TIME_IN_SECONDS 来改变这个限制，该参数优先于 `transactionMsgTimeout` 参数。 +4. 事务性消息可能不止一次被检查或消费。 +5. 提交给用户的目标主题消息可能会失败，目前这依日志的记录而定。它的高可用性通过 RocketMQ 本身的高可用性机制来保证，如果希望确保事务消息不丢失、并且事务完整性得到保证，建议使用同步的双重写入机制。 +6. 事务消息的生产者 ID 不能与其他类型消息的生产者 ID 共享。与其他类型的消息不同，事务消息允许反向查询、MQ服务器能通过它们的生产者 ID 查询到消费者。 + +7 Logappender样例 +----------------- + +RocketMQ日志提供log4j、log4j2和logback日志框架作为业务应用，下面是配置样例 + +### 7.1 log4j样例 + +按下面样例使用log4j属性配置 +``` +log4j.appender.mq=org.apache.rocketmq.logappender.log4j.RocketmqLog4jAppender +log4j.appender.mq.Tag=yourTag +log4j.appender.mq.Topic=yourLogTopic +log4j.appender.mq.ProducerGroup=yourLogGroup +log4j.appender.mq.NameServerAddress=yourRocketmqNameserverAddress +log4j.appender.mq.layout=org.apache.log4j.PatternLayout +log4j.appender.mq.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-4r [%t] (%F:%L) %-5p - %m%n +``` +按下面样例使用log4j xml配置来使用异步添加日志 +``` + +  +  +  +  +  +      +  + + +  +  +  + +``` +### 7.2 log4j2样例 + +用log4j2时，配置如下，如果想要非阻塞，只需要使用异步添加引用即可 +``` + +  + +``` +### 7.3 logback样例 +``` + +  yourTag +  yourLogTopic +  yourLogGroup +  yourRocketmqNameserverAddress +  +      %date %p %t - %m%n +  + + +  1024 +  80 +  2000 +  true +  + +``` + +8 OpenMessaging样例 +--------------- + + [OpenMessaging](https://www.google.com/url?q=http://openmessaging.cloud/&sa=D&ust=1546524111089000)旨在建立消息和流处理规范，以为金融、电子商务、物联网和大数据领域提供通用框架及工业级指导方案。在分布式异构环境中，设计原则是面向云、简单、灵活和独立于语言。符合这些规范将帮助企业方便的开发跨平台和操作系统的异构消息传递应用程序。提供了openmessaging-api 0.3.0-alpha的部分实现，下面的示例演示如何基于OpenMessaging访问RocketMQ。 + +### 8.1 OMSProducer样例 + +下面的示例演示如何在同步、异步或单向传输中向RocketMQ代理发送消息。 + +```java +import io.openmessaging.Future; +import io.openmessaging.FutureListener; +import io.openmessaging.Message; +import io.openmessaging.MessagingAccessPoint; +import io.openmessaging.OMS; +import io.openmessaging.producer.Producer; +import io.openmessaging.producer.SendResult; +import java.nio.charset.Charset; +import java.util.concurrent.CountDownLatch; + +public class SimpleProducer { + public static void main(String[] args) { + final MessagingAccessPoint messagingAccessPoint = + OMS.getMessagingAccessPoint("oms:rocketmq://localhost:9876/default:default"); + final Producer producer = messagingAccessPoint.createProducer(); + messagingAccessPoint.startup(); + System.out.printf("MessagingAccessPoint startup OK%n"); + producer.startup(); + System.out.printf("Producer startup OK%n"); + { + Message message = producer.createBytesMessage("OMS_HELLO_TOPIC", "OMS_HELLO_BODY".getBytes(Charset.forName("UTF-8"))); + SendResult sendResult = producer.send(message); + //final Void aVoid = result.get(3000L); + System.out.printf("Send async message OK, msgId: %s%n", sendResult.messageId()); + } + final CountDownLatch countDownLatch = new CountDownLatch(1); + { + final Future result = producer.sendAsync(producer.createBytesMessage("OMS_HELLO_TOPIC", "OMS_HELLO_BODY".getBytes(Charset.forName("UTF-8")))); + result.addListener(new FutureListener() { + @Override + public void operationComplete(Future future) { + if (future.getThrowable() != null) { + System.out.printf("Send async message Failed, error: %s%n", future.getThrowable().getMessage()); + } else { + System.out.printf("Send async message OK, msgId: %s%n", future.get().messageId()); + } + countDownLatch.countDown(); + } + }); + } + { + producer.sendOneway(producer.createBytesMessage("OMS_HELLO_TOPIC", "OMS_HELLO_BODY".getBytes(Charset.forName("UTF-8")))); + System.out.printf("Send oneway message OK%n"); + } + try { + countDownLatch.await(); + Thread.sleep(500); // 等一些时间来发送消息 + } catch (InterruptedException ignore) { + } + producer.shutdown(); + } +} +``` + +### 8.2 OMSPullConsumer + +用OMS PullConsumer 来从指定的队列中拉取消息 + +```java +import io.openmessaging.Message; +import io.openmessaging.MessagingAccessPoint; +import io.openmessaging.OMS; +import io.openmessaging.OMSBuiltinKeys; +import io.openmessaging.consumer.PullConsumer; +import io.openmessaging.producer.Producer; +import io.openmessaging.producer.SendResult; + +public class SimplePullConsumer { + public static void main(String[] args) { + final MessagingAccessPoint messagingAccessPoint = + OMS.getMessagingAccessPoint("oms:rocketmq://localhost:9876/default:default"); + messagingAccessPoint.startup(); + final Producer producer = messagingAccessPoint.createProducer(); + final PullConsumer consumer = messagingAccessPoint.createPullConsumer( + OMS.newKeyValue().put(OMSBuiltinKeys.CONSUMER_ID, "OMS_CONSUMER")); + messagingAccessPoint.startup(); + System.out.printf("MessagingAccessPoint startup OK%n"); + final String queueName = "TopicTest"; + producer.startup(); + Message msg = producer.createBytesMessage(queueName, "Hello Open Messaging".getBytes()); + SendResult sendResult = producer.send(msg); + System.out.printf("Send Message OK. MsgId: %s%n", sendResult.messageId()); + producer.shutdown(); + consumer.attachQueue(queueName); + consumer.startup(); + System.out.printf("Consumer startup OK%n"); + // 运行直到发现一个消息被发送了 + boolean stop = false; + while (!stop) { + Message message = consumer.receive(); + if (message != null) { + String msgId = message.sysHeaders().getString(Message.BuiltinKeys.MESSAGE_ID); + System.out.printf("Received one message: %s%n", msgId); + consumer.ack(msgId); + if (!stop) { + stop = msgId.equalsIgnoreCase(sendResult.messageId()); + } + } else { + System.out.printf("Return without any message%n"); + } + } + consumer.shutdown(); + messagingAccessPoint.shutdown(); + } +} +``` + +### 8.3 OMSPushConsumer + +以下示范如何将 OMS PushConsumer 添加到指定的队列，并通过 MessageListener 消费这些消息。 + +```java +import io.openmessaging.Message; +import io.openmessaging.MessagingAccessPoint; +import io.openmessaging.OMS; +import io.openmessaging.OMSBuiltinKeys; +import io.openmessaging.consumer.MessageListener; +import io.openmessaging.consumer.PushConsumer; + +public class SimplePushConsumer { + public static void main(String[] args) { + final MessagingAccessPoint messagingAccessPoint = OMS + .getMessagingAccessPoint("oms:rocketmq://localhost:9876/default:default"); + final PushConsumer consumer = messagingAccessPoint. + createPushConsumer(OMS.newKeyValue().put(OMSBuiltinKeys.CONSUMER_ID, "OMS_CONSUMER")); + messagingAccessPoint.startup(); + System.out.printf("MessagingAccessPoint startup OK%n"); + Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { + @Override + public void run() { + consumer.shutdown(); + messagingAccessPoint.shutdown(); + } + })); + consumer.attachQueue("OMS_HELLO_TOPIC", new MessageListener() { + @Override + public void onReceived(Message message, Context context) { + System.out.printf("Received one message: %s%n", message.sysHeaders().getString(Message.BuiltinKeys.MESSAGE_ID)); + context.ack(); + } + }); + consumer.startup(); + System.out.printf("Consumer startup OK%n"); + } +} +``` \ No newline at end of file diff --git a/docs/cn/acl/user_guide.md b/docs/cn/acl/user_guide.md new file mode 100644 index 00000000000..deeb2fa56af --- /dev/null +++ b/docs/cn/acl/user_guide.md @@ -0,0 +1,82 @@ +# 权限控制 +## 前言 +该文档主要介绍如何快速部署和使用支持权限控制特性的RocketMQ 集群。 + +## 1.权限控制特性介绍 +权限控制（ACL）主要为RocketMQ提供Topic资源级别的用户访问控制。用户在使用RocketMQ权限控制时，可以在Client客户端通过 RPCHook注入AccessKey和SecretKey签名；同时，将对应的权限控制属性（包括Topic访问权限、IP白名单和AccessKey和SecretKey签名等）设置在distribution/conf/plain_acl.yml的配置文件中。Broker端对AccessKey所拥有的权限进行校验，校验不过，抛出异常； +ACL客户端可以参考：**org.apache.rocketmq.example.simple**包下面的**AclClient**代码。 + +## 2. 权限控制的定义与属性值 +### 2.1权限定义 +对RocketMQ的Topic资源访问权限控制定义主要如下表所示，分为以下四种 + + +| 权限 | 含义 | +| --- | --- | +| DENY | 拒绝 | +| ANY | PUB 或者 SUB 权限 | +| PUB | 发送权限 | +| SUB | 订阅权限 | + +### 2.2 权限定义的关键属性 +| 字段 | 取值 | 含义 | +| --- | --- | --- | +| globalWhiteRemoteAddresses | \*;192.168.\*.\*;192.168.0.1 | 全局IP白名单 | +| accessKey | 字符串 | Access Key | +| secretKey | 字符串 | Secret Key | +| whiteRemoteAddress | \*;192.168.\*.\*;192.168.0.1 | 用户IP白名单 | +| admin | true;false | 是否管理员账户 | +| defaultTopicPerm | DENY;PUB;SUB;PUB\|SUB | 默认的Topic权限 | +| defaultGroupPerm | DENY;PUB;SUB;PUB\|SUB | 默认的ConsumerGroup权限 | +| topicPerms | topic=权限 | 各个Topic的权限 | +| groupPerms | group=权限 | 各个ConsumerGroup的权限 | + +具体可以参考**distribution/conf/plain_acl.yml**配置文件 + +## 3. 支持权限控制的集群部署 +在**distribution/conf/plain_acl.yml**配置文件中按照上述说明定义好权限属性后，打开**aclEnable**开关变量即可开启RocketMQ集群的ACL特性。这里贴出Broker端开启ACL特性的properties配置文件内容： +``` +brokerClusterName=DefaultCluster +brokerName=broker-a +brokerId=0 +deleteWhen=04 +fileReservedTime=48 +brokerRole=ASYNC_MASTER +flushDiskType=ASYNC_FLUSH +storePathRootDir=/data/rocketmq/rootdir-a-m +storePathCommitLog=/data/rocketmq/commitlog-a-m +autoCreateSubscriptionGroup=true +## if acl is open,the flag will be true +aclEnable=true +listenPort=10911 +brokerIP1=XX.XX.XX.XX1 +namesrvAddr=XX.XX.XX.XX:9876 +``` + +## 4. 权限控制主要流程 +ACL主要流程分为两部分，主要包括权限解析和权限校验。 + +### 4.1 权限解析 +Broker端对客户端的RequestCommand请求进行解析，拿到需要鉴权的属性字段。 +主要包括： +（1）AccessKey：类似于用户名，代指用户主体，权限数据与之对应； +（2）Signature：客户根据 SecretKey 签名得到的串，服务端再用SecretKey进行签名验证； + +### 4.2 权限校验 +Broker端对权限的校验逻辑主要分为以下几步： +（1）检查是否命中全局 IP 白名单；如果是，则认为校验通过；否则走 2； +（2）检查是否命中用户 IP 白名单；如果是，则认为校验通过；否则走 3； +（3）校验签名，校验不通过，抛出异常；校验通过，则走 4； +（4）对用户请求所需的权限 和 用户所拥有的权限进行校验；不通过，抛出异常； +用户所需权限的校验需要注意已下内容： +（1）特殊的请求例如 UPDATE_AND_CREATE_TOPIC 等，只能由 admin 账户进行操作； +（2）对于某个资源，如果有显性配置权限，则采用配置的权限；如果没有显性配置权限，则采用默认的权限； + +## 5. 热加载修改后权限控制定义 +RocketrMQ的权限控制存储的默认实现是基于yml配置文件。用户可以动态修改权限控制定义的属性，而不需重新启动Broker服务节点。 + + + + + + diff --git a/docs/cn/architecture.md b/docs/cn/architecture.md new file mode 100644 index 00000000000..51d8a0f7d98 --- /dev/null +++ b/docs/cn/architecture.md @@ -0,0 +1,46 @@ +# 架构设计 + +## 技术架构 +![](image/rocketmq_architecture_1.png) + +RocketMQ架构上主要分为四部分，如上图所示: + + +- Producer：消息发布的角色，支持分布式集群方式部署。Producer通过MQ的负载均衡模块选择相应的Broker集群队列进行消息投递，投递的过程支持快速失败并且低延迟。 + +- Consumer：消息消费的角色，支持分布式集群方式部署。支持以push推，pull拉两种模式对消息进行消费。同时也支持集群方式和广播方式的消费，它提供实时消息订阅机制，可以满足大多数用户的需求。 + +- NameServer：NameServer是一个非常简单的Topic路由注册中心，其角色类似Dubbo中的zookeeper，支持Broker的动态注册与发现。主要包括两个功能：Broker管理，NameServer接受Broker集群的注册信息并且保存下来作为路由信息的基本数据。然后提供心跳检测机制，检查Broker是否还存活；路由信息管理，每个NameServer将保存关于Broker集群的整个路由信息和用于客户端查询的队列信息。然后Producer和Conumser通过NameServer就可以知道整个Broker集群的路由信息，从而进行消息的投递和消费。NameServer通常也是集群的方式部署，各实例间相互不进行信息通讯。Broker是向每一台NameServer注册自己的路由信息，所以每一个NameServer实例上面都保存一份完整的路由信息。当某个NameServer因某种原因下线了，Broker仍然可以向其它NameServer同步其路由信息，Producer,Consumer仍然可以动态感知Broker的路由的信息。 + +- BrokerServer：Broker主要负责消息的存储、投递和查询以及服务高可用保证，为了实现这些功能，Broker包含了以下几个重要子模块。 +1. Remoting Module：整个Broker的实体，负责处理来自clients端的请求。 +2. Client Manager：负责管理客户端(Producer/Consumer)和维护Consumer的Topic订阅信息 +3. Store Service：提供方便简单的API接口处理消息存储到物理硬盘和查询功能。 +4. HA Service：高可用服务，提供Master Broker 和 Slave Broker之间的数据同步功能。 +5. Index Service：根据特定的Message key对投递到Broker的消息进行索引服务，以提供消息的快速查询。 + +![](image/rocketmq_architecture_2.png) + +## 部署架构 + + +![](image/rocketmq_architecture_3.png) + + +### RocketMQ 网络部署特点 + +- NameServer是一个几乎无状态节点，可集群部署，节点之间无任何信息同步。 + +- Broker部署相对复杂，Broker分为Master与Slave，一个Master可以对应多个Slave，但是一个Slave只能对应一个Master，Master与Slave 的对应关系通过指定相同的BrokerName，不同的BrokerId 来定义，BrokerId为0表示Master，非0表示Slave。Master也可以部署多个。每个Broker与NameServer集群中的所有节点建立长连接，定时注册Topic信息到所有NameServer。 注意：当前RocketMQ版本在部署架构上支持一Master多Slave，但只有BrokerId=1的从服务器才会参与消息的读负载。 + +- Producer与NameServer集群中的其中一个节点（随机选择）建立长连接，定期从NameServer获取Topic路由信息，并向提供Topic 服务的Master建立长连接，且定时向Master发送心跳。Producer完全无状态，可集群部署。 + +- Consumer与NameServer集群中的其中一个节点（随机选择）建立长连接，定期从NameServer获取Topic路由信息，并向提供Topic服务的Master、Slave建立长连接，且定时向Master、Slave发送心跳。Consumer既可以从Master订阅消息，也可以从Slave订阅消息，消费者在向Master拉取消息时，Master服务器会根据拉取偏移量与最大偏移量的距离（判断是否读老消息，产生读I/O），以及从服务器是否可读等因素建议下一次是从Master还是Slave拉取。 + +结合部署架构图，描述集群工作流程： + +- 启动NameServer，NameServer起来后监听端口，等待Broker、Producer、Consumer连上来，相当于一个路由控制中心。 +- Broker启动，跟所有的NameServer保持长连接，定时发送心跳包。心跳包中包含当前Broker信息(IP+端口等)以及存储所有Topic信息。注册成功后，NameServer集群中就有Topic跟Broker的映射关系。 +- 收发消息前，先创建Topic，创建Topic时需要指定该Topic要存储在哪些Broker上，也可以在发送消息时自动创建Topic。 +- Producer发送消息，启动时先跟NameServer集群中的其中一台建立长连接，并从NameServer中获取当前发送的Topic存在哪些Broker上，轮询从队列列表中选择一个队列，然后与队列所在的Broker建立长连接从而向Broker发消息。 +- Consumer跟Producer类似，跟其中一台NameServer建立长连接，获取当前订阅Topic存在哪些Broker上，然后直接跟Broker建立连接通道，开始消费消息。 diff --git a/docs/cn/best_practice.md b/docs/cn/best_practice.md new file mode 100755 index 00000000000..287469fa4c4 --- /dev/null +++ b/docs/cn/best_practice.md @@ -0,0 +1,369 @@ +# 最佳实践(best practice) + + +## 1 生产者 + +### 1.1 发送消息注意事项 + +#### 1 Tags的使用 + +一个应用尽可能用一个Topic，而消息子类型则可以用tags来标识。tags可以由应用自由设置，只有生产者在发送消息设置了tags，消费方在订阅消息时才可以利用tags通过broker做消息过滤：message.setTags("TagA")。 + +#### 2 Keys的使用 + +每个消息在业务层面的唯一标识码要设置到keys字段，方便将来定位消息丢失问题。服务器会为每个消息创建索引（哈希索引），应用可以通过topic、key来查询这条消息内容，以及消息被谁消费。由于是哈希索引，请务必保证key尽可能唯一，这样可以避免潜在的哈希冲突。 + + +```java + // 订单Id + String orderId = "20034568923546"; + message.setKeys(orderId); +``` +#### 3 日志的打印 + +​消息发送成功或者失败要打印消息日志，务必要打印SendResult和key字段。send消息方法只要不抛异常，就代表发送成功。发送成功会有多个状态，在sendResult里定义。以下对每个状态进行说明： + +- **SEND_OK** + +消息发送成功。要注意的是消息发送成功也不意味着它是可靠的。要确保不会丢失任何消息，还应启用同步Master服务器或同步刷盘，即SYNC_MASTER或SYNC_FLUSH。 + + +- **FLUSH_DISK_TIMEOUT** + +消息发送成功但是服务器刷盘超时。此时消息已经进入服务器队列（内存），只有服务器宕机，消息才会丢失。消息存储配置参数中可以设置刷盘方式和同步刷盘时间长度，如果Broker服务器设置了刷盘方式为同步刷盘，即FlushDiskType=SYNC_FLUSH（默认为异步刷盘方式），当Broker服务器未在同步刷盘时间内（默认为5s）完成刷盘，则将返回该状态——刷盘超时。 + +- **FLUSH_SLAVE_TIMEOUT** + +消息发送成功，但是服务器同步到Slave时超时。此时消息已经进入服务器队列，只有服务器宕机，消息才会丢失。如果Broker服务器的角色是同步Master，即SYNC_MASTER（默认是异步Master即ASYNC_MASTER），并且从Broker服务器未在同步刷盘时间（默认为5秒）内完成与主服务器的同步，则将返回该状态——数据同步到Slave服务器超时。 + +- **SLAVE_NOT_AVAILABLE** + +消息发送成功，但是此时Slave不可用。此时消息已经进入Master服务器队列，只有Master服务器宕机，消息才会丢失。如果Broker服务器的角色是同步Master，即SYNC_MASTER（默认是异步Master服务器即ASYNC_MASTER），但没有配置slave Broker服务器，则将返回该状态——无Slave服务器可用。 + + +### 1.2 消息发送失败处理方式 + +Producer的send方法本身支持内部重试，重试逻辑如下： + +- 至多重试2次（同步发送为2次，异步发送为0次）。 +- 如果发送失败，则轮转到下一个Broker。这个方法的总耗时时间不超过sendMsgTimeout设置的值，默认10s。 +- 如果本身向broker发送消息产生超时异常，就不会再重试。 + +以上策略也是在一定程度上保证了消息可以发送成功。如果业务对消息可靠性要求比较高，建议应用增加相应的重试逻辑：比如调用send同步方法发送失败时，则尝试将消息存储到db，然后由后台线程定时重试，确保消息一定到达Broker。 + +上述db重试方式为什么没有集成到MQ客户端内部做，而是要求应用自己去完成，主要基于以下几点考虑：首先，MQ的客户端设计为无状态模式，方便任意的水平扩展，且对机器资源的消耗仅仅是cpu、内存、网络。其次，如果MQ客户端内部集成一个KV存储模块，那么数据只有同步落盘才能较可靠，而同步落盘本身性能开销较大，所以通常会采用异步落盘，又由于应用关闭过程不受MQ运维人员控制，可能经常会发生 kill -9 这样暴力方式关闭，造成数据没有及时落盘而丢失。第三，Producer所在机器的可靠性较低，一般为虚拟机，不适合存储重要数据。综上，建议重试过程交由应用来控制。 + +### 1.3选择oneway形式发送 +通常消息的发送是这样一个过程： + +- 客户端发送请求到服务器 +- 服务器处理请求 +- 服务器向客户端返回应答 + +所以，一次消息发送的耗时时间是上述三个步骤的总和，而某些场景要求耗时非常短，但是对可靠性要求并不高，例如日志收集类应用，此类应用可以采用oneway形式调用，oneway形式只发送请求不等待应答，而发送请求在客户端实现层面仅仅是一个操作系统系统调用的开销，即将数据写入客户端的socket缓冲区，此过程耗时通常在微秒级。 + + +## 2 消费者 + +### 2.1 消费过程幂等 + +RocketMQ无法避免消息重复（Exactly-Once），所以如果业务对消费重复非常敏感，务必要在业务层面进行去重处理。可以借助关系数据库进行去重。首先需要确定消息的唯一键，可以是msgId，也可以是消息内容中的唯一标识字段，例如订单Id等。在消费之前判断唯一键是否在关系数据库中存在。如果不存在则插入，并消费，否则跳过。（实际过程要考虑原子性问题，判断是否存在可以尝试插入，如果报主键冲突，则插入失败，直接跳过） + +msgId一定是全局唯一标识符，但是实际使用中，可能会存在相同的消息有两个不同msgId的情况（消费者主动重发、因客户端重投机制导致的重复等），这种情况就需要使业务字段进行重复消费。 + +### 2.2 消费速度慢的处理方式 + +#### 1 提高消费并行度 + +绝大部分消息消费行为都属于 IO 密集型，即可能是操作数据库，或者调用 RPC，这类消费行为的消费速度在于后端数据库或者外系统的吞吐量，通过增加消费并行度，可以提高总的消费吞吐量，但是并行度增加到一定程度，反而会下降。所以，应用必须要设置合理的并行度。 如下有几种修改消费并行度的方法： + +- 同一个 ConsumerGroup 下，通过增加 Consumer 实例数量来提高并行度（需要注意的是超过订阅队列数的 Consumer 实例无效）。可以通过加机器，或者在已有机器启动多个进程的方式。 +- 提高单个 Consumer 的消费并行线程，通过修改参数 consumeThreadMin、consumeThreadMax实现。 + +#### 2 批量方式消费 + +某些业务流程如果支持批量方式消费，则可以很大程度上提高消费吞吐量，例如订单扣款类应用，一次处理一个订单耗时 1 s，一次处理 10 个订单可能也只耗时 2 s，这样即可大幅度提高消费的吞吐量，通过设置 consumer的 consumeMessageBatchMaxSize 返个参数，默认是 1，即一次只消费一条消息，例如设置为 N，那么每次消费的消息数小于等于 N。 + +#### 3 跳过非重要消息 + +发生消息堆积时，如果消费速度一直追不上发送速度，如果业务对数据要求不高的话，可以选择丢弃不重要的消息。例如，当某个队列的消息数堆积到100000条以上，则尝试丢弃部分或全部消息，这样就可以快速追上发送消息的速度。示例代码如下： + +```java + public ConsumeConcurrentlyStatus consumeMessage( + List msgs, + ConsumeConcurrentlyContext context) { + long offset = msgs.get(0).getQueueOffset(); + String maxOffset = + msgs.get(0).getProperty(Message.PROPERTY_MAX_OFFSET); + long diff = Long.parseLong(maxOffset) - offset; + if (diff > 100000) { + // TODO 消息堆积情况的特殊处理 + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + // TODO 正常消费过程 + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } +``` + + +#### 4 优化每条消息消费过程 + +举例如下，某条消息的消费过程如下： + +- 根据消息从 DB 查询【数据 1】 +- 根据消息从 DB 查询【数据 2】 +- 复杂的业务计算 +- 向 DB 插入【数据 3】 +- 向 DB 插入【数据 4】 + +这条消息的消费过程中有4次与 DB的 交互，如果按照每次 5ms 计算，那么总共耗时 20ms，假设业务计算耗时 5ms，那么总过耗时 25ms，所以如果能把 4 次 DB 交互优化为 2 次，那么总耗时就可以优化到 15ms，即总体性能提高了 40%。所以应用如果对时延敏感的话，可以把DB部署在SSD硬盘，相比于SCSI磁盘，前者的RT会小很多。 + +### 2.3 消费打印日志 + +如果消息量较少，建议在消费入口方法打印消息，消费耗时等，方便后续排查问题。 + + +```java + public ConsumeConcurrentlyStatus consumeMessage( + List msgs, + ConsumeConcurrentlyContext context) { + log.info("RECEIVE_MSG_BEGIN: " + msgs.toString()); + // TODO 正常消费过程 + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } +``` + +如果能打印每条消息消费耗时，那么在排查消费慢等线上问题时，会更方便。 + +### 2.4 其他消费建议 + +#### 1 关于消费者和订阅 + +​第一件需要注意的事情是，不同的消费者组可以独立的消费一些 topic，并且每个消费者组都有自己的消费偏移量，请确保同一组内的每个消费者订阅信息保持一致。 + +#### 2 关于有序消息 + +消费者将锁定每个消息队列，以确保他们被逐个消费，虽然这将会导致性能下降，但是当你关心消息顺序的时候会很有用。我们不建议抛出异常，你可以返回 ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT 作为替代。 + +#### 3 关于并发消费 + +顾名思义，消费者将并发消费这些消息，建议你使用它来获得良好性能，我们不建议抛出异常，你可以返回 ConsumeConcurrentlyStatus.RECONSUME_LATER 作为替代。 + +#### 4 关于消费状态Consume Status + +对于并发的消费监听器，你可以返回 RECONSUME_LATER 来通知消费者现在不能消费这条消息，并且希望可以稍后重新消费它。然后，你可以继续消费其他消息。对于有序的消息监听器，因为你关心它的顺序，所以不能跳过消息，但是你可以返回SUSPEND_CURRENT_QUEUE_A_MOMENT 告诉消费者等待片刻。 + +#### 5 关于Blocking + +不建议阻塞监听器，因为它会阻塞线程池，并最终可能会终止消费进程 + +#### 6 关于线程数设置 + +消费者使用 ThreadPoolExecutor 在内部对消息进行消费，所以你可以通过设置 setConsumeThreadMin 或 setConsumeThreadMax 来改变它。 + +#### 7 关于消费位点 + +当建立一个新的消费者组时，需要决定是否需要消费已经存在于 Broker 中的历史消息CONSUME_FROM_LAST_OFFSET 将会忽略历史消息，并消费之后生成的任何消息。CONSUME_FROM_FIRST_OFFSET 将会消费每个存在于 Broker 中的信息。你也可以使用 CONSUME_FROM_TIMESTAMP 来消费在指定时间戳后产生的消息。 + + + + + +## 3 Broker + +### 3.1 Broker 角色 +​ Broker 角色分为 ASYNC_MASTER（异步主机）、SYNC_MASTER（同步主机）以及SLAVE（从机）。如果对消息的可靠性要求比较严格，可以采用 SYNC_MASTER加SLAVE的部署方式。如果对消息可靠性要求不高，可以采用ASYNC_MASTER加SLAVE的部署方式。如果只是测试方便，则可以选择仅ASYNC_MASTER或仅SYNC_MASTER的部署方式。 +### 3.2 FlushDiskType +​ SYNC_FLUSH（同步刷新）相比于ASYNC_FLUSH（异步处理）会损失很多性能，但是也更可靠，所以需要根据实际的业务场景做好权衡。 +​ +​ +​ +​ +​ +​ + +## 4 NameServer + +​RocketMQ 中，Name Servers 被设计用来做简单的路由管理。其职责包括： + +- Brokers 定期向每个名称服务器注册路由数据。 +- 名称服务器为客户端，包括生产者，消费者和命令行客户端提供最新的路由信息。 +​ +​ + + + + +## 5 客户端配置 + +​ 相对于RocketMQ的Broker集群，生产者和消费者都是客户端。本小节主要描述生产者和消费者公共的行为配置。 + +### 5.1 客户端寻址方式 + +RocketMQ可以令客户端找到Name Server, 然后通过Name Server再找到Broker。如下所示有多种配置方式，优先级由高到低，高优先级会覆盖低优先级。 + +- 代码中指定Name Server地址，多个namesrv地址之间用分号分割 + +```java +producer.setNamesrvAddr("192.168.0.1:9876;192.168.0.2:9876"); + +consumer.setNamesrvAddr("192.168.0.1:9876;192.168.0.2:9876"); +``` +- Java启动参数中指定Name Server地址 + +```text +-Drocketmq.namesrv.addr=192.168.0.1:9876;192.168.0.2:9876 +``` +- 环境变量指定Name Server地址 + +```text +export NAMESRV_ADDR=192.168.0.1:9876;192.168.0.2:9876 +``` +- HTTP静态服务器寻址（默认） + +客户端启动后，会定时访问一个静态HTTP服务器，地址如下：，这个URL的返回内容如下： + +```text +192.168.0.1:9876;192.168.0.2:9876 +``` +客户端默认每隔2分钟访问一次这个HTTP服务器，并更新本地的Name Server地址。URL已经在代码中硬编码，可通过修改/etc/hosts文件来改变要访问的服务器，例如在/etc/hosts增加如下配置： +```text +10.232.22.67 jmenv.taobao.net +``` +推荐使用HTTP静态服务器寻址方式，好处是客户端部署简单，且Name Server集群可以热升级。 + +### 5.2 客户端配置 + +DefaultMQProducer、TransactionMQProducer、DefaultMQPushConsumer、DefaultMQPullConsumer都继承于ClientConfig类，ClientConfig为客户端的公共配置类。客户端的配置都是get、set形式，每个参数都可以用spring来配置，也可以在代码中配置，例如namesrvAddr这个参数可以这样配置，producer.setNamesrvAddr("192.168.0.1:9876")，其他参数同理。 + +#### 1 客户端的公共配置 + +| 参数名 | 默认值 | 说明 | +| ----------------------------- | ------- | ------------------------------------------------------------ | +| namesrvAddr | | Name Server地址列表，多个NameServer地址用分号隔开 | +| clientIP | 本机IP | 客户端本机IP地址，某些机器会发生无法识别客户端IP地址情况，需要应用在代码中强制指定 | +| instanceName | DEFAULT | 客户端实例名称，客户端创建的多个Producer、Consumer实际是共用一个内部实例（这个实例包含网络连接、线程资源等） | +| clientCallbackExecutorThreads | 4 | 通信层异步回调线程数 | +| pollNameServerInteval | 30000 | 轮询Name Server间隔时间，单位毫秒 | +| heartbeatBrokerInterval | 30000 | 向Broker发送心跳间隔时间，单位毫秒 | +| persistConsumerOffsetInterval | 5000 | 持久化Consumer消费进度间隔时间，单位毫秒 | + +#### 2 Producer配置 + +| 参数名 | 默认值 | 说明 | +| -------------------------------- | ---------------- | ------------------------------------------------------------ | +| producerGroup | DEFAULT_PRODUCER | Producer组名，多个Producer如果属于一个应用，发送同样的消息，则应该将它们归为同一组 | +| createTopicKey | TBW102 | 在发送消息时，自动创建服务器不存在的topic，需要指定Key，该Key可用于配置发送消息所在topic的默认路由。 | +| defaultTopicQueueNums | 4 | 在发送消息，自动创建服务器不存在的topic时，默认创建的队列数 | +| sendMsgTimeout | 10000 | 发送消息超时时间，单位毫秒 | +| compressMsgBodyOverHowmuch | 4096 | 消息Body超过多大开始压缩（Consumer收到消息会自动解压缩），单位字节 | +| retryAnotherBrokerWhenNotStoreOK | FALSE | 如果发送消息返回sendResult，但是sendStatus!=SEND_OK，是否重试发送 | +| retryTimesWhenSendFailed | 2 | 如果消息发送失败，最大重试次数，该参数只对同步发送模式起作用 | +| maxMessageSize | 4MB | 客户端限制的消息大小，超过报错，同时服务端也会限制，所以需要跟服务端配合使用。 | +| transactionCheckListener | | 事务消息回查监听器，如果发送事务消息，必须设置 | +| checkThreadPoolMinSize | 1 | Broker回查Producer事务状态时，线程池最小线程数 | +| checkThreadPoolMaxSize | 1 | Broker回查Producer事务状态时，线程池最大线程数 | +| checkRequestHoldMax | 2000 | Broker回查Producer事务状态时，Producer本地缓冲请求队列大小 | +| RPCHook | null | 该参数是在Producer创建时传入的，包含消息发送前的预处理和消息响应后的处理两个接口，用户可以在第一个接口中做一些安全控制或者其他操作。 | + +#### 3 PushConsumer配置 + +| 参数名 | 默认值 | 说明 | +| ---------------------------- | ----------------------------- | ------------------------------------------------------------ | +| consumerGroup | DEFAULT_CONSUMER | Consumer组名，多个Consumer如果属于一个应用，订阅同样的消息，且消费逻辑一致，则应该将它们归为同一组 | +| messageModel | CLUSTERING | 消费模型支持集群消费和广播消费两种 | +| consumeFromWhere | CONSUME_FROM_LAST_OFFSET | Consumer启动后，默认从上次消费的位置开始消费，这包含两种情况：一种是上次消费的位置未过期，则消费从上次中止的位置进行；一种是上次消费位置已经过期，则从当前队列第一条消息开始消费 | +| consumeTimestamp | 半个小时前 | 只有当consumeFromWhere值为CONSUME_FROM_TIMESTAMP时才起作用。 | +| allocateMessageQueueStrategy | AllocateMessageQueueAveragely | Rebalance算法实现策略 | +| subscription | | 订阅关系 | +| messageListener | | 消息监听器 | +| offsetStore | | 消费进度存储 | +| consumeThreadMin | 10 | 消费线程池最小线程数 | +| consumeThreadMax | 20 | 消费线程池最大线程数 | +| | | | +| consumeConcurrentlyMaxSpan | 2000 | 单队列并行消费允许的最大跨度 | +| pullThresholdForQueue | 1000 | 拉消息本地队列缓存消息最大数 | +| pullInterval | 0 | 拉消息间隔，由于是长轮询，所以为0，但是如果应用为了流控，也可以设置大于0的值，单位毫秒 | +| consumeMessageBatchMaxSize | 1 | 批量消费，一次消费多少条消息 | +| pullBatchSize | 32 | 批量拉消息，一次最多拉多少条 | + +#### 4 PullConsumer配置 + +| 参数名 | 默认值 | 说明 | +| -------------------------------- | ----------------------------- | ------------------------------------------------------------ | +| consumerGroup | DEFAULT_CONSUMER | Consumer组名，多个Consumer如果属于一个应用，订阅同样的消息，且消费逻辑一致，则应该将它们归为同一组 | +| brokerSuspendMaxTimeMillis | 20000 | 长轮询，Consumer拉消息请求在Broker挂起最长时间，单位毫秒 | +| consumerTimeoutMillisWhenSuspend | 30000 | 长轮询，Consumer拉消息请求在Broker挂起超过指定时间，客户端认为超时，单位毫秒 | +| consumerPullTimeoutMillis | 10000 | 非长轮询，拉消息超时时间，单位毫秒 | +| messageModel | BROADCASTING | 消息支持两种模式：集群消费和广播消费 | +| messageQueueListener | | 监听队列变化 | +| offsetStore | | 消费进度存储 | +| registerTopics | | 注册的topic集合 | +| allocateMessageQueueStrategy | AllocateMessageQueueAveragely | Rebalance算法实现策略 | + +#### 5 Message数据结构 + +| 字段名 | 默认值 | 说明 | +| -------------- | ------ | ------------------------------------------------------------ | +| Topic | null | 必填，消息所属topic的名称 | +| Body | null | 必填，消息体 | +| Tags | null | 选填，消息标签，方便服务器过滤使用。目前只支持每个消息设置一个tag | +| Keys | null | 选填，代表这条消息的业务关键词，服务器会根据keys创建哈希索引，设置后，可以在Console系统根据Topic、Keys来查询消息，由于是哈希索引，请尽可能保证key唯一，例如订单号，商品Id等。 | +| Flag | 0 | 选填，完全由应用来设置，RocketMQ不做干预 | +| DelayTimeLevel | 0 | 选填，消息延时级别，0表示不延时，大于0会延时特定的时间才会被消费 | +| WaitStoreMsgOK | TRUE | 选填，表示消息是否在服务器落盘后才返回应答。 | + + +​ +​ + +## 6 系统配置 + +本小节主要介绍系统（JVM/OS）相关的配置。 + +### 6.1 JVM选项 + +​ 推荐使用最新发布的JDK 1.8版本。通过设置相同的Xms和Xmx值来防止JVM调整堆大小以获得更好的性能。简单的JVM配置如下所示： +​ +​``` +​ +​-server -Xms8g -Xmx8g -Xmn4g +​ ``` +​ +​ +如果您不关心RocketMQ Broker的启动时间，还有一种更好的选择，就是通过“预触摸”Java堆以确保在JVM初始化期间每个页面都将被分配。那些不关心启动时间的人可以启用它： +​ -XX:+AlwaysPreTouch +禁用偏置锁定可能会减少JVM暂停， +​ -XX:-UseBiasedLocking +至于垃圾回收，建议使用带JDK 1.8的G1收集器。 + +```text +-XX:+UseG1GC -XX:G1HeapRegionSize=16m +-XX:G1ReservePercent=25 +-XX:InitiatingHeapOccupancyPercent=30 +``` + +​ 这些GC选项看起来有点激进，但事实证明它在我们的生产环境中具有良好的性能。另外不要把-XX:MaxGCPauseMillis的值设置太小，否则JVM将使用一个小的年轻代来实现这个目标，这将导致非常频繁的minor GC，所以建议使用rolling GC日志文件： + +```text +-XX:+UseGCLogFileRotation +-XX:NumberOfGCLogFiles=5 +-XX:GCLogFileSize=30m +``` + +如果写入GC文件会增加代理的延迟，可以考虑将GC日志文件重定向到内存文件系统： + +```text +-Xloggc:/dev/shm/mq_gc_%p.log123 +``` +### 6.2 Linux内核参数 + +​ os.sh脚本在bin文件夹中列出了许多内核参数，可以进行微小的更改然后用于生产用途。下面的参数需要注意，更多细节请参考/proc/sys/vm/*的[文档](https://www.kernel.org/doc/Documentation/sysctl/vm.txt) + +- **vm.extra_free_kbytes**，告诉VM在后台回收（kswapd）启动的阈值与直接回收（通过分配进程）的阈值之间保留额外的可用内存。RocketMQ使用此参数来避免内存分配中的长延迟。（与具体内核版本相关） +- **vm.min_free_kbytes**，如果将其设置为低于1024KB，将会巧妙的将系统破坏，并且系统在高负载下容易出现死锁。 +- **vm.max_map_count**，限制一个进程可能具有的最大内存映射区域数。RocketMQ将使用mmap加载CommitLog和ConsumeQueue，因此建议将为此参数设置较大的值。（agressiveness --> aggressiveness） +- **vm.swappiness**，定义内核交换内存页面的积极程度。较高的值会增加攻击性，较低的值会减少交换量。建议将值设置为10来避免交换延迟。 +- **File descriptor limits**，RocketMQ需要为文件（CommitLog和ConsumeQueue）和网络连接打开文件描述符。我们建议设置文件描述符的值为655350。 +- [Disk scheduler](https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/6/html/Performance_Tuning_Guide/ch06s04s02.html)，RocketMQ建议使用I/O截止时间调度器，它试图为请求提供有保证的延迟。 +[]([]()) diff --git a/docs/cn/concept.md b/docs/cn/concept.md new file mode 100644 index 00000000000..38036f22f66 --- /dev/null +++ b/docs/cn/concept.md @@ -0,0 +1,39 @@ +# 基本概念 +## 消息模型（Message Model） + + + +RocketMQ 消息模型如图1所示，主要由 Producer、Broker、Consumer 三部分组成，其中Producer 负责生产消息，Consumer 负责消费消息，Broker 负责存储消息。Broker 在实际部署过程中对应一台服务器，每个 Broker 可以存储多个Topic的消息，每个Topic的消息也可以分片存储于不同的 Broker。Message Queue 用于存储消息的物理地址，每个Topic中的消息地址存储于多个 Message Queue 中。ConsumerGroup 由多个Consumer 实例构成。 +## 消息生产者（Producer） + 负责生产消息，一般由业务系统负责生产消息。一个消息生产者会把业务应用系统里产生的消息发送到broker服务器。RocketMQ提供多种发送方式，同步发送、异步发送、顺序发送、单向发送。同步和异步方式均需要Broker返回确认信息，单向发送不需要。 +## 消息消费者（Consumer） + 负责消费消息，一般是后台系统负责异步消费。一个消息消费者会从Broker服务器拉取消息、并将其提供给应用程序。从用户应用的角度而言提供了两种消费形式：拉取式消费、推动式消费。 +## 主题（Topic） + 表示一类消息的集合，每个主题包含若干条消息，每条消息只能属于一个主题，是RocketMQ进行消息订阅的基本单位。 +##代理服务器（Broker Server） +消息中转角色，负责存储消息、转发消息。代理服务器在RocketMQ系统中负责接收从生产者发送来的消息并存储、同时为消费者的拉取请求作准备。代理服务器也存储消息相关的元数据，包括消费者组、消费进度偏移和主题和队列消息等。 +## 名字服务（Name Server） + 名称服务充当路由消息的提供者。生产者或消费者能够通过名字服务查找各主题相应的Broker IP列表。多个Namesrv实例组成集群，但相互独立，没有信息交换。 +## 拉取式消费（Pull Consumer） + Consumer消费的一种类型，应用通常主动调用Consumer的拉消息方法从Broker服务器拉消息、主动权由应用控制。一旦获取了批量消息，应用就会启动消费过程。 +## 推动式消费（Push Consumer） + Consumer消费的一种类型，该模式下Broker收到数据后会主动推送给消费端，该消费模式一般实时性较高。 +## 生产者组（Producer Group） + 同一类Producer的集合，这类Producer发送同一类消息且发送逻辑一致。如果发送的是事物消息且原始生产者在发送之后崩溃，则Broker服务器会联系同一生产者组的其他生产者实例以提交或回溯消费。 +## 消费者组（Consumer Group） + 同一类Consumer的集合，这类Consumer通常消费同一类消息且消费逻辑一致。消费者组使得在消息消费方面，实现负载均衡和容错的目标变得非常容易。要注意的是，消费者组的消费者实例必须订阅完全相同的Topic。RocketMQ 支持两种消息模式：集群消费（Clustering）和广播消费（Broadcasting）。 +## 集群消费（Clustering） +集群消费模式下,相同Consumer Group的每个Consumer实例平均分摊消息。 +## 广播消费（Broadcasting） +广播消费模式下，相同Consumer Group的每个Consumer实例都接收全量的消息。 +## 普通顺序消息（Normal Ordered Message） +普通顺序消费模式下，消费者通过同一个消费队列收到的消息是有顺序的，不同消息队列收到的消息则可能是无顺序的。 +## 严格顺序消息（Strictly Ordered Message） +严格顺序消息模式下，消费者收到的所有消息均是有顺序的。 +## 代理服务器（Broker Server） +消息中转角色，负责存储消息、转发消息。代理服务器在RocketMQ系统中负责接收从生产者发送来的消息并存储、同时为消费者的拉取请求作准备。代理服务器也存储消息相关的元数据，包括消费者组、消费进度偏移和主题和队列消息等。 +## 消息（Message） +消息系统所传输信息的物理载体，生产和消费数据的最小单位，每条消息必须属于一个主题。RocketMQ中每个消息拥有唯一的Message ID，且可以携带具有业务标识的Key。系统提供了通过Message ID和Key查询消息的功能。 +## 标签（Tag） + 为消息设置的标志，用于同一主题下区分不同类型的消息。来自同一业务单元的消息，可以根据不同业务目的在同一主题下设置不同标签。标签能够有效地保持代码的清晰度和连贯性，并优化RocketMQ提供的查询系统。消费者可以根据Tag实现对不同子主题的不同消费逻辑，实现更好的扩展性。 + diff --git a/docs/cn/image/rocketmq_architecture_1.png b/docs/cn/image/rocketmq_architecture_1.png new file mode 100644 index 00000000000..548c8752a84 Binary files /dev/null and b/docs/cn/image/rocketmq_architecture_1.png differ diff --git a/docs/cn/image/rocketmq_architecture_2.png b/docs/cn/image/rocketmq_architecture_2.png new file mode 100644 index 00000000000..b2ab8d34c6c Binary files /dev/null and b/docs/cn/image/rocketmq_architecture_2.png differ diff --git a/docs/cn/image/rocketmq_architecture_3.png b/docs/cn/image/rocketmq_architecture_3.png new file mode 100644 index 00000000000..aa74ed7ac15 Binary files /dev/null and b/docs/cn/image/rocketmq_architecture_3.png differ diff --git a/docs/cn/index.md b/docs/cn/index.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/docs/cn/msg_trace/user_guide.md b/docs/cn/msg_trace/user_guide.md new file mode 100644 index 00000000000..0320e16e006 --- /dev/null +++ b/docs/cn/msg_trace/user_guide.md @@ -0,0 +1,106 @@ +# 消息轨迹 +## 前言 +该文档主要介绍如何快速部署和使用支持消息轨迹特性的RocketMQ 集群。 + +## 1. 消息轨迹数据关键属性 +| Producer端| Consumer端 | Broker端 | +| --- | --- | --- | +| 生产实例信息 | 消费实例信息 | 消息的Topic | +| 发送消息时间 | 投递时间,投递轮次  | 消息存储位置 | +| 消息是否发送成功 | 消息是否消费成功 | 消息的Key值 | +| 发送耗时 | 消费耗时 | 消息的Tag值 | + +## 2. 支持消息轨迹集群部署 + +### 2.1 Broker端配置文件 +这里贴出Broker端开启消息轨迹特性的properties配置文件内容： +``` +brokerClusterName=DefaultCluster +brokerName=broker-a +brokerId=0 +deleteWhen=04 +fileReservedTime=48 +brokerRole=ASYNC_MASTER +flushDiskType=ASYNC_FLUSH +storePathRootDir=/data/rocketmq/rootdir-a-m +storePathCommitLog=/data/rocketmq/commitlog-a-m +autoCreateSubscriptionGroup=true +## if msg tracing is open,the flag will be true +traceTopicEnable=true +listenPort=10911 +brokerIP1=XX.XX.XX.XX1 +namesrvAddr=XX.XX.XX.XX:9876 +``` + +### 2.2 普通模式 +RocketMQ集群中每一个Broker节点均用于存储Client端收集并发送过来的消息轨迹数据。因此，对于RocketMQ集群中的Broker节点数量并无要求和限制。 + +### 2.3 物理IO隔离模式 +对于消息轨迹数据量较大的场景，可以在RocketMQ集群中选择其中一个Broker节点专用于存储消息轨迹，使得用户普通的消息数据与消息轨迹数据的物理IO完全隔离，互不影响。在该模式下，RockeMQ集群中至少有两个Broker节点，其中一个Broker节点定义为存储消息轨迹数据的服务端。 + +### 2.4 启动开启消息轨迹的Broker +`nohup sh mqbroker -c ../conf/2m-noslave/broker-a.properties &` + +## 3. 保存消息轨迹的Topic定义 +RocketMQ的消息轨迹特性支持两种存储轨迹数据的方式： + +### 3.1 系统级的TraceTopic +在默认情况下，消息轨迹数据是存储于系统级的TraceTopic中(其名称为：**RMQ_SYS_TRACE_TOPIC**)。该Topic在Broker节点启动时，会自动创建出来（如上所叙，需要在Broker端的配置文件中将**traceTopicEnable**的开关变量设置为**true**）。 + +### 3.2 用户自定义的TraceTopic +如果用户不准备将消息轨迹的数据存储于系统级的默认TraceTopic，也可以自己定义并创建用户级的Topic来保存轨迹（即为创建普通的Topic用于保存消息轨迹数据）。下面一节会介绍Client客户端的接口如何支持用户自定义的TraceTopic。 + +## 4. 支持消息轨迹的Client客户端实践 +为了尽可能地减少用户业务系统使用RocketMQ消息轨迹特性的改造工作量，作者在设计时候采用对原来接口增加一个开关参数(**enableMsgTrace**)来实现消息轨迹是否开启；并新增一个自定义参(**customizedTraceTopic**)数来实现用户存储消息轨迹数据至自己创建的用户级Topic。 + +### 4.1 发送消息时开启消息轨迹 +``` + DefaultMQProducer producer = new DefaultMQProducer("ProducerGroupName",true); + producer.setNamesrvAddr("XX.XX.XX.XX1"); + producer.start(); + try { + { + Message msg = new Message("TopicTest", + "TagA", + "OrderID188", + "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET)); + SendResult sendResult = producer.send(msg); + System.out.printf("%s%n", sendResult); + } + + } catch (Exception e) { + e.printStackTrace(); + } +``` + +### 4.2 订阅消息时开启消息轨迹 +``` + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("CID_JODIE_1",true); + consumer.subscribe("TopicTest", "*"); + consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET); + consumer.setConsumeTimestamp("20181109221800"); + consumer.registerMessageListener(new MessageListenerConcurrently() { + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, ConsumeConcurrentlyContext context) { + System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs); + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + consumer.start(); + System.out.printf("Consumer Started.%n"); +``` + +### 4.3 支持自定义存储消息轨迹Topic +在上面的发送和订阅消息时候分别将DefaultMQProducer和DefaultMQPushConsumer实例的初始化修改为如下即可支持自定义存储消息轨迹Topic。 +``` + ##其中Topic_test11111需要用户自己预先创建，来保存消息轨迹； + DefaultMQProducer producer = new DefaultMQProducer("ProducerGroupName",true,"Topic_test11111"); + ...... + + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("CID_JODIE_1",true,"Topic_test11111"); + ...... +``` + + + + diff --git a/docs/cn/operation.md b/docs/cn/operation.md new file mode 100644 index 00000000000..c00be933a7f --- /dev/null +++ b/docs/cn/operation.md @@ -0,0 +1,1410 @@ +## 运维管理(operation） + +### 1 集群搭建 + +#### 1.1 单Master模式 + +这种方式风险较大，一旦Broker重启或者宕机时，会导致整个服务不可用。不建议线上环境使用,可以用于本地测试。 + +##### 1）启动 NameServer + +```bash +### 首先启动Name Server +$ nohup sh mqnamesrv & + +### 验证Name Server 是否启动成功 +$ tail -f ~/logs/rocketmqlogs/namesrv.log +The Name Server boot success... +``` + +##### 2）启动 Broker + +```bash +### 启动Broker +$ nohup sh bin/mqbroker -n localhost:9876 & + +### 验证Name Server 是否启动成功，例如Broker的IP为：192.168.1.2，且名称为broker-a +$ tail -f ~/logs/rocketmqlogs/Broker.log +The broker[broker-a, 192.169.1.2:10911] boot success... +``` + +#### 1.2 多Master模式 + +一个集群无Slave，全是Master，例如2个Master或者3个Master，这种模式的优缺点如下： + +- 优点：配置简单，单个Master宕机或重启维护对应用无影响，在磁盘配置为RAID10时，即使机器宕机不可恢复情况下，由于RAID10磁盘非常可靠，消息也不会丢（异步刷盘丢失少量消息，同步刷盘一条不丢），性能最高； + +- 缺点：单台机器宕机期间，这台机器上未被消费的消息在机器恢复之前不可订阅，消息实时性会受到影响。 + +##### 1）启动NameServer + +NameServer需要先于Broker启动，且如果在生产环境使用，为了保证高可用，建议一般规模的集群启动3个NameServer，各节点的启动命令相同，如下： + +```bash +### 首先启动Name Server +$ nohup sh mqnamesrv & + +### 验证Name Server 是否启动成功 +$ tail -f ~/logs/rocketmqlogs/namesrv.log +The Name Server boot success... +``` + +##### 2）启动Broker集群 + +```bash +### 在机器A，启动第一个Master，例如NameServer的IP为：192.168.1.1 +$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-noslave/broker-a.properties & + +### 在机器B，启动第二个Master，例如NameServer的IP为：192.168.1.1 +$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-noslave/broker-b.properties & + +... +``` + +如上启动命令是在单个NameServer情况下使用的。对于多个NameServer的集群，Broker启动命令中`-n`后面的地址列表用分号隔开即可，例如 `192.168.1.1:9876;192.161.2:9876`。 + +#### 1.3 多Master多Slave模式-异步复制 + +每个Master配置一个Slave，有多对Master-Slave，HA采用异步复制方式，主备有短暂消息延迟（毫秒级），这种模式的优缺点如下： + +- 优点：即使磁盘损坏，消息丢失的非常少，且消息实时性不会受影响，同时Master宕机后，消费者仍然可以从Slave消费，而且此过程对应用透明，不需要人工干预，性能同多Master模式几乎一样； + +- 缺点：Master宕机，磁盘损坏情况下会丢失少量消息。 + +##### 1）启动NameServer + +```bash +### 首先启动Name Server +$ nohup sh mqnamesrv & + +### 验证Name Server 是否启动成功 +$ tail -f ~/logs/rocketmqlogs/namesrv.log +The Name Server boot success... +``` + +##### 2）启动Broker集群 + +```bash +### 在机器A，启动第一个Master，例如NameServer的IP为：192.168.1.1 +$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-a.properties & + +### 在机器B，启动第二个Master，例如NameServer的IP为：192.168.1.1 +$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-b.properties & + +### 在机器C，启动第一个Slave，例如NameServer的IP为：192.168.1.1 +$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-a-s.properties & + +### 在机器D，启动第二个Slave，例如NameServer的IP为：192.168.1.1 +$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-b-s.properties & +``` + +#### 1.4 多Master多Slave模式-同步双写 + +每个Master配置一个Slave，有多对Master-Slave，HA采用同步双写方式，即只有主备都写成功，才向应用返回成功，这种模式的优缺点如下： + +- 优点：数据与服务都无单点故障，Master宕机情况下，消息无延迟，服务可用性与数据可用性都非常高； + +- 缺点：性能比异步复制模式略低（大约低10%左右），发送单个消息的RT会略高，且目前版本在主节点宕机后，备机不能自动切换为主机。 + +##### 1）启动NameServer + +```bash +### 首先启动Name Server +$ nohup sh mqnamesrv & + +### 验证Name Server 是否启动成功 +$ tail -f ~/logs/rocketmqlogs/namesrv.log +The Name Server boot success... +``` + +##### 2）启动Broker集群 + +```bash +### 在机器A，启动第一个Master，例如NameServer的IP为：192.168.1.1 +$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-a.properties & + +### 在机器B，启动第二个Master，例如NameServer的IP为：192.168.1.1 +$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-b.properties & + +### 在机器C，启动第一个Slave，例如NameServer的IP为：192.168.1.1 +$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-a-s.properties & + +### 在机器D，启动第二个Slave，例如NameServer的IP为：192.168.1.1 +$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-b-s.properties & +``` + +以上Broker与Slave配对是通过指定相同的BrokerName参数来配对，Master的BrokerId必须是0，Slave的BrokerId必须是大于0的数。另外一个Master下面可以挂载多个Slave，同一Master下的多个Slave通过指定不同的BrokerId来区分。$ROCKETMQ_HOME指的RocketMQ安装目录，需要用户自己设置此环境变量。 + +### 2 mqadmin管理工具 + +> 注意： +> +> 1. 执行命令方法：`./mqadmin {command} {args}` +> 2. 几乎所有命令都需要配置-n表示NameServer地址，格式为ip:port +> 3. 几乎所有命令都可以通过-h获取帮助 +> 4. 如果既有Broker地址（-b）配置项又有clusterName（-c）配置项，则优先以Broker地址执行命令，如果不配置Broker地址，则对集群中所有主机执行命令，只支持一个Broker地址。-b格式为ip:port，port默认是10911 +> 5. 在tools下可以看到很多命令，但并不是所有命令都能使用，只有在MQAdminStartup中初始化的命令才能使用，你也可以修改这个类，增加或自定义命令 +> 6. 由于版本更新问题，少部分命令可能未及时更新，遇到错误请直接阅读相关命令源码 + +#### 2.1 Topic相关 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
名称含义命令选项说明
updateTopic创建更新Topic配置-bBroker 地址，表示 topic 所在 + Broker，只支持单台Broker，地址为ip:port
-ccluster 名称，表示 topic 所在集群（集群可通过 + clusterList 查询）
-h-打印帮助
-nNameServer服务地址，格式 ip:port
-p指定新topic的读写权限( W=2|R=4|WR=6 )
-r可读队列数（默认为 8）
-w可写队列数（默认为 8）
-ttopic 名称（名称只能使用字符 + ^[a-zA-Z0-9_-]+$ ）
deleteTopic删除Topic-ccluster 名称，表示删除某集群下的某个 topic （集群 + 可通过 clusterList 查询）
-h打印帮助
-nNameServer 服务地址，格式 ip:port
-ttopic 名称（名称只能使用字符 + ^[a-zA-Z0-9_-]+$ ）
topicList查看 Topic 列表信息-h打印帮助
-c不配置-c只返回topic列表，增加-c返回clusterName, + topic, consumerGroup信息，即topic的所属集群和订阅关系，没有参数
-nNameServer 服务地址，格式 ip:port
topicRoute查看 Topic 路由信息-ttopic 名称
-h打印帮助
-nNameServer 服务地址，格式 ip:port
topicStatus查看 Topic 消息队列offset-ttopic 名称
-h打印帮助
-nNameServer 服务地址，格式 ip:port
topicClusterList查看 Topic 所在集群列表-ttopic 名称
-h打印帮助
-nNameServer 服务地址，格式 ip:port
updateTopicPerm更新 Topic 读写权限-ttopic 名称
-h打印帮助
-nNameServer 服务地址，格式 ip:port
-bBroker 地址，表示 topic 所在 + Broker，只支持单台Broker，地址为ip:port
-p指定新 topic 的读写权限( W=2|R=4|WR=6 )
-ccluster 名称，表示 topic 所在集群（集群可通过 + clusterList 查询），-b优先，如果没有-b，则对集群中所有Broker执行命令
updateOrderConf从NameServer上创建、删除、获取特定命名空间的kv配置，目前还未启用-h打印帮助
-nNameServer 服务地址，格式 ip:port
-ttopic，键
-vorderConf，值
-mmethod，可选get、put、delete
allocateMQ以平均负载算法计算消费者列表负载消息队列的负载结果-ttopic 名称
-h打印帮助
-nNameServer 服务地址，格式 ip:port
-iipList，用逗号分隔，计算这些ip去负载Topic的消息队列
statsAll打印Topic订阅关系、TPS、积累量、24h读写总量等信息-h打印帮助
-nNameServer 服务地址，格式 ip:port
-a是否只打印活跃topic
-t指定topic
+ + + +#### 2.2 集群相关 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
名称含义命令选项说明
clusterList查看集群信息，集群、BrokerName、BrokerId、TPS等信息-m打印更多信息 (增加打印出如下信息 #InTotalYest, + #OutTotalYest, #InTotalToday ,#OutTotalToday)
-h打印帮助
-nNameServer 服务地址，格式 ip:port
-i打印间隔，单位秒
clusterRT发送消息检测集群各Broker RT。消息发往${BrokerName} Topic。-aamount，每次探测的总数，RT = 总时间 / + amount
-s消息大小，单位B
-c探测哪个集群
-p是否打印格式化日志，以|分割，默认不打印
-h打印帮助
-m所属机房，打印使用
-i发送间隔，单位秒
-nNameServer 服务地址，格式 ip:port
+ + +#### 2.3 Broker相关 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
名称含义命令选项说明
updateBrokerConfig更新 Broker 配置文件，会修改Broker.conf-bBroker 地址，格式为ip:port
-ccluster 名称
-kkey 值
-vvalue 值
-h打印帮助
-nNameServer 服务地址，格式 ip:port
brokerStatus查看 Broker 统计信息、运行状态（你想要的信息几乎都在里面）-bBroker 地址，地址为ip:port
-h打印帮助
-nNameServer 服务地址，格式 ip:port
brokerConsumeStatsBroker中各个消费者的消费情况，按Message Queue维度返回Consume + Offset，Broker Offset，Diff，TImestamp等信息-bBroker 地址，地址为ip:port
-t请求超时时间
-ldiff阈值，超过阈值才打印
-o是否为顺序topic，一般为false
-h打印帮助
-nNameServer 服务地址，格式 ip:port
getBrokerConfig获取Broker配置-bBroker 地址，地址为ip:port
-nNameServer 服务地址，格式 ip:port
wipeWritePerm从NameServer上清除 Broker写权限-bBroker 地址，地址为ip:port
-nNameServer 服务地址，格式 ip:port
-h打印帮助
cleanExpiredCQ清理Broker上过期的Consume Queue，如果手动减少对列数可能产生过期队列-nNameServer 服务地址，格式 ip:port
-h打印帮助
-bBroker 地址，地址为ip:port
-c集群名称
cleanUnusedTopic清理Broker上不使用的Topic，从内存中释放Topic的Consume + Queue，如果手动删除Topic会产生不使用的Topic-nNameServer 服务地址，格式 ip:port
-h打印帮助
-bBroker 地址，地址为ip:port
-c集群名称
sendMsgStatus向Broker发消息，返回发送状态和RT-nNameServer 服务地址，格式 ip:port
-h打印帮助
-bBrokerName，注意不同于Broker地址
-s消息大小，单位B
-c发送次数
+ + +#### 2.4 消息相关 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
名称含义命令选项说明
queryMsgById根据offsetMsgId查询msg，如果使用开源控制台，应使用offsetMsgId，此命令还有其他参数，具体作用请阅读QueryMsgByIdSubCommand。-imsgId
-h打印帮助
-nNameServer 服务地址，格式 ip:port
queryMsgByKey根据消息 Key 查询消息-kmsgKey
-tTopic 名称
-h打印帮助
-nNameServer 服务地址，格式 ip:port
queryMsgByOffset根据 Offset 查询消息-bBroker 名称，（这里需要注意 + 填写的是 Broker 的名称，不是 Broker 的地址，Broker 名称可以在 clusterList 查到）
-iquery 队列 id
-ooffset 值
-ttopic 名称
-h打印帮助
-nNameServer 服务地址，格式 ip:port
queryMsgByUniqueKey根据msgId查询，msgId不同于offsetMsgId，区别详见常见运维问题。-g，-d配合使用，查到消息后尝试让特定的消费者消费消息并返回消费结果-h打印帮助
-nNameServer 服务地址，格式 ip:port
-iuniqe msg id
-gconsumerGroup
-dclientId
-ttopic名称
checkMsgSendRT检测向topic发消息的RT，功能类似clusterRT-h打印帮助
-nNameServer 服务地址，格式 ip:port
-ttopic名称
-a探测次数
-s消息大小
sendMessage发送一条消息，可以根据配置发往特定Message Queue，或普通发送。-h打印帮助
-nNameServer 服务地址，格式 ip:port
-ttopic名称
-pbody，消息体
-kkeys
-ctags
-bBrokerName
-iqueueId
consumeMessage消费消息。可以根据offset、开始&结束时间戳、消息队列消费消息，配置不同执行不同消费逻辑，详见ConsumeMessageCommand。-h打印帮助
-nNameServer 服务地址，格式 ip:port
-ttopic名称
-bBrokerName
-o从offset开始消费
-iqueueId
-g消费者分组
-s开始时间戳，格式详见-h
-d结束时间戳
-c消费多少条消息
printMsg从Broker消费消息并打印，可选时间段-h打印帮助
-nNameServer 服务地址，格式 ip:port
-ttopic名称
-c字符集，例如UTF-8
-ssubExpress，过滤表达式
-b开始时间戳，格式参见-h
-e结束时间戳
-d是否打印消息体
printMsgByQueue类似printMsg，但指定Message Queue-h打印帮助
-nNameServer 服务地址，格式 ip:port
-ttopic名称
-iqueueId
-aBrokerName
-c字符集，例如UTF-8
-ssubExpress，过滤表达式
-b开始时间戳，格式参见-h
-e结束时间戳
-p是否打印消息
-d是否打印消息体
-f是否统计tag数量并打印
resetOffsetByTime按时间戳重置offset，Broker和consumer都会重置-h打印帮助
-nNameServer 服务地址，格式 ip:port
-g消费者分组
-ttopic名称
-s重置为此时间戳对应的offset
-f是否强制重置，如果false，只支持回溯offset，如果true，不管时间戳对应offset与consumeOffset关系
-c是否重置c++客户端offset
+ + +#### 2.5 消费者、消费组相关 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
名称含义命令选项说明
consumerProgress查看订阅组消费状态，可以查看具体的client IP的消息积累量-g消费者所属组名
-s是否打印client IP
-h打印帮助
-nNameServer 服务地址，格式 ip:port
consumerStatus查看消费者状态，包括同一个分组中是否都是相同的订阅，分析Process + Queue是否堆积，返回消费者jstack结果，内容较多，使用者参见ConsumerStatusSubCommand-h打印帮助
-nNameServer 服务地址，格式 ip:port
-gconsumer group
-iclientId
-s是否执行jstack
getConsumerStatus获取 Consumer 消费进度-g消费者所属组名
-t查询主题
-iConsumer 客户端 ip
-nNameServer 服务地址，格式 ip:port
-h打印帮助
updateSubGroup更新或创建订阅关系-nNameServer 服务地址，格式 ip:port
-h打印帮助
-bBroker地址
-c集群名称
-g消费者分组名称
-s分组是否允许消费
-m是否从最小offset开始消费
-d是否是广播模式
-q重试队列数量
-r最大重试次数
-i当slaveReadEnable开启时有效，且还未达到从slave消费时建议从哪个BrokerId消费，可以配置备机id，主动从备机消费
-w如果Broker建议从slave消费，配置决定从哪个slave消费，配置BrokerId，例如1
-a当消费者数量变化时是否通知其他消费者负载均衡
deleteSubGroup从Broker删除订阅关系-nNameServer 服务地址，格式 ip:port
-h打印帮助
-bBroker地址
-c集群名称
-g消费者分组名称
cloneGroupOffset在目标群组中使用源群组的offset-nNameServer 服务地址，格式 ip:port
-h打印帮助
-s源消费者组
-d目标消费者组
-ttopic名称
-o暂未使用
+ + + + +#### 2.6 连接相关 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
名称含义命令选项说明
consumerConnec tion查询 Consumer 的网络连接-g消费者所属组名
-nNameServer 服务地址，格式 ip:port
-h打印帮助
producerConnec tion查询 Producer 的网络连接-g生产者所属组名
-t主题名称
-nNameServer 服务地址，格式 ip:port
-h打印帮助
+ + + + +#### 2.7 NameServer相关 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
名称含义命令选项说明
updateKvConfig更新NameServer的kv配置，目前还未使用-s命名空间
-kkey
-vvalue
-nNameServer 服务地址，格式 ip:port
-h打印帮助
deleteKvConfig删除NameServer的kv配置-s命名空间
-kkey
-nNameServer 服务地址，格式 ip:port
-h打印帮助
getNamesrvConfig获取NameServer配置-nNameServer 服务地址，格式 ip:port
-h打印帮助
updateNamesrvConfig修改NameServer配置-nNameServer 服务地址，格式 ip:port
-h打印帮助
-kkey
-vvalue
+ + + + +#### 2.8 其他 + + + + + + + + + + + + + + + + + + + + + + +
名称含义命令选项说明
startMonitoring开启监控进程，监控消息误删、重试队列消息数等-nNameServer 服务地址，格式 ip:port
-h打印帮助
+ + +### 3 运维常见问题 + +#### 3.1 RocketMQ的mqadmin命令报错问题 + +> 问题描述：有时候在部署完RocketMQ集群后，尝试执行“mqadmin”一些运维命令，会出现下面的异常信息： +> +> ```java +> org.apache.rocketmq.remoting.exception.RemotingConnectException: connect to failed +> ``` + +解决方法：可以在部署RocketMQ集群的虚拟机上执行`export NAMESRV_ADDR=ip:9876`（ip指的是集群中部署NameServer组件的机器ip地址）命令之后再使用“mqadmin”的相关命令进行查询，即可得到结果。 + +#### 3.2 RocketMQ生产端和消费端版本不一致导致不能正常消费的问题 + +> 问题描述：同一个生产端发出消息，A消费端可消费，B消费端却无法消费，rocketMQ Console中出现： +> +> ```java +> Not found the consumer group consume stats, because return offset table is empty, maybe the consumer not consume any message的异常消息。 +> ``` + + 解决方案：RocketMQ 的jar包：rocketmq-client等包应该保持生产端，消费端使用相同的version。 + +#### 3.3 新增一个topic的消费组时，无法消费历史消息的问题 + +> 问题描述：当同一个topic的新增消费组启动时，消费的消息是当前的offset的消息，并未获取历史消息。 + +解决方案：rocketmq默认策略是从消息队列尾部，即跳过历史消息。如果想消费历史消息，则需要设置：`org.apache.rocketmq.client.consumer.DefaultMQPushConsumer#setConsumeFromWhere`。常用的有以下三种配置： + +- 默认配置,一个新的订阅组第一次启动从队列的最后位置开始消费，后续再启动接着上次消费的进度开始消费,即跳过历史消息； + +```java +consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET); +``` + +- 一个新的订阅组第一次启动从队列的最前位置开始消费，后续再启动接着上次消费的进度开始消费,即消费Broker未过期的历史消息； + +```java +consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET); +``` + +- 一个新的订阅组第一次启动从指定时间点开始消费，后续再启动接着上次消费的进度开始消费，和consumer.setConsumeTimestamp()配合使用，默认是半个小时以前； + +```java +consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_TIMESTAMP); +``` + +#### 3.4 如何开启从Slave读数据功能 + +在某些情况下，Consumer需要将消费位点重置到1-2天前，这时在内存有限的Master Broker上，CommitLog会承载比较重的IO压力，影响到该Broker的其它消息的读与写。可以开启`slaveReadEnable=true`，当Master Broker发现Consumer的消费位点与CommitLog的最新值的差值的容量超过该机器内存的百分比（`accessMessageInMemoryMaxRatio=40%`），会推荐Consumer从Slave Broker中去读取数据，降低Master Broker的IO。 + +#### 3.5 性能调优问题 + +异步刷盘建议使用自旋锁，同步刷盘建议使用重入锁，调整Broker配置项`useReentrantLockWhenPutMessage`，默认为false；异步刷盘建议开启`TransientStorePoolEnable`；建议关闭transferMsgByHeap，提高拉消息效率；同步刷盘建议适当增大`sendMessageThreadPoolNums`，具体配置需要经过压测。 + +#### 3.6 在RocketMQ中msgId和offsetMsgId的含义与区别 + +使用RocketMQ完成生产者客户端消息发送后，通常会看到如下日志打印信息： + +```java +SendResult [sendStatus=SEND_OK, msgId=0A42333A0DC818B4AAC246C290FD0000, offsetMsgId=0A42333A00002A9F000000000134F1F5, messageQueue=MessageQueue [topic=topicTest1, BrokerName=mac.local, queueId=3], queueOffset=4] +``` + +- msgId，对于客户端来说msgId是由客户端producer实例端生成的，具体来说，调用方法`MessageClientIDSetter.createUniqIDBuffer()`生成唯一的Id； +- offsetMsgId，offsetMsgId是由Broker服务端在写入消息时生成的（采用”IP地址+Port端口”与“CommitLog的物理偏移量地址”做了一个字符串拼接），其中offsetMsgId就是在RocketMQ控制台直接输入查询的那个messageId。 diff --git a/docs/en/index.md b/docs/en/index.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/example/pom.xml b/example/pom.xml index 28dfe922fb1..c43ff785e1e 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.4.0-SNAPSHOT + 4.4.1-SNAPSHOT 4.0.0 @@ -51,7 +51,12 @@ org.apache.rocketmq rocketmq-openmessaging - 4.4.0-SNAPSHOT + 4.4.1-SNAPSHOT + + + org.apache.rocketmq + rocketmq-acl + 4.4.1-SNAPSHOT diff --git a/example/src/main/java/org/apache/rocketmq/example/filter/SqlConsumer.java b/example/src/main/java/org/apache/rocketmq/example/filter/SqlFilterConsumer.java similarity index 74% rename from example/src/main/java/org/apache/rocketmq/example/filter/SqlConsumer.java rename to example/src/main/java/org/apache/rocketmq/example/filter/SqlFilterConsumer.java index c41c9c14c3c..8dd6d20eb79 100644 --- a/example/src/main/java/org/apache/rocketmq/example/filter/SqlConsumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/filter/SqlFilterConsumer.java @@ -17,28 +17,24 @@ package org.apache.rocketmq.example.filter; +import java.util.List; import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; import org.apache.rocketmq.client.consumer.MessageSelector; import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; -import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.common.message.MessageExt; -import java.util.List; +public class SqlFilterConsumer { + + public static void main(String[] args) throws Exception { -public class SqlConsumer { + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name"); - public static void main(String[] args) { - DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name_4"); - try { - consumer.subscribe("TopicTest", - MessageSelector.bySql("(TAGS is not null and TAGS in ('TagA', 'TagB'))" + - "and (a is not null and a between 0 3)")); - } catch (MQClientException e) { - e.printStackTrace(); - return; - } + // Don't forget to set enablePropertyFilter=true in broker + consumer.subscribe("SqlFilterTest", + MessageSelector.bySql("(TAGS is not null and TAGS in ('TagA', 'TagB'))" + + "and (a is not null and a between 0 and 3)")); consumer.registerMessageListener(new MessageListenerConcurrently() { @@ -50,12 +46,7 @@ public ConsumeConcurrentlyStatus consumeMessage(List msgs, } }); - try { - consumer.start(); - } catch (MQClientException e) { - e.printStackTrace(); - return; - } + consumer.start(); System.out.printf("Consumer Started.%n"); } } diff --git a/example/src/main/java/org/apache/rocketmq/example/filter/SqlProducer.java b/example/src/main/java/org/apache/rocketmq/example/filter/SqlFilterProducer.java similarity index 51% rename from example/src/main/java/org/apache/rocketmq/example/filter/SqlProducer.java rename to example/src/main/java/org/apache/rocketmq/example/filter/SqlFilterProducer.java index 3f3a0e65215..001827068a0 100644 --- a/example/src/main/java/org/apache/rocketmq/example/filter/SqlProducer.java +++ b/example/src/main/java/org/apache/rocketmq/example/filter/SqlFilterProducer.java @@ -17,51 +17,32 @@ package org.apache.rocketmq.example.filter; -import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.remoting.common.RemotingHelper; -public class SqlProducer { +public class SqlFilterProducer { + + public static void main(String[] args) throws Exception { - public static void main(String[] args) { DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name"); - try { - producer.start(); - } catch (MQClientException e) { - e.printStackTrace(); - return; - } + + producer.start(); + + String[] tags = new String[] {"TagA", "TagB", "TagC"}; for (int i = 0; i < 10; i++) { - try { - String tag; - int div = i % 3; - if (div == 0) { - tag = "TagA"; - } else if (div == 1) { - tag = "TagB"; - } else { - tag = "TagC"; - } - Message msg = new Message("TopicTest", - tag, - ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET) - ); - msg.putUserProperty("a", String.valueOf(i)); + Message msg = new Message("SqlFilterTest", + tags[i % tags.length], + ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET) + ); + msg.putUserProperty("a", String.valueOf(i)); - SendResult sendResult = producer.send(msg); - System.out.printf("%s%n", sendResult); - } catch (Exception e) { - e.printStackTrace(); - try { - Thread.sleep(1000); - } catch (InterruptedException e1) { - e1.printStackTrace(); - } - } + SendResult sendResult = producer.send(msg); + System.out.printf("%s%n", sendResult); } + producer.shutdown(); } } diff --git a/example/src/main/java/org/apache/rocketmq/example/filter/Consumer.java b/example/src/main/java/org/apache/rocketmq/example/filter/TagFilterConsumer.java similarity index 81% rename from example/src/main/java/org/apache/rocketmq/example/filter/Consumer.java rename to example/src/main/java/org/apache/rocketmq/example/filter/TagFilterConsumer.java index bb491ac40af..ba3723cdfef 100644 --- a/example/src/main/java/org/apache/rocketmq/example/filter/Consumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/filter/TagFilterConsumer.java @@ -16,7 +16,6 @@ */ package org.apache.rocketmq.example.filter; -import java.io.File; import java.io.IOException; import java.util.List; import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; @@ -24,20 +23,15 @@ import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.message.MessageExt; -public class Consumer { +public class TagFilterConsumer { public static void main(String[] args) throws InterruptedException, MQClientException, IOException { - DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("ConsumerGroupNamecc4"); - ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); - File classFile = new File(classLoader.getResource("MessageFilterImpl.java").getFile()); + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name"); - String filterCode = MixAll.file2String(classFile); - consumer.subscribe("TopicTest", "org.apache.rocketmq.example.filter.MessageFilterImpl", - filterCode); + consumer.subscribe("TagFilterTest", "TagA || TagC"); consumer.registerMessageListener(new MessageListenerConcurrently() { diff --git a/example/src/main/java/org/apache/rocketmq/example/filter/TagFilterProducer.java b/example/src/main/java/org/apache/rocketmq/example/filter/TagFilterProducer.java new file mode 100644 index 00000000000..b0a9e2dd023 --- /dev/null +++ b/example/src/main/java/org/apache/rocketmq/example/filter/TagFilterProducer.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.example.filter; + +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.remoting.common.RemotingHelper; + +public class TagFilterProducer { + + public static void main(String[] args) throws Exception { + + DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name"); + producer.start(); + + String[] tags = new String[] {"TagA", "TagB", "TagC"}; + + for (int i = 0; i < 60; i++) { + Message msg = new Message("TagFilterTest", + tags[i % tags.length], + "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET)); + + SendResult sendResult = producer.send(msg); + System.out.printf("%s%n", sendResult); + } + + producer.shutdown(); + } +} diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/AclClient.java b/example/src/main/java/org/apache/rocketmq/example/simple/AclClient.java new file mode 100644 index 00000000000..0c97cd33210 --- /dev/null +++ b/example/src/main/java/org/apache/rocketmq/example/simple/AclClient.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.example.simple; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.rocketmq.acl.common.AclClientRPCHook; +import org.apache.rocketmq.acl.common.SessionCredentials; +import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.common.RemotingHelper; + + +public class AclClient { + + private static final Map OFFSE_TABLE = new HashMap(); + + private static final String ACL_ACCESS_KEY = "RocketMQ"; + + private static final String ACL_SECRET_KEY = "1234567"; + + public static void main(String[] args) throws MQClientException, InterruptedException { + producer(); + pushConsumer(); + pullConsumer(); + } + + public static void producer() throws MQClientException { + DefaultMQProducer producer = new DefaultMQProducer("ProducerGroupName", getAclRPCHook()); + producer.setNamesrvAddr("127.0.0.1:9876"); + producer.start(); + + for (int i = 0; i < 128; i++) + try { + { + Message msg = new Message("TopicTest", + "TagA", + "OrderID188", + "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET)); + SendResult sendResult = producer.send(msg); + System.out.printf("%s%n", sendResult); + } + + } catch (Exception e) { + e.printStackTrace(); + } + + producer.shutdown(); + } + + public static void pushConsumer() throws MQClientException { + + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name_5", getAclRPCHook(), new AllocateMessageQueueAveragely()); + consumer.setNamesrvAddr("127.0.0.1:9876"); + consumer.subscribe("TopicTest", "*"); + consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET); + // Wrong time format 2017_0422_221800 + consumer.setConsumeTimestamp("20180422221800"); + consumer.registerMessageListener(new MessageListenerConcurrently() { + + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, ConsumeConcurrentlyContext context) { + System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs); + printBody(msgs); + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + consumer.start(); + System.out.printf("Consumer Started.%n"); + } + + public static void pullConsumer() throws MQClientException { + DefaultMQPullConsumer consumer = new DefaultMQPullConsumer("please_rename_unique_group_name_6", getAclRPCHook()); + consumer.setNamesrvAddr("127.0.0.1:9876"); + consumer.start(); + + Set mqs = consumer.fetchSubscribeMessageQueues("TopicTest"); + for (MessageQueue mq : mqs) { + System.out.printf("Consume from the queue: %s%n", mq); + SINGLE_MQ: + while (true) { + try { + PullResult pullResult = + consumer.pullBlockIfNotFound(mq, null, getMessageQueueOffset(mq), 32); + System.out.printf("%s%n", pullResult); + putMessageQueueOffset(mq, pullResult.getNextBeginOffset()); + printBody(pullResult); + switch (pullResult.getPullStatus()) { + case FOUND: + break; + case NO_MATCHED_MSG: + break; + case NO_NEW_MSG: + break SINGLE_MQ; + case OFFSET_ILLEGAL: + break; + default: + break; + } + } catch (Exception e) { + e.printStackTrace(); + } + } + } + + consumer.shutdown(); + } + + private static void printBody(PullResult pullResult) { + printBody(pullResult.getMsgFoundList()); + } + + private static void printBody(List msg) { + if (msg == null || msg.size() == 0) + return; + for (MessageExt m : msg) { + if (m != null) { + System.out.printf("msgId : %s body : %s \n\r", m.getMsgId(), new String(m.getBody())); + } + } + } + + private static long getMessageQueueOffset(MessageQueue mq) { + Long offset = OFFSE_TABLE.get(mq); + if (offset != null) + return offset; + + return 0; + } + + private static void putMessageQueueOffset(MessageQueue mq, long offset) { + OFFSE_TABLE.put(mq, offset); + } + + static RPCHook getAclRPCHook() { + return new AclClientRPCHook(new SessionCredentials(ACL_ACCESS_KEY,ACL_SECRET_KEY)); + } +} diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/Producer.java b/example/src/main/java/org/apache/rocketmq/example/simple/Producer.java index 7b504dd2a3e..448f8ee9f45 100644 --- a/example/src/main/java/org/apache/rocketmq/example/simple/Producer.java +++ b/example/src/main/java/org/apache/rocketmq/example/simple/Producer.java @@ -26,7 +26,6 @@ public class Producer { public static void main(String[] args) throws MQClientException, InterruptedException { DefaultMQProducer producer = new DefaultMQProducer("ProducerGroupName"); - producer.start(); for (int i = 0; i < 128; i++) diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/PullConsumer.java b/example/src/main/java/org/apache/rocketmq/example/simple/PullConsumer.java index efffa36d59d..8aec7e30934 100644 --- a/example/src/main/java/org/apache/rocketmq/example/simple/PullConsumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/simple/PullConsumer.java @@ -29,10 +29,10 @@ public class PullConsumer { public static void main(String[] args) throws MQClientException { DefaultMQPullConsumer consumer = new DefaultMQPullConsumer("please_rename_unique_group_name_5"); - + consumer.setNamesrvAddr("127.0.0.1:9876"); consumer.start(); - Set mqs = consumer.fetchSubscribeMessageQueues("TopicTest1"); + Set mqs = consumer.fetchSubscribeMessageQueues("broker-a"); for (MessageQueue mq : mqs) { System.out.printf("Consume from the queue: %s%n", mq); SINGLE_MQ: diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/PullConsumerTest.java b/example/src/main/java/org/apache/rocketmq/example/simple/PullConsumerTest.java index 16108b8c6a8..f12595a903b 100644 --- a/example/src/main/java/org/apache/rocketmq/example/simple/PullConsumerTest.java +++ b/example/src/main/java/org/apache/rocketmq/example/simple/PullConsumerTest.java @@ -24,6 +24,7 @@ public class PullConsumerTest { public static void main(String[] args) throws MQClientException { DefaultMQPullConsumer consumer = new DefaultMQPullConsumer("please_rename_unique_group_name_5"); + consumer.setNamesrvAddr("127.0.0.1:9876"); consumer.start(); try { diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/PushConsumer.java b/example/src/main/java/org/apache/rocketmq/example/simple/PushConsumer.java index c6c7e39d174..abbfbdffcdd 100644 --- a/example/src/main/java/org/apache/rocketmq/example/simple/PushConsumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/simple/PushConsumer.java @@ -29,10 +29,10 @@ public class PushConsumer { public static void main(String[] args) throws InterruptedException, MQClientException { DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("CID_JODIE_1"); - consumer.subscribe("Jodie_topic_1023", "*"); + consumer.subscribe("TopicTest", "*"); consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET); //wrong time format 2017_0422_221800 - consumer.setConsumeTimestamp("20170422221800"); + consumer.setConsumeTimestamp("20181109221800"); consumer.registerMessageListener(new MessageListenerConcurrently() { @Override diff --git a/example/src/main/java/org/apache/rocketmq/example/filter/Producer.java b/example/src/main/java/org/apache/rocketmq/example/tracemessage/TraceProducer.java similarity index 68% rename from example/src/main/java/org/apache/rocketmq/example/filter/Producer.java rename to example/src/main/java/org/apache/rocketmq/example/tracemessage/TraceProducer.java index 2a0da6546c8..fb8e37fd2b7 100644 --- a/example/src/main/java/org/apache/rocketmq/example/filter/Producer.java +++ b/example/src/main/java/org/apache/rocketmq/example/tracemessage/TraceProducer.java @@ -14,7 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.example.filter; + +package org.apache.rocketmq.example.tracemessage; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.producer.DefaultMQProducer; @@ -22,25 +23,27 @@ import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.remoting.common.RemotingHelper; -public class Producer { +public class TraceProducer { public static void main(String[] args) throws MQClientException, InterruptedException { - DefaultMQProducer producer = new DefaultMQProducer("ProducerGroupName"); + + DefaultMQProducer producer = new DefaultMQProducer("ProducerGroupName",true); producer.start(); - try { - for (int i = 0; i < 6000000; i++) { - Message msg = new Message("TopicFilter7", - "TagA", - "OrderID001", - "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET)); + for (int i = 0; i < 128; i++) + try { + { + Message msg = new Message("TopicTest", + "TagA", + "OrderID188", + "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET)); + SendResult sendResult = producer.send(msg); + System.out.printf("%s%n", sendResult); + } - msg.putUserProperty("SequenceId", String.valueOf(i)); - SendResult sendResult = producer.send(msg); - System.out.printf("%s%n", sendResult); + } catch (Exception e) { + e.printStackTrace(); } - } catch (Exception e) { - e.printStackTrace(); - } + producer.shutdown(); } } diff --git a/example/src/main/java/org/apache/rocketmq/example/tracemessage/TracePushConsumer.java b/example/src/main/java/org/apache/rocketmq/example/tracemessage/TracePushConsumer.java new file mode 100644 index 00000000000..473351963f5 --- /dev/null +++ b/example/src/main/java/org/apache/rocketmq/example/tracemessage/TracePushConsumer.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.example.tracemessage; + +import java.util.List; +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.message.MessageExt; + +public class TracePushConsumer { + public static void main(String[] args) throws InterruptedException, MQClientException { + // Here,we use the default message track trace topic name + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("CID_JODIE_1",true); + consumer.subscribe("TopicTest", "*"); + consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET); + // Wrong time format 2017_0422_221800 + consumer.setConsumeTimestamp("20181109221800"); + consumer.registerMessageListener(new MessageListenerConcurrently() { + + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, ConsumeConcurrentlyContext context) { + System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs); + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + consumer.start(); + System.out.printf("Consumer Started.%n"); + } +} diff --git a/filter/pom.xml b/filter/pom.xml index 0134f03d0e5..86f1bbb5d94 100644 --- a/filter/pom.xml +++ b/filter/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.4.0-SNAPSHOT + 4.4.1-SNAPSHOT 4.0.0 diff --git a/logappender/pom.xml b/logappender/pom.xml index c250e59e5b6..5388be6a26d 100644 --- a/logappender/pom.xml +++ b/logappender/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.4.0-SNAPSHOT + 4.4.1-SNAPSHOT 4.0.0 rocketmq-logappender diff --git a/logging/pom.xml b/logging/pom.xml index a52b9d0ac6c..ac435e7f111 100644 --- a/logging/pom.xml +++ b/logging/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.4.0-SNAPSHOT + 4.4.1-SNAPSHOT 4.0.0 diff --git a/namesrv/pom.xml b/namesrv/pom.xml index 79f0abdc818..eea7f85456f 100644 --- a/namesrv/pom.xml +++ b/namesrv/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.4.0-SNAPSHOT + 4.4.1-SNAPSHOT 4.0.0 diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml index 2850b1f2e17..4843662e4df 100644 --- a/openmessaging/pom.xml +++ b/openmessaging/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.4.0-SNAPSHOT + 4.4.1-SNAPSHOT 4.0.0 diff --git a/pom.xml b/pom.xml index 148cc99b299..83b564a01a3 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ 2012 org.apache.rocketmq rocketmq-all - 4.4.0-SNAPSHOT + 4.4.1-SNAPSHOT pom Apache RocketMQ ${project.version} http://rocketmq.apache.org/ @@ -125,6 +125,7 @@ distribution openmessaging logging + acl @@ -214,9 +215,6 @@ generate-effective-dependencies-pom generate-resources - - effective-pom - ${project.build.directory}/effective-pom/effective-dependencies.xml @@ -257,8 +255,10 @@ src/test/resources/certs/* src/test/**/*.log src/test/resources/META-INF/service/* + src/main/resources/META-INF/service/* */target/** */*.iml + docs/** @@ -521,6 +521,11 @@ rocketmq-example ${project.version} + + ${project.groupId} + rocketmq-acl + ${project.version} + org.slf4j slf4j-api @@ -581,6 +586,16 @@ log4j 1.2.17 + + org.yaml + snakeyaml + 1.19 + + + commons-codec + commons-codec + 1.9 + org.apache.logging.log4j log4j-core diff --git a/remoting/pom.xml b/remoting/pom.xml index 55d92f37051..84c2654e235 100644 --- a/remoting/pom.xml +++ b/remoting/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.4.0-SNAPSHOT + 4.4.1-SNAPSHOT 4.0.0 diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java index 8dccebc0457..d190e00f44f 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java @@ -23,6 +23,7 @@ import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslHandler; import java.net.SocketAddress; +import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; @@ -35,6 +36,8 @@ import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.ChannelEventListener; import org.apache.rocketmq.remoting.InvokeCallback; import org.apache.rocketmq.remoting.RPCHook; @@ -45,8 +48,6 @@ import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode; @@ -95,6 +96,13 @@ public abstract class NettyRemotingAbstract { */ protected volatile SslContext sslContext; + /** + * custom rpc hooks + */ + protected List rpcHooks = new ArrayList(); + + + static { NettyLogger.initNettyLogger(); } @@ -158,6 +166,23 @@ public void processMessageReceived(ChannelHandlerContext ctx, RemotingCommand ms } } + protected void doBeforeRpcHooks(String addr, RemotingCommand request) { + if (rpcHooks.size() > 0) { + for (RPCHook rpcHook: rpcHooks) { + rpcHook.doBeforeRequest(addr, request); + } + } + } + + protected void doAfterRpcHooks(String addr, RemotingCommand request, RemotingCommand response) { + if (rpcHooks.size() > 0) { + for (RPCHook rpcHook: rpcHooks) { + rpcHook.doAfterResponse(addr, request, response); + } + } + } + + /** * Process incoming request command issued by remote peer. * @@ -174,15 +199,9 @@ public void processRequestCommand(final ChannelHandlerContext ctx, final Remotin @Override public void run() { try { - RPCHook rpcHook = NettyRemotingAbstract.this.getRPCHook(); - if (rpcHook != null) { - rpcHook.doBeforeRequest(RemotingHelper.parseChannelRemoteAddr(ctx.channel()), cmd); - } - + doBeforeRpcHooks(RemotingHelper.parseChannelRemoteAddr(ctx.channel()), cmd); final RemotingCommand response = pair.getObject1().processRequest(ctx, cmd); - if (rpcHook != null) { - rpcHook.doAfterResponse(RemotingHelper.parseChannelRemoteAddr(ctx.channel()), cmd, response); - } + doAfterRpcHooks(RemotingHelper.parseChannelRemoteAddr(ctx.channel()), cmd, response); if (!cmd.isOnewayRPC()) { if (response != null) { @@ -314,12 +333,29 @@ public void run() { } } + + /** * Custom RPC hook. + * Just be compatible with the previous version, use getRPCHooks instead. + */ + @Deprecated + protected RPCHook getRPCHook() { + if (rpcHooks.size() > 0) { + return rpcHooks.get(0); + } + return null; + } + + /** + * Custom RPC hooks. * - * @return RPC hook if specified; null otherwise. + * @return RPC hooks if specified; null otherwise. */ - public abstract RPCHook getRPCHook(); + public List getRPCHooks() { + return rpcHooks; + } + /** * This method specifies thread pool to use while invoking callback methods. @@ -410,7 +446,8 @@ public void invokeAsyncImpl(final Channel channel, final RemotingCommand request final SemaphoreReleaseOnlyOnce once = new SemaphoreReleaseOnlyOnce(this.semaphoreAsync); long costTime = System.currentTimeMillis() - beginStartTime; if (timeoutMillis < costTime) { - throw new RemotingTooMuchRequestException("invokeAsyncImpl call timeout"); + once.release(); + throw new RemotingTimeoutException("invokeAsyncImpl call timeout"); } final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, timeoutMillis - costTime, invokeCallback, once); diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java index 33c2eed8de1..fc9df37c652 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java @@ -53,6 +53,8 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.ChannelEventListener; import org.apache.rocketmq.remoting.InvokeCallback; import org.apache.rocketmq.remoting.RPCHook; @@ -64,8 +66,6 @@ import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.protocol.RemotingCommand; public class NettyRemotingClient extends NettyRemotingAbstract implements RemotingClient { @@ -94,7 +94,6 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti private ExecutorService callbackExecutor; private final ChannelEventListener channelEventListener; private DefaultEventExecutorGroup defaultEventExecutorGroup; - private RPCHook rpcHook; public NettyRemotingClient(final NettyClientConfig nettyClientConfig) { this(nettyClientConfig, null); @@ -283,7 +282,9 @@ public void closeChannel(final String addr, final Channel channel) { @Override public void registerRPCHook(RPCHook rpcHook) { - this.rpcHook = rpcHook; + if (rpcHook != null && !rpcHooks.contains(rpcHook)) { + rpcHooks.add(rpcHook); + } } public void closeChannel(final Channel channel) { @@ -357,6 +358,8 @@ public void updateNameServerAddressList(List addrs) { } } + + @Override public RemotingCommand invokeSync(String addr, final RemotingCommand request, long timeoutMillis) throws InterruptedException, RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException { @@ -364,17 +367,13 @@ public RemotingCommand invokeSync(String addr, final RemotingCommand request, lo final Channel channel = this.getAndCreateChannel(addr); if (channel != null && channel.isActive()) { try { - if (this.rpcHook != null) { - this.rpcHook.doBeforeRequest(addr, request); - } + doBeforeRpcHooks(addr, request); long costTime = System.currentTimeMillis() - beginStartTime; if (timeoutMillis < costTime) { throw new RemotingTimeoutException("invokeSync call timeout"); } RemotingCommand response = this.invokeSyncImpl(channel, request, timeoutMillis - costTime); - if (this.rpcHook != null) { - this.rpcHook.doAfterResponse(RemotingHelper.parseChannelRemoteAddr(channel), request, response); - } + doAfterRpcHooks(RemotingHelper.parseChannelRemoteAddr(channel), request, response); return response; } catch (RemotingSendRequestException e) { log.warn("invokeSync: send request exception, so close the channel[{}]", addr); @@ -522,9 +521,7 @@ public void invokeAsync(String addr, RemotingCommand request, long timeoutMillis final Channel channel = this.getAndCreateChannel(addr); if (channel != null && channel.isActive()) { try { - if (this.rpcHook != null) { - this.rpcHook.doBeforeRequest(addr, request); - } + doBeforeRpcHooks(addr, request); long costTime = System.currentTimeMillis() - beginStartTime; if (timeoutMillis < costTime) { throw new RemotingTooMuchRequestException("invokeAsync call timeout"); @@ -547,9 +544,7 @@ public void invokeOneway(String addr, RemotingCommand request, long timeoutMilli final Channel channel = this.getAndCreateChannel(addr); if (channel != null && channel.isActive()) { try { - if (this.rpcHook != null) { - this.rpcHook.doBeforeRequest(addr, request); - } + doBeforeRpcHooks(addr, request); this.invokeOnewayImpl(channel, request, timeoutMillis); } catch (RemotingSendRequestException e) { log.warn("invokeOneway: send request exception, so close the channel[{}]", addr); @@ -592,10 +587,6 @@ public ChannelEventListener getChannelEventListener() { return channelEventListener; } - @Override - public RPCHook getRPCHook() { - return this.rpcHook; - } @Override public ExecutorService getCallbackExecutor() { diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java index 198484251c0..7f6284ea0b0 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java @@ -47,6 +47,8 @@ import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.ChannelEventListener; import org.apache.rocketmq.remoting.InvokeCallback; import org.apache.rocketmq.remoting.RPCHook; @@ -58,8 +60,6 @@ import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.protocol.RemotingCommand; public class NettyRemotingServer extends NettyRemotingAbstract implements RemotingServer { @@ -75,7 +75,6 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti private final Timer timer = new Timer("ServerHouseKeepingService", true); private DefaultEventExecutorGroup defaultEventExecutorGroup; - private RPCHook rpcHook; private int port = 0; @@ -108,16 +107,16 @@ public Thread newThread(Runnable r) { } }); - this.eventLoopGroupBoss = new NioEventLoopGroup(1, new ThreadFactory() { - private AtomicInteger threadIndex = new AtomicInteger(0); + if (useEpoll()) { + this.eventLoopGroupBoss = new EpollEventLoopGroup(1, new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); - @Override - public Thread newThread(Runnable r) { - return new Thread(r, String.format("NettyBoss_%d", this.threadIndex.incrementAndGet())); - } - }); + @Override + public Thread newThread(Runnable r) { + return new Thread(r, String.format("NettyEPOLLBoss_%d", this.threadIndex.incrementAndGet())); + } + }); - if (useEpoll()) { this.eventLoopGroupSelector = new EpollEventLoopGroup(nettyServerConfig.getServerSelectorThreads(), new ThreadFactory() { private AtomicInteger threadIndex = new AtomicInteger(0); private int threadTotal = nettyServerConfig.getServerSelectorThreads(); @@ -128,6 +127,15 @@ public Thread newThread(Runnable r) { } }); } else { + this.eventLoopGroupBoss = new NioEventLoopGroup(1, new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, String.format("NettyNIOBoss_%d", this.threadIndex.incrementAndGet())); + } + }); + this.eventLoopGroupSelector = new NioEventLoopGroup(nettyServerConfig.getServerSelectorThreads(), new ThreadFactory() { private AtomicInteger threadIndex = new AtomicInteger(0); private int threadTotal = nettyServerConfig.getServerSelectorThreads(); @@ -266,7 +274,9 @@ public void shutdown() { @Override public void registerRPCHook(RPCHook rpcHook) { - this.rpcHook = rpcHook; + if (rpcHook != null && !rpcHooks.contains(rpcHook)) { + rpcHooks.add(rpcHook); + } } @Override @@ -318,10 +328,6 @@ public ChannelEventListener getChannelEventListener() { return channelEventListener; } - @Override - public RPCHook getRPCHook() { - return this.rpcHook; - } @Override public ExecutorService getCallbackExecutor() { diff --git a/srvutil/pom.xml b/srvutil/pom.xml index 73229ac9b1e..e175088cb44 100644 --- a/srvutil/pom.xml +++ b/srvutil/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.4.0-SNAPSHOT + 4.4.1-SNAPSHOT 4.0.0 diff --git a/store/pom.xml b/store/pom.xml index 0d7681d34f8..13e3b7d5900 100644 --- a/store/pom.xml +++ b/store/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.4.0-SNAPSHOT + 4.4.1-SNAPSHOT 4.0.0 diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 75d2c8eb3ec..cb17345c3a1 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -609,7 +609,7 @@ public void setDefaultQueryMaxNum(int defaultQueryMaxNum) { } /** - * Enable transient commitLog store poll only if transientStorePoolEnable is true and the FlushDiskType is + * Enable transient commitLog store pool only if transientStorePoolEnable is true and the FlushDiskType is * ASYNC_FLUSH * * @return true or false diff --git a/test/pom.xml b/test/pom.xml index 08d26b3e29a..ba603c815d4 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.4.0-SNAPSHOT + 4.4.1-SNAPSHOT 4.0.0 diff --git a/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java b/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java index d9955c04de3..45c6750dfdf 100644 --- a/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java +++ b/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java @@ -19,9 +19,12 @@ import java.util.ArrayList; import java.util.List; + import org.apache.log4j.Logger; import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.MQVersion; import org.apache.rocketmq.namesrv.NamesrvController; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.test.client.rmq.RMQAsyncSendProducer; import org.apache.rocketmq.test.client.rmq.RMQNormalConsumer; import org.apache.rocketmq.test.client.rmq.RMQNormalProducer; @@ -48,6 +51,7 @@ public class BaseConf { private static Logger log = Logger.getLogger(BaseConf.class); static { + System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, Integer.toString(MQVersion.CURRENT_VERSION)); namesrvController = IntegrationTestBase.createAndStartNamesrv(); nsAddr = "127.0.0.1:" + namesrvController.getNettyServerConfig().getListenPort(); brokerController1 = IntegrationTestBase.createAndStartBroker(nsAddr); diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/filter/SqlFilterIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/filter/SqlFilterIT.java index 79f15dcc99d..a0f6555ced0 100644 --- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/filter/SqlFilterIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/filter/SqlFilterIT.java @@ -17,12 +17,18 @@ package org.apache.rocketmq.test.client.consumer.filter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; import org.apache.log4j.Logger; +import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; import org.apache.rocketmq.client.consumer.MessageSelector; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.test.base.BaseConf; -import org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadCastIT; -import org.apache.rocketmq.test.client.consumer.broadcast.normal.NormalMsgTwoSameGroupConsumerIT; -import org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer; import org.apache.rocketmq.test.client.rmq.RMQNormalProducer; import org.apache.rocketmq.test.client.rmq.RMQSqlConsumer; import org.apache.rocketmq.test.factory.ConsumerFactory; @@ -39,12 +45,14 @@ public class SqlFilterIT extends BaseConf { private static Logger logger = Logger.getLogger(SqlFilterIT.class); private RMQNormalProducer producer = null; private String topic = null; + private static final Map OFFSE_TABLE = new HashMap(); @Before public void setUp() { topic = initTopic(); logger.info(String.format("use topic: %s;", topic)); producer = getProducer(nsAddr, topic); + OFFSE_TABLE.clear(); } @After @@ -71,4 +79,65 @@ public void testFilterConsumer() throws Exception { assertThat(consumer.getListener().getAllMsgBody().size()).isEqualTo(msgSize * 2); } + + @Test + public void testFilterPullConsumer() throws Exception { + int msgSize = 16; + + String group = initConsumerGroup(); + MessageSelector selector = MessageSelector.bySql("(TAGS is not null and TAGS in ('TagA', 'TagB'))"); + DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(group); + consumer.setNamesrvAddr(nsAddr); + consumer.start(); + Thread.sleep(3000); + producer.send("TagA", msgSize); + producer.send("TagB", msgSize); + producer.send("TagC", msgSize); + Assert.assertEquals("Not all sent succeeded", msgSize * 3, producer.getAllUndupMsgBody().size()); + + List receivedMessage = new ArrayList<>(2); + Set mqs = consumer.fetchSubscribeMessageQueues(topic); + for (MessageQueue mq : mqs) { + SINGLE_MQ: + while (true) { + try { + PullResult pullResult = + consumer.pull(mq, selector, getMessageQueueOffset(mq), 32); + putMessageQueueOffset(mq, pullResult.getNextBeginOffset()); + switch (pullResult.getPullStatus()) { + case FOUND: + List msgs = pullResult.getMsgFoundList(); + for (MessageExt msg : msgs) { + receivedMessage.add(new String(msg.getBody())); + } + break; + case NO_MATCHED_MSG: + break; + case NO_NEW_MSG: + break SINGLE_MQ; + case OFFSET_ILLEGAL: + break; + default: + break; + } + } catch (Exception e) { + e.printStackTrace(); + } + } + } + + assertThat(receivedMessage.size()).isEqualTo(msgSize * 2); + } + + private static long getMessageQueueOffset(MessageQueue mq) { + Long offset = OFFSE_TABLE.get(mq); + if (offset != null) + return offset; + + return 0; + } + + private static void putMessageQueueOffset(MessageQueue mq, long offset) { + OFFSE_TABLE.put(mq, offset); + } } diff --git a/tools/pom.xml b/tools/pom.xml index dc0e256ed46..9d36e7093a4 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.4.0-SNAPSHOT + 4.4.1-SNAPSHOT 4.0.0 @@ -36,6 +36,10 @@ ${project.groupId} rocketmq-client + + ${project.groupId} + rocketmq-acl + ${project.groupId} rocketmq-store @@ -60,5 +64,9 @@ org.apache.commons commons-lang3 + + org.yaml + snakeyaml + diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java index 6a51b7b4b92..2ca60aa860c 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java @@ -19,13 +19,16 @@ import ch.qos.logback.classic.LoggerContext; import ch.qos.logback.classic.joran.JoranConfigurator; import ch.qos.logback.core.joran.spi.JoranException; - import java.util.ArrayList; import java.util.List; - +import com.alibaba.fastjson.JSONObject; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.commons.cli.PosixParser; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.acl.common.AclClientRPCHook; +import org.apache.rocketmq.acl.common.AclUtils; +import org.apache.rocketmq.acl.common.SessionCredentials; import org.apache.rocketmq.common.MQVersion; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.remoting.RPCHook; @@ -129,7 +132,7 @@ public static void main0(String[] args, RPCHook rpcHook) { System.setProperty(MixAll.NAMESRV_ADDR_PROPERTY, namesrvAddr); } - cmd.execute(commandLine, options, rpcHook); + cmd.execute(commandLine, options, getAclRPCHook()); } else { System.out.printf("The sub command %s not exist.%n", args[0]); } @@ -157,7 +160,7 @@ public static void initCommand() { initCommand(new QueryMsgByKeySubCommand()); initCommand(new QueryMsgByUniqueKeySubCommand()); initCommand(new QueryMsgByOffsetSubCommand()); - + initCommand(new PrintMessageSubCommand()); initCommand(new PrintMessageByQueueCommand()); initCommand(new SendMsgStatusCommand()); @@ -211,7 +214,6 @@ private static void initLogback() throws JoranException { private static void printHelp() { System.out.printf("The most commonly used mqadmin commands are:%n"); - for (SubCommand cmd : subCommandList) { System.out.printf(" %-20s %s%n", cmd.commandName(), cmd.commandDesc()); } @@ -243,4 +245,25 @@ private static String[] parseSubArgs(String[] args) { public static void initCommand(SubCommand command) { subCommandList.add(command); } + + public static RPCHook getAclRPCHook() { + String fileHome = System.getProperty(MixAll.ROCKETMQ_HOME_PROPERTY, System.getenv(MixAll.ROCKETMQ_HOME_ENV)); + String fileName = "/conf/tools.yml"; + JSONObject yamlDataObject = AclUtils.getYamlDataObject(fileHome + fileName , + JSONObject.class); + + if (yamlDataObject == null || yamlDataObject.isEmpty()) { + System.out.printf(" Cannot find conf file %s, acl is not be enabled.%n" ,fileHome + fileName); + return null; + } + + String accessKey = yamlDataObject.getString("accessKey"); + String secretKey = yamlDataObject.getString("secretKey"); + + if (StringUtils.isBlank(accessKey) || StringUtils.isBlank(secretKey)) { + System.out.printf("AccessKey or secretKey is blank, the acl is not enabled.%n"); + return null; + } + return new AclClientRPCHook(new SessionCredentials(accessKey,secretKey)); + } } diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java index a1b3c1a227d..b946ee141eb 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java @@ -155,7 +155,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t } System.out.printf("%n"); - System.out.printf("Consume TPS: %s%n", consumeStats.getConsumeTps()); + System.out.printf("Consume TPS: %.2f%n", consumeStats.getConsumeTps()); System.out.printf("Diff Total: %d%n", diffTotal); } else { System.out.printf("%-32s %-6s %-24s %-5s %-14s %-7s %s%n",