From 2fd56e0320573b205b5630b3a3a3877dd5c2df97 Mon Sep 17 00:00:00 2001 From: daizhenyu <1449308021@qq.com> Date: Sat, 14 Sep 2024 21:47:25 +0800 Subject: [PATCH] router plugin: interceptor with xDS router Signed-off-by: daizhenyu <1449308021@qq.com> --- .../sermant-router/config/config.yaml | 4 + .../router/common/config/RouterConfig.java | 28 ++ .../BaseRegistryPluginAdaptationDeclarer.java | 2 +- .../declarer/OkHttp3ClientDeclarer.java | 3 +- .../OkHttpClientInterceptorChainDeclarer.java | 47 ++++ .../BaseLoadBalancerInterceptor.java | 47 +++- .../interceptor/HttpClient4xInterceptor.java | 92 +++++-- .../HttpUrlConnectionConnectInterceptor.java | 83 ++++-- .../interceptor/LoadBalancerInterceptor.java | 24 ++ .../interceptor/OkHttp3ClientInterceptor.java | 48 +++- ...HttpClientInterceptorChainInterceptor.java | 117 +++++++++ ...erviceInstanceListSupplierInterceptor.java | 26 +- .../spring/utils/BaseHttpRouterUtils.java | 144 +++++++++++ .../spring/utils/SpringRouterUtils.java | 42 ++- ....core.plugin.agent.declarer.PluginDeclarer | 1 + .../router/spring/TestServiceInstance.java | 98 +++++++ .../LoadBalancerInterceptorTest.java | 4 + ...ClientInterceptorChainInterceptorTest.java | 105 ++++++++ .../spring/utils/BaseHttpRouterUtilsTest.java | 243 ++++++++++++++++++ 19 files changed, 1088 insertions(+), 70 deletions(-) create mode 100644 sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/declarer/OkHttpClientInterceptorChainDeclarer.java create mode 100644 sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/OkHttpClientInterceptorChainInterceptor.java create mode 100644 sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/utils/BaseHttpRouterUtils.java create mode 100644 sermant-plugins/sermant-router/spring-router-plugin/src/test/java/io/sermant/router/spring/TestServiceInstance.java create mode 100644 sermant-plugins/sermant-router/spring-router-plugin/src/test/java/io/sermant/router/spring/interceptor/OkHttpClientInterceptorChainInterceptorTest.java create mode 100644 sermant-plugins/sermant-router/spring-router-plugin/src/test/java/io/sermant/router/spring/utils/BaseHttpRouterUtilsTest.java diff --git a/sermant-plugins/sermant-router/config/config.yaml b/sermant-plugins/sermant-router/config/config.yaml index b02f2a79e7..22d028fdf3 100644 --- a/sermant-plugins/sermant-router/config/config.yaml +++ b/sermant-plugins/sermant-router/config/config.yaml @@ -1,6 +1,10 @@ router.plugin: # whether compatible with the sermant-springboot-registry plugin enabled-registry-plugin-adaptation: false + # whether to use xds route + enabled-xds-route: false + # whether to use secure protocol to invoke spring cloud downstream service with xds route, example: http or https + enabled-springcloud-xds-route-secure: false # Whether to use request information for routing use-request-router: false # Use request information as tags when routing diff --git a/sermant-plugins/sermant-router/router-common/src/main/java/io/sermant/router/common/config/RouterConfig.java b/sermant-plugins/sermant-router/router-common/src/main/java/io/sermant/router/common/config/RouterConfig.java index b6c8fcfbcf..5f92c59ca6 100644 --- a/sermant-plugins/sermant-router/router-common/src/main/java/io/sermant/router/common/config/RouterConfig.java +++ b/sermant-plugins/sermant-router/router-common/src/main/java/io/sermant/router/common/config/RouterConfig.java @@ -66,6 +66,18 @@ public class RouterConfig implements PluginConfig { @ConfigFieldKey("enabled-registry-plugin-adaptation") private boolean enabledRegistryPluginAdaptation; + /** + * whether to use xds route + */ + @ConfigFieldKey("enabled-xds-route") + private boolean enabledXdsRoute; + + /** + * whether to use secure protocol to invoke downstream service with xds route, example: http or https + */ + @ConfigFieldKey("enabled-springcloud-xds-route-secure") + private boolean enabledSpringCloudXdsRouteSecure; + /** * whether to use the request information for routing */ @@ -197,4 +209,20 @@ public boolean isEnabledPreviousRule() { public void setEnabledPreviousRule(boolean enabledPreviousRule) { this.enabledPreviousRule = enabledPreviousRule; } + + public boolean isEnabledXdsRoute() { + return enabledXdsRoute; + } + + public void setEnabledXdsRoute(boolean enabledXdsRoute) { + this.enabledXdsRoute = enabledXdsRoute; + } + + public boolean isEnabledSpringCloudXdsRouteSecure() { + return enabledSpringCloudXdsRouteSecure; + } + + public void setEnabledSpringCloudXdsRouteSecure(boolean enabledSpringCloudXdsRouteSecure) { + this.enabledSpringCloudXdsRouteSecure = enabledSpringCloudXdsRouteSecure; + } } \ No newline at end of file diff --git a/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/declarer/BaseRegistryPluginAdaptationDeclarer.java b/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/declarer/BaseRegistryPluginAdaptationDeclarer.java index ce86a7d570..230205bca2 100644 --- a/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/declarer/BaseRegistryPluginAdaptationDeclarer.java +++ b/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/declarer/BaseRegistryPluginAdaptationDeclarer.java @@ -38,6 +38,6 @@ public BaseRegistryPluginAdaptationDeclarer() { @Override public boolean isEnabled() { - return routerConfig.isEnabledRegistryPluginAdaptation(); + return routerConfig.isEnabledXdsRoute() || routerConfig.isEnabledRegistryPluginAdaptation(); } } diff --git a/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/declarer/OkHttp3ClientDeclarer.java b/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/declarer/OkHttp3ClientDeclarer.java index ae14e5f25b..4382289585 100644 --- a/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/declarer/OkHttp3ClientDeclarer.java +++ b/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/declarer/OkHttp3ClientDeclarer.java @@ -32,7 +32,8 @@ public class OkHttp3ClientDeclarer extends BaseRegistryPluginAdaptationDeclarer * The fully qualified name of the enhanced okhttp request */ private static final String[] ENHANCE_CLASSES = { - "okhttp3.RealCall" + "okhttp3.RealCall", + "okhttp3.internal.connection.RealCall" }; /** diff --git a/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/declarer/OkHttpClientInterceptorChainDeclarer.java b/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/declarer/OkHttpClientInterceptorChainDeclarer.java new file mode 100644 index 0000000000..aa5128aea5 --- /dev/null +++ b/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/declarer/OkHttpClientInterceptorChainDeclarer.java @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2024-2024 Sermant Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.sermant.router.spring.declarer; + +import io.sermant.core.plugin.agent.declarer.InterceptDeclarer; +import io.sermant.core.plugin.agent.matcher.ClassMatcher; +import io.sermant.core.plugin.agent.matcher.MethodMatcher; +import io.sermant.router.spring.interceptor.OkHttpClientInterceptorChainInterceptor; + +/** + * For OKHTTP requests, modify the URL of request + * + * @author daizhenyu + * @since 2024-09-06 + */ +public class OkHttpClientInterceptorChainDeclarer extends BaseRegistryPluginAdaptationDeclarer { + private static final String[] ENHANCE_CLASSES = { + "com.squareup.okhttp.Call$ApplicationInterceptorChain" + }; + + @Override + public ClassMatcher getClassMatcher() { + return ClassMatcher.nameContains(ENHANCE_CLASSES); + } + + @Override + public InterceptDeclarer[] getInterceptDeclarers(ClassLoader classLoader) { + return new InterceptDeclarer[]{ + InterceptDeclarer.build(MethodMatcher.nameContains("proceed"), + new OkHttpClientInterceptorChainInterceptor()) + }; + } +} diff --git a/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/BaseLoadBalancerInterceptor.java b/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/BaseLoadBalancerInterceptor.java index 6b13a42ddb..a0266d852e 100644 --- a/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/BaseLoadBalancerInterceptor.java +++ b/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/BaseLoadBalancerInterceptor.java @@ -21,13 +21,19 @@ import io.sermant.core.plugin.agent.entity.ExecuteContext; import io.sermant.core.plugin.agent.interceptor.AbstractInterceptor; +import io.sermant.core.plugin.config.PluginConfigManager; import io.sermant.core.plugin.service.PluginServiceManager; +import io.sermant.core.service.xds.entity.ServiceInstance; +import io.sermant.router.common.config.RouterConfig; import io.sermant.router.common.request.RequestData; import io.sermant.router.common.request.RequestTag; import io.sermant.router.common.utils.CollectionUtils; import io.sermant.router.common.utils.ReflectUtils; import io.sermant.router.common.utils.ThreadLocalUtils; +import io.sermant.router.common.xds.XdsRouterHandler; import io.sermant.router.spring.service.LoadBalancerService; +import io.sermant.router.spring.utils.BaseHttpRouterUtils; +import io.sermant.router.spring.utils.SpringRouterUtils; import java.util.ArrayList; import java.util.Collections; @@ -36,6 +42,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import javax.servlet.http.HttpServletRequest; @@ -50,28 +57,47 @@ public class BaseLoadBalancerInterceptor extends AbstractInterceptor { private final boolean canLoadZuul; + private final RouterConfig routerConfig; + /** * Constructor */ public BaseLoadBalancerInterceptor() { loadBalancerService = PluginServiceManager.getPluginService(LoadBalancerService.class); canLoadZuul = canLoadZuul(); + routerConfig = PluginConfigManager.getPluginConfig(RouterConfig.class); } @Override public ExecuteContext before(ExecuteContext context) { Object object = context.getObject(); - if (object instanceof BaseLoadBalancer) { - List serverList = getServerList(context.getMethod().getName(), object); - if (CollectionUtils.isEmpty(serverList)) { - return context; - } - BaseLoadBalancer loadBalancer = (BaseLoadBalancer) object; - String name = loadBalancer.getName(); - RequestData requestData = getRequestData().orElse(null); - List targetInstances = loadBalancerService.getTargetInstances(name, serverList, requestData); - context.skip(Collections.unmodifiableList(targetInstances)); + if (!(object instanceof BaseLoadBalancer)) { + return context; + } + BaseLoadBalancer loadBalancer = (BaseLoadBalancer) object; + String name = loadBalancer.getName(); + RequestData requestData = getRequestData().orElse(null); + + // use xds route to find service instances + Set serviceInstanceByXdsRoute = null; + if (routerConfig.isEnabledXdsRoute()) { + serviceInstanceByXdsRoute = XdsRouterHandler.INSTANCE + .getServiceInstanceByXdsRoute(name, requestData.getPath(), + BaseHttpRouterUtils.processHeaders(requestData.getTag())); + } + if (serviceInstanceByXdsRoute != null && !serviceInstanceByXdsRoute.isEmpty()) { + context.skip(Collections.unmodifiableList(SpringRouterUtils + .getSpringCloudServiceInstanceByXds(serviceInstanceByXdsRoute))); + return context; } + + List serverList = getServerList(context.getMethod().getName(), object); + if (CollectionUtils.isEmpty(serverList)) { + return context; + } + + List targetInstances = loadBalancerService.getTargetInstances(name, serverList, requestData); + context.skip(Collections.unmodifiableList(targetInstances)); return context; } @@ -104,6 +130,7 @@ private Optional getRequestData() { header.putAll(requestTag.getTag()); } HttpServletRequest request = context.getRequest(); + String scheme = request.getScheme(); Enumeration headerNames = request.getHeaderNames(); while (headerNames.hasMoreElements()) { String name = (String) headerNames.nextElement(); diff --git a/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/HttpClient4xInterceptor.java b/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/HttpClient4xInterceptor.java index ed1dccdbb3..5973b1af84 100644 --- a/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/HttpClient4xInterceptor.java +++ b/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/HttpClient4xInterceptor.java @@ -16,22 +16,30 @@ package io.sermant.router.spring.interceptor; +import io.sermant.core.common.LoggerFactory; import io.sermant.core.plugin.agent.entity.ExecuteContext; +import io.sermant.core.plugin.config.PluginConfigManager; +import io.sermant.core.service.xds.entity.ServiceInstance; import io.sermant.core.utils.LogUtils; import io.sermant.core.utils.StringUtils; +import io.sermant.router.common.config.RouterConfig; import io.sermant.router.common.request.RequestData; import io.sermant.router.common.utils.FlowContextUtils; import io.sermant.router.common.utils.ThreadLocalUtils; -import io.sermant.router.spring.utils.RequestInterceptorUtils; +import io.sermant.router.spring.utils.BaseHttpRouterUtils; import org.apache.http.Header; -import org.apache.http.HttpRequest; +import org.apache.http.HttpHost; +import org.apache.http.client.methods.HttpRequestBase; import java.net.URI; +import java.net.URISyntaxException; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; /** * HTTP interception only for version 4. x @@ -40,6 +48,10 @@ * @since 2022-10-25 */ public class HttpClient4xInterceptor extends MarkInterceptor { + private static final Logger LOGGER = LoggerFactory.getLogger(); + + private RouterConfig routerConfig = PluginConfigManager.getPluginConfig(RouterConfig.class); + /** * Pre-trigger point * @@ -50,31 +62,33 @@ public class HttpClient4xInterceptor extends MarkInterceptor { @Override public ExecuteContext doBefore(ExecuteContext context) throws Exception { LogUtils.printHttpRequestBeforePoint(context); - Object httpRequestObject = context.getArguments()[1]; - if (httpRequestObject instanceof HttpRequest) { - final HttpRequest httpRequest = (HttpRequest) httpRequestObject; - final Optional optionalUri = RequestInterceptorUtils.formatUri(httpRequest.getRequestLine().getUri()); - if (!optionalUri.isPresent()) { - return context; - } - URI uri = optionalUri.get(); - if (StringUtils.isBlank(FlowContextUtils.getTagName())) { - return context; - } - Header[] headers = httpRequest.getHeaders(FlowContextUtils.getTagName()); - Map> flowTags = new HashMap<>(); - if (headers != null && headers.length > 0) { - for (Header header : headers) { - String headerValue = header.getValue(); - Map> stringListMap = FlowContextUtils.decodeTags(headerValue); - flowTags.putAll(stringListMap); - } - } - if (flowTags.size() > 0) { - ThreadLocalUtils.setRequestData(new RequestData( - flowTags, uri.getPath(), httpRequest.getRequestLine().getMethod())); + Object[] arguments = context.getArguments(); + + Object httpRequestObject = arguments[1]; + if (!(httpRequestObject instanceof HttpRequestBase)) { + return context; + } + final HttpRequestBase httpRequest = (HttpRequestBase) httpRequestObject; + URI uri = httpRequest.getURI(); + + handleXdsRouterAndUpdateHttpRequest(arguments, httpRequest); + + if (StringUtils.isBlank(FlowContextUtils.getTagName())) { + return context; + } + Header[] headers = httpRequest.getHeaders(FlowContextUtils.getTagName()); + Map> flowTags = new HashMap<>(); + if (headers != null && headers.length > 0) { + for (Header header : headers) { + String headerValue = header.getValue(); + Map> stringListMap = FlowContextUtils.decodeTags(headerValue); + flowTags.putAll(stringListMap); } } + if (flowTags.size() > 0) { + ThreadLocalUtils.setRequestData(new RequestData( + flowTags, uri.getPath(), httpRequest.getRequestLine().getMethod())); + } return context; } @@ -98,4 +112,32 @@ public ExecuteContext onThrow(ExecuteContext context) { LogUtils.printHttpRequestOnThrowPoint(context); return context; } + + private Map getHeaders(HttpRequestBase httpRequest) { + Map headerMap = new HashMap<>(); + for (Header header : httpRequest.getAllHeaders()) { + headerMap.putIfAbsent(header.getName(), header.getValue()); + } + return headerMap; + } + + private void handleXdsRouterAndUpdateHttpRequest(Object[] arguments, HttpRequestBase httpRequest) { + URI uri = httpRequest.getURI(); + String host = uri.getHost(); + + // use xds route to find a service instance, and modify url by it + if (routerConfig.isEnabledXdsRoute() && BaseHttpRouterUtils.isXdsRouteRequired(host)) { + Optional serviceInstanceOptional = BaseHttpRouterUtils + .chooseServiceInstanceByXds(host.split("\\.")[0], uri.getPath(), getHeaders(httpRequest)); + if (serviceInstanceOptional.isPresent()) { + ServiceInstance instance = serviceInstanceOptional.get(); + try { + httpRequest.setURI(new URI(BaseHttpRouterUtils.rebuildUrlByXdsServiceInstance(uri, instance))); + arguments[0] = new HttpHost(instance.getHost(), instance.getPort()); + } catch (URISyntaxException e) { + LOGGER.log(Level.WARNING, "Create uri using xds service instance failed.", e.getMessage()); + } + } + } + } } diff --git a/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/HttpUrlConnectionConnectInterceptor.java b/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/HttpUrlConnectionConnectInterceptor.java index e7d7919b3e..ada0d91c1a 100644 --- a/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/HttpUrlConnectionConnectInterceptor.java +++ b/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/HttpUrlConnectionConnectInterceptor.java @@ -16,22 +16,29 @@ package io.sermant.router.spring.interceptor; +import io.sermant.core.common.LoggerFactory; import io.sermant.core.plugin.agent.entity.ExecuteContext; import io.sermant.core.plugin.agent.interceptor.AbstractInterceptor; +import io.sermant.core.plugin.config.PluginConfigManager; +import io.sermant.core.service.xds.entity.ServiceInstance; import io.sermant.core.utils.LogUtils; import io.sermant.core.utils.ReflectUtils; import io.sermant.core.utils.StringUtils; +import io.sermant.router.common.config.RouterConfig; import io.sermant.router.common.request.RequestData; import io.sermant.router.common.utils.CollectionUtils; import io.sermant.router.common.utils.FlowContextUtils; import io.sermant.router.common.utils.ThreadLocalUtils; -import sun.net.www.MessageHeader; +import io.sermant.router.spring.utils.BaseHttpRouterUtils; import java.net.HttpURLConnection; +import java.net.MalformedURLException; import java.net.URL; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; /** * An enhanced interceptor for java.net.HttpURLConnection in JDK version 1.8
@@ -40,33 +47,37 @@ * @since 2022-10-25 */ public class HttpUrlConnectionConnectInterceptor extends AbstractInterceptor { + private static final Logger LOGGER = LoggerFactory.getLogger(); + + private RouterConfig routerConfig = PluginConfigManager.getPluginConfig(RouterConfig.class); + @Override public ExecuteContext before(ExecuteContext context) { LogUtils.printHttpRequestBeforePoint(context); - if (context.getObject() instanceof HttpURLConnection) { - HttpURLConnection connection = (HttpURLConnection) context.getObject(); - Optional requests = ReflectUtils.getFieldValue(connection, "requests"); - if (!requests.isPresent()) { - return context; - } - Map> headers = ((MessageHeader) requests.get()).getHeaders(null); - String method = connection.getRequestMethod(); - if (StringUtils.isBlank(FlowContextUtils.getTagName()) || CollectionUtils - .isEmpty(headers.get(FlowContextUtils.getTagName()))) { - ThreadLocalUtils.setRequestData(new RequestData(headers, getPath(connection), method)); - return context; - } - String encodeTag = headers.get(FlowContextUtils.getTagName()).get(0); - if (StringUtils.isBlank(encodeTag)) { - ThreadLocalUtils.setRequestData(new RequestData(headers, getPath(connection), method)); - return context; - } - Map> tags = FlowContextUtils.decodeTags(encodeTag); - if (!tags.isEmpty()) { - ThreadLocalUtils.setRequestData(new RequestData(tags, getPath(connection), method)); - } else { - ThreadLocalUtils.setRequestData(new RequestData(headers, getPath(connection), method)); - } + if (!(context.getObject() instanceof HttpURLConnection)) { + return context; + } + HttpURLConnection connection = (HttpURLConnection) context.getObject(); + Map> headers = connection.getRequestProperties(); + + handleXdsRouterAndUpdateHttpRequest(connection); + + String method = connection.getRequestMethod(); + if (StringUtils.isBlank(FlowContextUtils.getTagName()) || CollectionUtils + .isEmpty(headers.get(FlowContextUtils.getTagName()))) { + ThreadLocalUtils.setRequestData(new RequestData(headers, getPath(connection), method)); + return context; + } + String encodeTag = headers.get(FlowContextUtils.getTagName()).get(0); + if (StringUtils.isBlank(encodeTag)) { + ThreadLocalUtils.setRequestData(new RequestData(headers, getPath(connection), method)); + return context; + } + Map> tags = FlowContextUtils.decodeTags(encodeTag); + if (!tags.isEmpty()) { + ThreadLocalUtils.setRequestData(new RequestData(tags, getPath(connection), method)); + } else { + ThreadLocalUtils.setRequestData(new RequestData(headers, getPath(connection), method)); } return context; } @@ -88,4 +99,26 @@ public ExecuteContext onThrow(ExecuteContext context) throws Exception { LogUtils.printHttpRequestOnThrowPoint(context); return super.onThrow(context); } + + private void handleXdsRouterAndUpdateHttpRequest(HttpURLConnection connection) { + Map> headers = connection.getRequestProperties(); + URL url = connection.getURL(); + String host = url.getHost(); + + // use xds route to find a service instance, and modify url by it + if (routerConfig.isEnabledXdsRoute() && BaseHttpRouterUtils.isXdsRouteRequired(host)) { + Optional serviceInstanceOptional = BaseHttpRouterUtils + .chooseServiceInstanceByXds(host.split("\\.")[0], url.getPath(), + BaseHttpRouterUtils.processHeaders(headers)); + if (serviceInstanceOptional.isPresent()) { + ServiceInstance instance = serviceInstanceOptional.get(); + try { + ReflectUtils.setFieldValue(connection, "url", + new URL(BaseHttpRouterUtils.rebuildUrlByXdsServiceInstance(url, instance))); + } catch (MalformedURLException e) { + LOGGER.log(Level.WARNING, "Create url using xds service instance failed.", e.getMessage()); + } + } + } + } } diff --git a/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/LoadBalancerInterceptor.java b/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/LoadBalancerInterceptor.java index 76b3949fa5..1a8d5ad999 100644 --- a/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/LoadBalancerInterceptor.java +++ b/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/LoadBalancerInterceptor.java @@ -18,16 +18,23 @@ import io.sermant.core.plugin.agent.entity.ExecuteContext; import io.sermant.core.plugin.agent.interceptor.AbstractInterceptor; +import io.sermant.core.plugin.config.PluginConfigManager; import io.sermant.core.service.ServiceManager; +import io.sermant.core.service.xds.entity.ServiceInstance; import io.sermant.core.utils.StringUtils; +import io.sermant.router.common.config.RouterConfig; import io.sermant.router.common.request.RequestData; import io.sermant.router.common.utils.CollectionUtils; import io.sermant.router.common.utils.ReflectUtils; import io.sermant.router.common.utils.ThreadLocalUtils; +import io.sermant.router.common.xds.XdsRouterHandler; import io.sermant.router.spring.service.LoadBalancerService; +import io.sermant.router.spring.utils.BaseHttpRouterUtils; +import io.sermant.router.spring.utils.SpringRouterUtils; import java.util.List; import java.util.Optional; +import java.util.Set; /** * spring cloud loadbalancer Interception points @@ -38,11 +45,14 @@ public class LoadBalancerInterceptor extends AbstractInterceptor { private final LoadBalancerService loadBalancerService; + private final RouterConfig routerConfig; + /** * Constructor */ public LoadBalancerInterceptor() { loadBalancerService = ServiceManager.getService(LoadBalancerService.class); + routerConfig = PluginConfigManager.getPluginConfig(RouterConfig.class); } @Override @@ -58,6 +68,20 @@ public ExecuteContext before(ExecuteContext context) { return context; } RequestData requestData = ThreadLocalUtils.getRequestData(); + + // use xds route to find service instances + Set serviceInstanceByXdsRoute = null; + if (routerConfig.isEnabledXdsRoute()) { + serviceInstanceByXdsRoute = XdsRouterHandler.INSTANCE + .getServiceInstanceByXdsRoute(serviceId, requestData.getPath(), + BaseHttpRouterUtils.processHeaders(requestData.getTag())); + } + if (serviceInstanceByXdsRoute != null && !serviceInstanceByXdsRoute.isEmpty()) { + arguments[0] = SpringRouterUtils + .getSpringCloudServiceInstanceByXds(serviceInstanceByXdsRoute); + return context; + } + List targetInstances = loadBalancerService.getTargetInstances(serviceId, instances, requestData); arguments[0] = targetInstances; return context; diff --git a/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/OkHttp3ClientInterceptor.java b/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/OkHttp3ClientInterceptor.java index dd53aa273f..5de39be9f1 100644 --- a/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/OkHttp3ClientInterceptor.java +++ b/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/OkHttp3ClientInterceptor.java @@ -17,16 +17,22 @@ package io.sermant.router.spring.interceptor; import io.sermant.core.plugin.agent.entity.ExecuteContext; +import io.sermant.core.plugin.config.PluginConfigManager; +import io.sermant.core.service.xds.entity.ServiceInstance; import io.sermant.core.utils.LogUtils; import io.sermant.core.utils.ReflectUtils; import io.sermant.core.utils.StringUtils; +import io.sermant.router.common.config.RouterConfig; import io.sermant.router.common.request.RequestData; import io.sermant.router.common.utils.FlowContextUtils; import io.sermant.router.common.utils.ThreadLocalUtils; +import io.sermant.router.spring.utils.BaseHttpRouterUtils; import okhttp3.Headers; +import okhttp3.HttpUrl; import okhttp3.Request; import java.net.URI; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -40,6 +46,8 @@ public class OkHttp3ClientInterceptor extends MarkInterceptor { private static final String FIELD_NAME = "originalRequest"; + private RouterConfig routerConfig = PluginConfigManager.getPluginConfig(RouterConfig.class); + /** * Pre-trigger point * @@ -51,12 +59,18 @@ public class OkHttp3ClientInterceptor extends MarkInterceptor { public ExecuteContext doBefore(ExecuteContext context) throws Exception { LogUtils.printHttpRequestBeforePoint(context); final Optional rawRequest = getRequest(context); - if (!rawRequest.isPresent() || StringUtils.isBlank(FlowContextUtils.getTagName())) { + if (!rawRequest.isPresent()) { return context; } Request request = rawRequest.get(); URI uri = request.url().uri(); Headers headers = request.headers(); + + handleXdsRouterAndUpdateHttpRequest(context.getObject(), request); + + if (StringUtils.isBlank(FlowContextUtils.getTagName())) { + return context; + } String str = headers.get(FlowContextUtils.getTagName()); Map> decodeTags = FlowContextUtils.decodeTags(str); if (decodeTags.size() > 0) { @@ -93,4 +107,36 @@ private Optional getRequest(ExecuteContext context) { } return Optional.empty(); } + + private Map getHeaders(Headers headers) { + Map headerMap = new HashMap<>(); + for (String name : headers.names()) { + headerMap.putIfAbsent(name, headers.get(name)); + } + return headerMap; + } + + private Request rebuildRequest(Request request, ServiceInstance serviceInstance) { + return request.newBuilder() + .url(HttpUrl + .parse(BaseHttpRouterUtils + .rebuildUrlByXdsServiceInstance(request.url().uri(), serviceInstance))) + .build(); + } + + private void handleXdsRouterAndUpdateHttpRequest(Object obj, Request request) { + Headers headers = request.headers(); + URI uri = request.url().uri(); + String host = uri.getHost(); + + // use xds route to find a service instance, and modify url by it + if (routerConfig.isEnabledXdsRoute() && BaseHttpRouterUtils.isXdsRouteRequired(host)) { + Optional serviceInstanceOptional = BaseHttpRouterUtils + .chooseServiceInstanceByXds(host.split("\\.")[0], uri.getPath(), getHeaders(headers)); + if (serviceInstanceOptional.isPresent()) { + ServiceInstance instance = serviceInstanceOptional.get(); + ReflectUtils.setFieldValue(obj, "originalRequest", rebuildRequest(request, instance)); + } + } + } } diff --git a/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/OkHttpClientInterceptorChainInterceptor.java b/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/OkHttpClientInterceptorChainInterceptor.java new file mode 100644 index 0000000000..e3907e9728 --- /dev/null +++ b/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/OkHttpClientInterceptorChainInterceptor.java @@ -0,0 +1,117 @@ +/* + * Copyright (C) 2024-2024 Sermant Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.sermant.router.spring.interceptor; + +import com.squareup.okhttp.Headers; +import com.squareup.okhttp.HttpUrl; +import com.squareup.okhttp.Request; + +import io.sermant.core.common.LoggerFactory; +import io.sermant.core.plugin.agent.entity.ExecuteContext; +import io.sermant.core.plugin.agent.interceptor.Interceptor; +import io.sermant.core.plugin.config.PluginConfigManager; +import io.sermant.core.service.xds.entity.ServiceInstance; +import io.sermant.router.common.config.RouterConfig; +import io.sermant.router.spring.utils.BaseHttpRouterUtils; + +import java.io.IOException; +import java.net.URI; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Intercept for versions below okHttp3.1 + * + * @author daizhenyu + * @since 2024-09-06 + */ +public class OkHttpClientInterceptorChainInterceptor implements Interceptor { + private static final Logger LOGGER = LoggerFactory.getLogger(); + + private RouterConfig routerConfig = PluginConfigManager.getPluginConfig(RouterConfig.class); + + /** + * Pre-trigger point + * + * @param context Execution context + * @return Execution context + * @throws Exception Execution Exception + */ + @Override + public ExecuteContext before(ExecuteContext context) throws Exception { + Object[] arguments = context.getArguments(); + Request request = (Request) arguments[0]; + handleXdsRouterAndUpdateHttpRequest(arguments, request); + return context; + } + + /** + * Rear trigger point + * + * @param context Execution context + * @return Execution context + * @throws Exception Execution Exception + */ + @Override + public ExecuteContext after(ExecuteContext context) throws Exception { + return context; + } + + @Override + public ExecuteContext onThrow(ExecuteContext context) { + return context; + } + + private Map getHeaders(Request request) { + Map headerMap = new HashMap<>(); + Headers headers = request.headers(); + for (String name : request.headers().names()) { + headerMap.putIfAbsent(name, headers.get(name)); + } + return headerMap; + } + + private Request rebuildRequest(Request request, URI uri, ServiceInstance serviceInstance) { + return request.newBuilder() + .url(HttpUrl.parse(BaseHttpRouterUtils.rebuildUrlByXdsServiceInstance(uri, serviceInstance))) + .build(); + } + + private void handleXdsRouterAndUpdateHttpRequest(Object[] arguments, Request request) { + URI uri = null; + try { + uri = request.uri(); + } catch (IOException e) { + LOGGER.log(Level.WARNING, "Get uri from okhttp request failed.", e.getMessage()); + return; + } + String host = uri.getHost(); + + // use xds route to find a service instance, and modify url by it + if (routerConfig.isEnabledXdsRoute() && BaseHttpRouterUtils.isXdsRouteRequired(host)) { + Optional serviceInstanceOptional = BaseHttpRouterUtils + .chooseServiceInstanceByXds(host.split("\\.")[0], uri.getPath(), getHeaders(request)); + if (serviceInstanceOptional.isPresent()) { + ServiceInstance instance = serviceInstanceOptional.get(); + arguments[0] = rebuildRequest(request, uri, instance); + } + } + } +} diff --git a/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/ServiceInstanceListSupplierInterceptor.java b/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/ServiceInstanceListSupplierInterceptor.java index 03995d49a4..8d3fa52347 100644 --- a/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/ServiceInstanceListSupplierInterceptor.java +++ b/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/interceptor/ServiceInstanceListSupplierInterceptor.java @@ -18,12 +18,18 @@ import io.sermant.core.plugin.agent.entity.ExecuteContext; import io.sermant.core.plugin.agent.interceptor.AbstractInterceptor; +import io.sermant.core.plugin.config.PluginConfigManager; import io.sermant.core.plugin.service.PluginServiceManager; +import io.sermant.core.service.xds.entity.ServiceInstance; import io.sermant.core.utils.StringUtils; +import io.sermant.router.common.config.RouterConfig; import io.sermant.router.common.request.RequestData; import io.sermant.router.common.utils.CollectionUtils; import io.sermant.router.common.utils.ThreadLocalUtils; +import io.sermant.router.common.xds.XdsRouterHandler; import io.sermant.router.spring.service.LoadBalancerService; +import io.sermant.router.spring.utils.BaseHttpRouterUtils; +import io.sermant.router.spring.utils.SpringRouterUtils; import reactor.core.publisher.Flux; import org.springframework.cloud.loadbalancer.core.DiscoveryClientServiceInstanceListSupplier; @@ -32,6 +38,7 @@ import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.Set; /** * CachingServiceInstance List Supplier/DiscoveryClientServiceInstance List Supplier enhanced class, filtering @@ -43,11 +50,14 @@ public class ServiceInstanceListSupplierInterceptor extends AbstractInterceptor { private final LoadBalancerService loadBalancerService; + private final RouterConfig routerConfig; + /** * Constructor */ public ServiceInstanceListSupplierInterceptor() { loadBalancerService = PluginServiceManager.getPluginService(LoadBalancerService.class); + routerConfig = PluginConfigManager.getPluginConfig(RouterConfig.class); } @Override @@ -57,13 +67,27 @@ public ExecuteContext before(ExecuteContext context) { if (StringUtils.isBlank(serviceId)) { return context; } + RequestData requestData = ThreadLocalUtils.getRequestData(); + + // use xds route to find service instances + Set serviceInstanceByXdsRoute = null; + if (routerConfig.isEnabledXdsRoute()) { + serviceInstanceByXdsRoute = XdsRouterHandler.INSTANCE + .getServiceInstanceByXdsRoute(serviceId, requestData.getPath(), + BaseHttpRouterUtils.processHeaders(requestData.getTag())); + } + if (serviceInstanceByXdsRoute != null && !serviceInstanceByXdsRoute.isEmpty()) { + context.skip(Flux.just(SpringRouterUtils + .getSpringCloudServiceInstanceByXds(serviceInstanceByXdsRoute))); + return context; + } + Object obj = context.getMemberFieldValue("serviceInstances"); if (obj instanceof Flux) { List instances = getInstances((Flux) obj, object); if (CollectionUtils.isEmpty(instances)) { return context; } - RequestData requestData = ThreadLocalUtils.getRequestData(); List targetInstances = loadBalancerService.getTargetInstances(serviceId, instances, requestData); context.skip(Flux.just(targetInstances)); } diff --git a/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/utils/BaseHttpRouterUtils.java b/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/utils/BaseHttpRouterUtils.java new file mode 100644 index 0000000000..bef8188451 --- /dev/null +++ b/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/utils/BaseHttpRouterUtils.java @@ -0,0 +1,144 @@ +/* + * Copyright (C) 2024-2024 Sermant Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.sermant.router.spring.utils; + +import io.sermant.core.common.LoggerFactory; +import io.sermant.core.service.xds.entity.ServiceInstance; +import io.sermant.core.utils.CollectionUtils; +import io.sermant.core.utils.StringUtils; +import io.sermant.router.common.xds.XdsRouterHandler; +import io.sermant.router.common.xds.lb.XdsLoadBalancer; +import io.sermant.router.common.xds.lb.XdsLoadBalancerFactory; + +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * BaseHttpUtils + * + * @author daizhenyu + * @since 2024-09-09 + **/ +public class BaseHttpRouterUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(); + + private static final Pattern IP_PATTERN = Pattern.compile("^([0-9]{1,3}\\.){3}[0-9]{1,3}$"); + + private static final String LOCAL_HOST = "localhost"; + + private BaseHttpRouterUtils() { + } + + /** + * rebuild new url by XdsServiceInstance + * + * @param oldUrl old url + * @param serviceInstance xds service instance + * @return new url + */ + public static String rebuildUrlByXdsServiceInstance(URL oldUrl, ServiceInstance serviceInstance) { + try { + return rebuildUrlByXdsServiceInstance(oldUrl.toURI(), serviceInstance); + } catch (URISyntaxException e) { + LOGGER.log(Level.WARNING, "Convert url to uri failed.", e.getMessage()); + return StringUtils.EMPTY; + } + } + + /** + * rebuild new url by XdsServiceInstance + * + * @param oldUri old uri + * @param serviceInstance xds service instance + * @return new url + */ + public static String rebuildUrlByXdsServiceInstance(URI oldUri, ServiceInstance serviceInstance) { + StringBuilder builder = new StringBuilder(); + builder.append(oldUri.getScheme()) + .append("://") + .append(serviceInstance.getHost()) + .append(":") + .append(serviceInstance.getPort()) + .append(oldUri.getPath()); + String query = oldUri.getQuery(); + if (StringUtils.isEmpty(query)) { + return builder.toString(); + } + builder.append("?").append(query); + return builder.toString(); + } + + /** + * choose service instance by xds for http call + * + * @param serviceName service name + * @param path path + * @param headers headers + * @return ServiceInstance + */ + public static Optional chooseServiceInstanceByXds(String serviceName, String path, + Map headers) { + Set serviceInstanceByXdsRoute = XdsRouterHandler.INSTANCE + .getServiceInstanceByXdsRoute(serviceName, path, headers); + if (serviceInstanceByXdsRoute.isEmpty()) { + return Optional.empty(); + } + List serviceInstanceList = new ArrayList<>(serviceInstanceByXdsRoute); + XdsLoadBalancer loadBalancer = XdsLoadBalancerFactory + .getLoadBalancer(serviceInstanceList.get(0).getClusterName()); + return Optional.of(loadBalancer.selectInstance(serviceInstanceList)); + } + + /** + * isXdsRouteRequired + * + * @param host host + * @return isXdsRouteRequired + */ + public static boolean isXdsRouteRequired(String host) { + // if host is ip, so no xds routing required + if (StringUtils.isEmpty(host) || host.equals(LOCAL_HOST) || IP_PATTERN.matcher(host).matches()) { + return false; + } + return true; + } + + /** + * process headers, just get first value for every header + * + * @param headers headers + * @return processed headers + */ + public static Map processHeaders(Map> headers) { + return headers.entrySet().stream() + .collect(Collectors.toMap( + Entry::getKey, + entry -> CollectionUtils.isEmpty(entry.getValue()) ? "" : entry.getValue().get(0) + )); + } +} diff --git a/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/utils/SpringRouterUtils.java b/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/utils/SpringRouterUtils.java index e641dcb17a..a279c89ce0 100644 --- a/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/utils/SpringRouterUtils.java +++ b/sermant-plugins/sermant-router/spring-router-plugin/src/main/java/io/sermant/router/spring/utils/SpringRouterUtils.java @@ -16,14 +16,21 @@ package io.sermant.router.spring.utils; +import io.sermant.core.plugin.config.PluginConfigManager; +import io.sermant.core.service.xds.entity.ServiceInstance; import io.sermant.core.utils.StringUtils; import io.sermant.router.common.config.RouterConfig; import io.sermant.router.common.utils.CollectionUtils; import io.sermant.router.common.utils.ReflectUtils; import io.sermant.router.spring.cache.AppCache; +import org.springframework.cloud.client.DefaultServiceInstance; + +import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; /** * Reflection tool class @@ -36,9 +43,24 @@ public class SpringRouterUtils { private static final String ZONE_KEY = "zone"; + private static RouterConfig routerConfig = PluginConfigManager.getPluginConfig(RouterConfig.class); + private SpringRouterUtils() { } + /** + * get SpringCloud ServiceInstance By XdsServiceInstance + * + * @param xdsServiceInstances + * @return spring cloud service instance + */ + public static List getSpringCloudServiceInstanceByXds( + Set xdsServiceInstances) { + return xdsServiceInstances.stream() + .map(SpringRouterUtils::convertServiceInstance) + .collect(Collectors.toList()); + } + /** * Get metadata * @@ -53,21 +75,29 @@ public static Map getMetadata(Object obj) { * Deposit metadata * * @param metadata Metadata - * @param routerConfig Route configuration + * @param config Route configuration */ - public static void putMetaData(Map metadata, RouterConfig routerConfig) { + public static void putMetaData(Map metadata, RouterConfig config) { if (metadata == null) { return; } - metadata.putIfAbsent(VERSION_KEY, routerConfig.getRouterVersion()); - if (StringUtils.isExist(routerConfig.getZone())) { - metadata.putIfAbsent(ZONE_KEY, routerConfig.getZone()); + metadata.putIfAbsent(VERSION_KEY, config.getRouterVersion()); + if (StringUtils.isExist(config.getZone())) { + metadata.putIfAbsent(ZONE_KEY, config.getZone()); } - Map parameters = routerConfig.getParameters(); + Map parameters = config.getParameters(); if (!CollectionUtils.isEmpty(parameters)) { // The request header is changed to lowercase in the HTTP request parameters.forEach((key, value) -> metadata.putIfAbsent(key.toLowerCase(Locale.ROOT), value)); } AppCache.INSTANCE.setMetadata(metadata); } + + private static org.springframework.cloud.client.ServiceInstance convertServiceInstance( + io.sermant.core.service.xds.entity.ServiceInstance xdsServiceInstance) { + String instanceId = xdsServiceInstance.getHost() + xdsServiceInstance.getPort(); + return new DefaultServiceInstance( + instanceId, xdsServiceInstance.getServiceName(), xdsServiceInstance.getHost(), + xdsServiceInstance.getPort(), routerConfig.isEnabledSpringCloudXdsRouteSecure()); + } } diff --git a/sermant-plugins/sermant-router/spring-router-plugin/src/main/resources/META-INF/services/io.sermant.core.plugin.agent.declarer.PluginDeclarer b/sermant-plugins/sermant-router/spring-router-plugin/src/main/resources/META-INF/services/io.sermant.core.plugin.agent.declarer.PluginDeclarer index 975f20710e..9576a11665 100644 --- a/sermant-plugins/sermant-router/spring-router-plugin/src/main/resources/META-INF/services/io.sermant.core.plugin.agent.declarer.PluginDeclarer +++ b/sermant-plugins/sermant-router/spring-router-plugin/src/main/resources/META-INF/services/io.sermant.core.plugin.agent.declarer.PluginDeclarer @@ -12,6 +12,7 @@ io.sermant.router.spring.declarer.ServiceInstanceListSupplierDeclarer io.sermant.router.spring.declarer.ServiceRegistryDeclarer io.sermant.router.spring.declarer.OkHttp3ClientDeclarer io.sermant.router.spring.declarer.OkHttpClientDeclarer +io.sermant.router.spring.declarer.OkHttpClientInterceptorChainDeclarer io.sermant.router.spring.declarer.HttpClient4xDeclarer io.sermant.router.spring.declarer.HttpUrlConnectionConnectDeclarer io.sermant.router.spring.declarer.RestTemplateDeclarer diff --git a/sermant-plugins/sermant-router/spring-router-plugin/src/test/java/io/sermant/router/spring/TestServiceInstance.java b/sermant-plugins/sermant-router/spring-router-plugin/src/test/java/io/sermant/router/spring/TestServiceInstance.java new file mode 100644 index 0000000000..b882f1a008 --- /dev/null +++ b/sermant-plugins/sermant-router/spring-router-plugin/src/test/java/io/sermant/router/spring/TestServiceInstance.java @@ -0,0 +1,98 @@ +/* + * Copyright (C) 2024-2024 Sermant Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.sermant.router.spring; + +import io.sermant.core.service.xds.entity.ServiceInstance; + +import java.util.Map; + +/** + * TestServiceInstance + * + * @author daizhenyu + * @since 2024-09-10 + **/ +public class TestServiceInstance implements ServiceInstance { + public TestServiceInstance() { + } + + private String cluster; + + private String service; + + private String host; + + private int port; + + private Map metaData; + + private boolean healthy; + + @Override + public String getClusterName() { + return cluster; + } + + @Override + public String getServiceName() { + return service; + } + + @Override + public String getHost() { + return host; + } + + @Override + public int getPort() { + return port; + } + + @Override + public Map getMetaData() { + return metaData; + } + + @Override + public boolean isHealthy() { + return healthy; + } + + public void setCluster(String cluster) { + this.cluster = cluster; + } + + public void setService(String service) { + this.service = service; + } + + public void setHost(String host) { + this.host = host; + } + + public void setPort(int port) { + this.port = port; + } + + public void setMetaData(Map metaData) { + this.metaData = metaData; + } + + public void setHealthy(boolean healthy) { + this.healthy = healthy; + } +} diff --git a/sermant-plugins/sermant-router/spring-router-plugin/src/test/java/io/sermant/router/spring/interceptor/LoadBalancerInterceptorTest.java b/sermant-plugins/sermant-router/spring-router-plugin/src/test/java/io/sermant/router/spring/interceptor/LoadBalancerInterceptorTest.java index 7ff7465c9f..e8ba438583 100644 --- a/sermant-plugins/sermant-router/spring-router-plugin/src/test/java/io/sermant/router/spring/interceptor/LoadBalancerInterceptorTest.java +++ b/sermant-plugins/sermant-router/spring-router-plugin/src/test/java/io/sermant/router/spring/interceptor/LoadBalancerInterceptorTest.java @@ -17,7 +17,9 @@ package io.sermant.router.spring.interceptor; import io.sermant.core.plugin.agent.entity.ExecuteContext; +import io.sermant.core.plugin.config.PluginConfigManager; import io.sermant.core.service.ServiceManager; +import io.sermant.router.common.config.RouterConfig; import io.sermant.router.common.request.RequestData; import io.sermant.router.common.utils.ThreadLocalUtils; import io.sermant.router.spring.BaseTransmitConfigTest; @@ -66,6 +68,8 @@ public static void before() { mockServiceManager = Mockito.mockStatic(ServiceManager.class); mockServiceManager.when(() -> ServiceManager.getService(LoadBalancerService.class)) .thenReturn(new TestLoadBalancerService()); + mockPluginConfigManager.when(() -> PluginConfigManager.getPluginConfig(RouterConfig.class)) + .thenReturn(new RouterConfig()); } /** diff --git a/sermant-plugins/sermant-router/spring-router-plugin/src/test/java/io/sermant/router/spring/interceptor/OkHttpClientInterceptorChainInterceptorTest.java b/sermant-plugins/sermant-router/spring-router-plugin/src/test/java/io/sermant/router/spring/interceptor/OkHttpClientInterceptorChainInterceptorTest.java new file mode 100644 index 0000000000..9448922f68 --- /dev/null +++ b/sermant-plugins/sermant-router/spring-router-plugin/src/test/java/io/sermant/router/spring/interceptor/OkHttpClientInterceptorChainInterceptorTest.java @@ -0,0 +1,105 @@ +/* + * Copyright (C) 2024-2024 Sermant Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.sermant.router.spring.interceptor; + +import com.squareup.okhttp.HttpUrl; +import com.squareup.okhttp.Request; + +import io.sermant.core.plugin.agent.entity.ExecuteContext; +import io.sermant.core.plugin.config.PluginConfigManager; +import io.sermant.router.common.config.RouterConfig; +import io.sermant.router.spring.TestServiceInstance; +import io.sermant.router.spring.utils.BaseHttpRouterUtils; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.net.URI; +import java.util.Optional; + +/** + * @author daizhenyu + * @since 2024-09-10 + **/ +public class OkHttpClientInterceptorChainInterceptorTest { + private static MockedStatic mockPluginConfigManager; + + private static MockedStatic mockedUtils; + + private static OkHttpClientInterceptorChainInterceptor interceptor; + + @BeforeClass + public static void setUp() { + RouterConfig routerConfig = new RouterConfig(); + routerConfig.setEnabledXdsRoute(true); + mockPluginConfigManager = Mockito.mockStatic(PluginConfigManager.class); + mockPluginConfigManager.when(() -> PluginConfigManager.getPluginConfig(RouterConfig.class)) + .thenReturn(routerConfig); + mockedUtils = Mockito.mockStatic(BaseHttpRouterUtils.class); + + mockedUtils.when(() -> BaseHttpRouterUtils.isXdsRouteRequired(Mockito.any())).thenCallRealMethod(); + mockedUtils + .when(() -> BaseHttpRouterUtils.rebuildUrlByXdsServiceInstance(Mockito.any(URI.class), Mockito.any())) + .thenCallRealMethod(); + interceptor = new OkHttpClientInterceptorChainInterceptor(); + } + + @AfterClass + public static void tearDown() { + mockPluginConfigManager.close(); + mockedUtils.close(); + } + + @Test + public void testBefore() throws Exception { + Object obj = new Object(); + Object[] arguments = new Object[1]; + arguments[0] = new Request.Builder() + .url("http://example.default.svc.cluster.local/test") + .header("Header1", "Value1") + .build(); + ExecuteContext context = ExecuteContext.forMemberMethod(obj, null, arguments, null, null); + + // service instance is null + ExecuteContext result = interceptor.before(context); + Request newRequest = (Request) result.getArguments()[0]; + Assert.assertNotNull(newRequest); + HttpUrl newUrl = newRequest.httpUrl(); + Assert.assertEquals("http://example.default.svc.cluster.local/test", newUrl.toString()); + mockedUtils + .when(() -> BaseHttpRouterUtils.chooseServiceInstanceByXds(Mockito.any(), Mockito.any(), Mockito.any())) + .thenReturn(Optional.ofNullable(null)); + + // service instance is not empty + TestServiceInstance serviceInstance = new TestServiceInstance(); + serviceInstance.setService("serviceA"); + serviceInstance.setHost("127.0.0.1"); + serviceInstance.setPort(8080); + mockedUtils + .when(() -> BaseHttpRouterUtils.chooseServiceInstanceByXds(Mockito.any(), Mockito.any(), Mockito.any())) + .thenReturn(Optional.of(serviceInstance)); + result = interceptor.before(context); + newRequest = (Request) result.getArguments()[0]; + Assert.assertNotNull(newRequest); + newUrl = newRequest.httpUrl(); + Assert.assertEquals("http://127.0.0.1:8080/test", newUrl.toString()); + } +} \ No newline at end of file diff --git a/sermant-plugins/sermant-router/spring-router-plugin/src/test/java/io/sermant/router/spring/utils/BaseHttpRouterUtilsTest.java b/sermant-plugins/sermant-router/spring-router-plugin/src/test/java/io/sermant/router/spring/utils/BaseHttpRouterUtilsTest.java new file mode 100644 index 0000000000..bf5c2e392e --- /dev/null +++ b/sermant-plugins/sermant-router/spring-router-plugin/src/test/java/io/sermant/router/spring/utils/BaseHttpRouterUtilsTest.java @@ -0,0 +1,243 @@ +/* + * Copyright (C) 2024-2024 Sermant Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.sermant.router.spring.utils; + +import io.sermant.core.service.ServiceManager; +import io.sermant.core.service.xds.XdsCoreService; +import io.sermant.core.service.xds.XdsRouteService; +import io.sermant.core.service.xds.XdsServiceDiscovery; +import io.sermant.core.service.xds.entity.ServiceInstance; +import io.sermant.core.service.xds.entity.XdsClusterLoadAssigment; +import io.sermant.core.service.xds.entity.XdsHeaderMatcher; +import io.sermant.core.service.xds.entity.XdsLocality; +import io.sermant.core.service.xds.entity.XdsPathMatcher; +import io.sermant.core.service.xds.entity.XdsRoute; +import io.sermant.core.service.xds.entity.XdsRouteAction; +import io.sermant.core.service.xds.entity.XdsRouteAction.XdsClusterWeight; +import io.sermant.core.service.xds.entity.XdsRouteAction.XdsWeightedClusters; +import io.sermant.core.service.xds.entity.XdsRouteMatch; +import io.sermant.core.service.xds.entity.match.ExactMatchStrategy; +import io.sermant.router.common.xds.XdsRouterHandler; +import io.sermant.router.common.xds.lb.XdsLoadBalancer; +import io.sermant.router.common.xds.lb.XdsLoadBalancerFactory; +import io.sermant.router.common.xds.lb.XdsRoundRobinLoadBalancer; +import io.sermant.router.spring.TestServiceInstance; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * BaseHttpRouterUtilsTest + * + * @author daizhenyu + * @since 2024-09-10 + **/ +public class BaseHttpRouterUtilsTest { + private static final String CLUSTER_NAME = "outbound|8080||serviceA.default.svc.cluster.local"; + + private static MockedStatic serviceManager; + + private static XdsServiceDiscovery serviceDiscovery; + + @BeforeClass + public static void setUp() { + XdsRouteService routeService = Mockito.mock(XdsRouteService.class); + Mockito.when(routeService.isLocalityRoute(CLUSTER_NAME)).thenReturn(false); + Mockito.when(routeService.getServiceRoute("serviceA")).thenReturn(createXdsRoute()); + serviceDiscovery = Mockito.mock(XdsServiceDiscovery.class); + Mockito.when(serviceDiscovery.getClusterServiceInstance(CLUSTER_NAME)) + .thenReturn(Optional.of(createXdsClusterInstance(CLUSTER_NAME, + Arrays.asList("test-region-1")))); + XdsCoreService xdsCoreService = Mockito.mock(XdsCoreService.class); + Mockito.when(xdsCoreService.getXdsRouteService()).thenReturn(routeService); + Mockito.when(xdsCoreService.getXdsServiceDiscovery()).thenReturn(serviceDiscovery); + serviceManager = Mockito.mockStatic(ServiceManager.class); + Mockito.when(ServiceManager.getService(XdsCoreService.class)).thenReturn(xdsCoreService); + } + + @AfterClass + public static void tearDown() { + serviceManager.close(); + } + + @Test + public void testRebuildUrlByXdsServiceInstance() throws MalformedURLException, URISyntaxException { + // prepare data + TestServiceInstance testServiceInstance = new TestServiceInstance(); + testServiceInstance.setHost("127.0.0.1"); + testServiceInstance.setPort(8080); + + // use URL with query + URL oldUrl = new URL("http://example.com/test?param=value"); + Assert.assertEquals("http://127.0.0.1:8080/test?param=value", + BaseHttpRouterUtils.rebuildUrlByXdsServiceInstance(oldUrl, testServiceInstance)); + + // use URL without query + oldUrl = new URL("http://example.com/test"); + Assert.assertEquals("http://127.0.0.1:8080/test", + BaseHttpRouterUtils.rebuildUrlByXdsServiceInstance(oldUrl, testServiceInstance)); + + // use invalid URL + URL invalidOldUrl = new URL("http://invalid url"); + Assert.assertEquals("", BaseHttpRouterUtils.rebuildUrlByXdsServiceInstance(invalidOldUrl, testServiceInstance)); + + // use URI + URI oldUri = new URI("http://example.com/test?param=value"); + Assert.assertEquals("http://127.0.0.1:8080/test?param=value", + BaseHttpRouterUtils.rebuildUrlByXdsServiceInstance(oldUri, testServiceInstance)); + } + + @Test + public void testChooseServiceInstanceByXds() { + XdsLoadBalancer loadBalancer = new XdsRoundRobinLoadBalancer(); + MockedStatic xdsLoadBalancerFactory = Mockito.mockStatic(XdsLoadBalancerFactory.class); + Mockito.when(XdsLoadBalancerFactory.getLoadBalancer(Mockito.any())).thenReturn(loadBalancer); + + Map headers = new HashMap<>(); + headers.put("version", "v1"); + + // service instance is empty + Mockito.when(serviceDiscovery.getClusterServiceInstance(CLUSTER_NAME)) + .thenReturn(Optional.of(createXdsClusterInstance(CLUSTER_NAME, new ArrayList<>()))); + Optional result = BaseHttpRouterUtils + .chooseServiceInstanceByXds("serviceA", "/test", headers); + Assert.assertFalse(result.isPresent()); + + // route not match and service instance is not empty + result = BaseHttpRouterUtils + .chooseServiceInstanceByXds("serviceA", "/test-invalid", Collections.emptyMap()); + Assert.assertFalse(result.isPresent()); + + // route match and service instance is not empty + Mockito.when(serviceDiscovery.getClusterServiceInstance(CLUSTER_NAME)) + .thenReturn(Optional.of(createXdsClusterInstance(CLUSTER_NAME, Arrays.asList("region-1")))); + result = BaseHttpRouterUtils + .chooseServiceInstanceByXds("serviceA", "/test", headers); + Assert.assertTrue(result.isPresent()); + Assert.assertEquals("serviceA", result.get().getServiceName()); + } + + @Test + public void testIsXdsRouteRequired() { + // empty string + Assert.assertFalse(BaseHttpRouterUtils.isXdsRouteRequired("")); + + // null + Assert.assertFalse(BaseHttpRouterUtils.isXdsRouteRequired(null)); + + // IPv4 + Assert.assertFalse(BaseHttpRouterUtils.isXdsRouteRequired("192.168.1.1")); + Assert.assertFalse(BaseHttpRouterUtils.isXdsRouteRequired("10.0.0.1")); + Assert.assertFalse(BaseHttpRouterUtils.isXdsRouteRequired("255.255.255.255")); + + // host + Assert.assertTrue(BaseHttpRouterUtils.isXdsRouteRequired("example.com")); + Assert.assertFalse(BaseHttpRouterUtils.isXdsRouteRequired("localhost")); + } + + @Test + public void testProcessHeaders() { + // every header has value + Map> headers = new HashMap<>(); + headers.put("Header1", Arrays.asList("Value1", "Value2")); + headers.put("Header2", Collections.singletonList("Value3")); + + Map result = BaseHttpRouterUtils.processHeaders(headers); + Assert.assertEquals(2, result.size()); + Assert.assertEquals("Value1", result.get("Header1")); + Assert.assertEquals("Value3", result.get("Header2")); + + // header has empty value + headers = new HashMap<>(); + headers.put("Header1", Arrays.asList("Value1", "Value2")); + headers.put("Header2", Collections.emptyList()); + + result = BaseHttpRouterUtils.processHeaders(headers); + Assert.assertEquals(2, result.size()); + Assert.assertEquals("Value1", result.get("Header1")); + Assert.assertEquals("", result.get("Header2")); + + // header has null value + headers = new HashMap<>(); + headers.put("Header1", Arrays.asList("Value1", "Value2")); + headers.put("Header2", null); + + result = BaseHttpRouterUtils.processHeaders(headers); + Assert.assertEquals(2, result.size()); + Assert.assertEquals("Value1", result.get("Header1")); + Assert.assertEquals("", result.get("Header2")); + } + + private static List createXdsRoute() { + XdsRoute route = new XdsRoute(); + route.setName("test-route"); + + XdsRouteMatch xdsRouteMatch = new XdsRouteMatch(); + XdsPathMatcher pathMatcher = new XdsPathMatcher(new ExactMatchStrategy("/test"), true); + XdsHeaderMatcher headerMatcher = new XdsHeaderMatcher("version", new ExactMatchStrategy("v1")); + xdsRouteMatch.setCaseSensitive(true); + xdsRouteMatch.setPathMatcher(pathMatcher); + xdsRouteMatch.setHeaderMatchers(Arrays.asList(headerMatcher)); + + XdsRouteAction xdsRouteAction = new XdsRouteAction(); + xdsRouteAction.setCluster("outbound|8080||serviceA.default.svc.cluster.local"); + + route.setRouteMatch(xdsRouteMatch); + route.setRouteAction(xdsRouteAction); + return Arrays.asList(route); + } + + private static XdsClusterLoadAssigment createXdsClusterInstance(String clusterName, List localityList) { + Map> localityInstances = new HashMap<>(); + for (String region : localityList) { + Set instances = new HashSet<>(); + TestServiceInstance testServiceInstance = new TestServiceInstance(); + testServiceInstance.setService("serviceA"); + Map metaData = new HashMap<>(); + metaData.put("region", region); + testServiceInstance.setMetaData(metaData); + instances.add(testServiceInstance); + XdsLocality locality = new XdsLocality(); + locality.setRegion(region); + localityInstances.put(locality, instances); + } + + XdsClusterLoadAssigment clusterInstance = new XdsClusterLoadAssigment(); + clusterInstance.setClusterName(clusterName); + clusterInstance.setLocalityInstances(localityInstances); + + return clusterInstance; + } +} \ No newline at end of file