From f718f39961763ccf9174edbdc27c593985baf06a Mon Sep 17 00:00:00 2001 From: wankai123 Date: Fri, 18 Jun 2021 08:53:20 +0800 Subject: [PATCH 01/14] Support endpoint name grouping by OpenAPI definitions --- .../apm/util/StringFormatGroup.java | 7 +- .../setup/backend/configuration-vocabulary.md | 1 + .../setup/backend/endpoint-grouping-rules.md | 311 +++++++++++++++++- .../src/main/resources/application.yml | 2 + oap-server/server-core/pom.xml | 11 +- .../oap/server/core/CoreModuleConfig.java | 4 + .../oap/server/core/CoreModuleProvider.java | 6 + .../config/group/EndpointNameGrouping.java | 34 +- .../openapi/EndpointGroupingRule4Openapi.java | 121 +++++++ .../EndpointGroupingRuleReader4Openapi.java | 135 ++++++++ .../EndpointGroupingBenchmark4Openapi.java | 172 ++++++++++ ...ndpointGroupingRuleReader4OpenapiTest.java | 97 ++++++ .../serviceA-api-v1/customerAPI-v1.yaml | 172 ++++++++++ .../serviceA-api-v1/productAPI-v1.yaml | 181 ++++++++++ .../serviceB-api-v2/productAPI-v2.yaml | 211 ++++++++++++ .../server/library/util/ResourceUtils.java | 33 +- 16 files changed, 1488 insertions(+), 10 deletions(-) create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRule4Openapi.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java create mode 100644 oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingBenchmark4Openapi.java create mode 100644 oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4OpenapiTest.java create mode 100644 oap-server/server-core/src/test/resources/openapi-definitions/serviceA-api-v1/customerAPI-v1.yaml create mode 100644 oap-server/server-core/src/test/resources/openapi-definitions/serviceA-api-v1/productAPI-v1.yaml create mode 100644 oap-server/server-core/src/test/resources/openapi-definitions/serviceB-api-v2/productAPI-v2.yaml diff --git a/apm-commons/apm-util/src/main/java/org/apache/skywalking/apm/util/StringFormatGroup.java b/apm-commons/apm-util/src/main/java/org/apache/skywalking/apm/util/StringFormatGroup.java index ab30cdb45149..9f22dafd2a64 100644 --- a/apm-commons/apm-util/src/main/java/org/apache/skywalking/apm/util/StringFormatGroup.java +++ b/apm-commons/apm-util/src/main/java/org/apache/skywalking/apm/util/StringFormatGroup.java @@ -19,6 +19,7 @@ package org.apache.skywalking.apm.util; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.regex.Pattern; import lombok.Getter; @@ -68,6 +69,10 @@ public FormatResult format(String string) { return new FormatResult(false, string, string); } + public void sortRules(Comparator comparator) { + rules.sort(comparator); + } + @Getter @RequiredArgsConstructor public static class FormatResult { @@ -78,7 +83,7 @@ public static class FormatResult { @Getter @ToString - private static class PatternRule { + public static class PatternRule { private final String name; private final Pattern pattern; diff --git a/docs/en/setup/backend/configuration-vocabulary.md b/docs/en/setup/backend/configuration-vocabulary.md index 5becb12ee5cd..1132aa2bbb42 100644 --- a/docs/en/setup/backend/configuration-vocabulary.md +++ b/docs/en/setup/backend/configuration-vocabulary.md @@ -42,6 +42,7 @@ core|default|role|Option values, `Mixed/Receiver/Aggregator`. **Receiver** mode | - | - | maxSizeOfAnalyzeProfileSnapshot|The max number of snapshots analyzed by OAP| - | 12000 | | - | - | syncThreads|The number of threads used to synchronously refresh the metrics data to the storage.| SW_CORE_SYNC_THREADS | 2 | | - | - | maxSyncOperationNum|The maximum number of processes supported for each synchronous storage operation. When the number of the flush data is greater than this value, it will be assigned to multiple cores for execution.| SW_CORE_MAX_SYNC_OPERATION_NUM | 50000 | +| - | - | enableEndpointNameGroupingByOpenapi |Turn it on then automatically grouping endpoint by the given OpenAPI definitions.| SW_CORE_ENABLE_ENDPOINT_NAME_GROUPING_BY_OPAENAPI | false | |cluster|standalone| - | standalone is not suitable for one node running, no available configuration.| - | - | | - | zookeeper|nameSpace|The namespace, represented by root path, isolates the configurations in the zookeeper.|SW_NAMESPACE| `/`, root path| | - | - | hostPort|hosts and ports of Zookeeper Cluster|SW_CLUSTER_ZK_HOST_PORT| localhost:2181| diff --git a/docs/en/setup/backend/endpoint-grouping-rules.md b/docs/en/setup/backend/endpoint-grouping-rules.md index dd623beacb35..5050449c1a04 100644 --- a/docs/en/setup/backend/endpoint-grouping-rules.md +++ b/docs/en/setup/backend/endpoint-grouping-rules.md @@ -1,15 +1,320 @@ # Group Parameterized Endpoints -In most cases, the endpoint should be detected automatically through the language agents, service mesh observability solution, +In most cases, the endpoint should be detected automatically through the language agents, service mesh observability solution, or configuration of meter system. -There are some special cases, especially when people use REST style URI, the application codes put the parameter in the endpoint name, +There are some special cases, especially when people use REST style URI, the application codes put the parameter in the endpoint name, such as putting order id in the URI, like `/prod/ORDER123` and `/prod/ORDER123`. But logically, people expect they could have an endpoint name like `prod/{order-id}`. This is the feature of parameterized endpoint grouping designed for. +If the incoming endpoint name hit the rules, SkyWalking will grouping the endpoint by rules. + +SkyWalking provides 2 ways to support endpoint grouping: +1. Endpoint name grouping by OpenAPI definitions. +2. Endpoint name grouping by custom configuration. + +The 2 grouping features can work together in sequence. +## Endpoint name grouping by OpenAPI definitions +The OpenAPI definitions are the documents based on The [OpenAPI Specification (OAS)](https://github.com/OAI/OpenAPI-Specification) which used to define a standard, language-agnostic interface for HTTP APIs. +SkyWalking now support `OAS v2.0+)`, could parse the documents `(yaml)` and build the grouping rules from them automatically. + + +### How to use +1. Add some `Specification Extensions` for SkyWalking in the OpenAPI definition documents:
+ \${METHOD} is a reserved placeholder which represent HTTP method eg. `POST/GET...`
+ \${PATH} is a reserved placeholder which represent the path eg. `/products/{id}`. + + | Extension Name | Required | Description | Default Value | + |-----|-----|-----|-----| + | x-sw-service-name | true | The service name which these endpoints belong | | + | x-sw-endpoint-name-match-rule | false | The rule use to match the endpoint.| \${METHOD}:\${PATH} | + | x-sw-endpoint-name-format | false | The endpoint name after grouping.| \${METHOD}:\${PATH} | + + These extensions are under `OpenAPI Object`. + +2. Put the OpenAPI definition documents into folder `openapi-definitions`, SkyWalking could read all documents or documents in subfolders from it，so you can organize these documents by yourself. For example: + ``` +├── openapi-definitions +│   ├── serviceA-api-v1 +│   │   ├── customerAPI-v1.yaml +│   │   └── productAPI-v1.yaml +│   └── serviceB-api-v2 +│   └── productAPI-v2.yaml +``` +3. Turn the feature on by set the `Core Module` configuration `${SW_CORE_ENABLE_ENDPOINT_NAME_GROUPING_BY_OPAENAPI:true}` +
+ +### Rules match priority +We recommend designing the API path as clear as possible. If the API path is fuzzy and a endpoint name might match multiple paths, SkyWalking would follow the match priority to select one as below orders: +1. The exact path matched first. + Eg. `/products or /products/inventory` +2. The the path which has the less variables. + Eg. `/products/{var1}/{var2} and /products/{var1}/abc`, endpoint name `/products/123/abc` will match the second one. +3. If the paths have the same number of variables, match the longest path, and the vars are considered to be `1`. + Eg. `/products/abc/{var1} and products/{var12345}/ef`, endpoint name `/products/abc/ef` will match the first one. +### Examples +If we have a OpenAPI definition doc `productAPI-v2.yaml` like this: +```yaml + +openapi: 3.0.0 +x-sw-service-name: serviceB +x-sw-endpoint-name-format: "${PATH}:<${METHOD}>" + +info: + description: OpenAPI definition for SkyWalking test. + version: v2 + title: Product API + +tags: + - name: product + description: product + - name: relatedProducts + description: Related Products + +paths: + /products: + get: + tags: + - product + summary: Get all products list + description: Get all products list. + operationId: getProducts + responses: + "200": + description: Success + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Product" + /products/{region}/{country}: + get: + tags: + - product + summary: Get products regional + description: Get products regional with the given id. + operationId: getProductRegional + parameters: + - name: region + in: path + description: Products region + required: true + schema: + type: string + - name: country + in: path + description: Products country + required: true + schema: + type: string + responses: + "200": + description: successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/Product" + "400": + description: Invalid parameters supplied + /products/{id}: + get: + tags: + - product + summary: Get product details + description: Get product details with the given id. + operationId: getProduct + parameters: + - name: id + in: path + description: Product id + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/ProductDetails" + "400": + description: Invalid product id + post: + tags: + - product + summary: Update product details + description: Update product details with the given id. + operationId: updateProduct + parameters: + - name: id + in: path + description: Product id + required: true + schema: + type: integer + format: int64 + - name: name + in: query + description: Product name + required: true + schema: + type: string + responses: + "200": + description: successful operation + delete: + tags: + - product + summary: Delete product details + description: Delete product details with the given id. + operationId: deleteProduct + parameters: + - name: id + in: path + description: Product id + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: successful operation + /products/{id}/relatedProducts: + get: + tags: + - relatedProducts + summary: Get related products + description: Get related products with the given product id. + operationId: getRelatedProducts + parameters: + - name: id + in: path + description: Product id + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/RelatedProducts" + "400": + description: Invalid product id + +components: + schemas: + Product: + type: object + description: Product id and name + properties: + id: + type: integer + format: int64 + description: Product id + name: + type: string + description: Product name + required: + - id + - name + ProductDetails: + type: object + description: Product details + properties: + id: + type: integer + format: int64 + description: Product id + name: + type: string + description: Product name + description: + type: string + description: Product description + required: + - id + - name + RelatedProducts: + type: object + description: Related Products + properties: + id: + type: integer + format: int32 + description: Product id + relatedProducts: + type: array + description: List of related products + items: + $ref: "#/components/schemas/Product" + + +``` + +Here give some scenario we might use: +1. Only set the `x-sw-service-name`, `x-sw-endpoint-name-match-rule` and `x-sw-endpoint-name-format` are default: +``` yaml +openapi: 3.0.0 +x-sw-service-name: serviceB + +info: + description: OpenAPI definition for SkyWalking test. + version: v2 + title: Product API + ... +``` +2. Set the `x-sw-service-name` , `x-sw-endpoint-name-match-rule` and `x-sw-endpoint-name-format` : +``` yaml +openapi: 3.0.0 +x-sw-service-name: serviceB +x-sw-endpoint-name-match-rule: "<${METHOD}>:${PATH}" +x-sw-endpoint-name-format: "<${METHOD}>:${PATH}" + +info: + description: OpenAPI definition for SkyWalking test. + version: v2 + title: Product API + ... +``` +3. Set the `x-sw-service-name` , `x-sw-endpoint-name-match-rule` and `x-sw-endpoint-name-format` : +``` yaml +openapi: 3.0.0 +x-sw-service-name: serviceB +x-sw-endpoint-name-match-rule: "<${METHOD}>:${PATH}" +x-sw-endpoint-name-format: "<${METHOD}>:${PATH}" + +info: + description: OpenAPI definition for SkyWalking test. + version: v2 + title: Product API + ... +``` + + | Incoming Endpiont | Incoming Service | x-sw-endpoint-name-match-rule | x-sw-endpoint-name-format | Matched | Grouping Result | + |-----|-----|-----|-----|-----|-----| + | GET:/products | serviceB | default | default | true | GET:/products | + | GET:/products/123 | serviceB | default |default | true | GET:/products{id} | + | GET:/products/asia/cn | serviceB | default | default | true | GET:/products/{region}/{country} | + | GET:/products/123/abc/efg | serviceB | default |default | false | GET:/products/123/abc/efg | + | \:/products/123 | serviceB | default | default | false | \:/products/123| + | GET:/products/123 | serviceC | default | default | false | GET:/products/123 | + | \:/products/123 | serviceB | \<\${METHOD}\>:\${PATH} | \<\${METHOD}>:\${PATH} | true | \:/products/{id} | + | GET:/products/123 | serviceB | default | ${PATH}:\<\${METHOD}\> | true | /products/{id}:\ | + | /products/123: | serviceB | ${PATH}:\<\${METHOD}\> | default | true | \:/products/{id} | +
+ + + + +## Endpoint name grouping by custom configuration Current, user could set up grouping rules through the static YAML file, named `endpoint-name-grouping.yml`, or use [Dynamic Configuration](dynamic-config.md) to initial and update the endpoint grouping rule. -## Configuration Format +### Configuration Format No matter in static local file or dynamic configuration value, they are sharing the same YAML format. ```yaml diff --git a/oap-server/server-bootstrap/src/main/resources/application.yml b/oap-server/server-bootstrap/src/main/resources/application.yml index f3ac4d1a5e30..dbc3f275b5c5 100755 --- a/oap-server/server-bootstrap/src/main/resources/application.yml +++ b/oap-server/server-bootstrap/src/main/resources/application.yml @@ -108,6 +108,8 @@ core: syncThreads: ${SW_CORE_SYNC_THREADS:2} # The maximum number of processes supported for each synchronous storage operation. When the number of the flush data is greater than this value, it will be assigned to multiple cores for execution. maxSyncOperationNum: ${SW_CORE_MAX_SYNC_OPERATION_NUM:50000} + # Turn it on then automatically grouping endpoint by the given OpenAPI definitions. + enableEndpointNameGroupingByOpenapi: ${SW_CORE_ENABLE_ENDPOINT_NAME_GROUPING_BY_OPAENAPI:true} storage: selector: ${SW_STORAGE:h2} elasticsearch: diff --git a/oap-server/server-core/pom.xml b/oap-server/server-core/pom.xml index 098d2f8e9e83..382bf17a2ceb 100644 --- a/oap-server/server-core/pom.xml +++ b/oap-server/server-core/pom.xml @@ -82,7 +82,11 @@ io.vavr vavr - + + io.swagger.parser.v3 + swagger-parser + 2.0.26 + org.apache.skywalking server-testing @@ -94,6 +98,11 @@ grpc-testing test + + org.openjdk.jmh + jmh-generator-annprocess + test + diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java index 22f75a9df85f..bed117dfe22c 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java @@ -158,6 +158,10 @@ public class CoreModuleConfig extends ModuleConfig { @Setter private int maxSyncOperationNum = 50000; + @Getter + @Setter + private boolean enableEndpointNameGroupingByOpenapi = false; + public CoreModuleConfig() { this.downsampling = new ArrayList<>(); } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java index 38929892f913..9e99bd1f09ea 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java @@ -46,6 +46,7 @@ import org.apache.skywalking.oap.server.core.config.DownSamplingConfigService; import org.apache.skywalking.oap.server.core.config.IComponentLibraryCatalogService; import org.apache.skywalking.oap.server.core.config.NamingControl; +import org.apache.skywalking.oap.server.core.config.group.openapi.EndpointGroupingRuleReader4Openapi; import org.apache.skywalking.oap.server.core.config.group.EndpointNameGrouping; import org.apache.skywalking.oap.server.core.config.group.EndpointNameGroupingRuleWatcher; import org.apache.skywalking.oap.server.core.management.ui.template.UITemplateInitializer; @@ -159,6 +160,11 @@ public void prepare() throws ServiceNotProvidedException, ModuleStartException { try { endpointNameGroupingRuleWatcher = new EndpointNameGroupingRuleWatcher( this, endpointNameGrouping); + + if (moduleConfig.isEnableEndpointNameGroupingByOpenapi()) { + endpointNameGrouping.setEndpointGroupingRule4Openapi( + new EndpointGroupingRuleReader4Openapi("openapi-definitions").read()); + } } catch (FileNotFoundException e) { throw new ModuleStartException(e.getMessage(), e); } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java index da952d67d2c2..f5ad43d39770 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java @@ -21,20 +21,48 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.apm.util.StringFormatGroup; +import org.apache.skywalking.oap.server.core.config.group.openapi.EndpointGroupingRule4Openapi; @Slf4j public class EndpointNameGrouping { @Setter private volatile EndpointGroupingRule endpointGroupingRule; + @Setter + private volatile EndpointGroupingRule4Openapi endpointGroupingRule4Openapi; public String format(String serviceName, String endpointName) { - if (endpointGroupingRule == null) { - return endpointName; + String formattedName = endpointName; + if (endpointGroupingRule4Openapi != null) { + formattedName = formatByOpenapi(serviceName, formattedName.toString()); + } + + if (endpointGroupingRule != null) { + formattedName = formatByCustom(serviceName, formattedName.toString()); } + + return formattedName; + } + + private String formatByCustom(String serviceName, String endpointName) { final StringFormatGroup.FormatResult formatResult = endpointGroupingRule.format(serviceName, endpointName); if (log.isDebugEnabled() || log.isTraceEnabled()) { if (formatResult.isMatch()) { - log.debug("Endpoint {} of Service {} has been renamed in group {}", + log.debug("Endpoint {} of Service {} has been renamed in group {} by endpointGroupingRule", + endpointName, serviceName, formatResult.getName() + ); + } else { + log.trace("Endpoint {} of Service {} keeps unchanged.", endpointName, serviceName); + } + } + return formatResult.getName(); + } + + private String formatByOpenapi(String serviceName, String endpointName) { + final StringFormatGroup.FormatResult formatResult = endpointGroupingRule4Openapi.format( + serviceName, endpointName); + if (log.isDebugEnabled() || log.isTraceEnabled()) { + if (formatResult.isMatch()) { + log.debug("Endpoint {} of Service {} has been renamed in group {} by endpointGroupingRule4Openapi", endpointName, serviceName, formatResult.getName() ); } else { diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRule4Openapi.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRule4Openapi.java new file mode 100644 index 000000000000..52510fd5654e --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRule4Openapi.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.config.group.openapi; + +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; +import lombok.Getter; +import org.apache.skywalking.apm.util.StringFormatGroup; + +public class EndpointGroupingRule4Openapi { + private final Map> directLookup = new HashMap<>(); + @Getter + private final Map> groupedRules = new HashMap<>(); + + void addDirectLookup(String serviceName, String endpointName, String endpointGroupName) { + Map endpointNameLookup = directLookup.computeIfAbsent(serviceName, name -> new HashMap<>()); + endpointNameLookup.put(endpointName, endpointGroupName); + } + + void addGroupedRule(String serviceName, String endpointGroupName, String ruleRegex) { + String rulesGroupkey = getGroupedRulesKey(ruleRegex); + Map rules = groupedRules.computeIfAbsent(serviceName, name -> new HashMap<>()); + StringFormatGroup formatGroup = rules.computeIfAbsent(rulesGroupkey, name -> new StringFormatGroup()); + formatGroup.addRule(endpointGroupName, ruleRegex); + } + + public StringFormatGroup.FormatResult format(String service, String endpointName) { + Map endpointNameLookup = directLookup.get(service); + if (endpointNameLookup != null && endpointNameLookup.get(endpointName) != null) { + return new StringFormatGroup.FormatResult(true, endpointNameLookup.get(endpointName), endpointName); + } + + Map rules = groupedRules.get(service); + if (rules != null) { + final StringFormatGroup stringFormatGroup = rules.get(getGroupedRulesKey(endpointName)); + if (stringFormatGroup != null) { + return stringFormatGroup.format(endpointName); + } + } + + return new StringFormatGroup.FormatResult(false, endpointName, endpointName); + } + + void sortRulesAll() { + groupedRules.entrySet().forEach(rules -> { + sortRulesByService(rules.getKey()); + }); + } + + void sortRulesByService(String serviceName) { + Map rules = groupedRules.get(serviceName); + if (rules != null) { + rules.entrySet().forEach(stringFormatGroup -> { + stringFormatGroup.getValue() + .sortRules(new EndpointGroupingRule4Openapi.EndpointGroupingRulesComparator()); + }); + } + } + + String getGroupedRulesKey(String string) { + String[] ss = string.split("/"); + if (ss.length == 1) { //eg. POST:/ + return ss[0] + "/"; + } + if (ss.length > 1) { + return ss[0] + "/" + ss[1]; + } + return "/"; + } + + static class EndpointGroupingRulesComparator implements Comparator { + private static final String VAR_PATTERN = "\\(\\[\\^\\/\\]\\+\\)"; + + @Override + public int compare(final StringFormatGroup.PatternRule rule1, final StringFormatGroup.PatternRule rule2) { + + String pattern1 = rule1.getPattern().pattern(); + String pattern2 = rule2.getPattern().pattern(); + + if (getPatternVarsCount(pattern1) < getPatternVarsCount(pattern2)) { + return -1; + } else if (getPatternVarsCount(pattern1) > getPatternVarsCount(pattern2)) { + return 1; + } + + int length1 = getPatternLength(pattern1); + int length2 = getPatternLength(pattern2); + if (length1 != length2) { + return length2 - length1; + } + + return 0; + } + + private int getPatternVarsCount(String pattern) { + return ",".concat(pattern).concat(",").split(VAR_PATTERN).length - 1; + } + + private int getPatternLength(String pattern) { + return pattern.replaceAll(VAR_PATTERN, "#").length(); + } + + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java new file mode 100644 index 000000000000..7ccf30c49330 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.config.group.openapi; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.Reader; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.skywalking.apm.util.StringUtil; +import org.apache.skywalking.oap.server.library.util.ResourceUtils; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.SafeConstructor; + +public class EndpointGroupingRuleReader4Openapi { + + private final String openapiDefPath; + private final static String DEFAULT_ENDPOINT_NAME_FORMAT = "${METHOD}:${PATH}"; + private final static String DEFAULT_ENDPOINT_NAME_MATCH_RULE = "${METHOD}:${PATH}"; + private final Map requestMethodsMap = new HashMap() { + { + put("get", "GET"); + put("post", "POST"); + put("put", "PUT"); + put("delete", "DELETE"); + put("trace", "TRACE"); + put("options", "OPTIONS"); + put("head", "HEAD"); + put("patch", "PATCH"); + } + }; + + public EndpointGroupingRuleReader4Openapi(final String openapiDefPath) { + + this.openapiDefPath = openapiDefPath; + } + + public EndpointGroupingRule4Openapi read() throws FileNotFoundException { + EndpointGroupingRule4Openapi endpointGroupingRule = new EndpointGroupingRule4Openapi(); + + List fileList = ResourceUtils.getPathFilesRecursive(openapiDefPath); + for (File file : fileList) { + Reader reader = new FileReader(file); + Yaml yaml = new Yaml(new SafeConstructor()); + Map openapiData = yaml.load(reader); + if (openapiData != null) { + String serviceName = getServiceName(openapiData, file); + LinkedHashMap> paths = + (LinkedHashMap>) openapiData.get("paths"); + + if (paths != null) { + paths.forEach((pathString, pathItem) -> { + pathItem.keySet().forEach(key -> { + String requestMethod = requestMethodsMap.get(key); + if (!StringUtil.isEmpty(requestMethod)) { + String endpointGroupName = formatEndPointName(pathString, requestMethod, openapiData); + String groupRegex = getGroupRegex(pathString, requestMethod, openapiData); + if (isTemplatePath(pathString)) { + endpointGroupingRule.addGroupedRule(serviceName, endpointGroupName, groupRegex); + } else { + endpointGroupingRule.addDirectLookup(serviceName, groupRegex, endpointGroupName); + } + } + }); + }); + } + } + } + endpointGroupingRule.sortRulesAll(); + return endpointGroupingRule; + } + + private String getServiceName(Map openapiData, File file) { + String serviceName = (String) openapiData.get("x-sw-service-name"); + if (StringUtil.isEmpty(serviceName)) { + throw new IllegalArgumentException( + "OpenAPI definition: " + file.getAbsolutePath() + " x-sw-service-name can't be empty"); + } + + return serviceName; + } + + private boolean isTemplatePath(String pathString) { + return pathString.matches("(.*)\\{(.+?)}(.*)"); + } + + private String getGroupRegex(String pathString, String requstMathod, Map openapiData) { + String endPointNameMatchRuleTemplate = (String) openapiData.get("x-sw-endpoint-name-match-rule"); + String endPointNameMatchRule = replaceTemplateVars(DEFAULT_ENDPOINT_NAME_MATCH_RULE, pathString, requstMathod); + + if (!StringUtil.isEmpty(endPointNameMatchRuleTemplate)) { + endPointNameMatchRule = replaceTemplateVars(endPointNameMatchRuleTemplate, pathString, requstMathod); + } + + if (isTemplatePath(endPointNameMatchRule)) { + return endPointNameMatchRule.replaceAll("\\{(.+?)}", "([^/]+)"); + } + return endPointNameMatchRule; + } + + private String formatEndPointName(String pathString, String requstMathod, Map openapiData) { + String endPointNameFormat = (String) openapiData.get("x-sw-endpoint-name-format"); + + if (!StringUtil.isEmpty(endPointNameFormat)) { + return replaceTemplateVars(endPointNameFormat, pathString, requstMathod); + } + + return replaceTemplateVars(DEFAULT_ENDPOINT_NAME_FORMAT, pathString, requstMathod); + } + + private String replaceTemplateVars(String template, String pathString, String requstMathod) { + return template.replaceAll("\\$\\{METHOD}", requstMathod) + .replaceAll("\\$\\{PATH}", pathString); + } + +} diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingBenchmark4Openapi.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingBenchmark4Openapi.java new file mode 100644 index 000000000000..de1fa377ed74 --- /dev/null +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingBenchmark4Openapi.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.config.group.openapi; + +import java.io.FileNotFoundException; +import lombok.SneakyThrows; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.profile.GCProfiler; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +@BenchmarkMode({Mode.Throughput}) +@Threads(4) +public class EndpointGroupingBenchmark4Openapi { + + @State(Scope.Benchmark) + public static class FormatClassPaths20 { + private EndpointGroupingRule4Openapi rule; + + @SneakyThrows + public FormatClassPaths20() { + rule = new EndpointGroupingRule4Openapi(); + for (int i = 0; i <= 3; i++) { + rule.addGroupedRule("serviceA", "GET:/products1/{id}/" + +i, "GET:/products1/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "POST:/products1/{id}/" + +i, "POST:/products1/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "GET:/products2/{id}/" + +i, "GET:/products2/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "POST:/products3/{id}/" + +i, "POST:/products3/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "GET:/products3/{id}/" + +i, "GET:/products3/([^/]+)/" + i); + } + } + + public void format(String serviceName, String endpointName) { + rule.format(serviceName, endpointName); + } + } + + @State(Scope.Benchmark) + public static class FormatClassPaths50 { + private EndpointGroupingRule4Openapi rule; + + @SneakyThrows + public FormatClassPaths50() { + rule = new EndpointGroupingRule4Openapi(); + for (int i = 0; i <= 9; i++) { + rule.addGroupedRule("serviceA", "GET:/products1/{id}/" + +i, "GET:/products1/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "POST:/products1/{id}/" + +i, "POST:/products1/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "GET:/products2/{id}/" + +i, "GET:/products2/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "POST:/products3/{id}/" + +i, "POST:/products3/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "GET:/products3/{id}/" + +i, "GET:/products3/([^/]+)/" + i); + } + } + + public void format(String serviceName, String endpointName) { + rule.format(serviceName, endpointName); + } + } + + @State(Scope.Benchmark) + public static class FormatClassPaths200 { + private EndpointGroupingRule4Openapi rule; + + @SneakyThrows + public FormatClassPaths200() { + rule = new EndpointGroupingRule4Openapi(); + for (int i = 0; i <= 39; i++) { + rule.addGroupedRule("serviceA", "GET:/products1/{id}/" + +i, "GET:/products1/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "POST:/products1/{id}/" + +i, "POST:/products1/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "GET:/products2/{id}/" + +i, "GET:/products2/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "POST:/products3/{id}/" + +i, "POST:/products3/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "GET:/products3/{id}/" + +i, "GET:/products3/([^/]+)/" + i); + } + } + + public void format(String serviceName, String endpointName) { + rule.format(serviceName, endpointName); + } + + } + + @Benchmark + public void formatEndpointNameMatchedPaths20(FormatClassPaths20 formatClass) { + formatClass.format("serviceA", "GET:/products1/123"); + } + + @Benchmark + public void formatEndpointNameMatchedPaths50(FormatClassPaths50 formatClass) { + formatClass.format("serviceA", "GET:/products1/123"); + } + + @Benchmark + public void formatEndpointNameMatchedPaths200(FormatClassPaths200 formatClass) { + formatClass.format("serviceA", "GET:/products1/123"); + } + + public static void main(String[] args) throws RunnerException, FileNotFoundException { + + Options opt = new OptionsBuilder() + .include(EndpointGroupingBenchmark4Openapi.class.getName()) + .addProfiler(GCProfiler.class) + .jvmArgsAppend("-Xmx512m", "-Xms512m") + .forks(1) + .build(); + + new Runner(opt).run(); + } +} + +/* +* The test is assumed each endpoint need to run all match within it's rules group. +* +# JMH version: 1.21 +# VM version: JDK 1.8.0_271, Java HotSpot(TM) 64-Bit Server VM, 25.271-b09 +# VM invoker: /Library/Java/JavaVirtualMachines/jdk1.8.0_271.jdk/Contents/Home/jre/bin/java +# VM options: -javaagent:/Applications/IntelliJ IDEA CE.app/Contents/lib/idea_rt.jar=51431:/Applications/IntelliJ IDEA CE.app/Contents/bin -Dfile.encoding=UTF-8 -Xmx512m -Xms512m +# Warmup: 5 iterations, 10 s each +# Measurement: 5 iterations, 10 s each +# Timeout: 10 min per iteration +# Threads: 4 threads, will synchronize iterations +# Benchmark mode: Throughput, ops/time + +Benchmark Mode Cnt Score Error Units +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20 thrpt 5 4052121.622 ± 427561.892 ops/s +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.alloc.rate thrpt 5 4386.461 ± 465.129 MB/sec +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.alloc.rate.norm thrpt 5 1192.000 ± 0.001 B/op +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.churn.PS_Eden_Space thrpt 5 4411.321 ± 445.136 MB/sec +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.churn.PS_Eden_Space.norm thrpt 5 1198.792 ± 6.537 B/op +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.churn.PS_Survivor_Space thrpt 5 0.475 ± 0.164 MB/sec +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.churn.PS_Survivor_Space.norm thrpt 5 0.129 ± 0.036 B/op +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.count thrpt 5 1367.000 counts +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.time thrpt 5 777.000 ms +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200 thrpt 5 494580.800 ± 13430.644 ops/s +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.alloc.rate thrpt 5 3510.759 ± 95.418 MB/sec +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.alloc.rate.norm thrpt 5 7816.000 ± 0.001 B/op +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.churn.PS_Eden_Space thrpt 5 3531.052 ± 101.381 MB/sec +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.churn.PS_Eden_Space.norm thrpt 5 7861.170 ± 52.430 B/op +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.churn.PS_Survivor_Space thrpt 5 0.289 ± 0.092 MB/sec +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.churn.PS_Survivor_Space.norm thrpt 5 0.644 ± 0.217 B/op +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.count thrpt 5 1094.000 counts +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.time thrpt 5 604.000 ms +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50 thrpt 5 1834653.371 ± 128456.874 ops/s +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.alloc.rate thrpt 5 3784.855 ± 263.420 MB/sec +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.alloc.rate.norm thrpt 5 2272.000 ± 0.001 B/op +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.churn.PS_Eden_Space thrpt 5 3807.122 ± 288.032 MB/sec +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.churn.PS_Eden_Space.norm thrpt 5 2285.315 ± 16.947 B/op +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.churn.PS_Survivor_Space thrpt 5 0.365 ± 0.080 MB/sec +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.churn.PS_Survivor_Space.norm thrpt 5 0.219 ± 0.047 B/op +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.count thrpt 5 1180.000 counts +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.time thrpt 5 690.000 ms + */ \ No newline at end of file diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4OpenapiTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4OpenapiTest.java new file mode 100644 index 000000000000..248d4dafcbda --- /dev/null +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4OpenapiTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.config.group.openapi; + +import java.io.IOException; +import org.apache.skywalking.oap.server.core.config.group.EndpointNameGrouping; +import org.junit.Assert; +import org.junit.Test; + +public class EndpointGroupingRuleReader4OpenapiTest { + + @Test + public void testReadingRule() throws IOException { + + EndpointGroupingRuleReader4Openapi reader = new EndpointGroupingRuleReader4Openapi("openapi-definitions"); + EndpointGroupingRule4Openapi rule = reader.read(); + EndpointNameGrouping nameGrouping = new EndpointNameGrouping(); + nameGrouping.setEndpointGroupingRule4Openapi(rule); + + //default x-sw-endpoint-name-match-rule and x-sw-endpoint-name-format + // test direct lookup + String endpointName = nameGrouping.format("serviceA", "GET:/products"); + Assert.assertEquals("GET:/products", endpointName); + + endpointName = nameGrouping.format("serviceA", "GET:/products/123"); + Assert.assertEquals("GET:/products/{id}", endpointName); + + endpointName = nameGrouping.format("serviceA", "GET:/products/123/abc/ef"); + Assert.assertEquals("GET:/products/123/abc/ef", endpointName); + + endpointName = nameGrouping.format("serviceA", "GET:/products/123/relatedProducts"); + Assert.assertEquals("GET:/products/{id}/relatedProducts", endpointName); + + endpointName = nameGrouping.format("serviceA", "GET:/products/1/relatedProducts"); + Assert.assertEquals("GET:/products/{id}/relatedProducts", endpointName); + + //test same x-sw-endpoint-name-match-rule and x-sw-endpoint-name-format + endpointName = nameGrouping.format("serviceA", "POST:/customer"); + Assert.assertEquals("POST:/customer", endpointName); + + endpointName = nameGrouping.format("serviceA", ":/customers/1"); + Assert.assertEquals(":/customers/{id}", endpointName); + + //test different x-sw-endpoint-name-match-rule and x-sw-endpoint-name-format + endpointName = nameGrouping.format("serviceB", "GET:/products"); + Assert.assertEquals("/products:", endpointName); + + endpointName = nameGrouping.format("serviceB", "GET:/products/asia/cn"); + Assert.assertEquals("/products/{region}/{country}:", endpointName); + + //test match priority, not match /products/{region}/{country}: + endpointName = nameGrouping.format("serviceB", "GET:/products/12/relatedProducts"); + Assert.assertEquals("/products/{id}/relatedProducts:", endpointName); + + //test not match, return the origin + endpointName = nameGrouping.format("serviceA", "GET:/products/"); + Assert.assertNotEquals("GET:/products", endpointName); + + endpointName = nameGrouping.format("serviceA", "GET:/products/123/"); + Assert.assertEquals("GET:/products/123/", endpointName); + + endpointName = nameGrouping.format("serviceC", "GET:/products/123"); + Assert.assertEquals("GET:/products/123", endpointName); + + endpointName = nameGrouping.format("serviceA", "GET:/products/1/ratings/123"); + Assert.assertEquals("GET:/products/1/ratings/123", endpointName); + + endpointName = nameGrouping.format("serviceA", ":/customers/1/123"); + Assert.assertEquals(":/customers/1/123", endpointName); + + endpointName = nameGrouping.format("serviceB", "/products/:"); + Assert.assertEquals("/products/:", endpointName); + + endpointName = nameGrouping.format("serviceB", "{GET}:/products"); + Assert.assertEquals("{GET}:/products", endpointName); + + endpointName = nameGrouping.format("serviceB", "/products/1/2/3:"); + Assert.assertEquals("/products/1/2/3:", endpointName); + + } +} diff --git a/oap-server/server-core/src/test/resources/openapi-definitions/serviceA-api-v1/customerAPI-v1.yaml b/oap-server/server-core/src/test/resources/openapi-definitions/serviceA-api-v1/customerAPI-v1.yaml new file mode 100644 index 000000000000..23bd2d311c41 --- /dev/null +++ b/oap-server/server-core/src/test/resources/openapi-definitions/serviceA-api-v1/customerAPI-v1.yaml @@ -0,0 +1,172 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +openapi: 3.0.0 +x-sw-service-name: serviceA +x-sw-endpoint-name-match-rule: "<${METHOD}>:${PATH}" +x-sw-endpoint-name-format: "<${METHOD}>:${PATH}" +info: + description: OpenAPI definition for SkyWalking test. + version: v1 + title: Customer API + +tags: + - name: customer + description: customer + +paths: + /customers: + get: + tags: + - customer + summary: Get all customers list + description: Get all customers list. + operationId: getCustomers + responses: + "200": + description: Success + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Customer" + /customers/{id}: + get: + tags: + - customer + summary: Get customer details + description: Get customer details with the given id. + operationId: getCustomer + parameters: + - name: id + in: path + description: Customer id + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/CustomerDetails" + "400": + description: Invalid customer id + post: + tags: + - customer + summary: Update customer details + description: Update customer details with the given id. + operationId: updateCustomer + parameters: + - name: id + in: path + description: Customer id + required: true + schema: + type: integer + format: int64 + - name: name + in: query + description: Customer name + required: true + schema: + type: string + responses: + "200": + description: successful operation + delete: + tags: + - customer + summary: Delete customer details + description: Delete customer details with the given id. + operationId: deleteCustomer + parameters: + - name: id + in: path + description: Customer id + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: successful operation + + /customer/{region}/{country}: + get: + tags: + - customer + summary: Get customers regional + description: Get customers regional with the given id. + operationId: getCustomersRegional + parameters: + - name: region + in: path + description: Customers region + required: true + schema: + type: string + - name: country + in: path + description: Customers country + required: true + schema: + type: string + responses: + "200": + description: successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/Customer" + "400": + description: Invalid parameters supplied +components: + schemas: + Customer: + type: object + description: Customer id and name + properties: + id: + type: integer + format: int64 + description: Customer id + name: + type: string + description: Customer name + required: + - id + - name + CustomerDetails: + type: object + description: Customer details + properties: + id: + type: integer + format: int64 + description: Customer id + name: + type: string + description: Customer name + description: + type: string + description: Customer description + required: + - id + - name diff --git a/oap-server/server-core/src/test/resources/openapi-definitions/serviceA-api-v1/productAPI-v1.yaml b/oap-server/server-core/src/test/resources/openapi-definitions/serviceA-api-v1/productAPI-v1.yaml new file mode 100644 index 000000000000..61d6c565df4a --- /dev/null +++ b/oap-server/server-core/src/test/resources/openapi-definitions/serviceA-api-v1/productAPI-v1.yaml @@ -0,0 +1,181 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +openapi: 3.0.0 +x-sw-service-name: serviceA + +info: + description: OpenAPI definition for SkyWalking test. + version: v1 + title: Product API + +tags: + - name: product + description: product + - name: relatedProducts + description: Related Products + +paths: + /products: + get: + tags: + - product + summary: Get all products list + description: Get all products list. + operationId: getProducts + responses: + "200": + description: Success + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Product" + /products/{id}: + get: + tags: + - product + summary: Get product details + description: Get product details with the given id. + operationId: getProduct + parameters: + - name: id + in: path + description: Product id + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/ProductDetails" + "400": + description: Invalid product id + post: + tags: + - product + summary: Update product details + description: Update product details with the given id. + operationId: updateProduct + parameters: + - name: id + in: path + description: Product id + required: true + schema: + type: integer + format: int64 + - name: name + in: query + description: Product name + required: true + schema: + type: string + responses: + "200": + description: successful operation + delete: + tags: + - product + summary: Delete product details + description: Delete product details with the given id. + operationId: deleteProduct + parameters: + - name: id + in: path + description: Product id + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: successful operation + /products/{id}/relatedProducts: + get: + tags: + - relatedProducts + summary: Get related products + description: Get related products with the given product id. + operationId: getRelatedProducts + parameters: + - name: id + in: path + description: Product id + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/RelatedProducts" + "400": + description: Invalid product id + +components: + schemas: + Product: + type: object + description: Product id and name + properties: + id: + type: integer + format: int64 + description: Product id + name: + type: string + description: Product name + required: + - id + - name + ProductDetails: + type: object + description: Product details + properties: + id: + type: integer + format: int64 + description: Product id + name: + type: string + description: Product name + description: + type: string + description: Product description + required: + - id + - name + RelatedProducts: + type: object + description: Related Products + properties: + id: + type: integer + format: int32 + description: Product id + relatedProducts: + type: array + description: List of related products + items: + $ref: "#/components/schemas/Product" diff --git a/oap-server/server-core/src/test/resources/openapi-definitions/serviceB-api-v2/productAPI-v2.yaml b/oap-server/server-core/src/test/resources/openapi-definitions/serviceB-api-v2/productAPI-v2.yaml new file mode 100644 index 000000000000..b502c83192c0 --- /dev/null +++ b/oap-server/server-core/src/test/resources/openapi-definitions/serviceB-api-v2/productAPI-v2.yaml @@ -0,0 +1,211 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +/* +openapi: 3.0.0 +x-sw-service-name: serviceB +x-sw-endpoint-name-format: "${PATH}:<${METHOD}>" + +info: + description: OpenAPI definition for SkyWalking test. + version: v2 + title: Product API + +tags: + - name: product + description: product + - name: relatedProducts + description: Related Products + +paths: + /products: + get: + tags: + - product + summary: Get all products list + description: Get all products list. + operationId: getProducts + responses: + "200": + description: Success + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Product" + /products/{region}/{country}: + get: + tags: + - product + summary: Get products regional + description: Get products regional with the given id. + operationId: getProductRegional + parameters: + - name: region + in: path + description: Products region + required: true + schema: + type: string + - name: country + in: path + description: Products country + required: true + schema: + type: string + responses: + "200": + description: successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/Product" + "400": + description: Invalid parameters supplied + /products/{id}: + get: + tags: + - product + summary: Get product details + description: Get product details with the given id. + operationId: getProduct + parameters: + - name: id + in: path + description: Product id + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/ProductDetails" + "400": + description: Invalid product id + post: + tags: + - product + summary: Update product details + description: Update product details with the given id. + operationId: updateProduct + parameters: + - name: id + in: path + description: Product id + required: true + schema: + type: integer + format: int64 + - name: name + in: query + description: Product name + required: true + schema: + type: string + responses: + "200": + description: successful operation + delete: + tags: + - product + summary: Delete product details + description: Delete product details with the given id. + operationId: deleteProduct + parameters: + - name: id + in: path + description: Product id + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: successful operation + /products/{id}/relatedProducts: + get: + tags: + - relatedProducts + summary: Get related products + description: Get related products with the given product id. + operationId: getRelatedProducts + parameters: + - name: id + in: path + description: Product id + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/RelatedProducts" + "400": + description: Invalid product id + +components: + schemas: + Product: + type: object + description: Product id and name + properties: + id: + type: integer + format: int64 + description: Product id + name: + type: string + description: Product name + required: + - id + - name + ProductDetails: + type: object + description: Product details + properties: + id: + type: integer + format: int64 + description: Product id + name: + type: string + description: Product name + description: + type: string + description: Product description + required: + - id + - name + RelatedProducts: + type: object + description: Related Products + properties: + id: + type: integer + format: int32 + description: Product id + relatedProducts: + type: array + description: List of related products + items: + $ref: "#/components/schemas/Product" diff --git a/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/ResourceUtils.java b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/ResourceUtils.java index 1381090a218e..a327aadc0cc4 100644 --- a/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/ResourceUtils.java +++ b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/ResourceUtils.java @@ -24,8 +24,10 @@ import java.io.InputStreamReader; import java.io.Reader; import java.net.URL; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; +import java.util.List; import java.util.Objects; import java.util.Set; @@ -58,12 +60,39 @@ public static File[] getPathFiles(String parentPath, String[] fileNames) throws throw new FileNotFoundException("path not found: " + parentPath); } final Set nameSet = new HashSet<>(Arrays.asList(fileNames)); - final File[] listFiles = Objects.requireNonNull(new File(url.getPath()) - .listFiles((dir, name) -> nameSet.contains(name)), "No files in " + parentPath); + final File[] listFiles = Objects.requireNonNull( + new File(url.getPath()) + .listFiles((dir, name) -> nameSet.contains(name)), "No files in " + parentPath); if (listFiles.length == 0) { throw new FileNotFoundException("files not found:" + nameSet); } return listFiles; } + + public static List getPathFilesRecursive(String path) throws FileNotFoundException { + URL url = ResourceUtils.class.getClassLoader().getResource(path); + if (url == null) { + throw new FileNotFoundException("path not found: " + path); + } + List fileList = new ArrayList<>(); + return getPathFilesRecursive(url.getPath(), fileList); + } + + private static List getPathFilesRecursive(String filepath, List fileList) { + File file = new File(filepath); + if (file.isDirectory()) { + File[] subFiles = file.listFiles(); + if (subFiles != null) { + for (File subFile : subFiles) { + if (subFile.isDirectory()) { + getPathFilesRecursive(subFile.getPath(), fileList); + } else { + fileList.add(subFile); + } + } + } + } + return fileList; + } } From 4bd111e6bbd43e0dea9120ad4c4077c2e329fd9c Mon Sep 17 00:00:00 2001 From: wankai123 Date: Fri, 18 Jun 2021 09:52:37 +0800 Subject: [PATCH 02/14] polish some docs and configs --- CHANGES.md | 1 + .../setup/backend/endpoint-grouping-rules.md | 4 +- .../serviceA-api-v1/productAPI-v1.yaml | 202 ++++++++++++++++++ .../serviceA-api-v1/customerAPI-v1.yaml | 13 +- .../serviceA-api-v1/productAPI-v1.yaml | 13 +- .../serviceB-api-v2/productAPI-v2.yaml | 14 +- 6 files changed, 243 insertions(+), 4 deletions(-) create mode 100644 oap-server/server-bootstrap/src/main/resources/openapi-definitions/serviceA-api-v1/productAPI-v1.yaml diff --git a/CHANGES.md b/CHANGES.md index 0273b9658086..3dfa7381b801 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -34,6 +34,7 @@ Release Notes. * Upgrade zookeeper caused by CVE-2019-0201. * Upgrade snake yaml caused by CVE-2017-18640. * Upgrade embed tomcat caused by CVE-2020-13935. +* Support endpoint name grouping by OpenAPI definitions. #### UI diff --git a/docs/en/setup/backend/endpoint-grouping-rules.md b/docs/en/setup/backend/endpoint-grouping-rules.md index 5050449c1a04..4afd89805954 100644 --- a/docs/en/setup/backend/endpoint-grouping-rules.md +++ b/docs/en/setup/backend/endpoint-grouping-rules.md @@ -30,8 +30,10 @@ SkyWalking now support `OAS v2.0+)`, could parse the documents `(yaml)` and buil | x-sw-endpoint-name-format | false | The endpoint name after grouping.| \${METHOD}:\${PATH} | These extensions are under `OpenAPI Object`. + We highly recommend using the default config, the custom config would be considered as part of the match rules (regex pattern). + We provide some cases in `org.apache.skywalking.oap.server.core.config.group.openapi.EndpointGroupingRuleReader4OpenapiTest`, you could validate your custom config as well. -2. Put the OpenAPI definition documents into folder `openapi-definitions`, SkyWalking could read all documents or documents in subfolders from it，so you can organize these documents by yourself. For example: +1. Put the OpenAPI definition documents into folder `openapi-definitions`, SkyWalking could read all documents or documents in subfolders from it，so you can organize these documents by yourself. For example: ``` ├── openapi-definitions │   ├── serviceA-api-v1 diff --git a/oap-server/server-bootstrap/src/main/resources/openapi-definitions/serviceA-api-v1/productAPI-v1.yaml b/oap-server/server-bootstrap/src/main/resources/openapi-definitions/serviceA-api-v1/productAPI-v1.yaml new file mode 100644 index 000000000000..06adcb176aa6 --- /dev/null +++ b/oap-server/server-bootstrap/src/main/resources/openapi-definitions/serviceA-api-v1/productAPI-v1.yaml @@ -0,0 +1,202 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# ${METHOD} is a reserved placeholder which represent HTTP method eg. `POST/GET...` +# ${PATH} is a reserved placeholder which represent the path eg. `/products/{id}`. +# +# | Extension Name | Required | Description | Default Value | +# |-----|-----|-----|-----| +# | x-sw-service-name | true | The service name which these endpoints belong | | +# | x-sw-endpoint-name-match-rule | false | The rule use to match the endpoint.| ${METHOD}:${PATH} | +# | x-sw-endpoint-name-format | false | The endpoint name after grouping.| ${METHOD}:${PATH} | +# +# These extensions are under `OpenAPI Object`. +# We highly recommend using the default config, the custom config would be considered as part of the match rules (regex pattern). +# We provide some cases in org.apache.skywalking.oap.server.core.config.group.openapi.EndpointGroupingRuleReader4OpenapiTest, +# you could validate your custom config as well. + + +################################################### +# This is only a example using the default config.# +################################################### + + +#openapi: 3.0.0 +#x-sw-service-name: serviceA +# +#info: +# description: OpenAPI definition for SkyWalking test. +# version: v1 +# title: Product API +# +#tags: +# - name: product +# description: product +# - name: relatedProducts +# description: Related Products +# +#paths: +# /products: +# get: +# tags: +# - product +# summary: Get all products list +# description: Get all products list. +# operationId: getProducts +# responses: +# "200": +# description: Success +# content: +# application/json: +# schema: +# type: array +# items: +# $ref: "#/components/schemas/Product" +# /products/{id}: +# get: +# tags: +# - product +# summary: Get product details +# description: Get product details with the given id. +# operationId: getProduct +# parameters: +# - name: id +# in: path +# description: Product id +# required: true +# schema: +# type: integer +# format: int64 +# responses: +# "200": +# description: successful operation +# content: +# application/json: +# schema: +# $ref: "#/components/schemas/ProductDetails" +# "400": +# description: Invalid product id +# post: +# tags: +# - product +# summary: Update product details +# description: Update product details with the given id. +# operationId: updateProduct +# parameters: +# - name: id +# in: path +# description: Product id +# required: true +# schema: +# type: integer +# format: int64 +# - name: name +# in: query +# description: Product name +# required: true +# schema: +# type: string +# responses: +# "200": +# description: successful operation +# delete: +# tags: +# - product +# summary: Delete product details +# description: Delete product details with the given id. +# operationId: deleteProduct +# parameters: +# - name: id +# in: path +# description: Product id +# required: true +# schema: +# type: integer +# format: int64 +# responses: +# "200": +# description: successful operation +# /products/{id}/relatedProducts: +# get: +# tags: +# - relatedProducts +# summary: Get related products +# description: Get related products with the given product id. +# operationId: getRelatedProducts +# parameters: +# - name: id +# in: path +# description: Product id +# required: true +# schema: +# type: integer +# format: int64 +# responses: +# "200": +# description: successful operation +# content: +# application/json: +# schema: +# $ref: "#/components/schemas/RelatedProducts" +# "400": +# description: Invalid product id +# +#components: +# schemas: +# Product: +# type: object +# description: Product id and name +# properties: +# id: +# type: integer +# format: int64 +# description: Product id +# name: +# type: string +# description: Product name +# required: +# - id +# - name +# ProductDetails: +# type: object +# description: Product details +# properties: +# id: +# type: integer +# format: int64 +# description: Product id +# name: +# type: string +# description: Product name +# description: +# type: string +# description: Product description +# required: +# - id +# - name +# RelatedProducts: +# type: object +# description: Related Products +# properties: +# id: +# type: integer +# format: int32 +# description: Product id +# relatedProducts: +# type: array +# description: List of related products +# items: +# $ref: "#/components/schemas/Product" \ No newline at end of file diff --git a/oap-server/server-core/src/test/resources/openapi-definitions/serviceA-api-v1/customerAPI-v1.yaml b/oap-server/server-core/src/test/resources/openapi-definitions/serviceA-api-v1/customerAPI-v1.yaml index 23bd2d311c41..dd58eae83b11 100644 --- a/oap-server/server-core/src/test/resources/openapi-definitions/serviceA-api-v1/customerAPI-v1.yaml +++ b/oap-server/server-core/src/test/resources/openapi-definitions/serviceA-api-v1/customerAPI-v1.yaml @@ -12,7 +12,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +# +# ${METHOD} is a reserved placeholder which represent HTTP method eg. `POST/GET...` +# ${PATH} is a reserved placeholder which represent the path eg. `/products/{id}`. +# +# | Extension Name | Required | Description | Default Value | +# |-----|-----|-----|-----| +# | x-sw-service-name | true | The service name which these endpoints belong | | +# | x-sw-endpoint-name-match-rule | false | The rule use to match the endpoint.| ${METHOD}:${PATH} | +# | x-sw-endpoint-name-format | false | The endpoint name after grouping.| ${METHOD}:${PATH} | +# +# These extensions are under `OpenAPI Object`. +# We highly recommend using the default config, the custom config would be considered as part of the match rules (regex pattern). openapi: 3.0.0 x-sw-service-name: serviceA x-sw-endpoint-name-match-rule: "<${METHOD}>:${PATH}" diff --git a/oap-server/server-core/src/test/resources/openapi-definitions/serviceA-api-v1/productAPI-v1.yaml b/oap-server/server-core/src/test/resources/openapi-definitions/serviceA-api-v1/productAPI-v1.yaml index 61d6c565df4a..01f795fc7af0 100644 --- a/oap-server/server-core/src/test/resources/openapi-definitions/serviceA-api-v1/productAPI-v1.yaml +++ b/oap-server/server-core/src/test/resources/openapi-definitions/serviceA-api-v1/productAPI-v1.yaml @@ -12,7 +12,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +# +# ${METHOD} is a reserved placeholder which represent HTTP method eg. `POST/GET...` +# ${PATH} is a reserved placeholder which represent the path eg. `/products/{id}`. +# +# | Extension Name | Required | Description | Default Value | +# |-----|-----|-----|-----| +# | x-sw-service-name | true | The service name which these endpoints belong | | +# | x-sw-endpoint-name-match-rule | false | The rule use to match the endpoint.| ${METHOD}:${PATH} | +# | x-sw-endpoint-name-format | false | The endpoint name after grouping.| ${METHOD}:${PATH} | +# +# These extensions are under `OpenAPI Object`. +# We highly recommend using the default config, the custom config would be considered as part of the match rules (regex pattern). openapi: 3.0.0 x-sw-service-name: serviceA diff --git a/oap-server/server-core/src/test/resources/openapi-definitions/serviceB-api-v2/productAPI-v2.yaml b/oap-server/server-core/src/test/resources/openapi-definitions/serviceB-api-v2/productAPI-v2.yaml index b502c83192c0..7c58e44a593d 100644 --- a/oap-server/server-core/src/test/resources/openapi-definitions/serviceB-api-v2/productAPI-v2.yaml +++ b/oap-server/server-core/src/test/resources/openapi-definitions/serviceB-api-v2/productAPI-v2.yaml @@ -12,7 +12,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -/* +# +# ${METHOD} is a reserved placeholder which represent HTTP method eg. `POST/GET...` +# ${PATH} is a reserved placeholder which represent the path eg. `/products/{id}`. +# +# | Extension Name | Required | Description | Default Value | +# |-----|-----|-----|-----| +# | x-sw-service-name | true | The service name which these endpoints belong | | +# | x-sw-endpoint-name-match-rule | false | The rule use to match the endpoint.| ${METHOD}:${PATH} | +# | x-sw-endpoint-name-format | false | The endpoint name after grouping.| ${METHOD}:${PATH} | +# +# These extensions are under `OpenAPI Object`. +# We highly recommend using the default config, the custom config would be considered as part of the match rules (regex pattern). + openapi: 3.0.0 x-sw-service-name: serviceB x-sw-endpoint-name-format: "${PATH}:<${METHOD}>" From 83007d86234c797dc2334b99ad0068ad0647f4d9 Mon Sep 17 00:00:00 2001 From: wankai123 Date: Fri, 18 Jun 2021 09:54:45 +0800 Subject: [PATCH 03/14] turn off enableEndpointNameGroupingByOpenapi --- oap-server/server-bootstrap/src/main/resources/application.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oap-server/server-bootstrap/src/main/resources/application.yml b/oap-server/server-bootstrap/src/main/resources/application.yml index 4c39a68b8e46..2988bf31da01 100755 --- a/oap-server/server-bootstrap/src/main/resources/application.yml +++ b/oap-server/server-bootstrap/src/main/resources/application.yml @@ -109,7 +109,7 @@ core: # The maximum number of processes supported for each synchronous storage operation. When the number of the flush data is greater than this value, it will be assigned to multiple cores for execution. maxSyncOperationNum: ${SW_CORE_MAX_SYNC_OPERATION_NUM:50000} # Turn it on then automatically grouping endpoint by the given OpenAPI definitions. - enableEndpointNameGroupingByOpenapi: ${SW_CORE_ENABLE_ENDPOINT_NAME_GROUPING_BY_OPAENAPI:true} + enableEndpointNameGroupingByOpenapi: ${SW_CORE_ENABLE_ENDPOINT_NAME_GROUPING_BY_OPAENAPI:false} storage: selector: ${SW_STORAGE:h2} elasticsearch: From a55560131a3dc444db79798fcc6e55fd0286fa01 Mon Sep 17 00:00:00 2001 From: wankai123 Date: Fri, 18 Jun 2021 10:22:10 +0800 Subject: [PATCH 04/14] polish docs --- docs/en/setup/backend/endpoint-grouping-rules.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/en/setup/backend/endpoint-grouping-rules.md b/docs/en/setup/backend/endpoint-grouping-rules.md index 4afd89805954..072f85ed31a2 100644 --- a/docs/en/setup/backend/endpoint-grouping-rules.md +++ b/docs/en/setup/backend/endpoint-grouping-rules.md @@ -58,8 +58,6 @@ If we have a OpenAPI definition doc `productAPI-v2.yaml` like this: ```yaml openapi: 3.0.0 -x-sw-service-name: serviceB -x-sw-endpoint-name-format: "${PATH}:<${METHOD}>" info: description: OpenAPI definition for SkyWalking test. From 73c887ff5d09cd1b4b6b4b2b08b7f0a378c4a5c0 Mon Sep 17 00:00:00 2001 From: wankai123 Date: Fri, 18 Jun 2021 10:41:18 +0800 Subject: [PATCH 05/14] polish docs --- docs/en/setup/backend/endpoint-grouping-rules.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/setup/backend/endpoint-grouping-rules.md b/docs/en/setup/backend/endpoint-grouping-rules.md index 072f85ed31a2..4a5682732aa1 100644 --- a/docs/en/setup/backend/endpoint-grouping-rules.md +++ b/docs/en/setup/backend/endpoint-grouping-rules.md @@ -33,7 +33,7 @@ SkyWalking now support `OAS v2.0+)`, could parse the documents `(yaml)` and buil We highly recommend using the default config, the custom config would be considered as part of the match rules (regex pattern). We provide some cases in `org.apache.skywalking.oap.server.core.config.group.openapi.EndpointGroupingRuleReader4OpenapiTest`, you could validate your custom config as well. -1. Put the OpenAPI definition documents into folder `openapi-definitions`, SkyWalking could read all documents or documents in subfolders from it，so you can organize these documents by yourself. For example: +2. Put the OpenAPI definition documents into folder `openapi-definitions`, SkyWalking could read all documents or documents in subfolders from it, so you can organize these documents by yourself. For example: ``` ├── openapi-definitions │   ├── serviceA-api-v1 @@ -304,7 +304,7 @@ info: | GET:/products/123 | serviceC | default | default | false | GET:/products/123 | | \:/products/123 | serviceB | \<\${METHOD}\>:\${PATH} | \<\${METHOD}>:\${PATH} | true | \:/products/{id} | | GET:/products/123 | serviceB | default | ${PATH}:\<\${METHOD}\> | true | /products/{id}:\ | - | /products/123: | serviceB | ${PATH}:\<\${METHOD}\> | default | true | \:/products/{id} | + | /products/123:\ | serviceB | ${PATH}:\<\${METHOD}\> | default | true | GET:/products/{id} |
From d41c5cc7116f944a39e4f451443feea3b0eda7c3 Mon Sep 17 00:00:00 2001 From: wankai123 Date: Fri, 18 Jun 2021 12:46:22 +0800 Subject: [PATCH 06/14] Apply suggestions from code review fix/polish some typo and codes Co-authored-by: kezhenxu94 --- .../setup/backend/endpoint-grouping-rules.md | 21 +++++++++---------- .../config/group/EndpointNameGrouping.java | 4 ++-- .../openapi/EndpointGroupingRule4Openapi.java | 6 +----- .../EndpointGroupingRuleReader4Openapi.java | 2 +- 4 files changed, 14 insertions(+), 19 deletions(-) diff --git a/docs/en/setup/backend/endpoint-grouping-rules.md b/docs/en/setup/backend/endpoint-grouping-rules.md index 4a5682732aa1..41a05bd83b14 100644 --- a/docs/en/setup/backend/endpoint-grouping-rules.md +++ b/docs/en/setup/backend/endpoint-grouping-rules.md @@ -20,13 +20,13 @@ SkyWalking now support `OAS v2.0+)`, could parse the documents `(yaml)` and buil ### How to use 1. Add some `Specification Extensions` for SkyWalking in the OpenAPI definition documents:
- \${METHOD} is a reserved placeholder which represent HTTP method eg. `POST/GET...`
- \${PATH} is a reserved placeholder which represent the path eg. `/products/{id}`. + \${METHOD} is a reserved placeholder which represents the HTTP method eg. `POST/GET...`
+ \${PATH} is a reserved placeholder which represents the path eg. `/products/{id}`. | Extension Name | Required | Description | Default Value | |-----|-----|-----|-----| - | x-sw-service-name | true | The service name which these endpoints belong | | - | x-sw-endpoint-name-match-rule | false | The rule use to match the endpoint.| \${METHOD}:\${PATH} | + | x-sw-service-name | true | The service name to which these endpoints belong | | + | x-sw-endpoint-name-match-rule | false | The rule used to match the endpoint.| \${METHOD}:\${PATH} | | x-sw-endpoint-name-format | false | The endpoint name after grouping.| \${METHOD}:\${PATH} | These extensions are under `OpenAPI Object`. @@ -42,19 +42,18 @@ SkyWalking now support `OAS v2.0+)`, could parse the documents `(yaml)` and buil │   └── serviceB-api-v2 │   └── productAPI-v2.yaml ``` -3. Turn the feature on by set the `Core Module` configuration `${SW_CORE_ENABLE_ENDPOINT_NAME_GROUPING_BY_OPAENAPI:true}` -
+3. Turn the feature on by setting the `Core Module` configuration `${SW_CORE_ENABLE_ENDPOINT_NAME_GROUPING_BY_OPAENAPI:true}` ### Rules match priority -We recommend designing the API path as clear as possible. If the API path is fuzzy and a endpoint name might match multiple paths, SkyWalking would follow the match priority to select one as below orders: +We recommend designing the API path as clear as possible. If the API path is fuzzy and an endpoint name might match multiple paths, SkyWalking would follow the match priority to select one as below orders: 1. The exact path matched first. Eg. `/products or /products/inventory` -2. The the path which has the less variables. +2. The path which has the less variables. Eg. `/products/{var1}/{var2} and /products/{var1}/abc`, endpoint name `/products/123/abc` will match the second one. 3. If the paths have the same number of variables, match the longest path, and the vars are considered to be `1`. - Eg. `/products/abc/{var1} and products/{var12345}/ef`, endpoint name `/products/abc/ef` will match the first one. + Eg. `/products/abc/{var1} and products/{var12345}/ef`, endpoint name `/products/abc/ef` will match the first one, because `length("abc") = 3` is larger than `length("ef") = 2`. ### Examples -If we have a OpenAPI definition doc `productAPI-v2.yaml` like this: +If we have an OpenAPI definition doc `productAPI-v2.yaml` like this: ```yaml openapi: 3.0.0 @@ -325,4 +324,4 @@ grouping: # Logic name when the regex expression matched. - endpoint-name: /prod/{id} regex: \/prod\/.+ -``` \ No newline at end of file +``` diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java index f5ad43d39770..8c460a1f056a 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java @@ -33,11 +33,11 @@ public class EndpointNameGrouping { public String format(String serviceName, String endpointName) { String formattedName = endpointName; if (endpointGroupingRule4Openapi != null) { - formattedName = formatByOpenapi(serviceName, formattedName.toString()); + formattedName = formatByOpenapi(serviceName, formattedName); } if (endpointGroupingRule != null) { - formattedName = formatByCustom(serviceName, formattedName.toString()); + formattedName = formatByCustom(serviceName, formattedName); } return formattedName; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRule4Openapi.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRule4Openapi.java index 52510fd5654e..ea3b0ebb6fd5 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRule4Openapi.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRule4Openapi.java @@ -102,11 +102,7 @@ public int compare(final StringFormatGroup.PatternRule rule1, final StringFormat int length1 = getPatternLength(pattern1); int length2 = getPatternLength(pattern2); - if (length1 != length2) { - return length2 - length1; - } - - return 0; + return length2 - length1; } private int getPatternVarsCount(String pattern) { diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java index 7ccf30c49330..afc942014e19 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java @@ -117,7 +117,7 @@ private String getGroupRegex(String pathString, String requstMathod, Map openapi return endPointNameMatchRule; } - private String formatEndPointName(String pathString, String requstMathod, Map openapiData) { + private String formatEndPointName(String pathString, String requstMethod, Map openapiData) { String endPointNameFormat = (String) openapiData.get("x-sw-endpoint-name-format"); if (!StringUtil.isEmpty(endPointNameFormat)) { From 5ac177e9594fcf5720fb2edeab5b8d415aba6ddc Mon Sep 17 00:00:00 2001 From: wankai123 Date: Fri, 18 Jun 2021 12:50:31 +0800 Subject: [PATCH 07/14] remove swagger dependency --- oap-server/server-core/pom.xml | 5 ----- .../group/openapi/EndpointGroupingRuleReader4Openapi.java | 4 ++-- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/oap-server/server-core/pom.xml b/oap-server/server-core/pom.xml index 382bf17a2ceb..1135336b56d0 100644 --- a/oap-server/server-core/pom.xml +++ b/oap-server/server-core/pom.xml @@ -82,11 +82,6 @@ io.vavr vavr - - io.swagger.parser.v3 - swagger-parser - 2.0.26 - org.apache.skywalking server-testing diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java index afc942014e19..5239fb36642a 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java @@ -121,10 +121,10 @@ private String formatEndPointName(String pathString, String requstMethod, Map op String endPointNameFormat = (String) openapiData.get("x-sw-endpoint-name-format"); if (!StringUtil.isEmpty(endPointNameFormat)) { - return replaceTemplateVars(endPointNameFormat, pathString, requstMathod); + return replaceTemplateVars(endPointNameFormat, pathString, requstMethod); } - return replaceTemplateVars(DEFAULT_ENDPOINT_NAME_FORMAT, pathString, requstMathod); + return replaceTemplateVars(DEFAULT_ENDPOINT_NAME_FORMAT, pathString, requstMethod); } private String replaceTemplateVars(String template, String pathString, String requstMathod) { From 8a4e0a3c13f0ac8d22e0540d453265dd83f23116 Mon Sep 17 00:00:00 2001 From: wankai123 Date: Fri, 18 Jun 2021 17:50:48 +0800 Subject: [PATCH 08/14] polish docs --- .../setup/backend/endpoint-grouping-rules.md | 60 +++++-------------- 1 file changed, 16 insertions(+), 44 deletions(-) diff --git a/docs/en/setup/backend/endpoint-grouping-rules.md b/docs/en/setup/backend/endpoint-grouping-rules.md index 41a05bd83b14..01cde71cf622 100644 --- a/docs/en/setup/backend/endpoint-grouping-rules.md +++ b/docs/en/setup/backend/endpoint-grouping-rules.md @@ -24,16 +24,28 @@ SkyWalking now support `OAS v2.0+)`, could parse the documents `(yaml)` and buil \${PATH} is a reserved placeholder which represents the path eg. `/products/{id}`. | Extension Name | Required | Description | Default Value | - |-----|-----|-----|-----| + |-----|-----|-----|-----| | x-sw-service-name | true | The service name to which these endpoints belong | | | x-sw-endpoint-name-match-rule | false | The rule used to match the endpoint.| \${METHOD}:\${PATH} | | x-sw-endpoint-name-format | false | The endpoint name after grouping.| \${METHOD}:\${PATH} | - These extensions are under `OpenAPI Object`. + These extensions are under `OpenAPI Object`. For example, the document below has a full custom config. +``` yaml +openapi: 3.0.0 +x-sw-service-name: serviceB +x-sw-endpoint-name-match-rule: "<${METHOD}>:${PATH}" +x-sw-endpoint-name-format: "<${METHOD}>:${PATH}" + +info: + description: OpenAPI definition for SkyWalking test. + version: v2 + title: Product API + ... +``` We highly recommend using the default config, the custom config would be considered as part of the match rules (regex pattern). We provide some cases in `org.apache.skywalking.oap.server.core.config.group.openapi.EndpointGroupingRuleReader4OpenapiTest`, you could validate your custom config as well. -2. Put the OpenAPI definition documents into folder `openapi-definitions`, SkyWalking could read all documents or documents in subfolders from it, so you can organize these documents by yourself. For example: +1. Put the OpenAPI definition documents into folder `openapi-definitions`, SkyWalking could read all documents or documents in subfolders from it, so you can organize these documents by yourself. For example: ``` ├── openapi-definitions │   ├── serviceA-api-v1 @@ -254,44 +266,7 @@ components: ``` -Here give some scenario we might use: -1. Only set the `x-sw-service-name`, `x-sw-endpoint-name-match-rule` and `x-sw-endpoint-name-format` are default: -``` yaml -openapi: 3.0.0 -x-sw-service-name: serviceB - -info: - description: OpenAPI definition for SkyWalking test. - version: v2 - title: Product API - ... -``` -2. Set the `x-sw-service-name` , `x-sw-endpoint-name-match-rule` and `x-sw-endpoint-name-format` : -``` yaml -openapi: 3.0.0 -x-sw-service-name: serviceB -x-sw-endpoint-name-match-rule: "<${METHOD}>:${PATH}" -x-sw-endpoint-name-format: "<${METHOD}>:${PATH}" - -info: - description: OpenAPI definition for SkyWalking test. - version: v2 - title: Product API - ... -``` -3. Set the `x-sw-service-name` , `x-sw-endpoint-name-match-rule` and `x-sw-endpoint-name-format` : -``` yaml -openapi: 3.0.0 -x-sw-service-name: serviceB -x-sw-endpoint-name-match-rule: "<${METHOD}>:${PATH}" -x-sw-endpoint-name-format: "<${METHOD}>:${PATH}" - -info: - description: OpenAPI definition for SkyWalking test. - version: v2 - title: Product API - ... -``` +Here are some cases: | Incoming Endpiont | Incoming Service | x-sw-endpoint-name-match-rule | x-sw-endpoint-name-format | Matched | Grouping Result | |-----|-----|-----|-----|-----|-----| @@ -304,9 +279,6 @@ info: | \:/products/123 | serviceB | \<\${METHOD}\>:\${PATH} | \<\${METHOD}>:\${PATH} | true | \:/products/{id} | | GET:/products/123 | serviceB | default | ${PATH}:\<\${METHOD}\> | true | /products/{id}:\ | | /products/123:\ | serviceB | ${PATH}:\<\${METHOD}\> | default | true | GET:/products/{id} | -
- - ## Endpoint name grouping by custom configuration From 2c3fc5f506aeb8281ad6614a2b2dc5725e7438ba Mon Sep 17 00:00:00 2001 From: wankai123 Date: Fri, 18 Jun 2021 18:31:16 +0800 Subject: [PATCH 09/14] polish doc use `` instead \ to escape characters --- .../setup/backend/endpoint-grouping-rules.md | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/docs/en/setup/backend/endpoint-grouping-rules.md b/docs/en/setup/backend/endpoint-grouping-rules.md index 01cde71cf622..d8568f69796b 100644 --- a/docs/en/setup/backend/endpoint-grouping-rules.md +++ b/docs/en/setup/backend/endpoint-grouping-rules.md @@ -20,16 +20,17 @@ SkyWalking now support `OAS v2.0+)`, could parse the documents `(yaml)` and buil ### How to use 1. Add some `Specification Extensions` for SkyWalking in the OpenAPI definition documents:
- \${METHOD} is a reserved placeholder which represents the HTTP method eg. `POST/GET...`
- \${PATH} is a reserved placeholder which represents the path eg. `/products/{id}`. + `${METHOD}` is a reserved placeholder which represents the HTTP method eg. `POST/GET...`
+ `${PATH}` is a reserved placeholder which represents the path eg. `/products/{id}`. | Extension Name | Required | Description | Default Value | |-----|-----|-----|-----| | x-sw-service-name | true | The service name to which these endpoints belong | | - | x-sw-endpoint-name-match-rule | false | The rule used to match the endpoint.| \${METHOD}:\${PATH} | - | x-sw-endpoint-name-format | false | The endpoint name after grouping.| \${METHOD}:\${PATH} | + | x-sw-endpoint-name-match-rule | false | The rule used to match the endpoint.| `${METHOD}:${PATH}` | + | x-sw-endpoint-name-format | false | The endpoint name after grouping.| `${METHOD}:${PATH}` | + + These extensions are under `OpenAPI Object`. For example, the document below has a full custom config: - These extensions are under `OpenAPI Object`. For example, the document below has a full custom config. ``` yaml openapi: 3.0.0 x-sw-service-name: serviceB @@ -42,6 +43,7 @@ info: title: Product API ... ``` + We highly recommend using the default config, the custom config would be considered as part of the match rules (regex pattern). We provide some cases in `org.apache.skywalking.oap.server.core.config.group.openapi.EndpointGroupingRuleReader4OpenapiTest`, you could validate your custom config as well. @@ -270,15 +272,15 @@ Here are some cases: | Incoming Endpiont | Incoming Service | x-sw-endpoint-name-match-rule | x-sw-endpoint-name-format | Matched | Grouping Result | |-----|-----|-----|-----|-----|-----| - | GET:/products | serviceB | default | default | true | GET:/products | - | GET:/products/123 | serviceB | default |default | true | GET:/products{id} | - | GET:/products/asia/cn | serviceB | default | default | true | GET:/products/{region}/{country} | - | GET:/products/123/abc/efg | serviceB | default |default | false | GET:/products/123/abc/efg | - | \:/products/123 | serviceB | default | default | false | \:/products/123| - | GET:/products/123 | serviceC | default | default | false | GET:/products/123 | - | \:/products/123 | serviceB | \<\${METHOD}\>:\${PATH} | \<\${METHOD}>:\${PATH} | true | \:/products/{id} | - | GET:/products/123 | serviceB | default | ${PATH}:\<\${METHOD}\> | true | /products/{id}:\ | - | /products/123:\ | serviceB | ${PATH}:\<\${METHOD}\> | default | true | GET:/products/{id} | + | `GET:/products` | serviceB | default | default | true | `GET:/products` | + | `GET:/products/123` | serviceB | default |default | true | `GET:/products{id}` | + | `GET:/products/asia/cn` | serviceB | default | default | true | `GET:/products/{region}/{country}` | + | `GET:/products/123/abc/efg` | serviceB | default |default | false | `GET:/products/123/abc/efg` | + | `:/products/123` | serviceB | default | default | false | `:/products/123`| + | `GET:/products/123` | serviceC | default | default | false | `GET:/products/123` | + | `:/products/123` | serviceB | `<${METHOD}>:${PATH}` | `<${METHOD}>:${PATH}` | true | <`GET>:/products/{id}` | + | `GET:/products/123` | serviceB | default | `${PATH}:<${METHOD}>` | true | `/products/{id}:` | + | `/products/123:` | serviceB | `${PATH}:<${METHOD}>` | default | true | `GET:/products/{id}` | ## Endpoint name grouping by custom configuration From 9912a51eabbddfd8a73b4c638a92b74d9b9ad3a3 Mon Sep 17 00:00:00 2001 From: wankai123 Date: Sat, 19 Jun 2021 16:29:19 +0800 Subject: [PATCH 10/14] set directory name as the default service name, set the file reader read the max depth 1 of the directory --- .../setup/backend/endpoint-grouping-rules.md | 39 ++-- .../productAPI-v1.yaml | 37 ++-- .../EndpointGroupingRuleReader4Openapi.java | 9 +- ...ndpointGroupingRuleReader4OpenapiTest.java | 10 +- .../customerAPI-v1.yaml | 15 +- .../productAPI-v1.yaml | 14 +- .../serviceA-1/customerAPI-a-1-v1.yaml | 171 ++++++++++++++++++ .../productAPI-v2.yaml | 13 -- .../server/library/util/ResourceUtils.java | 26 ++- 9 files changed, 249 insertions(+), 85 deletions(-) rename oap-server/server-bootstrap/src/main/resources/openapi-definitions/{serviceA-api-v1 => serviceA}/productAPI-v1.yaml (77%) rename oap-server/server-core/src/test/resources/openapi-definitions/{serviceA-api-v1 => serviceA}/customerAPI-v1.yaml (86%) rename oap-server/server-core/src/test/resources/openapi-definitions/{serviceA-api-v1 => serviceA}/productAPI-v1.yaml (86%) create mode 100644 oap-server/server-core/src/test/resources/openapi-definitions/serviceA/serviceA-1/customerAPI-a-1-v1.yaml rename oap-server/server-core/src/test/resources/openapi-definitions/{serviceB-api-v2 => serviceB}/productAPI-v2.yaml (88%) diff --git a/docs/en/setup/backend/endpoint-grouping-rules.md b/docs/en/setup/backend/endpoint-grouping-rules.md index d8568f69796b..6172d917ff4d 100644 --- a/docs/en/setup/backend/endpoint-grouping-rules.md +++ b/docs/en/setup/backend/endpoint-grouping-rules.md @@ -15,17 +15,17 @@ SkyWalking provides 2 ways to support endpoint grouping: The 2 grouping features can work together in sequence. ## Endpoint name grouping by OpenAPI definitions The OpenAPI definitions are the documents based on The [OpenAPI Specification (OAS)](https://github.com/OAI/OpenAPI-Specification) which used to define a standard, language-agnostic interface for HTTP APIs. -SkyWalking now support `OAS v2.0+)`, could parse the documents `(yaml)` and build the grouping rules from them automatically. +SkyWalking now support `OAS v2.0+`, could parse the documents `(yaml)` and build the grouping rules from them automatically. ### How to use -1. Add some `Specification Extensions` for SkyWalking in the OpenAPI definition documents:
+1. Add some `Specification Extensions` for SkyWalking config in the OpenAPI definition documents, otherwise, all configs are default:
`${METHOD}` is a reserved placeholder which represents the HTTP method eg. `POST/GET...`
`${PATH}` is a reserved placeholder which represents the path eg. `/products/{id}`. | Extension Name | Required | Description | Default Value | |-----|-----|-----|-----| - | x-sw-service-name | true | The service name to which these endpoints belong | | + | x-sw-service-name | false | The service name to which these endpoints belong | The directory name which the definition documents belong to| | x-sw-endpoint-name-match-rule | false | The rule used to match the endpoint.| `${METHOD}:${PATH}` | | x-sw-endpoint-name-format | false | The endpoint name after grouping.| `${METHOD}:${PATH}` | @@ -44,16 +44,16 @@ info: ... ``` - We highly recommend using the default config, the custom config would be considered as part of the match rules (regex pattern). + We highly recommend using the default config, the custom config (`x-sw-endpoint-name-match-rule/x-sw-endpoint-name-format`) would be considered as part of the match rules (regex pattern). We provide some cases in `org.apache.skywalking.oap.server.core.config.group.openapi.EndpointGroupingRuleReader4OpenapiTest`, you could validate your custom config as well. -1. Put the OpenAPI definition documents into folder `openapi-definitions`, SkyWalking could read all documents or documents in subfolders from it, so you can organize these documents by yourself. For example: +1. Put the OpenAPI definition documents into directory `openapi-definitions`, SkyWalking could read all documents or documents in this subDirectorys from it, you can organize these documents by yourself. Recommend using the service name as the subDirectory name then you are not necessary to set `x-sw-service-name`. For example: ``` ├── openapi-definitions -│   ├── serviceA-api-v1 +│   ├── serviceA │   │   ├── customerAPI-v1.yaml │   │   └── productAPI-v1.yaml -│   └── serviceB-api-v2 +│   └── serviceB │   └── productAPI-v2.yaml ``` 3. Turn the feature on by setting the `Core Module` configuration `${SW_CORE_ENABLE_ENDPOINT_NAME_GROUPING_BY_OPAENAPI:true}` @@ -67,7 +67,7 @@ We recommend designing the API path as clear as possible. If the API path is fuz 3. If the paths have the same number of variables, match the longest path, and the vars are considered to be `1`. Eg. `/products/abc/{var1} and products/{var12345}/ef`, endpoint name `/products/abc/ef` will match the first one, because `length("abc") = 3` is larger than `length("ef") = 2`. ### Examples -If we have an OpenAPI definition doc `productAPI-v2.yaml` like this: +If we have an OpenAPI definition doc `productAPI-v2.yaml` in directory `serviceB` like this: ```yaml openapi: 3.0.0 @@ -270,17 +270,18 @@ components: Here are some cases: - | Incoming Endpiont | Incoming Service | x-sw-endpoint-name-match-rule | x-sw-endpoint-name-format | Matched | Grouping Result | - |-----|-----|-----|-----|-----|-----| - | `GET:/products` | serviceB | default | default | true | `GET:/products` | - | `GET:/products/123` | serviceB | default |default | true | `GET:/products{id}` | - | `GET:/products/asia/cn` | serviceB | default | default | true | `GET:/products/{region}/{country}` | - | `GET:/products/123/abc/efg` | serviceB | default |default | false | `GET:/products/123/abc/efg` | - | `:/products/123` | serviceB | default | default | false | `:/products/123`| - | `GET:/products/123` | serviceC | default | default | false | `GET:/products/123` | - | `:/products/123` | serviceB | `<${METHOD}>:${PATH}` | `<${METHOD}>:${PATH}` | true | <`GET>:/products/{id}` | - | `GET:/products/123` | serviceB | default | `${PATH}:<${METHOD}>` | true | `/products/{id}:` | - | `/products/123:` | serviceB | `${PATH}:<${METHOD}>` | default | true | `GET:/products/{id}` | + | Incoming Endpiont | Incoming Service | x-sw-service-name | x-sw-endpoint-name-match-rule | x-sw-endpoint-name-format | Matched | Grouping Result | + |-----|-----|-----|-----|-----|-----|-----| + | `GET:/products` | serviceB | default | default | default | true | `GET:/products` | + | `GET:/products/123` | serviceB | default | default | default | true | `GET:/products{id}` | + | `GET:/products/asia/cn` | serviceB | default | default | default | true | `GET:/products/{region}/{country}` | + | `GET:/products/123/abc/efg` | serviceB | default | default | default | false | `GET:/products/123/abc/efg` | + | `:/products/123` | serviceB | default | default | default | false | `:/products/123`| + | `GET:/products/123` | serviceC | default | default | default | false | `GET:/products/123` | + | `GET:/products/123` | serviceC | serviceC | default | default | true | `GET:/products/123` | + | `:/products/123` | serviceB | default | `<${METHOD}>:${PATH}` | `<${METHOD}>:${PATH}` | true | `:/products/{id}` | + | `GET:/products/123` | serviceB | default | default | `${PATH}:<${METHOD}>` | true | `/products/{id}:` | + | `/products/123:` | serviceB | default | `${PATH}:<${METHOD}>` | default | true | `GET:/products/{id}` | ## Endpoint name grouping by custom configuration diff --git a/oap-server/server-bootstrap/src/main/resources/openapi-definitions/serviceA-api-v1/productAPI-v1.yaml b/oap-server/server-bootstrap/src/main/resources/openapi-definitions/serviceA/productAPI-v1.yaml similarity index 77% rename from oap-server/server-bootstrap/src/main/resources/openapi-definitions/serviceA-api-v1/productAPI-v1.yaml rename to oap-server/server-bootstrap/src/main/resources/openapi-definitions/serviceA/productAPI-v1.yaml index 06adcb176aa6..89e85ec01987 100644 --- a/oap-server/server-bootstrap/src/main/resources/openapi-definitions/serviceA-api-v1/productAPI-v1.yaml +++ b/oap-server/server-bootstrap/src/main/resources/openapi-definitions/serviceA/productAPI-v1.yaml @@ -14,19 +14,33 @@ # limitations under the License. # # -# ${METHOD} is a reserved placeholder which represent HTTP method eg. `POST/GET...` -# ${PATH} is a reserved placeholder which represent the path eg. `/products/{id}`. +# Add some `Specification Extensions` for SkyWalking config in the OpenAPI definition documents, otherwise, all configs are default: +# `${METHOD}` is a reserved placeholder which represents the HTTP method eg. `POST/GET...` +# `${PATH}` is a reserved placeholder which represents the path eg. `/products/{id}`. # -# | Extension Name | Required | Description | Default Value | -# |-----|-----|-----|-----| -# | x-sw-service-name | true | The service name which these endpoints belong | | -# | x-sw-endpoint-name-match-rule | false | The rule use to match the endpoint.| ${METHOD}:${PATH} | -# | x-sw-endpoint-name-format | false | The endpoint name after grouping.| ${METHOD}:${PATH} | +# | Extension Name | Required | Description | Default Value | +# |-----|-----|-----|-----| +# | x-sw-service-name | false | The service name to which these endpoints belong | The directory name which the definition documents belong to| +# | x-sw-endpoint-name-match-rule | false | The rule used to match the endpoint.| `${METHOD}:${PATH}` | +# | x-sw-endpoint-name-format | false | The endpoint name after grouping.| `${METHOD}:${PATH}` | # -# These extensions are under `OpenAPI Object`. -# We highly recommend using the default config, the custom config would be considered as part of the match rules (regex pattern). -# We provide some cases in org.apache.skywalking.oap.server.core.config.group.openapi.EndpointGroupingRuleReader4OpenapiTest, -# you could validate your custom config as well. +# These extensions are under `OpenAPI Object`. For example, the document below has a full custom config: +# +#``` yaml +#openapi: 3.0.0 +#x-sw-service-name: serviceB +#x-sw-endpoint-name-match-rule: "<${METHOD}>:${PATH}" +#x-sw-endpoint-name-format: "<${METHOD}>:${PATH}" +# +#info: +# description: OpenAPI definition for SkyWalking test. +# version: v2 +# title: Product API +# ... +#``` +# +# We highly recommend using the default config, the custom config (`x-sw-endpoint-name-match-rule/x-sw-endpoint-name-format`) would be considered as part of the match rules (regex pattern). +# We provide some cases in `org.apache.skywalking.oap.server.core.config.group.openapi.EndpointGroupingRuleReader4OpenapiTest`, you could validate your custom config as well. ################################################### @@ -35,7 +49,6 @@ #openapi: 3.0.0 -#x-sw-service-name: serviceA # #info: # description: OpenAPI definition for SkyWalking test. diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java index 5239fb36642a..f9f5f3ba47e4 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java @@ -57,8 +57,11 @@ public EndpointGroupingRuleReader4Openapi(final String openapiDefPath) { public EndpointGroupingRule4Openapi read() throws FileNotFoundException { EndpointGroupingRule4Openapi endpointGroupingRule = new EndpointGroupingRule4Openapi(); - List fileList = ResourceUtils.getPathFilesRecursive(openapiDefPath); + List fileList = ResourceUtils.getDirectoryFilesRecursive(openapiDefPath, 1); for (File file : fileList) { + if (!file.getName().endsWith(".yaml")) { + continue; + } Reader reader = new FileReader(file); Yaml yaml = new Yaml(new SafeConstructor()); Map openapiData = yaml.load(reader); @@ -92,8 +95,8 @@ public EndpointGroupingRule4Openapi read() throws FileNotFoundException { private String getServiceName(Map openapiData, File file) { String serviceName = (String) openapiData.get("x-sw-service-name"); if (StringUtil.isEmpty(serviceName)) { - throw new IllegalArgumentException( - "OpenAPI definition: " + file.getAbsolutePath() + " x-sw-service-name can't be empty"); + File directory = new File(file.getParent()); + serviceName = directory.getName(); } return serviceName; diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4OpenapiTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4OpenapiTest.java index 248d4dafcbda..b66df94c8cfe 100644 --- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4OpenapiTest.java +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4OpenapiTest.java @@ -33,7 +33,7 @@ public void testReadingRule() throws IOException { EndpointNameGrouping nameGrouping = new EndpointNameGrouping(); nameGrouping.setEndpointGroupingRule4Openapi(rule); - //default x-sw-endpoint-name-match-rule and x-sw-endpoint-name-format + //default x-sw-service-name x-sw-endpoint-name-match-rule and x-sw-endpoint-name-format // test direct lookup String endpointName = nameGrouping.format("serviceA", "GET:/products"); Assert.assertEquals("GET:/products", endpointName); @@ -50,11 +50,11 @@ public void testReadingRule() throws IOException { endpointName = nameGrouping.format("serviceA", "GET:/products/1/relatedProducts"); Assert.assertEquals("GET:/products/{id}/relatedProducts", endpointName); - //test same x-sw-endpoint-name-match-rule and x-sw-endpoint-name-format - endpointName = nameGrouping.format("serviceA", "POST:/customer"); + //test custom x-sw-service-name same x-sw-endpoint-name-match-rule and x-sw-endpoint-name-format + endpointName = nameGrouping.format("serviceA-1", "POST:/customer"); Assert.assertEquals("POST:/customer", endpointName); - endpointName = nameGrouping.format("serviceA", ":/customers/1"); + endpointName = nameGrouping.format("serviceA-1", ":/customers/1"); Assert.assertEquals(":/customers/{id}", endpointName); //test different x-sw-endpoint-name-match-rule and x-sw-endpoint-name-format @@ -81,7 +81,7 @@ public void testReadingRule() throws IOException { endpointName = nameGrouping.format("serviceA", "GET:/products/1/ratings/123"); Assert.assertEquals("GET:/products/1/ratings/123", endpointName); - endpointName = nameGrouping.format("serviceA", ":/customers/1/123"); + endpointName = nameGrouping.format("serviceA-1", ":/customers/1/123"); Assert.assertEquals(":/customers/1/123", endpointName); endpointName = nameGrouping.format("serviceB", "/products/:"); diff --git a/oap-server/server-core/src/test/resources/openapi-definitions/serviceA-api-v1/customerAPI-v1.yaml b/oap-server/server-core/src/test/resources/openapi-definitions/serviceA/customerAPI-v1.yaml similarity index 86% rename from oap-server/server-core/src/test/resources/openapi-definitions/serviceA-api-v1/customerAPI-v1.yaml rename to oap-server/server-core/src/test/resources/openapi-definitions/serviceA/customerAPI-v1.yaml index dd58eae83b11..20dc3f0db1a7 100644 --- a/oap-server/server-core/src/test/resources/openapi-definitions/serviceA-api-v1/customerAPI-v1.yaml +++ b/oap-server/server-core/src/test/resources/openapi-definitions/serviceA/customerAPI-v1.yaml @@ -12,20 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# ${METHOD} is a reserved placeholder which represent HTTP method eg. `POST/GET...` -# ${PATH} is a reserved placeholder which represent the path eg. `/products/{id}`. -# -# | Extension Name | Required | Description | Default Value | -# |-----|-----|-----|-----| -# | x-sw-service-name | true | The service name which these endpoints belong | | -# | x-sw-endpoint-name-match-rule | false | The rule use to match the endpoint.| ${METHOD}:${PATH} | -# | x-sw-endpoint-name-format | false | The endpoint name after grouping.| ${METHOD}:${PATH} | -# -# These extensions are under `OpenAPI Object`. -# We highly recommend using the default config, the custom config would be considered as part of the match rules (regex pattern). + openapi: 3.0.0 -x-sw-service-name: serviceA +x-sw-service-name: serviceA-1 x-sw-endpoint-name-match-rule: "<${METHOD}>:${PATH}" x-sw-endpoint-name-format: "<${METHOD}>:${PATH}" info: diff --git a/oap-server/server-core/src/test/resources/openapi-definitions/serviceA-api-v1/productAPI-v1.yaml b/oap-server/server-core/src/test/resources/openapi-definitions/serviceA/productAPI-v1.yaml similarity index 86% rename from oap-server/server-core/src/test/resources/openapi-definitions/serviceA-api-v1/productAPI-v1.yaml rename to oap-server/server-core/src/test/resources/openapi-definitions/serviceA/productAPI-v1.yaml index 01f795fc7af0..2a6d5722ad3f 100644 --- a/oap-server/server-core/src/test/resources/openapi-definitions/serviceA-api-v1/productAPI-v1.yaml +++ b/oap-server/server-core/src/test/resources/openapi-definitions/serviceA/productAPI-v1.yaml @@ -12,20 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# ${METHOD} is a reserved placeholder which represent HTTP method eg. `POST/GET...` -# ${PATH} is a reserved placeholder which represent the path eg. `/products/{id}`. -# -# | Extension Name | Required | Description | Default Value | -# |-----|-----|-----|-----| -# | x-sw-service-name | true | The service name which these endpoints belong | | -# | x-sw-endpoint-name-match-rule | false | The rule use to match the endpoint.| ${METHOD}:${PATH} | -# | x-sw-endpoint-name-format | false | The endpoint name after grouping.| ${METHOD}:${PATH} | -# -# These extensions are under `OpenAPI Object`. -# We highly recommend using the default config, the custom config would be considered as part of the match rules (regex pattern). + openapi: 3.0.0 -x-sw-service-name: serviceA info: description: OpenAPI definition for SkyWalking test. diff --git a/oap-server/server-core/src/test/resources/openapi-definitions/serviceA/serviceA-1/customerAPI-a-1-v1.yaml b/oap-server/server-core/src/test/resources/openapi-definitions/serviceA/serviceA-1/customerAPI-a-1-v1.yaml new file mode 100644 index 000000000000..d4a60a532d88 --- /dev/null +++ b/oap-server/server-core/src/test/resources/openapi-definitions/serviceA/serviceA-1/customerAPI-a-1-v1.yaml @@ -0,0 +1,171 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +openapi: 3.0.0 +x-sw-service-name: serviceA + +info: + description: OpenAPI definition for SkyWalking test. + version: v1 + title: Customer API + +tags: + - name: customer + description: customer + +paths: + /customers: + get: + tags: + - customer + summary: Get all customers list + description: Get all customers list. + operationId: getCustomers + responses: + "200": + description: Success + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Customer" + /customers/{id}: + get: + tags: + - customer + summary: Get customer details + description: Get customer details with the given id. + operationId: getCustomer + parameters: + - name: id + in: path + description: Customer id + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/CustomerDetails" + "400": + description: Invalid customer id + post: + tags: + - customer + summary: Update customer details + description: Update customer details with the given id. + operationId: updateCustomer + parameters: + - name: id + in: path + description: Customer id + required: true + schema: + type: integer + format: int64 + - name: name + in: query + description: Customer name + required: true + schema: + type: string + responses: + "200": + description: successful operation + delete: + tags: + - customer + summary: Delete customer details + description: Delete customer details with the given id. + operationId: deleteCustomer + parameters: + - name: id + in: path + description: Customer id + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: successful operation + + /customer/{region}/{country}: + get: + tags: + - customer + summary: Get customers regional + description: Get customers regional with the given id. + operationId: getCustomersRegional + parameters: + - name: region + in: path + description: Customers region + required: true + schema: + type: string + - name: country + in: path + description: Customers country + required: true + schema: + type: string + responses: + "200": + description: successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/Customer" + "400": + description: Invalid parameters supplied +components: + schemas: + Customer: + type: object + description: Customer id and name + properties: + id: + type: integer + format: int64 + description: Customer id + name: + type: string + description: Customer name + required: + - id + - name + CustomerDetails: + type: object + description: Customer details + properties: + id: + type: integer + format: int64 + description: Customer id + name: + type: string + description: Customer name + description: + type: string + description: Customer description + required: + - id + - name diff --git a/oap-server/server-core/src/test/resources/openapi-definitions/serviceB-api-v2/productAPI-v2.yaml b/oap-server/server-core/src/test/resources/openapi-definitions/serviceB/productAPI-v2.yaml similarity index 88% rename from oap-server/server-core/src/test/resources/openapi-definitions/serviceB-api-v2/productAPI-v2.yaml rename to oap-server/server-core/src/test/resources/openapi-definitions/serviceB/productAPI-v2.yaml index 7c58e44a593d..b30a8b36654f 100644 --- a/oap-server/server-core/src/test/resources/openapi-definitions/serviceB-api-v2/productAPI-v2.yaml +++ b/oap-server/server-core/src/test/resources/openapi-definitions/serviceB/productAPI-v2.yaml @@ -12,21 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# ${METHOD} is a reserved placeholder which represent HTTP method eg. `POST/GET...` -# ${PATH} is a reserved placeholder which represent the path eg. `/products/{id}`. -# -# | Extension Name | Required | Description | Default Value | -# |-----|-----|-----|-----| -# | x-sw-service-name | true | The service name which these endpoints belong | | -# | x-sw-endpoint-name-match-rule | false | The rule use to match the endpoint.| ${METHOD}:${PATH} | -# | x-sw-endpoint-name-format | false | The endpoint name after grouping.| ${METHOD}:${PATH} | -# -# These extensions are under `OpenAPI Object`. -# We highly recommend using the default config, the custom config would be considered as part of the match rules (regex pattern). openapi: 3.0.0 -x-sw-service-name: serviceB x-sw-endpoint-name-format: "${PATH}:<${METHOD}>" info: diff --git a/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/ResourceUtils.java b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/ResourceUtils.java index a327aadc0cc4..93b26207a7d0 100644 --- a/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/ResourceUtils.java +++ b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/ResourceUtils.java @@ -70,23 +70,35 @@ public static File[] getPathFiles(String parentPath, String[] fileNames) throws return listFiles; } - public static List getPathFilesRecursive(String path) throws FileNotFoundException { - URL url = ResourceUtils.class.getClassLoader().getResource(path); + /** + * @param directoryPath the directory path + * @param maxDepth the max directory depth to get the files, the given directory is 0 as the tree root + * @return all normal files which in this directory and subDirectory according to the maxDepth + * @throws FileNotFoundException the directory not exist in the given path + */ + public static List getDirectoryFilesRecursive(String directoryPath, + int maxDepth) throws FileNotFoundException { + + URL url = ResourceUtils.class.getClassLoader().getResource(directoryPath); if (url == null) { - throw new FileNotFoundException("path not found: " + path); + throw new FileNotFoundException("path not found: " + directoryPath); } List fileList = new ArrayList<>(); - return getPathFilesRecursive(url.getPath(), fileList); + return getDirectoryFilesRecursive(url.getPath(), fileList, maxDepth); } - private static List getPathFilesRecursive(String filepath, List fileList) { - File file = new File(filepath); + private static List getDirectoryFilesRecursive(String directoryPath, List fileList, int maxDepth) { + if (maxDepth < 0) { + return fileList; + } + maxDepth--; + File file = new File(directoryPath); if (file.isDirectory()) { File[] subFiles = file.listFiles(); if (subFiles != null) { for (File subFile : subFiles) { if (subFile.isDirectory()) { - getPathFilesRecursive(subFile.getPath(), fileList); + getDirectoryFilesRecursive(subFile.getPath(), fileList, maxDepth); } else { fileList.add(subFile); } From 90d7167e8063e258bdbda63a88414a93738f7be1 Mon Sep 17 00:00:00 2001 From: wankai123 Date: Sat, 19 Jun 2021 22:43:21 +0800 Subject: [PATCH 11/14] set this feature enabled by default, throw exception if openAPI file doesn't set service name and in root folder directly --- docs/en/setup/backend/configuration-vocabulary.md | 2 +- docs/en/setup/backend/endpoint-grouping-rules.md | 6 +++--- .../server-bootstrap/src/main/resources/application.yml | 2 +- .../group/openapi/EndpointGroupingRuleReader4Openapi.java | 5 +++++ 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/en/setup/backend/configuration-vocabulary.md b/docs/en/setup/backend/configuration-vocabulary.md index 1132aa2bbb42..33f406a2c9c8 100644 --- a/docs/en/setup/backend/configuration-vocabulary.md +++ b/docs/en/setup/backend/configuration-vocabulary.md @@ -42,7 +42,7 @@ core|default|role|Option values, `Mixed/Receiver/Aggregator`. **Receiver** mode | - | - | maxSizeOfAnalyzeProfileSnapshot|The max number of snapshots analyzed by OAP| - | 12000 | | - | - | syncThreads|The number of threads used to synchronously refresh the metrics data to the storage.| SW_CORE_SYNC_THREADS | 2 | | - | - | maxSyncOperationNum|The maximum number of processes supported for each synchronous storage operation. When the number of the flush data is greater than this value, it will be assigned to multiple cores for execution.| SW_CORE_MAX_SYNC_OPERATION_NUM | 50000 | -| - | - | enableEndpointNameGroupingByOpenapi |Turn it on then automatically grouping endpoint by the given OpenAPI definitions.| SW_CORE_ENABLE_ENDPOINT_NAME_GROUPING_BY_OPAENAPI | false | +| - | - | enableEndpointNameGroupingByOpenapi |Turn it on then automatically grouping endpoint by the given OpenAPI definitions.| SW_CORE_ENABLE_ENDPOINT_NAME_GROUPING_BY_OPAENAPI | true | |cluster|standalone| - | standalone is not suitable for one node running, no available configuration.| - | - | | - | zookeeper|nameSpace|The namespace, represented by root path, isolates the configurations in the zookeeper.|SW_NAMESPACE| `/`, root path| | - | - | hostPort|hosts and ports of Zookeeper Cluster|SW_CLUSTER_ZK_HOST_PORT| localhost:2181| diff --git a/docs/en/setup/backend/endpoint-grouping-rules.md b/docs/en/setup/backend/endpoint-grouping-rules.md index 6172d917ff4d..05351dd60765 100644 --- a/docs/en/setup/backend/endpoint-grouping-rules.md +++ b/docs/en/setup/backend/endpoint-grouping-rules.md @@ -25,7 +25,7 @@ SkyWalking now support `OAS v2.0+`, could parse the documents `(yaml)` and build | Extension Name | Required | Description | Default Value | |-----|-----|-----|-----| - | x-sw-service-name | false | The service name to which these endpoints belong | The directory name which the definition documents belong to| + | x-sw-service-name | false | The service name which these endpoints belong to | The directory name which the OpenAPI definition documents belong to | | x-sw-endpoint-name-match-rule | false | The rule used to match the endpoint.| `${METHOD}:${PATH}` | | x-sw-endpoint-name-format | false | The endpoint name after grouping.| `${METHOD}:${PATH}` | @@ -47,7 +47,7 @@ info: We highly recommend using the default config, the custom config (`x-sw-endpoint-name-match-rule/x-sw-endpoint-name-format`) would be considered as part of the match rules (regex pattern). We provide some cases in `org.apache.skywalking.oap.server.core.config.group.openapi.EndpointGroupingRuleReader4OpenapiTest`, you could validate your custom config as well. -1. Put the OpenAPI definition documents into directory `openapi-definitions`, SkyWalking could read all documents or documents in this subDirectorys from it, you can organize these documents by yourself. Recommend using the service name as the subDirectory name then you are not necessary to set `x-sw-service-name`. For example: +2. All OpenAPI definition documents should be located in the `openapi-definitions` directory, with at most two levels directories. Recommend using the service name as the subDirectory name then you are not required to set `x-sw-service-name`. For example: ``` ├── openapi-definitions │   ├── serviceA @@ -56,7 +56,7 @@ info: │   └── serviceB │   └── productAPI-v2.yaml ``` -3. Turn the feature on by setting the `Core Module` configuration `${SW_CORE_ENABLE_ENDPOINT_NAME_GROUPING_BY_OPAENAPI:true}` +3. The feature is enabled by default, you can disable it by setting the `Core Module` configuration `${SW_CORE_ENABLE_ENDPOINT_NAME_GROUPING_BY_OPAENAPI:false}` ### Rules match priority We recommend designing the API path as clear as possible. If the API path is fuzzy and an endpoint name might match multiple paths, SkyWalking would follow the match priority to select one as below orders: diff --git a/oap-server/server-bootstrap/src/main/resources/application.yml b/oap-server/server-bootstrap/src/main/resources/application.yml index 2988bf31da01..4c39a68b8e46 100755 --- a/oap-server/server-bootstrap/src/main/resources/application.yml +++ b/oap-server/server-bootstrap/src/main/resources/application.yml @@ -109,7 +109,7 @@ core: # The maximum number of processes supported for each synchronous storage operation. When the number of the flush data is greater than this value, it will be assigned to multiple cores for execution. maxSyncOperationNum: ${SW_CORE_MAX_SYNC_OPERATION_NUM:50000} # Turn it on then automatically grouping endpoint by the given OpenAPI definitions. - enableEndpointNameGroupingByOpenapi: ${SW_CORE_ENABLE_ENDPOINT_NAME_GROUPING_BY_OPAENAPI:false} + enableEndpointNameGroupingByOpenapi: ${SW_CORE_ENABLE_ENDPOINT_NAME_GROUPING_BY_OPAENAPI:true} storage: selector: ${SW_STORAGE:h2} elasticsearch: diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java index f9f5f3ba47e4..ab945e7aea7a 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java @@ -93,9 +93,14 @@ public EndpointGroupingRule4Openapi read() throws FileNotFoundException { } private String getServiceName(Map openapiData, File file) { + String serviceName = (String) openapiData.get("x-sw-service-name"); if (StringUtil.isEmpty(serviceName)) { File directory = new File(file.getParent()); + if (openapiDefPath.equals(directory.getName())) { + throw new IllegalArgumentException( + "OpenAPI definition file: " + file.getAbsolutePath() + " found in root directory, but doesn't include x-sw-service-name extensive definition in the file."); + } serviceName = directory.getName(); } From 026a6d5e783a78c7b47b324766c7c80d991222c3 Mon Sep 17 00:00:00 2001 From: wankai123 Date: Sat, 19 Jun 2021 22:47:22 +0800 Subject: [PATCH 12/14] polish doc --- docs/en/setup/backend/endpoint-grouping-rules.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/setup/backend/endpoint-grouping-rules.md b/docs/en/setup/backend/endpoint-grouping-rules.md index 05351dd60765..9aee0680bd30 100644 --- a/docs/en/setup/backend/endpoint-grouping-rules.md +++ b/docs/en/setup/backend/endpoint-grouping-rules.md @@ -34,8 +34,8 @@ SkyWalking now support `OAS v2.0+`, could parse the documents `(yaml)` and build ``` yaml openapi: 3.0.0 x-sw-service-name: serviceB -x-sw-endpoint-name-match-rule: "<${METHOD}>:${PATH}" -x-sw-endpoint-name-format: "<${METHOD}>:${PATH}" +x-sw-endpoint-name-match-rule: "${METHOD}:${PATH}" +x-sw-endpoint-name-format: "${METHOD}:${PATH}" info: description: OpenAPI definition for SkyWalking test. From 814eb19a8dd46fff3aed472517353ed287e8cdd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E6=99=9F=20Wu=20Sheng?= Date: Sat, 19 Jun 2021 23:14:53 +0800 Subject: [PATCH 13/14] Update docs/en/setup/backend/endpoint-grouping-rules.md --- docs/en/setup/backend/endpoint-grouping-rules.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/setup/backend/endpoint-grouping-rules.md b/docs/en/setup/backend/endpoint-grouping-rules.md index 9aee0680bd30..e4eda52b5115 100644 --- a/docs/en/setup/backend/endpoint-grouping-rules.md +++ b/docs/en/setup/backend/endpoint-grouping-rules.md @@ -14,7 +14,7 @@ SkyWalking provides 2 ways to support endpoint grouping: The 2 grouping features can work together in sequence. ## Endpoint name grouping by OpenAPI definitions -The OpenAPI definitions are the documents based on The [OpenAPI Specification (OAS)](https://github.com/OAI/OpenAPI-Specification) which used to define a standard, language-agnostic interface for HTTP APIs. +The OpenAPI definitions are the documents based on The [OpenAPI Specification (OAS)](https://www.openapis.org/) which used to define a standard, language-agnostic interface for HTTP APIs. SkyWalking now support `OAS v2.0+`, could parse the documents `(yaml)` and build the grouping rules from them automatically. From e014c0547862af0844aa97f089b9969b533558a1 Mon Sep 17 00:00:00 2001 From: wankai123 Date: Sun, 20 Jun 2021 00:14:14 +0800 Subject: [PATCH 14/14] polish codes --- .../EndpointGroupingBenchmark4Openapi.java | 82 +++++++++---------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingBenchmark4Openapi.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingBenchmark4Openapi.java index de1fa377ed74..d2ffdf3dea0f 100644 --- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingBenchmark4Openapi.java +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingBenchmark4Openapi.java @@ -44,11 +44,11 @@ public static class FormatClassPaths20 { public FormatClassPaths20() { rule = new EndpointGroupingRule4Openapi(); for (int i = 0; i <= 3; i++) { - rule.addGroupedRule("serviceA", "GET:/products1/{id}/" + +i, "GET:/products1/([^/]+)/" + i); - rule.addGroupedRule("serviceA", "POST:/products1/{id}/" + +i, "POST:/products1/([^/]+)/" + i); - rule.addGroupedRule("serviceA", "GET:/products2/{id}/" + +i, "GET:/products2/([^/]+)/" + i); - rule.addGroupedRule("serviceA", "POST:/products3/{id}/" + +i, "POST:/products3/([^/]+)/" + i); - rule.addGroupedRule("serviceA", "GET:/products3/{id}/" + +i, "GET:/products3/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "GET:/products1/{id}/" + i, "GET:/products1/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "POST:/products1/{id}/" + i, "POST:/products1/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "GET:/products2/{id}/" + i, "GET:/products2/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "POST:/products3/{id}/" + i, "POST:/products3/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "GET:/products3/{id}/" + i, "GET:/products3/([^/]+)/" + i); } } @@ -65,11 +65,11 @@ public static class FormatClassPaths50 { public FormatClassPaths50() { rule = new EndpointGroupingRule4Openapi(); for (int i = 0; i <= 9; i++) { - rule.addGroupedRule("serviceA", "GET:/products1/{id}/" + +i, "GET:/products1/([^/]+)/" + i); - rule.addGroupedRule("serviceA", "POST:/products1/{id}/" + +i, "POST:/products1/([^/]+)/" + i); - rule.addGroupedRule("serviceA", "GET:/products2/{id}/" + +i, "GET:/products2/([^/]+)/" + i); - rule.addGroupedRule("serviceA", "POST:/products3/{id}/" + +i, "POST:/products3/([^/]+)/" + i); - rule.addGroupedRule("serviceA", "GET:/products3/{id}/" + +i, "GET:/products3/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "GET:/products1/{id}/" + i, "GET:/products1/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "POST:/products1/{id}/" + i, "POST:/products1/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "GET:/products2/{id}/" + i, "GET:/products2/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "POST:/products3/{id}/" + i, "POST:/products3/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "GET:/products3/{id}/" + i, "GET:/products3/([^/]+)/" + i); } } @@ -86,11 +86,11 @@ public static class FormatClassPaths200 { public FormatClassPaths200() { rule = new EndpointGroupingRule4Openapi(); for (int i = 0; i <= 39; i++) { - rule.addGroupedRule("serviceA", "GET:/products1/{id}/" + +i, "GET:/products1/([^/]+)/" + i); - rule.addGroupedRule("serviceA", "POST:/products1/{id}/" + +i, "POST:/products1/([^/]+)/" + i); - rule.addGroupedRule("serviceA", "GET:/products2/{id}/" + +i, "GET:/products2/([^/]+)/" + i); - rule.addGroupedRule("serviceA", "POST:/products3/{id}/" + +i, "POST:/products3/([^/]+)/" + i); - rule.addGroupedRule("serviceA", "GET:/products3/{id}/" + +i, "GET:/products3/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "GET:/products1/{id}/" + i, "GET:/products1/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "POST:/products1/{id}/" + i, "POST:/products1/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "GET:/products2/{id}/" + i, "GET:/products2/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "POST:/products3/{id}/" + i, "POST:/products3/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "GET:/products3/{id}/" + i, "GET:/products3/([^/]+)/" + i); } } @@ -141,32 +141,32 @@ public static void main(String[] args) throws RunnerException, FileNotFoundExcep # Threads: 4 threads, will synchronize iterations # Benchmark mode: Throughput, ops/time -Benchmark Mode Cnt Score Error Units -EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20 thrpt 5 4052121.622 ± 427561.892 ops/s -EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.alloc.rate thrpt 5 4386.461 ± 465.129 MB/sec +Benchmark Mode Cnt Score Error Units +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20 thrpt 5 4180207.544 ± 833644.395 ops/s +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.alloc.rate thrpt 5 4524.954 ± 903.291 MB/sec EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.alloc.rate.norm thrpt 5 1192.000 ± 0.001 B/op -EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.churn.PS_Eden_Space thrpt 5 4411.321 ± 445.136 MB/sec -EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.churn.PS_Eden_Space.norm thrpt 5 1198.792 ± 6.537 B/op -EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.churn.PS_Survivor_Space thrpt 5 0.475 ± 0.164 MB/sec -EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.churn.PS_Survivor_Space.norm thrpt 5 0.129 ± 0.036 B/op -EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.count thrpt 5 1367.000 counts -EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.time thrpt 5 777.000 ms -EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200 thrpt 5 494580.800 ± 13430.644 ops/s -EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.alloc.rate thrpt 5 3510.759 ± 95.418 MB/sec +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.churn.PS_Eden_Space thrpt 5 4550.511 ± 916.117 MB/sec +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.churn.PS_Eden_Space.norm thrpt 5 1198.713 ± 10.572 B/op +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.churn.PS_Survivor_Space thrpt 5 0.493 ± 0.118 MB/sec +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.churn.PS_Survivor_Space.norm thrpt 5 0.130 ± 0.039 B/op +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.count thrpt 5 1410.000 counts +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.time thrpt 5 783.000 ms +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200 thrpt 5 600313.461 ± 58702.201 ops/s +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.alloc.rate thrpt 5 4260.484 ± 415.215 MB/sec EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.alloc.rate.norm thrpt 5 7816.000 ± 0.001 B/op -EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.churn.PS_Eden_Space thrpt 5 3531.052 ± 101.381 MB/sec -EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.churn.PS_Eden_Space.norm thrpt 5 7861.170 ± 52.430 B/op -EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.churn.PS_Survivor_Space thrpt 5 0.289 ± 0.092 MB/sec -EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.churn.PS_Survivor_Space.norm thrpt 5 0.644 ± 0.217 B/op -EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.count thrpt 5 1094.000 counts -EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.time thrpt 5 604.000 ms -EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50 thrpt 5 1834653.371 ± 128456.874 ops/s -EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.alloc.rate thrpt 5 3784.855 ± 263.420 MB/sec -EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.alloc.rate.norm thrpt 5 2272.000 ± 0.001 B/op -EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.churn.PS_Eden_Space thrpt 5 3807.122 ± 288.032 MB/sec -EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.churn.PS_Eden_Space.norm thrpt 5 2285.315 ± 16.947 B/op -EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.churn.PS_Survivor_Space thrpt 5 0.365 ± 0.080 MB/sec -EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.churn.PS_Survivor_Space.norm thrpt 5 0.219 ± 0.047 B/op -EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.count thrpt 5 1180.000 counts -EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.time thrpt 5 690.000 ms +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.churn.PS_Eden_Space thrpt 5 4285.685 ± 407.822 MB/sec +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.churn.PS_Eden_Space.norm thrpt 5 7862.339 ± 46.737 B/op +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.churn.PS_Survivor_Space thrpt 5 0.444 ± 0.061 MB/sec +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.churn.PS_Survivor_Space.norm thrpt 5 0.815 ± 0.062 B/op +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.count thrpt 5 1328.000 counts +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.time thrpt 5 729.000 ms +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50 thrpt 5 2001647.224 ± 139386.146 ops/s +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.alloc.rate thrpt 5 4173.062 ± 291.166 MB/sec +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.alloc.rate.norm thrpt 5 2296.000 ± 0.001 B/op +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.churn.PS_Eden_Space thrpt 5 4198.202 ± 271.551 MB/sec +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.churn.PS_Eden_Space.norm thrpt 5 2309.878 ± 14.994 B/op +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.churn.PS_Survivor_Space thrpt 5 0.393 ± 0.171 MB/sec +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.churn.PS_Survivor_Space.norm thrpt 5 0.216 ± 0.086 B/op +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.count thrpt 5 1301.000 counts +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.time thrpt 5 715.000 ms */ \ No newline at end of file