Skip to content

Commit

Permalink
Polish config, expose rpc config in application.yml
Browse files Browse the repository at this point in the history
  • Loading branch information
ruanwenjun committed Jul 10, 2023
1 parent 406aeaf commit a8324f9
Show file tree
Hide file tree
Showing 24 changed files with 298 additions and 298 deletions.
10 changes: 5 additions & 5 deletions docs/docs/en/architecture/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -259,11 +259,11 @@ Location: `api-server/conf/application.yaml`
|security.authentication.ldap.ssl.enable|false|LDAP switch|
|security.authentication.ldap.ssl.trust-store|ldapkeystore.jks|LDAP jks file absolute path|
|security.authentication.ldap.ssl.trust-store-password|password|LDAP jks password|
|traffic.control.global.switch|false|traffic control global switch|
|traffic.control.max-global-qps-rate|300|global max request number per second|
|traffic.control.tenant-switch|false|traffic control tenant switch|
|traffic.control.default-tenant-qps-rate|10|default tenant max request number per second|
|traffic.control.customize-tenant-qps-rate||customize tenant max request number per second|
|api.traffic.control.global.switch|false|traffic control global switch|
|api.traffic.control.max-global-qps-rate|300|global max request number per second|
|api.traffic.control.tenant-switch|false|traffic control tenant switch|
|api.traffic.control.default-tenant-qps-rate|10|default tenant max request number per second|
|api.traffic.control.customize-tenant-qps-rate||customize tenant max request number per second|

### Master Server related configuration

Expand Down
10 changes: 5 additions & 5 deletions docs/docs/zh/architecture/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -257,11 +257,11 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId
|security.authentication.ldap.ssl.enable|false|LDAP ssl开关|
|security.authentication.ldap.ssl.trust-store|ldapkeystore.jks|LDAP jks文件绝对路径|
|security.authentication.ldap.ssl.trust-store-password|password|LDAP jks密码|
|traffic.control.global.switch|false|流量控制全局开关|
|traffic.control.max-global-qps-rate|300|全局最大请求数/秒|
|traffic.control.tenant-switch|false|流量控制租户开关|
|traffic.control.default-tenant-qps-rate|10|默认租户最大请求数/秒限制|
|traffic.control.customize-tenant-qps-rate||自定义租户最大请求数/秒限制|
|api.traffic.control.global.switch|false|流量控制全局开关|
|api.traffic.control.max-global-qps-rate|300|全局最大请求数/秒|
|api.traffic.control.tenant-switch|false|流量控制租户开关|
|api.traffic.control.default-tenant-qps-rate|10|默认租户最大请求数/秒限制|
|api.traffic.control.customize-tenant-qps-rate||自定义租户最大请求数/秒限制|

## Master Server相关配置

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

package org.apache.dolphinscheduler.api.audit;

import org.apache.dolphinscheduler.api.configuration.AuditConfiguration;
import org.apache.dolphinscheduler.api.configuration.ApiConfig;

import java.util.List;
import java.util.concurrent.BlockingQueue;
Expand All @@ -34,20 +34,20 @@
@Slf4j
public class AuditPublishService {

private BlockingQueue<AuditMessage> auditMessageQueue = new LinkedBlockingQueue<>();
private final BlockingQueue<AuditMessage> auditMessageQueue = new LinkedBlockingQueue<>();

@Autowired
private List<AuditSubscriber> subscribers;

@Autowired
private AuditConfiguration auditConfiguration;
private ApiConfig apiConfig;

/**
* create a daemon thread to process the message queue
*/
@PostConstruct
private void init() {
if (auditConfiguration.getEnabled()) {
if (apiConfig.isAuditEnable()) {
Thread thread = new Thread(this::doPublish);
thread.setDaemon(true);
thread.setName("Audit-Log-Consume-Thread");
Expand All @@ -61,7 +61,7 @@ private void init() {
* @param message audit message
*/
public void publish(AuditMessage message) {
if (auditConfiguration.getEnabled() && !auditMessageQueue.offer(message)) {
if (apiConfig.isAuditEnable() && !auditMessageQueue.offer(message)) {
log.error("Publish audit message failed, message:{}", message);
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.dolphinscheduler.api.configuration;

import java.util.HashMap;
import java.util.Map;

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;

import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import org.springframework.validation.Errors;
import org.springframework.validation.Validator;
import org.springframework.validation.annotation.Validated;

@Slf4j
@Data
@Validated
@Configuration
@ConfigurationProperties(value = "api")
public class ApiConfig implements Validator {

private boolean auditEnable = false;

private TrafficConfiguration trafficControl = new TrafficConfiguration();

private PythonGatewayConfiguration pythonGateway = new PythonGatewayConfiguration();

@Override
public boolean supports(Class<?> clazz) {
return ApiConfig.class.isAssignableFrom(clazz);
}

@Override
public void validate(Object target, Errors errors) {
printConfig();
}

private void printConfig() {
log.info("API config: auditEnable -> {} ", auditEnable);
log.info("API config: trafficControl -> {} ", trafficControl);
log.info("API config: pythonGateway -> {} ", pythonGateway);
}

@Data
@NoArgsConstructor
@AllArgsConstructor
public static class TrafficConfiguration {

private boolean globalSwitch = false;
private Integer maxGlobalQpsRate = 300;
private boolean tenantSwitch = false;
private Integer defaultTenantQpsRate = 10;
private Map<String, Integer> customizeTenantQpsRate = new HashMap<>();
}

@Data
@NoArgsConstructor
@AllArgsConstructor
public static class PythonGatewayConfiguration {

private boolean enabled = true;
private String gatewayServerAddress = "0.0.0.0";
private int gatewayServerPort = 25333;
private String pythonAddress = "127.0.0.1";
private int pythonPort = 25334;
private int connectTimeout = 0;
private int readTimeout = 0;
private String authToken = "jwUDzpLsNKEFER4*a8gruBH_GsAurNxU7A@Xc";
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ public class AppConfiguration implements WebMvcConfigurer {
public static final String LOCALE_LANGUAGE_COOKIE = "language";

@Autowired
private TrafficConfiguration trafficConfiguration;
private ApiConfig apiConfig;

@Bean
public CorsFilter corsFilter() {
Expand Down Expand Up @@ -90,14 +90,15 @@ public LocaleChangeInterceptor localeChangeInterceptor() {

@Bean
public RateLimitInterceptor createRateLimitInterceptor() {
return new RateLimitInterceptor(trafficConfiguration);
return new RateLimitInterceptor(apiConfig.getTrafficControl());
}

@Override
public void addInterceptors(InterceptorRegistry registry) {
// i18n
registry.addInterceptor(localeChangeInterceptor());
if (trafficConfiguration.isGlobalSwitch() || trafficConfiguration.isTenantSwitch()) {
ApiConfig.TrafficConfiguration trafficControl = apiConfig.getTrafficControl();
if (trafficControl.isGlobalSwitch() || trafficControl.isTenantSwitch()) {
registry.addInterceptor(createRateLimitInterceptor());
}
registry.addInterceptor(loginInterceptor())
Expand Down

This file was deleted.

This file was deleted.

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

package org.apache.dolphinscheduler.api.interceptor;

import org.apache.dolphinscheduler.api.configuration.TrafficConfiguration;
import org.apache.dolphinscheduler.api.configuration.ApiConfig;

import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.StringUtils;
Expand Down Expand Up @@ -47,7 +47,7 @@
@Slf4j
public class RateLimitInterceptor implements HandlerInterceptor {

private TrafficConfiguration trafficConfiguration;
private ApiConfig.TrafficConfiguration trafficConfiguration;

private RateLimiter globalRateLimiter;

Expand Down Expand Up @@ -98,7 +98,7 @@ public boolean preHandle(HttpServletRequest request, HttpServletResponse respons
return true;
}

public RateLimitInterceptor(TrafficConfiguration trafficConfiguration) {
public RateLimitInterceptor(ApiConfig.TrafficConfiguration trafficConfiguration) {
this.trafficConfiguration = trafficConfiguration;
if (trafficConfiguration.isGlobalSwitch()) {
this.globalRateLimiter =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

package org.apache.dolphinscheduler.api.python;

import org.apache.dolphinscheduler.api.configuration.PythonGatewayConfiguration;
import org.apache.dolphinscheduler.api.configuration.ApiConfig;
import org.apache.dolphinscheduler.api.dto.EnvironmentDto;
import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
import org.apache.dolphinscheduler.api.enums.Status;
Expand Down Expand Up @@ -145,7 +145,7 @@ public class PythonGateway {
private DataSourceMapper dataSourceMapper;

@Autowired
private PythonGatewayConfiguration pythonGatewayConfiguration;
private ApiConfig apiConfig;

@Autowired
private ProjectUserMapper projectUserMapper;
Expand Down Expand Up @@ -689,13 +689,14 @@ public StorageEntity createOrUpdateResource(String userName, String fullName,

@PostConstruct
public void init() {
if (pythonGatewayConfiguration.isEnabled()) {
if (apiConfig.getPythonGateway().isEnabled()) {
this.start();
}
}

private void start() {
try {
ApiConfig.PythonGatewayConfiguration pythonGatewayConfiguration = apiConfig.getPythonGateway();
InetAddress gatewayHost = InetAddress.getByName(pythonGatewayConfiguration.getGatewayServerAddress());
GatewayServerBuilder serverBuilder = new GatewayServer.GatewayServerBuilder()
.entryPoint(this)
Expand Down
Loading

0 comments on commit a8324f9

Please sign in to comment.