Skip to content

Commit

Permalink
Merge pull request #227 from nacos-group/develop
Browse files Browse the repository at this point in the history 
Develop
  • Loading branch information
@paderlol
paderlol authored Apr 20, 2021
2 parents 144d5f7 + 7cb9d73 commit ef4398d
Show file tree
Hide file tree
Showing 26 changed files with 1,093 additions and 244 deletions.
2 changes: 1 addition & 1 deletion nacossync-console/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
<parent>
<artifactId>nacossync-parent</artifactId>
<groupId>com.alibaba.nacossync</groupId>
<version>0.4.4</version>
<version>0.4.5</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
1 change: 0 additions & 1 deletion nacossync-console/src/main/resources/static/console-fe/dist/css/main.68bce23a.css
View file

This file was deleted.

1 change: 1 addition & 0 deletions nacossync-console/src/main/resources/static/console-fe/dist/css/main.9093077e.css
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions nacossync-console/src/main/resources/static/console-fe/dist/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
<meta http-equiv="X-UA-Compatible" content="ie=edge">
<title>Nacos-Sync</title>
<link rel="shortcut icon" href="//www.aliyun.com/favicon.ico" type="image/x-icon">
<link href="./css/main.68bce23a.css" rel="stylesheet"></head>
<link href="./css/main.9093077e.css" rel="stylesheet"></head>
<body>
<div id="root"></div>
<script type="text/javascript" src="./js/main.1cd0c600.js"></script></body>
<script type="text/javascript" src="./js/main.cf091959.js"></script></body>
</html>
57 changes: 0 additions & 57 deletions nacossync-console/src/main/resources/static/console-fe/dist/js/main.1cd0c600.js
View file

This file was deleted.

328 changes: 328 additions & 0 deletions nacossync-console/src/main/resources/static/console-fe/dist/js/main.cf091959.js
View file

Large diffs are not rendered by default.

17 changes: 9 additions & 8 deletions ...-console/src/main/resources/static/console-fe/src/containers/ServiceSync/AddSyncDialog.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ class AddSyncDialog extends React.Component {
this.state = {
visible: false,
destClusterId: '',
groupName: '',
nameSpace: '',
groupName: '',
serviceName: '',
sourceClusterId: '',
version: '',
Expand All @@ -32,8 +32,8 @@ class AddSyncDialog extends React.Component {
}

save() {
const { destClusterId, groupName, nameSpace, serviceName, sourceClusterId, version } = this.state;
add({ destClusterId, groupName, nameSpace, serviceName, sourceClusterId, version })
const { destClusterId, nameSpace, groupName, serviceName, sourceClusterId, version } = this.state;
add({ destClusterId, nameSpace, groupName, serviceName, sourceClusterId, version })
.then(() => {
this.props.turnPage(1);
this.close();
Expand Down Expand Up @@ -65,6 +65,12 @@ class AddSyncDialog extends React.Component {
onClose={() => this.close()}
>
<Form>
<FormItem label={`${locale.nameSpace}:`}>
<Input
placeholder={locale.nameSpacePlaceholder}
onChange={nameSpace => this.setState({ nameSpace })}
/>
</FormItem>
<FormItem label={`${locale.serviceName}:`}>
<Input
placeholder={locale.serviceNamePlaceholder}
Expand All @@ -76,11 +82,6 @@ class AddSyncDialog extends React.Component {
placeholder={locale.groupNamePlaceholder}
onChange={groupName => this.setState({ groupName })}
/>
</FormItem> <FormItem label={`${locale.nameSpace}:`}>
<Input
placeholder={locale.nameSpacePlaceholder}
onChange={nameSpace => this.setState({ nameSpace })}
/>
</FormItem>
{
sourceCluster.clusterType === 'ZK' && (
Expand Down
4 changes: 2 additions & 2 deletions nacossync-console/src/main/resources/static/console-fe/src/locales/en-US.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,9 @@ const I18N_CONF = {
serviceNamePlaceholder: 'Please enter service name',
search: 'Search',
addSync: 'New Sync',
nameSpace: 'Namespace',
serviceName: 'Service Name',
groupName: 'Group',
nameSpace: 'Namespace',
sourceCluster: 'Source Cluster',
destCluster: 'Dest Cluster',
instancesCount: 'Instances Count',
Expand All @@ -62,7 +62,7 @@ const I18N_CONF = {
groupName: 'Group Name',
groupNamePlaceholder: 'Please enter group name',
nameSpace: 'Namespace',
nameSpacePlaceholder: 'Please enter namespace',
nameSpacePlaceholder: 'Please enter namespace id',
sourceCluster: 'Source Cluster',
destCluster: 'Dest Cluster',
version: 'Version',
Expand Down
3 changes: 3 additions & 0 deletions nacossync-console/src/main/resources/static/console-fe/src/locales/zh-CN.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ const I18N_CONF = {
serviceNamePlaceholder: '请输入服务名',
search: '搜索',
addSync: '新增同步',
nameSpace: '命名空间',
serviceName: '服务名',
groupName: '分组',
nameSpace: '命名空间',
Expand All @@ -57,6 +58,8 @@ const I18N_CONF = {
},
AddSyncDialog: {
title: '新增同步',
nameSpace: '命名空间',
nameSpacePlaceholder: '请输入命名空间ID',
serviceName: '服务名',
serviceNamePlaceholder: '请输入服务名',
groupName: '分组名',
Expand Down
1 change: 0 additions & 1 deletion nacossync-console/src/main/resources/static/css/main.68bce23a.css
View file

This file was deleted.

1 change: 1 addition & 0 deletions nacossync-console/src/main/resources/static/css/main.9093077e.css
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions nacossync-console/src/main/resources/static/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
<meta http-equiv="X-UA-Compatible" content="ie=edge">
<title>Nacos-Sync</title>
<link rel="shortcut icon" href="//www.aliyun.com/favicon.ico" type="image/x-icon">
<link href="./css/main.68bce23a.css" rel="stylesheet"></head>
<link href="./css/main.9093077e.css" rel="stylesheet"></head>
<body>
<div id="root"></div>
<script type="text/javascript" src="./js/main.1cd0c600.js"></script></body>
<script type="text/javascript" src="./js/main.cf091959.js"></script></body>
</html>
57 changes: 0 additions & 57 deletions nacossync-console/src/main/resources/static/js/main.1cd0c600.js
View file

This file was deleted.

328 changes: 328 additions & 0 deletions nacossync-console/src/main/resources/static/js/main.cf091959.js
View file

Large diffs are not rendered by default.

12 changes: 6 additions & 6 deletions nacossync-distribution/bin/nacosSync.sql
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/******************************************/
/* DB name = nacos_Sync */
/* DB name = nacos_sync */
/* Table name = cluster */
/******************************************/
CREATE TABLE `cluster` (
Expand All @@ -9,9 +9,9 @@ CREATE TABLE `cluster` (
`cluster_type` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL,
`connect_key_list` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=MyISAM AUTO_INCREMENT=5 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
) ENGINE=InnoDB AUTO_INCREMENT=5 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
/******************************************/
/* DB name = nacos_Sync */
/* DB name = nacos_sync */
/* Table name = system_config */
/******************************************/
CREATE TABLE `system_config` (
Expand All @@ -20,9 +20,9 @@ CREATE TABLE `system_config` (
`config_key` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL,
`config_value` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
/******************************************/
/* DB name = nacos_Sync */
/* DB name = nacos_sync */
/* Table name = task */
/******************************************/
CREATE TABLE `task` (
Expand All @@ -38,4 +38,4 @@ CREATE TABLE `task` (
`version` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL,
`worker_ip` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=MyISAM AUTO_INCREMENT=6 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
2 changes: 1 addition & 1 deletion nacossync-distribution/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<parent>
<artifactId>nacossync-parent</artifactId>
<groupId>com.alibaba.nacossync</groupId>
<version>0.4.4</version>
<version>0.4.5</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
Expand Down
4 changes: 2 additions & 2 deletions nacossync-test/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
<parent>
<artifactId>nacossync-parent</artifactId>
<groupId>com.alibaba.nacossync</groupId>
<version>0.4.4</version>
<version>0.4.5</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
Expand All @@ -39,7 +39,7 @@
<dependency>
<groupId>com.alibaba.nacossync</groupId>
<artifactId>nacossync-worker</artifactId>
<version>0.4.4</version>
<version>0.4.5</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
Expand Down
4 changes: 2 additions & 2 deletions nacossync-worker/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,11 @@
<parent>
<artifactId>nacossync-parent</artifactId>
<groupId>com.alibaba.nacossync</groupId>
<version>0.4.4</version>
<version>0.4.5</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>nacossync-worker</artifactId>
<version>0.4.4</version>
<version>0.4.5</version>
<properties>
<zookeeper.version>3.4.9</zookeeper.version>
<curator.version>4.1.0</curator.version>
Expand Down
94 changes: 61 additions & 33 deletions ...orker/src/main/java/com/alibaba/nacossync/extension/impl/NacosSyncToNacosServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
*/
package com.alibaba.nacossync.extension.impl;

import com.alibaba.nacos.api.common.Constants;
import com.alibaba.nacos.api.exception.NacosException;
import com.alibaba.nacos.api.naming.NamingService;
import com.alibaba.nacos.api.naming.listener.EventListener;
Expand All @@ -38,6 +39,7 @@
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;

/**
Expand Down Expand Up @@ -65,20 +67,25 @@ public class NacosSyncToNacosServiceImpl implements SyncService {
@Override
public boolean delete(TaskDO taskDO) {
try {

NamingService sourceNamingService =
nacosServerHolder.get(taskDO.getSourceClusterId(), taskDO.getGroupName());
NamingService destNamingService = nacosServerHolder.get(taskDO.getDestClusterId(), taskDO.getGroupName());
nacosServerHolder.get(taskDO.getSourceClusterId(), taskDO.getNameSpace());
NamingService destNamingService = nacosServerHolder.get(taskDO.getDestClusterId(), taskDO.getNameSpace());
//移除订阅
sourceNamingService.unsubscribe(taskDO.getServiceName(), listenerMap.remove(taskDO.getTaskId()));
sourceNamingService
.unsubscribe(taskDO.getServiceName(), getGroupNameOrDefault(taskDO.getGroupName()),
listenerMap.remove(taskDO.getTaskId()));
sourceInstanceSnapshot.remove(taskDO.getTaskId());

// 删除目标集群中同步的实例列表
List<Instance> sourceInstances = sourceNamingService.getAllInstances(taskDO.getServiceName(),
new ArrayList<>(), false);
List<Instance> sourceInstances = sourceNamingService
.getAllInstances(taskDO.getServiceName(), getGroupNameOrDefault(taskDO.getGroupName()),
new ArrayList<>(), false);
for (Instance instance : sourceInstances) {
if (needSync(instance.getMetadata())) {
destNamingService.deregisterInstance(taskDO.getServiceName(), instance.getIp(), instance.getPort());
destNamingService
.deregisterInstance(taskDO.getServiceName(), getGroupNameOrDefault(taskDO.getGroupName()),
instance.getIp(),
instance.getPort());
}
}
} catch (Exception e) {
Expand All @@ -94,8 +101,8 @@ public boolean sync(TaskDO taskDO) {
String taskId = taskDO.getTaskId();
try {
NamingService sourceNamingService =
nacosServerHolder.get(taskDO.getSourceClusterId(), taskDO.getGroupName());
NamingService destNamingService = nacosServerHolder.get(taskDO.getDestClusterId(), taskDO.getGroupName());
nacosServerHolder.get(taskDO.getSourceClusterId(), taskDO.getNameSpace());
NamingService destNamingService = nacosServerHolder.get(taskDO.getDestClusterId(), taskDO.getNameSpace());

this.listenerMap.putIfAbsent(taskId, event -> {
if (event instanceof NamingEvent) {
Expand All @@ -104,34 +111,23 @@ public boolean sync(TaskDO taskDO) {
List<Instance> sourceInstances = sourceNamingService.getAllInstances(taskDO.getServiceName(),
new ArrayList<>(), false);
// 先删除不存在的
this.removeInvalidInstance(taskDO, taskId, destNamingService, sourceInstances);
Set<String> latestSyncInstance = new TreeSet<>();
//再次添加新实例
Set<String> instanceKeys = sourceInstanceSnapshot.get(taskId);
for (Instance instance : sourceInstances) {
if (needSync(instance.getMetadata())) {
String instanceKey = composeInstanceKey(instance);
if (CollectionUtils.isEmpty(instanceKeys) || !instanceKeys.contains(instanceKey)) {
destNamingService.registerInstance(taskDO.getServiceName(),
buildSyncInstance(instance, taskDO));
}
latestSyncInstance.add(instanceKey);

}
}
if (CollectionUtils.isNotEmpty(latestSyncInstance)) {

log.info("任务Id:{},已同步实例个数:{}", taskId, latestSyncInstance.size());
sourceInstanceSnapshot.put(taskId, latestSyncInstance);
this.removeInvalidInstance(taskDO, destNamingService, sourceInstances);
// 如果同步实例已经为空代表该服务所有实例已经下线,清除本地持有快照
if (sourceInstances.isEmpty()) {
sourceInstanceSnapshot.remove(taskId);
return;
}
// 同步实例
this.syncNewInstance(taskDO, destNamingService, sourceInstances);
} catch (Exception e) {
log.error("event process fail, taskId:{}", taskId, e);
metricsManager.recordError(MetricsStatisticsType.SYNC_ERROR);
}
}
});

sourceNamingService.subscribe(taskDO.getServiceName(), listenerMap.get(taskId));
sourceNamingService.subscribe(taskDO.getServiceName(), getGroupNameOrDefault(taskDO.getGroupName()),
listenerMap.get(taskId));
} catch (Exception e) {
log.error("sync task from nacos to nacos was failed, taskId:{}", taskId, e);
metricsManager.recordError(MetricsStatisticsType.SYNC_ERROR);
Expand All @@ -140,8 +136,35 @@ public boolean sync(TaskDO taskDO) {
return true;
}

private void removeInvalidInstance(TaskDO taskDO, String taskId, NamingService destNamingService,
private void syncNewInstance(TaskDO taskDO, NamingService destNamingService,
List<Instance> sourceInstances) throws NacosException {
Set<String> latestSyncInstance = new TreeSet<>();
//再次添加新实例
String taskId = taskDO.getTaskId();
Set<String> instanceKeys = sourceInstanceSnapshot.get(taskId);
for (Instance instance : sourceInstances) {
if (needSync(instance.getMetadata())) {
String instanceKey = composeInstanceKey(instance);
if (CollectionUtils.isEmpty(instanceKeys) || !instanceKeys.contains(instanceKey)) {
destNamingService.registerInstance(taskDO.getServiceName(),
getGroupNameOrDefault(taskDO.getGroupName()),
buildSyncInstance(instance, taskDO));
}
latestSyncInstance.add(instanceKey);

}
}
if (CollectionUtils.isNotEmpty(latestSyncInstance)) {

log.info("任务Id:{},已同步实例个数:{}", taskId, latestSyncInstance.size());
sourceInstanceSnapshot.put(taskId, latestSyncInstance);
}
}


private void removeInvalidInstance(TaskDO taskDO, NamingService destNamingService,
List<Instance> sourceInstances) throws NacosException {
String taskId = taskDO.getTaskId();
if (this.sourceInstanceSnapshot.containsKey(taskId)) {
Set<String> oldInstanceKeys = this.sourceInstanceSnapshot.get(taskId);
List<String> newInstanceKeys = sourceInstances.stream().map(this::composeInstanceKey)
Expand All @@ -150,19 +173,24 @@ private void removeInvalidInstance(TaskDO taskDO, String taskId, NamingService d
for (String instanceKey : instanceKeys) {
log.info("任务Id:{},移除无效同步实例:{}", taskId, instanceKey);
String[] split = instanceKey.split(":", -1);
destNamingService.deregisterInstance(taskDO.getServiceName(), split[0],
Integer.parseInt(split[1]));
destNamingService
.deregisterInstance(taskDO.getServiceName(), getGroupNameOrDefault(taskDO.getGroupName()), split[0],
Integer.parseInt(split[1]));

}

}
}

private String composeInstanceKey(Instance instance) {
return instance.getIp() + ":" + instance.getPort();
}

private String getGroupNameOrDefault(String groupName) {
return StringUtils.defaultIfBlank(groupName, Constants.DEFAULT_GROUP);
}

public Instance buildSyncInstance(Instance instance, TaskDO taskDO) {
private Instance buildSyncInstance(Instance instance, TaskDO taskDO) {
Instance temp = new Instance();
temp.setIp(instance.getIp());
temp.setPort(instance.getPort());
Expand Down
1 change: 0 additions & 1 deletion nacossync-worker/src/main/resources/static/css/main.68bce23a.css
View file

This file was deleted.

1 change: 1 addition & 0 deletions nacossync-worker/src/main/resources/static/css/main.9093077e.css
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions nacossync-worker/src/main/resources/static/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
<meta http-equiv="X-UA-Compatible" content="ie=edge">
<title>Nacos-Sync</title>
<link rel="shortcut icon" href="//www.aliyun.com/favicon.ico" type="image/x-icon">
<link href="./css/main.68bce23a.css" rel="stylesheet"></head>
<link href="./css/main.9093077e.css" rel="stylesheet"></head>
<body>
<div id="root"></div>
<script type="text/javascript" src="./js/main.1cd0c600.js"></script></body>
<script type="text/javascript" src="./js/main.cf091959.js"></script></body>
</html>
57 changes: 0 additions & 57 deletions nacossync-worker/src/main/resources/static/js/main.1cd0c600.js
View file

This file was deleted.

328 changes: 328 additions & 0 deletions nacossync-worker/src/main/resources/static/js/main.cf091959.js
View file

Large diffs are not rendered by default.

Loading

0 comments on commit ef4398d

Please sign in to comment.