From 6613e4fa02d43900b6092f17d4ac3c01cdb66bdf Mon Sep 17 00:00:00 2001 From: ZhongJinHacker Date: Sun, 10 Mar 2024 11:04:49 +0800 Subject: [PATCH 01/10] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dpath=E4=B8=BAnull?= =?UTF-8?q?=E6=97=B6=EF=BC=8CserverHost=E4=B8=BAnull=E7=9A=84=E6=83=85?= =?UTF-8?q?=E5=86=B5=EF=BC=8C=E5=90=8E=E7=BB=ADfailoverServerWhenDown?= =?UTF-8?q?=E6=96=B9=E6=B3=95=E8=B0=83=E7=94=A8serverHost=E4=B8=BAnull?= =?UTF-8?q?=E4=BC=9A=E6=8A=A5=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../master/registry/MasterRegistryClient.java | 23 +++++++++++-------- .../registry/MasterRegistryClientTest.java | 10 ++++++++ 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java index 99731bbf0e9f..4468af495a9d 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java @@ -131,17 +131,20 @@ public void removeMasterNodePath(String path, RegistryNodeType nodeType, boolean public void removeWorkerNodePath(String path, RegistryNodeType nodeType, boolean failover) { log.info("{} node deleted : {}", nodeType, path); try { - String serverHost = null; - if (!StringUtils.isEmpty(path)) { - serverHost = registryClient.getHostByEventDataPath(path); - if (StringUtils.isEmpty(serverHost)) { - log.error("server down error: unknown path: {}", path); - return; - } - if (!registryClient.exists(path)) { - log.info("path: {} not exists", path); - } + if (StringUtils.isEmpty(path)) { + log.error("server down error: node empty path: {}, nodeType:{}", path, nodeType); + return; } + + String serverHost = registryClient.getHostByEventDataPath(path); + if (StringUtils.isEmpty(serverHost)) { + log.error("server down error: unknown path: {}", path); + return; + } + if (!registryClient.exists(path)) { + log.info("path: {} not exists", path); + } + // failover server if (failover) { failoverService.failoverServerWhenDown(serverHost, nodeType); diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java index 133ed4e4ffa1..bde4ccde8af5 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java @@ -28,6 +28,7 @@ import org.apache.dolphinscheduler.registry.api.RegistryClient; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; import org.apache.dolphinscheduler.server.master.config.MasterConfig; +import org.apache.dolphinscheduler.server.master.service.FailoverService; import org.apache.dolphinscheduler.server.master.task.MasterHeartBeatTask; import org.apache.dolphinscheduler.service.process.ProcessService; @@ -63,6 +64,9 @@ public class MasterRegistryClientTest { @Mock private MasterHeartBeatTask masterHeartBeatTask; + @Mock + private FailoverService failoverService; + @Mock private MasterConfig masterConfig; @@ -103,4 +107,10 @@ public void removeNodePathTest() { // Cannot mock static methods masterRegistryClient.removeWorkerNodePath("/path", RegistryNodeType.WORKER, true); } + + @Test + public void removeWorkNodePathTest() { + masterRegistryClient.removeWorkerNodePath("", RegistryNodeType.WORKER, true); + masterRegistryClient.removeWorkerNodePath(null, RegistryNodeType.WORKER, true); + } } From 82878d5aac4e9dfade8f4987d3e15af5e4f047a4 Mon Sep 17 00:00:00 2001 From: ZhongJinHacker Date: Sun, 10 Mar 2024 12:14:55 +0800 Subject: [PATCH 02/10] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dpath=E4=B8=BAnull?= =?UTF-8?q?=E6=97=B6=EF=BC=8CserverHost=E4=B8=BAnull=E7=9A=84=E6=83=85?= =?UTF-8?q?=E5=86=B5=EF=BC=8C=E5=90=8E=E7=BB=ADfailoverServerWhenDown?= =?UTF-8?q?=E6=96=B9=E6=B3=95=E8=B0=83=E7=94=A8serverHost=E4=B8=BAnull?= =?UTF-8?q?=E4=BC=9A=E6=8A=A5=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../server/master/registry/MasterRegistryClientTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java index bde4ccde8af5..eef8ff5f2290 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java @@ -64,9 +64,6 @@ public class MasterRegistryClientTest { @Mock private MasterHeartBeatTask masterHeartBeatTask; - @Mock - private FailoverService failoverService; - @Mock private MasterConfig masterConfig; From a500a0ef8f1bbc169c712c0ea49cda53616161f5 Mon Sep 17 00:00:00 2001 From: ZhongJinHacker Date: Sun, 10 Mar 2024 12:15:48 +0800 Subject: [PATCH 03/10] clean unused import --- .../master/registry/MasterRegistryClientTest.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java index eef8ff5f2290..2ed196c14ed9 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.server.master.registry; -import static org.mockito.BDDMockito.given; - import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.ServerStatus; import org.apache.dolphinscheduler.common.model.MasterHeartBeat; @@ -28,12 +26,8 @@ import org.apache.dolphinscheduler.registry.api.RegistryClient; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; import org.apache.dolphinscheduler.server.master.config.MasterConfig; -import org.apache.dolphinscheduler.server.master.service.FailoverService; import org.apache.dolphinscheduler.server.master.task.MasterHeartBeatTask; import org.apache.dolphinscheduler.service.process.ProcessService; - -import java.util.Date; - import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -45,6 +39,10 @@ import org.mockito.quality.Strictness; import org.springframework.test.util.ReflectionTestUtils; +import java.util.Date; + +import static org.mockito.BDDMockito.given; + /** * MasterRegistryClientTest */ From 099d251131ec90253523f4c48b5ce46079610d69 Mon Sep 17 00:00:00 2001 From: ZhongJinHacker Date: Sun, 10 Mar 2024 22:18:52 +0800 Subject: [PATCH 04/10] fix style check --- .../server/master/registry/MasterRegistryClientTest.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java index 2ed196c14ed9..2ff2b873e1b9 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.server.master.registry; +import static org.mockito.BDDMockito.given; + import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.ServerStatus; import org.apache.dolphinscheduler.common.model.MasterHeartBeat; @@ -28,6 +30,9 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.task.MasterHeartBeatTask; import org.apache.dolphinscheduler.service.process.ProcessService; + +import java.util.Date; + import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -39,10 +44,6 @@ import org.mockito.quality.Strictness; import org.springframework.test.util.ReflectionTestUtils; -import java.util.Date; - -import static org.mockito.BDDMockito.given; - /** * MasterRegistryClientTest */ From 47f4572ec6bf0f16277e09ba020cfc01e30e60e9 Mon Sep 17 00:00:00 2001 From: ZhongJinHacker Date: Sun, 10 Mar 2024 11:04:49 +0800 Subject: [PATCH 05/10] fix when path is null or empty, it will cause serverhost is null, --- .../master/registry/MasterRegistryClient.java | 23 +++++++++++-------- .../registry/MasterRegistryClientTest.java | 10 ++++++++ 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java index 99731bbf0e9f..4468af495a9d 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java @@ -131,17 +131,20 @@ public void removeMasterNodePath(String path, RegistryNodeType nodeType, boolean public void removeWorkerNodePath(String path, RegistryNodeType nodeType, boolean failover) { log.info("{} node deleted : {}", nodeType, path); try { - String serverHost = null; - if (!StringUtils.isEmpty(path)) { - serverHost = registryClient.getHostByEventDataPath(path); - if (StringUtils.isEmpty(serverHost)) { - log.error("server down error: unknown path: {}", path); - return; - } - if (!registryClient.exists(path)) { - log.info("path: {} not exists", path); - } + if (StringUtils.isEmpty(path)) { + log.error("server down error: node empty path: {}, nodeType:{}", path, nodeType); + return; } + + String serverHost = registryClient.getHostByEventDataPath(path); + if (StringUtils.isEmpty(serverHost)) { + log.error("server down error: unknown path: {}", path); + return; + } + if (!registryClient.exists(path)) { + log.info("path: {} not exists", path); + } + // failover server if (failover) { failoverService.failoverServerWhenDown(serverHost, nodeType); diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java index 133ed4e4ffa1..bde4ccde8af5 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java @@ -28,6 +28,7 @@ import org.apache.dolphinscheduler.registry.api.RegistryClient; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; import org.apache.dolphinscheduler.server.master.config.MasterConfig; +import org.apache.dolphinscheduler.server.master.service.FailoverService; import org.apache.dolphinscheduler.server.master.task.MasterHeartBeatTask; import org.apache.dolphinscheduler.service.process.ProcessService; @@ -63,6 +64,9 @@ public class MasterRegistryClientTest { @Mock private MasterHeartBeatTask masterHeartBeatTask; + @Mock + private FailoverService failoverService; + @Mock private MasterConfig masterConfig; @@ -103,4 +107,10 @@ public void removeNodePathTest() { // Cannot mock static methods masterRegistryClient.removeWorkerNodePath("/path", RegistryNodeType.WORKER, true); } + + @Test + public void removeWorkNodePathTest() { + masterRegistryClient.removeWorkerNodePath("", RegistryNodeType.WORKER, true); + masterRegistryClient.removeWorkerNodePath(null, RegistryNodeType.WORKER, true); + } } From 7462ea3d6656577ec902220574f95f53727f6134 Mon Sep 17 00:00:00 2001 From: caishunfeng Date: Fri, 8 Mar 2024 17:21:23 +0800 Subject: [PATCH 06/10] fix UT test (#15684) --- .../dolphinscheduler/api/service/ResourcesServiceTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java index fc1766214796..a01ce75a4c2b 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.api.service; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; @@ -494,9 +495,8 @@ public void testUpdateResourceContent() throws Exception { ServiceException serviceException = Assertions.assertThrows(ServiceException.class, () -> resourcesService.updateResourceContent(getUser(), "/dolphinscheduler/123/resources/ResourcesServiceTest.jar", "123", "content")); - assertEquals( - "Internal Server Error: Resource file: /dolphinscheduler/123/resources/ResourcesServiceTest.jar is illegal", - serviceException.getMessage()); + assertTrue(serviceException.getMessage() + .contains("Resource file: /dolphinscheduler/123/resources/ResourcesServiceTest.jar is illegal")); // RESOURCE_NOT_EXIST when(storageOperate.getResDir(Mockito.anyString())).thenReturn("/dolphinscheduler/123/resources"); From 479e0aceae95bfc0b7d7e1f6bf3cb7015708e814 Mon Sep 17 00:00:00 2001 From: lch Date: Fri, 8 Mar 2024 17:52:24 +0800 Subject: [PATCH 07/10] [Fix-15639] parameterPassing is null case NPE (#15678) Co-authored-by: caishunfeng --- .../dolphinscheduler/plugin/task/api/model/DependentItem.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DependentItem.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DependentItem.java index 360fac42b61c..47f42c416c20 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DependentItem.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DependentItem.java @@ -35,7 +35,7 @@ public class DependentItem { private String dateValue; private DependResult dependResult; private TaskExecutionStatus status; - private Boolean parameterPassing; + private Boolean parameterPassing = false; public String getKey() { return String.format("%d-%d-%s-%s", From d2c31ce8cd121f2ddb1dd908dde6089ef6d7d516 Mon Sep 17 00:00:00 2001 From: ZhongJinHacker Date: Sun, 10 Mar 2024 12:14:55 +0800 Subject: [PATCH 08/10] fix when path is null or empty, it will cause serverhost is null, --- .../server/master/registry/MasterRegistryClientTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java index bde4ccde8af5..eef8ff5f2290 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java @@ -64,9 +64,6 @@ public class MasterRegistryClientTest { @Mock private MasterHeartBeatTask masterHeartBeatTask; - @Mock - private FailoverService failoverService; - @Mock private MasterConfig masterConfig; From 92b3d1efcca2ffc9de622f64fd5da1536d8547d1 Mon Sep 17 00:00:00 2001 From: ZhongJinHacker Date: Sun, 10 Mar 2024 12:15:48 +0800 Subject: [PATCH 09/10] clean unused import --- .../master/registry/MasterRegistryClientTest.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java index eef8ff5f2290..2ed196c14ed9 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.server.master.registry; -import static org.mockito.BDDMockito.given; - import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.ServerStatus; import org.apache.dolphinscheduler.common.model.MasterHeartBeat; @@ -28,12 +26,8 @@ import org.apache.dolphinscheduler.registry.api.RegistryClient; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; import org.apache.dolphinscheduler.server.master.config.MasterConfig; -import org.apache.dolphinscheduler.server.master.service.FailoverService; import org.apache.dolphinscheduler.server.master.task.MasterHeartBeatTask; import org.apache.dolphinscheduler.service.process.ProcessService; - -import java.util.Date; - import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -45,6 +39,10 @@ import org.mockito.quality.Strictness; import org.springframework.test.util.ReflectionTestUtils; +import java.util.Date; + +import static org.mockito.BDDMockito.given; + /** * MasterRegistryClientTest */ From ee15b5b585b37f15152d3d9368e6180034a5e0aa Mon Sep 17 00:00:00 2001 From: ZhongJinHacker Date: Sun, 10 Mar 2024 22:18:52 +0800 Subject: [PATCH 10/10] fix style check --- .../server/master/registry/MasterRegistryClientTest.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java index 2ed196c14ed9..2ff2b873e1b9 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.server.master.registry; +import static org.mockito.BDDMockito.given; + import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.ServerStatus; import org.apache.dolphinscheduler.common.model.MasterHeartBeat; @@ -28,6 +30,9 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.task.MasterHeartBeatTask; import org.apache.dolphinscheduler.service.process.ProcessService; + +import java.util.Date; + import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -39,10 +44,6 @@ import org.mockito.quality.Strictness; import org.springframework.test.util.ReflectionTestUtils; -import java.util.Date; - -import static org.mockito.BDDMockito.given; - /** * MasterRegistryClientTest */