From 288beaf2697e755cb489c7691bfe09c69f120f4f Mon Sep 17 00:00:00 2001 From: BePPPower Date: Tue, 26 Mar 2024 18:04:11 +0800 Subject: [PATCH 1/9] debug --- .../main/java/org/apache/doris/hudi/Utils.java | 3 +++ .../doris/trinoconnector/TrinoConnectorCache.java | 15 +++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java index 5614f8bcc96eb1..3a8040ac1a2b53 100644 --- a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java +++ b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java @@ -44,13 +44,16 @@ public static long getCurrentProcId() { public static List getChildProcessIds(long pid) { try { Process pgrep = (new ProcessBuilder("pgrep", "-P", String.valueOf(pid))).start(); + System.out.println("--ftw: pgrep = " + pgrep); BufferedReader reader = new BufferedReader(new InputStreamReader(pgrep.getInputStream())); List result = new LinkedList<>(); String line; while ((line = reader.readLine()) != null) { result.add(Long.valueOf(line.trim())); } + System.out.println("--ftw: pgrep.waitFor();"); pgrep.waitFor(); + System.out.println("--ftw: get all result"); return result; } catch (Exception e) { throw new RuntimeException("Couldn't get child processes of PID " + pid, e); diff --git a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorCache.java b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorCache.java index 0a076716b2fe77..e2020037f4341b 100644 --- a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorCache.java +++ b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorCache.java @@ -112,26 +112,37 @@ private static TrinoConnectorCacheValue loadCache(TrinoConnectorCacheKey key) { // RecordReader will use ProcessBuilder to start a hotspot process, which may be stuck, // so use another process to kill this stuck process. AtomicBoolean isKilled = new AtomicBoolean(false); + LOG.info("--ftw: isKilled = " + isKilled); ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1); executorService.scheduleAtFixedRate(() -> { if (!isKilled.get()) { - List pids = Utils.getChildProcessIds( - Utils.getCurrentProcId()); + LOG.info("--ftw: isKilled is false."); + long currentPid = Utils.getCurrentProcId(); + LOG.info("--ftw: currentPid = " + currentPid); + List pids = Utils.getChildProcessIds(currentPid); + LOG.info("--ftw: get all pids"); for (long pid : pids) { + LOG.info("--ftw: pid = " + pid); String cmd = Utils.getCommandLine(pid); + LOG.info("--ftw: get cmd = " + cmd); if (cmd != null && cmd.contains("org.openjdk.jol.vm.sa.AttachMain")) { + LOG.info("--ftw: begin kill, cmd = " + cmd); Utils.killProcess(pid); + LOG.info("--ftw: finish killed, cmd = " + cmd); isKilled.set(true); } } } }, 100, 1000, TimeUnit.MILLISECONDS); + LOG.info("--ftw: isKill = " + isKilled); // create CatalogHandle CatalogHandle catalogHandle = CatalogHandle.createRootCatalogHandle(key.catalogName, new CatalogVersion("test")); + LOG.info("--ftw: finish create catalog handle."); isKilled.set(true); + LOG.info("--ftw: isKill = " + isKilled); executorService.shutdownNow(); CatalogFactory catalogFactory = createCatalogFactory(key.trinoConnectorPluginManager.getTypeRegistry(), From 56642a6bf70bb79c15d38eb119cd1b7458966fac Mon Sep 17 00:00:00 2001 From: BePPPower Date: Tue, 26 Mar 2024 20:42:33 +0800 Subject: [PATCH 2/9] debug 2 --- .../TrinoConnectorPluginLoader.java | 15 --------------- .../TrinoConnectorPluginLoader.java | 16 ---------------- 2 files changed, 31 deletions(-) diff --git a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorPluginLoader.java b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorPluginLoader.java index 1f08e28b005fa6..134f315c333bc7 100644 --- a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorPluginLoader.java +++ b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorPluginLoader.java @@ -30,9 +30,6 @@ import org.apache.logging.log4j.Logger; import java.io.File; -import java.util.logging.FileHandler; -import java.util.logging.Level; -import java.util.logging.SimpleFormatter; public class TrinoConnectorPluginLoader { private static final Logger LOG = LogManager.getLogger(TrinoConnectorPluginLoader.class); @@ -45,18 +42,6 @@ private static class TrinoConnectorPluginLoad { static { try { - // Trino uses jul as its own log system, so the attributes of JUL are configured here - System.setProperty("java.util.logging.SimpleFormatter.format", - "%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS %4$s: %5$s%6$s%n"); - java.util.logging.Logger logger = java.util.logging.Logger.getLogger(""); - logger.setUseParentHandlers(false); - FileHandler fileHandler = new FileHandler(EnvUtils.getDorisHome() + "/log/trinoconnector%g.log", - 500000000, 10, true); - fileHandler.setLevel(Level.INFO); - fileHandler.setFormatter(new SimpleFormatter()); - logger.addHandler(fileHandler); - java.util.logging.LogManager.getLogManager().addLogger(logger); - TypeOperators typeOperators = new TypeOperators(); featuresConfig = new FeaturesConfig(); TypeRegistry typeRegistry = new TypeRegistry(typeOperators, featuresConfig); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorPluginLoader.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorPluginLoader.java index 1e08c9effcb00f..a0d34ffa31e306 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorPluginLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorPluginLoader.java @@ -18,7 +18,6 @@ package org.apache.doris.datasource.trinoconnector; import org.apache.doris.common.Config; -import org.apache.doris.common.EnvUtils; import org.apache.doris.trinoconnector.TrinoConnectorPluginManager; import com.google.common.util.concurrent.MoreExecutors; @@ -32,9 +31,6 @@ import org.apache.logging.log4j.Logger; import java.io.File; -import java.util.logging.FileHandler; -import java.util.logging.Level; -import java.util.logging.SimpleFormatter; public class TrinoConnectorPluginLoader { private static final Logger LOG = LogManager.getLogger(TrinoConnectorPluginLoader.class); @@ -48,18 +44,6 @@ private static class TrinoConnectorPluginLoad { static { try { - // Trino uses jul as its own log system, so the attributes of JUL are configured here - System.setProperty("java.util.logging.SimpleFormatter.format", - "%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS %4$s: %5$s%6$s%n"); - java.util.logging.Logger logger = java.util.logging.Logger.getLogger(""); - logger.setUseParentHandlers(false); - FileHandler fileHandler = new FileHandler(EnvUtils.getDorisHome() + "/log/trinoconnector%g.log", - 500000000, 10, true); - fileHandler.setLevel(Level.INFO); - fileHandler.setFormatter(new SimpleFormatter()); - logger.addHandler(fileHandler); - java.util.logging.LogManager.getLogManager().addLogger(logger); - typeRegistry = new TypeRegistry(typeOperators, featuresConfig); ServerPluginsProviderConfig serverPluginsProviderConfig = new ServerPluginsProviderConfig() .setInstalledPluginsDir(new File(Config.trino_connector_plugin_dir)); From fd53906b3f16de04f745c38af6bdaaceb8ea3673 Mon Sep 17 00:00:00 2001 From: BePPPower Date: Wed, 27 Mar 2024 20:27:41 +0800 Subject: [PATCH 3/9] fix --- fe/be-java-extensions/hudi-scanner/pom.xml | 5 ++++ .../org/apache/doris/hudi/HudiJniScanner.java | 9 +++--- .../doris/preload/common/ProcessUtils.java} | 20 ++++--------- .../trino-connector-scanner/pom.xml | 2 +- .../trinoconnector/TrinoConnectorCache.java | 28 ++++++++----------- 5 files changed, 28 insertions(+), 36 deletions(-) rename fe/be-java-extensions/{hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java => preload-extensions/src/main/java/org/apache/doris/preload/common/ProcessUtils.java} (77%) diff --git a/fe/be-java-extensions/hudi-scanner/pom.xml b/fe/be-java-extensions/hudi-scanner/pom.xml index d4f7a458612c85..2cd5b7deee39b6 100644 --- a/fe/be-java-extensions/hudi-scanner/pom.xml +++ b/fe/be-java-extensions/hudi-scanner/pom.xml @@ -214,6 +214,11 @@ under the License. + + org.apache.doris + preload-extensions + ${project.version} + hudi-scanner diff --git a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java index a284c7adcdd105..4cf76dce07e1dc 100644 --- a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java +++ b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java @@ -22,6 +22,7 @@ import org.apache.doris.common.jni.vec.ColumnType; import org.apache.doris.common.security.authentication.AuthenticationConfig; import org.apache.doris.common.security.authentication.HadoopUGI; +import org.apache.doris.preload.common.ProcessUtils; import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.avro.generic.GenericDatumReader; @@ -143,12 +144,12 @@ public void open() throws IOException { executorService.scheduleAtFixedRate(() -> { if (!isKilled.get()) { synchronized (HudiJniScanner.class) { - List pids = Utils.getChildProcessIds( - Utils.getCurrentProcId()); + List pids = ProcessUtils.getChildProcessIds( + ProcessUtils.getCurrentProcId()); for (long pid : pids) { - String cmd = Utils.getCommandLine(pid); + String cmd = ProcessUtils.getCommandLine(pid); if (cmd != null && cmd.contains("org.openjdk.jol.vm.sa.AttachMain")) { - Utils.killProcess(pid); + ProcessUtils.killProcess(pid); isKilled.set(true); LOG.info("Kill hotspot debugger process " + pid); } diff --git a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java b/fe/be-java-extensions/preload-extensions/src/main/java/org/apache/doris/preload/common/ProcessUtils.java similarity index 77% rename from fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java rename to fe/be-java-extensions/preload-extensions/src/main/java/org/apache/doris/preload/common/ProcessUtils.java index 3a8040ac1a2b53..5823c7e6aa2062 100644 --- a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java +++ b/fe/be-java-extensions/preload-extensions/src/main/java/org/apache/doris/preload/common/ProcessUtils.java @@ -15,14 +15,9 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.hudi; - -import org.apache.doris.common.security.authentication.AuthenticationConfig; -import org.apache.doris.common.security.authentication.HadoopUGI; +package org.apache.doris.preload.common; import org.apache.commons.io.FileUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hudi.common.table.HoodieTableMetaClient; import java.io.BufferedReader; import java.io.File; @@ -32,7 +27,10 @@ import java.util.LinkedList; import java.util.List; -public class Utils { +/** + * Utils for handling processes + */ +public class ProcessUtils { public static long getCurrentProcId() { try { return ManagementFactory.getRuntimeMXBean().getPid(); @@ -44,16 +42,13 @@ public static long getCurrentProcId() { public static List getChildProcessIds(long pid) { try { Process pgrep = (new ProcessBuilder("pgrep", "-P", String.valueOf(pid))).start(); - System.out.println("--ftw: pgrep = " + pgrep); BufferedReader reader = new BufferedReader(new InputStreamReader(pgrep.getInputStream())); List result = new LinkedList<>(); String line; while ((line = reader.readLine()) != null) { result.add(Long.valueOf(line.trim())); } - System.out.println("--ftw: pgrep.waitFor();"); pgrep.waitFor(); - System.out.println("--ftw: get all result"); return result; } catch (Exception e) { throw new RuntimeException("Couldn't get child processes of PID " + pid, e); @@ -76,9 +71,4 @@ public static void killProcess(long pid) { throw new RuntimeException("Couldn't kill process PID " + pid, e); } } - - public static HoodieTableMetaClient getMetaClient(Configuration conf, String basePath) { - return HadoopUGI.ugiDoAs(AuthenticationConfig.getKerberosConfig(conf), () -> HoodieTableMetaClient.builder() - .setConf(conf).setBasePath(basePath).build()); - } } diff --git a/fe/be-java-extensions/trino-connector-scanner/pom.xml b/fe/be-java-extensions/trino-connector-scanner/pom.xml index c9c821820a20ce..d4b8556e67f45c 100644 --- a/fe/be-java-extensions/trino-connector-scanner/pom.xml +++ b/fe/be-java-extensions/trino-connector-scanner/pom.xml @@ -35,7 +35,7 @@ under the License. org.apache.doris - hudi-scanner + preload-extensions ${project.version} diff --git a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorCache.java b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorCache.java index e2020037f4341b..234c3fe1c76a40 100644 --- a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorCache.java +++ b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorCache.java @@ -17,7 +17,7 @@ package org.apache.doris.trinoconnector; -import org.apache.doris.hudi.Utils; +import org.apache.doris.preload.common.ProcessUtils; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; @@ -112,37 +112,33 @@ private static TrinoConnectorCacheValue loadCache(TrinoConnectorCacheKey key) { // RecordReader will use ProcessBuilder to start a hotspot process, which may be stuck, // so use another process to kill this stuck process. AtomicBoolean isKilled = new AtomicBoolean(false); - LOG.info("--ftw: isKilled = " + isKilled); ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1); executorService.scheduleAtFixedRate(() -> { if (!isKilled.get()) { - LOG.info("--ftw: isKilled is false."); - long currentPid = Utils.getCurrentProcId(); - LOG.info("--ftw: currentPid = " + currentPid); - List pids = Utils.getChildProcessIds(currentPid); - LOG.info("--ftw: get all pids"); + List pids; + try { + pids = ProcessUtils.getChildProcessIds( + ProcessUtils.getCurrentProcId()); + } catch (Exception e) { + System.out.println("get pid exception: " + e.getMessage()); + LOG.info("get pid exception: " + e.getMessage()); + throw new RuntimeException("get pid error.", e); + } for (long pid : pids) { - LOG.info("--ftw: pid = " + pid); - String cmd = Utils.getCommandLine(pid); - LOG.info("--ftw: get cmd = " + cmd); + String cmd = ProcessUtils.getCommandLine(pid); if (cmd != null && cmd.contains("org.openjdk.jol.vm.sa.AttachMain")) { - LOG.info("--ftw: begin kill, cmd = " + cmd); - Utils.killProcess(pid); - LOG.info("--ftw: finish killed, cmd = " + cmd); + ProcessUtils.killProcess(pid); isKilled.set(true); } } } }, 100, 1000, TimeUnit.MILLISECONDS); - LOG.info("--ftw: isKill = " + isKilled); // create CatalogHandle CatalogHandle catalogHandle = CatalogHandle.createRootCatalogHandle(key.catalogName, new CatalogVersion("test")); - LOG.info("--ftw: finish create catalog handle."); isKilled.set(true); - LOG.info("--ftw: isKill = " + isKilled); executorService.shutdownNow(); CatalogFactory catalogFactory = createCatalogFactory(key.trinoConnectorPluginManager.getTypeRegistry(), From c02a75c84e68314cc16eea1cbaf3783ee4bbe2c2 Mon Sep 17 00:00:00 2001 From: BePPPower Date: Wed, 27 Mar 2024 20:46:13 +0800 Subject: [PATCH 4/9] fix 2 --- .../TrinoConnectorPluginLoader.java | 15 +++++++++++++++ .../TrinoConnectorPluginLoader.java | 16 ++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorPluginLoader.java b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorPluginLoader.java index 134f315c333bc7..1f08e28b005fa6 100644 --- a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorPluginLoader.java +++ b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorPluginLoader.java @@ -30,6 +30,9 @@ import org.apache.logging.log4j.Logger; import java.io.File; +import java.util.logging.FileHandler; +import java.util.logging.Level; +import java.util.logging.SimpleFormatter; public class TrinoConnectorPluginLoader { private static final Logger LOG = LogManager.getLogger(TrinoConnectorPluginLoader.class); @@ -42,6 +45,18 @@ private static class TrinoConnectorPluginLoad { static { try { + // Trino uses jul as its own log system, so the attributes of JUL are configured here + System.setProperty("java.util.logging.SimpleFormatter.format", + "%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS %4$s: %5$s%6$s%n"); + java.util.logging.Logger logger = java.util.logging.Logger.getLogger(""); + logger.setUseParentHandlers(false); + FileHandler fileHandler = new FileHandler(EnvUtils.getDorisHome() + "/log/trinoconnector%g.log", + 500000000, 10, true); + fileHandler.setLevel(Level.INFO); + fileHandler.setFormatter(new SimpleFormatter()); + logger.addHandler(fileHandler); + java.util.logging.LogManager.getLogManager().addLogger(logger); + TypeOperators typeOperators = new TypeOperators(); featuresConfig = new FeaturesConfig(); TypeRegistry typeRegistry = new TypeRegistry(typeOperators, featuresConfig); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorPluginLoader.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorPluginLoader.java index a0d34ffa31e306..1e08c9effcb00f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorPluginLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorPluginLoader.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.trinoconnector; import org.apache.doris.common.Config; +import org.apache.doris.common.EnvUtils; import org.apache.doris.trinoconnector.TrinoConnectorPluginManager; import com.google.common.util.concurrent.MoreExecutors; @@ -31,6 +32,9 @@ import org.apache.logging.log4j.Logger; import java.io.File; +import java.util.logging.FileHandler; +import java.util.logging.Level; +import java.util.logging.SimpleFormatter; public class TrinoConnectorPluginLoader { private static final Logger LOG = LogManager.getLogger(TrinoConnectorPluginLoader.class); @@ -44,6 +48,18 @@ private static class TrinoConnectorPluginLoad { static { try { + // Trino uses jul as its own log system, so the attributes of JUL are configured here + System.setProperty("java.util.logging.SimpleFormatter.format", + "%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS %4$s: %5$s%6$s%n"); + java.util.logging.Logger logger = java.util.logging.Logger.getLogger(""); + logger.setUseParentHandlers(false); + FileHandler fileHandler = new FileHandler(EnvUtils.getDorisHome() + "/log/trinoconnector%g.log", + 500000000, 10, true); + fileHandler.setLevel(Level.INFO); + fileHandler.setFormatter(new SimpleFormatter()); + logger.addHandler(fileHandler); + java.util.logging.LogManager.getLogManager().addLogger(logger); + typeRegistry = new TypeRegistry(typeOperators, featuresConfig); ServerPluginsProviderConfig serverPluginsProviderConfig = new ServerPluginsProviderConfig() .setInstalledPluginsDir(new File(Config.trino_connector_plugin_dir)); From 4f92ea7c4a7f7c4cb8002526beae1d0e39fd7f66 Mon Sep 17 00:00:00 2001 From: BePPPower Date: Wed, 27 Mar 2024 20:50:43 +0800 Subject: [PATCH 5/9] fix 3 --- .../pipeline/external/conf/regression-conf.groovy | 5 ----- regression-test/pipeline/p0/conf/regression-conf.groovy | 3 --- 2 files changed, 8 deletions(-) diff --git a/regression-test/pipeline/external/conf/regression-conf.groovy b/regression-test/pipeline/external/conf/regression-conf.groovy index 3638891d75b61a..43994cccd8eb91 100644 --- a/regression-test/pipeline/external/conf/regression-conf.groovy +++ b/regression-test/pipeline/external/conf/regression-conf.groovy @@ -63,16 +63,11 @@ excludeSuites = "000_the_start_sentinel_do_not_touch," + // keep this line as th "test_cast_string_to_array," + "test_refresh_mtmv," + "test_spark_load," + - "test_trino_hive_orc," + - "test_trino_hive_other," + "test_hive_write_insert," + "test_broker_load_func," + "test_hive_write_partitions," + "zzz_the_end_sentinel_do_not_touch" // keep this line as the last line -// this directories will not be executed -excludeDirectories = "external_table_p0/trino_connector" // unstable - customConf1 = "test_custom_conf_value" // for test csv with header diff --git a/regression-test/pipeline/p0/conf/regression-conf.groovy b/regression-test/pipeline/p0/conf/regression-conf.groovy index c29b5f6bcf18ca..20238b26312afc 100644 --- a/regression-test/pipeline/p0/conf/regression-conf.groovy +++ b/regression-test/pipeline/p0/conf/regression-conf.groovy @@ -70,8 +70,6 @@ excludeSuites = "000_the_start_sentinel_do_not_touch," + // keep this line as th "test_profile," + "test_refresh_mtmv," + "test_spark_load," + - "test_trino_hive_orc," + - "test_trino_hive_other," + "test_hive_write_insert," + "test_hive_write_partitions," + "test_broker_load_func," + @@ -83,7 +81,6 @@ excludeDirectories = "000_the_start_sentinel_do_not_touch," + // keep this line "cloud," + "nereids_rules_p0/subquery," + "workload_manager_p1," + - "external_table_p0/trino_connector," + // unstable "zzz_the_end_sentinel_do_not_touch" // keep this line as the last line customConf1 = "test_custom_conf_value" From 9887e083933d45a80888e5635626b9c836cd78c4 Mon Sep 17 00:00:00 2001 From: BePPPower Date: Wed, 27 Mar 2024 20:52:42 +0800 Subject: [PATCH 6/9] fix4 --- .../doris/trinoconnector/TrinoConnectorCache.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorCache.java b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorCache.java index 234c3fe1c76a40..0bcbceea844199 100644 --- a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorCache.java +++ b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorCache.java @@ -115,15 +115,8 @@ private static TrinoConnectorCacheValue loadCache(TrinoConnectorCacheKey key) { ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1); executorService.scheduleAtFixedRate(() -> { if (!isKilled.get()) { - List pids; - try { - pids = ProcessUtils.getChildProcessIds( - ProcessUtils.getCurrentProcId()); - } catch (Exception e) { - System.out.println("get pid exception: " + e.getMessage()); - LOG.info("get pid exception: " + e.getMessage()); - throw new RuntimeException("get pid error.", e); - } + List pids; = ProcessUtils.getChildProcessIds( + ProcessUtils.getCurrentProcId()); for (long pid : pids) { String cmd = ProcessUtils.getCommandLine(pid); if (cmd != null && cmd.contains("org.openjdk.jol.vm.sa.AttachMain")) { From a7caac396cb5e0f2f6b42a84c1906005beaf4ded Mon Sep 17 00:00:00 2001 From: BePPPower Date: Wed, 27 Mar 2024 20:53:39 +0800 Subject: [PATCH 7/9] fix5 --- .../org/apache/doris/trinoconnector/TrinoConnectorCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorCache.java b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorCache.java index 0bcbceea844199..eb83f880a7bb3f 100644 --- a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorCache.java +++ b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorCache.java @@ -115,7 +115,7 @@ private static TrinoConnectorCacheValue loadCache(TrinoConnectorCacheKey key) { ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1); executorService.scheduleAtFixedRate(() -> { if (!isKilled.get()) { - List pids; = ProcessUtils.getChildProcessIds( + List pids = ProcessUtils.getChildProcessIds( ProcessUtils.getCurrentProcId()); for (long pid : pids) { String cmd = ProcessUtils.getCommandLine(pid); From cc2b1e46a39dbc48b1b6c4593ce75be6ed88a5c6 Mon Sep 17 00:00:00 2001 From: BePPPower Date: Wed, 27 Mar 2024 22:14:46 +0800 Subject: [PATCH 8/9] fix 6 --- .../java/org/apache/doris/hudi/Utils.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java diff --git a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java new file mode 100644 index 00000000000000..5d08a2eb44ef89 --- /dev/null +++ b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.hudi; + +import org.apache.doris.common.security.authentication.AuthenticationConfig; +import org.apache.doris.common.security.authentication.HadoopUGI; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hudi.common.table.HoodieTableMetaClient; + +public class Utils { + public static HoodieTableMetaClient getMetaClient(Configuration conf, String basePath) { + return HadoopUGI.ugiDoAs(AuthenticationConfig.getKerberosConfig(conf), () -> HoodieTableMetaClient.builder() + .setConf(conf).setBasePath(basePath).build()); + } +} From 42f0940c0a56b8cb015052437d15783542e5e133 Mon Sep 17 00:00:00 2001 From: BePPPower Date: Thu, 28 Mar 2024 11:46:41 +0800 Subject: [PATCH 9/9] fix 7 --- fe/be-java-extensions/hudi-scanner/pom.xml | 5 -- .../org/apache/doris/hudi/HudiJniScanner.java | 9 ++-- .../java/org/apache/doris/hudi/Utils.java | 50 +++++++++++++++++++ .../trino-connector-scanner/pom.xml | 10 ++-- .../doris/trinoconnector}/ProcessUtils.java | 2 +- .../trinoconnector/TrinoConnectorCache.java | 2 - 6 files changed, 60 insertions(+), 18 deletions(-) rename fe/be-java-extensions/{preload-extensions/src/main/java/org/apache/doris/preload/common => trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector}/ProcessUtils.java (98%) diff --git a/fe/be-java-extensions/hudi-scanner/pom.xml b/fe/be-java-extensions/hudi-scanner/pom.xml index 2cd5b7deee39b6..d4f7a458612c85 100644 --- a/fe/be-java-extensions/hudi-scanner/pom.xml +++ b/fe/be-java-extensions/hudi-scanner/pom.xml @@ -214,11 +214,6 @@ under the License. - - org.apache.doris - preload-extensions - ${project.version} - hudi-scanner diff --git a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java index 4cf76dce07e1dc..a284c7adcdd105 100644 --- a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java +++ b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java @@ -22,7 +22,6 @@ import org.apache.doris.common.jni.vec.ColumnType; import org.apache.doris.common.security.authentication.AuthenticationConfig; import org.apache.doris.common.security.authentication.HadoopUGI; -import org.apache.doris.preload.common.ProcessUtils; import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.avro.generic.GenericDatumReader; @@ -144,12 +143,12 @@ public void open() throws IOException { executorService.scheduleAtFixedRate(() -> { if (!isKilled.get()) { synchronized (HudiJniScanner.class) { - List pids = ProcessUtils.getChildProcessIds( - ProcessUtils.getCurrentProcId()); + List pids = Utils.getChildProcessIds( + Utils.getCurrentProcId()); for (long pid : pids) { - String cmd = ProcessUtils.getCommandLine(pid); + String cmd = Utils.getCommandLine(pid); if (cmd != null && cmd.contains("org.openjdk.jol.vm.sa.AttachMain")) { - ProcessUtils.killProcess(pid); + Utils.killProcess(pid); isKilled.set(true); LOG.info("Kill hotspot debugger process " + pid); } diff --git a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java index 5d08a2eb44ef89..5614f8bcc96eb1 100644 --- a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java +++ b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java @@ -20,10 +20,60 @@ import org.apache.doris.common.security.authentication.AuthenticationConfig; import org.apache.doris.common.security.authentication.HadoopUGI; +import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hudi.common.table.HoodieTableMetaClient; +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.lang.management.ManagementFactory; +import java.util.LinkedList; +import java.util.List; + public class Utils { + public static long getCurrentProcId() { + try { + return ManagementFactory.getRuntimeMXBean().getPid(); + } catch (Exception e) { + throw new RuntimeException("Couldn't find PID of current JVM process.", e); + } + } + + public static List getChildProcessIds(long pid) { + try { + Process pgrep = (new ProcessBuilder("pgrep", "-P", String.valueOf(pid))).start(); + BufferedReader reader = new BufferedReader(new InputStreamReader(pgrep.getInputStream())); + List result = new LinkedList<>(); + String line; + while ((line = reader.readLine()) != null) { + result.add(Long.valueOf(line.trim())); + } + pgrep.waitFor(); + return result; + } catch (Exception e) { + throw new RuntimeException("Couldn't get child processes of PID " + pid, e); + } + } + + public static String getCommandLine(long pid) { + try { + return FileUtils.readFileToString(new File(String.format("/proc/%d/cmdline", pid))).trim(); + } catch (IOException e) { + return null; + } + } + + public static void killProcess(long pid) { + try { + Process kill = (new ProcessBuilder("kill", "-9", String.valueOf(pid))).start(); + kill.waitFor(); + } catch (Exception e) { + throw new RuntimeException("Couldn't kill process PID " + pid, e); + } + } + public static HoodieTableMetaClient getMetaClient(Configuration conf, String basePath) { return HadoopUGI.ugiDoAs(AuthenticationConfig.getKerberosConfig(conf), () -> HoodieTableMetaClient.builder() .setConf(conf).setBasePath(basePath).build()); diff --git a/fe/be-java-extensions/trino-connector-scanner/pom.xml b/fe/be-java-extensions/trino-connector-scanner/pom.xml index d4b8556e67f45c..2c8b91e0a6c65a 100644 --- a/fe/be-java-extensions/trino-connector-scanner/pom.xml +++ b/fe/be-java-extensions/trino-connector-scanner/pom.xml @@ -33,11 +33,6 @@ under the License. - - org.apache.doris - preload-extensions - ${project.version} - org.apache.doris java-common @@ -47,6 +42,11 @@ under the License. io.trino trino-main + + commons-io + commons-io + ${commons-io.version} + diff --git a/fe/be-java-extensions/preload-extensions/src/main/java/org/apache/doris/preload/common/ProcessUtils.java b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/ProcessUtils.java similarity index 98% rename from fe/be-java-extensions/preload-extensions/src/main/java/org/apache/doris/preload/common/ProcessUtils.java rename to fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/ProcessUtils.java index 5823c7e6aa2062..429b268e01ba1a 100644 --- a/fe/be-java-extensions/preload-extensions/src/main/java/org/apache/doris/preload/common/ProcessUtils.java +++ b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/ProcessUtils.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.preload.common; +package org.apache.doris.trinoconnector; import org.apache.commons.io.FileUtils; diff --git a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorCache.java b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorCache.java index eb83f880a7bb3f..e9d92a98f788ce 100644 --- a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorCache.java +++ b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorCache.java @@ -17,8 +17,6 @@ package org.apache.doris.trinoconnector; -import org.apache.doris.preload.common.ProcessUtils; - import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache;