From 61cfd909aa04289e104ef20e245cb0de700c60f3 Mon Sep 17 00:00:00 2001 From: James Turton <9107319+jnturton@users.noreply.github.com> Date: Thu, 24 Aug 2023 05:39:04 +0200 Subject: [PATCH] DRILL-8452: Library upgrades (#2823) --- contrib/storage-elasticsearch/pom.xml | 2 +- .../elasticsearch/TestElasticsearchSuite.java | 5 +- contrib/storage-hive/core/pom.xml | 4 - contrib/storage-kafka/pom.xml | 2 +- .../store/kafka/KafkaFilterPushdownTest.java | 6 +- .../store/kafka/KafkaMessageGenerator.java | 2 +- .../exec/store/kafka/KafkaQueriesTest.java | 14 +- .../drill/exec/store/kafka/KafkaTestBase.java | 10 +- .../exec/store/kafka/MessageIteratorTest.java | 4 +- ...TestKafkaSuit.java => TestKafkaSuite.java} | 10 +- .../resources/{login.conf => login.jaasconf} | 2 +- .../Dockerfile | 0 .../README => docker-compose/README.txt} | 0 .../docker-compose.yml | 0 .../drill-override.conf | 0 .../logback.xml | 0 distribution/src/assemble/component.xml | 4 +- .../apache/drill/yarn/core/ClusterDef.java | 8 +- exec/java-exec/pom.xml | 9 +- .../src/main/codegen/templates/Parser.jj | 13 +- .../exec/expr/fn/FunctionInitializer.java | 2 +- .../rest/auth/DrillSpnegoLoginService.java | 5 +- .../ssl/SslContextFactoryConfigurator.java | 8 +- .../apache/drill/exec/ZookeeperTestUtil.java | 2 +- .../physical/unit/TestOutputBatchSize.java | 4 +- .../SslContextFactoryConfiguratorTest.java | 2 +- .../apache/drill/exec/sql/TestInfoSchema.java | 4 +- .../drill/exec/store/json/TestJsonReader.java | 4 +- .../resources/{login.conf => login.jaasconf} | 2 +- .../resources/vault/read-vault-secrets.hcl | 2 + exec/jdbc-all/pom.xml | 8 +- exec/jdbc/pom.xml | 11 +- .../templates/RepeatedValueVectors.java | 2 +- pom.xml | 559 +++++++++--------- 34 files changed, 369 insertions(+), 341 deletions(-) rename contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/{TestKafkaSuit.java => TestKafkaSuite.java} (97%) rename contrib/storage-kafka/src/test/resources/{login.conf => login.jaasconf} (99%) rename distribution/{docker-cluster-mode => docker-compose}/Dockerfile (100%) rename distribution/{docker-cluster-mode/README => docker-compose/README.txt} (100%) rename distribution/{docker-cluster-mode => docker-compose}/docker-compose.yml (100%) rename distribution/{docker-cluster-mode => docker-compose}/drill-override.conf (100%) rename distribution/{docker-cluster-mode => docker-compose}/logback.xml (100%) rename exec/java-exec/src/test/resources/{login.conf => login.jaasconf} (99%) diff --git a/contrib/storage-elasticsearch/pom.xml b/contrib/storage-elasticsearch/pom.xml index 40f4a869b58..74840fea79c 100644 --- a/contrib/storage-elasticsearch/pom.xml +++ b/contrib/storage-elasticsearch/pom.xml @@ -68,7 +68,7 @@ org.elasticsearch.client elasticsearch-rest-client - 8.6.0 + 8.9.1 commons-logging diff --git a/contrib/storage-elasticsearch/src/test/java/org/apache/drill/exec/store/elasticsearch/TestElasticsearchSuite.java b/contrib/storage-elasticsearch/src/test/java/org/apache/drill/exec/store/elasticsearch/TestElasticsearchSuite.java index 12fbe72e6f3..258034e9fde 100644 --- a/contrib/storage-elasticsearch/src/test/java/org/apache/drill/exec/store/elasticsearch/TestElasticsearchSuite.java +++ b/contrib/storage-elasticsearch/src/test/java/org/apache/drill/exec/store/elasticsearch/TestElasticsearchSuite.java @@ -68,7 +68,7 @@ public class TestElasticsearchSuite extends BaseTest { protected static ElasticsearchContainer elasticsearch; public static final String ELASTICSEARCH_USERNAME = "elastic"; public static final String ELASTICSEARCH_PASSWORD = "s3cret"; - private static final String IMAGE_NAME = "docker.elastic.co/elasticsearch/elasticsearch:8.6.0"; + private static final String IMAGE_NAME = "docker.elastic.co/elasticsearch/elasticsearch:8.9.1"; private static final AtomicInteger initCount = new AtomicInteger(0); @@ -164,7 +164,8 @@ private static void startElasticsearch() throws GeneralSecurityException { .withPassword(ELASTICSEARCH_PASSWORD) .withEnv("xpack.security.enabled", "true") .withEnv("xpack.security.transport.ssl.enabled", "false") - .withEnv("ES_JAVA_OPTS", "-Xmx1g"); // ES gobbles up lots of RAM under defaults. + .withEnv("discovery.type", "single-node") + .withEnv("ES_JAVA_OPTS", "-Xmx2g"); // ES gobbles up lots of RAM under defaults. HttpsURLConnection.setDefaultSSLSocketFactory(SslUtils.trustAllSSLContext().getSocketFactory()); elasticsearch.start(); diff --git a/contrib/storage-hive/core/pom.xml b/contrib/storage-hive/core/pom.xml index d491c972737..ce889521ca8 100644 --- a/contrib/storage-hive/core/pom.xml +++ b/contrib/storage-hive/core/pom.xml @@ -257,10 +257,6 @@ reload4j ch.qos.reload4j - - org.mortbay.jetty - servlet-api-2.5 - tomcat jasper-compiler diff --git a/contrib/storage-kafka/pom.xml b/contrib/storage-kafka/pom.xml index 040a2ecfb32..35cb6022e62 100644 --- a/contrib/storage-kafka/pom.xml +++ b/contrib/storage-kafka/pom.xml @@ -32,7 +32,7 @@ 2.8.2 - **/TestKafkaSuit.class + **/TestKafkaSuite.class diff --git a/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaFilterPushdownTest.java b/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaFilterPushdownTest.java index c9277f90a5a..f1dd78b8767 100644 --- a/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaFilterPushdownTest.java +++ b/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaFilterPushdownTest.java @@ -24,8 +24,8 @@ import org.junit.Test; import org.junit.experimental.categories.Category; -import static org.apache.drill.exec.store.kafka.TestKafkaSuit.NUM_JSON_MSG; -import static org.apache.drill.exec.store.kafka.TestKafkaSuit.embeddedKafkaCluster; +import static org.apache.drill.exec.store.kafka.TestKafkaSuite.NUM_JSON_MSG; +import static org.apache.drill.exec.store.kafka.TestKafkaSuite.embeddedKafkaCluster; import static org.junit.Assert.assertEquals; @Category({KafkaStorageTest.class, SlowTest.class}) @@ -36,7 +36,7 @@ public class KafkaFilterPushdownTest extends KafkaTestBase { @BeforeClass public static void setup() throws Exception { - TestKafkaSuit.createTopicHelper(TestQueryConstants.JSON_PUSHDOWN_TOPIC, NUM_PARTITIONS); + TestKafkaSuite.createTopicHelper(TestQueryConstants.JSON_PUSHDOWN_TOPIC, NUM_PARTITIONS); KafkaMessageGenerator generator = new KafkaMessageGenerator(embeddedKafkaCluster.getKafkaBrokerList(), StringSerializer.class); generator.populateJsonMsgWithTimestamps(TestQueryConstants.JSON_PUSHDOWN_TOPIC, NUM_JSON_MSG); diff --git a/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaMessageGenerator.java b/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaMessageGenerator.java index f4e4730b801..ef2ea8fc7fe 100644 --- a/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaMessageGenerator.java +++ b/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaMessageGenerator.java @@ -87,7 +87,7 @@ public void populateAvroMsgIntoKafka(String topic, int numMsg) { String key1Schema = "{\"type\":\"record\"," + "\"name\":\"key1record\"," + "\"fields\":[" + - "{\"name\":\"key1\",\"type\":\"string\"}]}\""; + "{\"name\":\"key1\",\"type\":\"string\"}]}"; Schema keySchema = parser.parse(key1Schema); GenericRecordBuilder keyBuilder = new GenericRecordBuilder(keySchema); diff --git a/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaQueriesTest.java b/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaQueriesTest.java index e91303610f8..327962f99ca 100644 --- a/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaQueriesTest.java +++ b/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaQueriesTest.java @@ -41,7 +41,7 @@ import java.util.Map; import java.util.Set; -import static org.apache.drill.exec.store.kafka.TestKafkaSuit.embeddedKafkaCluster; +import static org.apache.drill.exec.store.kafka.TestKafkaSuite.embeddedKafkaCluster; import static org.junit.Assert.fail; @FixMethodOrder(MethodSorters.JVM) @@ -76,7 +76,7 @@ public void testResultLimit() throws Exception { @Test public void testResultCount() { String queryString = String.format(TestQueryConstants.MSG_SELECT_QUERY, TestQueryConstants.JSON_TOPIC); - runKafkaSQLVerifyCount(queryString, TestKafkaSuit.NUM_JSON_MSG); + runKafkaSQLVerifyCount(queryString, TestKafkaSuite.NUM_JSON_MSG); } @Test @@ -91,7 +91,7 @@ public void testAvroResultCount() { KafkaAvroDeserializer.class.getName()); String queryString = String.format(TestQueryConstants.MSG_SELECT_QUERY, TestQueryConstants.AVRO_TOPIC); - runKafkaSQLVerifyCount(queryString, TestKafkaSuit.NUM_JSON_MSG); + runKafkaSQLVerifyCount(queryString, TestKafkaSuite.NUM_JSON_MSG); } finally { client.resetSession(ExecConstants.KAFKA_RECORD_READER); } @@ -209,7 +209,7 @@ public void testPhysicalPlanSubmissionAvro() throws Exception { @Test public void testOneMessageTopic() throws Exception { String topicName = "topicWithOneMessage"; - TestKafkaSuit.createTopicHelper(topicName, 1); + TestKafkaSuite.createTopicHelper(topicName, 1); KafkaMessageGenerator generator = new KafkaMessageGenerator(embeddedKafkaCluster.getKafkaBrokerList(), StringSerializer.class); generator.populateMessages(topicName, "{\"index\": 1}"); @@ -224,7 +224,7 @@ public void testOneMessageTopic() throws Exception { @Test public void testMalformedRecords() throws Exception { String topicName = "topicWithMalFormedMessages"; - TestKafkaSuit.createTopicHelper(topicName, 1); + TestKafkaSuite.createTopicHelper(topicName, 1); try { KafkaMessageGenerator generator = new KafkaMessageGenerator(embeddedKafkaCluster.getKafkaBrokerList(), StringSerializer.class); generator.populateMessages(topicName, "Test"); @@ -259,7 +259,7 @@ public void testMalformedRecords() throws Exception { @Test public void testNanInf() throws Exception { String topicName = "topicWithNanInf"; - TestKafkaSuit.createTopicHelper(topicName, 1); + TestKafkaSuite.createTopicHelper(topicName, 1); try { KafkaMessageGenerator generator = new KafkaMessageGenerator(embeddedKafkaCluster.getKafkaBrokerList(), StringSerializer.class); generator.populateMessages(topicName, "{\"nan_col\":NaN, \"inf_col\":Infinity}"); @@ -287,7 +287,7 @@ public void testNanInf() throws Exception { @Test public void testEscapeAnyChar() throws Exception { String topicName = "topicWithEscapeAnyChar"; - TestKafkaSuit.createTopicHelper(topicName, 1); + TestKafkaSuite.createTopicHelper(topicName, 1); try { KafkaMessageGenerator generator = new KafkaMessageGenerator(embeddedKafkaCluster.getKafkaBrokerList(), StringSerializer.class); generator.populateMessages(topicName, "{\"name\": \"AB\\\"\\C\"}"); diff --git a/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaTestBase.java b/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaTestBase.java index c426b061713..0af48213a41 100644 --- a/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaTestBase.java +++ b/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaTestBase.java @@ -43,11 +43,11 @@ public class KafkaTestBase extends ClusterTest { @BeforeClass public static void setUpBeforeClass() throws Exception { // Make sure this test is only running as part of the suit - Assume.assumeTrue(TestKafkaSuit.isRunningSuite()); + Assume.assumeTrue(TestKafkaSuite.isRunningSuite()); ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher); startCluster(builder); - TestKafkaSuit.initKafka(); - initKafkaStoragePlugin(TestKafkaSuit.embeddedKafkaCluster); + TestKafkaSuite.initKafka(); + initKafkaStoragePlugin(TestKafkaSuite.embeddedKafkaCluster); } public static void initKafkaStoragePlugin(EmbeddedKafkaCluster embeddedKafkaCluster) throws Exception { @@ -76,8 +76,8 @@ public static long testSql(String sql) { @AfterClass public static void tearDownKafkaTestBase() { - if (TestKafkaSuit.isRunningSuite()) { - TestKafkaSuit.tearDownCluster(); + if (TestKafkaSuite.isRunningSuite()) { + TestKafkaSuite.tearDownCluster(); } } } \ No newline at end of file diff --git a/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/MessageIteratorTest.java b/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/MessageIteratorTest.java index 12db4c3de1e..ed5f1e6874c 100644 --- a/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/MessageIteratorTest.java +++ b/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/MessageIteratorTest.java @@ -48,7 +48,7 @@ public void setUp() { consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class); consumerProps.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, "4"); kafkaConsumer = new KafkaConsumer<>(consumerProps); - subScanSpec = new KafkaPartitionScanSpec(TestQueryConstants.JSON_TOPIC, 0, 0, TestKafkaSuit.NUM_JSON_MSG); + subScanSpec = new KafkaPartitionScanSpec(TestQueryConstants.JSON_TOPIC, 0, 0, TestKafkaSuite.NUM_JSON_MSG); } @After @@ -105,6 +105,6 @@ public void testShouldReturnMessage2() { Assert.assertNotNull(consumerRecord); ++messageCount; } - Assert.assertEquals(TestKafkaSuit.NUM_JSON_MSG, messageCount); + Assert.assertEquals(TestKafkaSuite.NUM_JSON_MSG, messageCount); } } diff --git a/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/TestKafkaSuit.java b/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/TestKafkaSuite.java similarity index 97% rename from contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/TestKafkaSuit.java rename to contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/TestKafkaSuite.java index 73384d28d89..a8a75f1bcb1 100644 --- a/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/TestKafkaSuit.java +++ b/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/TestKafkaSuite.java @@ -54,11 +54,11 @@ @Category({KafkaStorageTest.class, SlowTest.class}) @RunWith(Suite.class) @SuiteClasses({KafkaQueriesTest.class, MessageIteratorTest.class, MessageReaderFactoryTest.class, KafkaFilterPushdownTest.class}) -public class TestKafkaSuit extends BaseTest { +public class TestKafkaSuite extends BaseTest { - private static final Logger logger = LoggerFactory.getLogger(TestKafkaSuit.class); + private static final Logger logger = LoggerFactory.getLogger(TestKafkaSuite.class); - private static final String LOGIN_CONF_RESOURCE_PATHNAME = "login.conf"; + private static final String LOGIN_CONF_RESOURCE_PATHNAME = "login.jaasconf"; public static EmbeddedKafkaCluster embeddedKafkaCluster; @@ -76,7 +76,7 @@ public class TestKafkaSuit extends BaseTest { @BeforeClass public static void initKafka() throws Exception { - synchronized (TestKafkaSuit.class) { + synchronized (TestKafkaSuite.class) { if (initCount.get() == 0) { ZookeeperTestUtil.setZookeeperSaslTestConfigProps(); System.setProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM, ClassLoader.getSystemResource(LOGIN_CONF_RESOURCE_PATHNAME).getFile()); @@ -104,7 +104,7 @@ public static boolean isRunningSuite() { @AfterClass public static void tearDownCluster() { - synchronized (TestKafkaSuit.class) { + synchronized (TestKafkaSuite.class) { if (initCount.decrementAndGet() == 0) { if (zkClient != null) { zkClient.close(); diff --git a/contrib/storage-kafka/src/test/resources/login.conf b/contrib/storage-kafka/src/test/resources/login.jaasconf similarity index 99% rename from contrib/storage-kafka/src/test/resources/login.conf rename to contrib/storage-kafka/src/test/resources/login.jaasconf index 091612028f3..56a10c4fd41 100644 --- a/contrib/storage-kafka/src/test/resources/login.conf +++ b/contrib/storage-kafka/src/test/resources/login.jaasconf @@ -22,4 +22,4 @@ hadoop_simple { org.apache.hadoop.security.login.GenericOSLoginModule required; org.apache.hadoop.security.login.HadoopLoginModule required; -}; \ No newline at end of file +}; diff --git a/distribution/docker-cluster-mode/Dockerfile b/distribution/docker-compose/Dockerfile similarity index 100% rename from distribution/docker-cluster-mode/Dockerfile rename to distribution/docker-compose/Dockerfile diff --git a/distribution/docker-cluster-mode/README b/distribution/docker-compose/README.txt similarity index 100% rename from distribution/docker-cluster-mode/README rename to distribution/docker-compose/README.txt diff --git a/distribution/docker-cluster-mode/docker-compose.yml b/distribution/docker-compose/docker-compose.yml similarity index 100% rename from distribution/docker-cluster-mode/docker-compose.yml rename to distribution/docker-compose/docker-compose.yml diff --git a/distribution/docker-cluster-mode/drill-override.conf b/distribution/docker-compose/drill-override.conf similarity index 100% rename from distribution/docker-cluster-mode/drill-override.conf rename to distribution/docker-compose/drill-override.conf diff --git a/distribution/docker-cluster-mode/logback.xml b/distribution/docker-compose/logback.xml similarity index 100% rename from distribution/docker-cluster-mode/logback.xml rename to distribution/docker-compose/logback.xml diff --git a/distribution/src/assemble/component.xml b/distribution/src/assemble/component.xml index e2b255269ca..6e6473a79e9 100644 --- a/distribution/src/assemble/component.xml +++ b/distribution/src/assemble/component.xml @@ -114,7 +114,7 @@ org.glassfish.jersey.core org.reflections org.glassfish.hk2.external - org.mortbay.jetty + org.eclipse.jetty javax.activation javax.annotation org.glassfish.jersey.containers @@ -155,7 +155,7 @@ org.glassfish.jersey.core org.reflections org.glassfish.hk2.external - org.mortbay.jetty + org.eclipse.jetty javax.activation javax.annotation org.glassfish.jersey.containers diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/core/ClusterDef.java b/drill-yarn/src/main/java/org/apache/drill/yarn/core/ClusterDef.java index 223b6068de8..825da7c9720 100644 --- a/drill-yarn/src/main/java/org/apache/drill/yarn/core/ClusterDef.java +++ b/drill-yarn/src/main/java/org/apache/drill/yarn/core/ClusterDef.java @@ -22,13 +22,17 @@ import java.util.Map; import org.apache.drill.yarn.appMaster.TaskSpec; -import org.mortbay.log.Log; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.typesafe.config.Config; import com.typesafe.config.ConfigList; import com.typesafe.config.ConfigValue; public class ClusterDef { + + private static Logger logger = LoggerFactory.getLogger(ClusterDef.class); + // The following keys are relative to the cluster group definition public static final String GROUP_NAME = "name"; @@ -139,7 +143,7 @@ public LabeledGroup(Map pool, int index) { super(pool, index, GroupType.LABELED); drillbitLabelExpr = (String) pool.get(DRILLBIT_LABEL); if (drillbitLabelExpr == null) { - Log.warn("Labeled pool is missing the drillbit label expression (" + logger.warn("Labeled pool is missing the drillbit label expression (" + DRILLBIT_LABEL + "), will treat pool as basic."); } } diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml index b2f4e1fae4b..6d0a06d4fd6 100644 --- a/exec/java-exec/pom.xml +++ b/exec/java-exec/pom.xml @@ -359,9 +359,8 @@ protobuf-java - org.mortbay.jetty + org.eclipse.jetty jetty-util - 6.1.26 joda-time @@ -442,10 +441,6 @@ org.eclipse.jetty jetty-servlet - - org.eclipse.jetty - jetty-servlets - org.eclipse.jetty jetty-security @@ -845,9 +840,9 @@ unpack - false + false ${calcite.groupId} calcite-core jar diff --git a/exec/java-exec/src/main/codegen/templates/Parser.jj b/exec/java-exec/src/main/codegen/templates/Parser.jj index ccd2091f7ec..0040e97b076 100644 --- a/exec/java-exec/src/main/codegen/templates/Parser.jj +++ b/exec/java-exec/src/main/codegen/templates/Parser.jj @@ -1,10 +1,11 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionInitializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionInitializer.java index dfaf380f04d..442c158f823 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionInitializer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionInitializer.java @@ -30,7 +30,7 @@ import org.codehaus.janino.Java.CompilationUnit; import org.codehaus.janino.Parser; import org.codehaus.janino.Scanner; -import org.mortbay.util.IO; +import org.eclipse.jetty.util.IO; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/auth/DrillSpnegoLoginService.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/auth/DrillSpnegoLoginService.java index 98a76cbe15d..c6ba0c18717 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/auth/DrillSpnegoLoginService.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/auth/DrillSpnegoLoginService.java @@ -28,7 +28,6 @@ import org.eclipse.jetty.security.DefaultIdentityService; import org.eclipse.jetty.security.SpnegoLoginService; import org.eclipse.jetty.server.UserIdentity; -import org.eclipse.jetty.util.B64Code; import org.ietf.jgss.GSSContext; import org.ietf.jgss.GSSCredential; import org.ietf.jgss.GSSException; @@ -42,6 +41,7 @@ import java.lang.reflect.Field; import java.security.Principal; import java.security.PrivilegedExceptionAction; +import java.util.Base64; /** * Custom implementation of DrillSpnegoLoginService to avoid the need of passing targetName in a config file, @@ -94,7 +94,7 @@ public UserIdentity login(final String username, final Object credentials, Servl private UserIdentity spnegoLogin(Object credentials, ServletRequest request) { String encodedAuthToken = (String) credentials; - byte[] authToken = B64Code.decode(encodedAuthToken); + byte[] authToken = Base64.getDecoder().decode(encodedAuthToken); GSSManager manager = GSSManager.getInstance(); try { @@ -149,4 +149,3 @@ private UserIdentity spnegoLogin(Object credentials, ServletRequest request) { return null; } } - diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ssl/SslContextFactoryConfigurator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ssl/SslContextFactoryConfigurator.java index 3a745491abc..89421c69194 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ssl/SslContextFactoryConfigurator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ssl/SslContextFactoryConfigurator.java @@ -66,14 +66,14 @@ public SslContextFactoryConfigurator(DrillConfig config, String drillbitEndpoint * @return new configured sslContextFactory * @throws Exception when generation of self-signed certificate failed */ - public SslContextFactory configureNewSslContextFactory() throws Exception { + public SslContextFactory.Server configureNewSslContextFactory() throws Exception { SSLConfig sslConf = new SSLConfigBuilder() .config(config) .mode(SSLConfig.Mode.SERVER) .initializeSSLContext(false) .validateKeyStore(true) .build(); - final SslContextFactory sslContextFactory = new SslContextFactory(); + SslContextFactory.Server sslContextFactory = new SslContextFactory.Server(); if (sslConf.isSslValid()) { useOptionsConfiguredByUser(sslContextFactory, sslConf); } else { @@ -82,7 +82,7 @@ public SslContextFactory configureNewSslContextFactory() throws Exception { return sslContextFactory; } - private void useOptionsConfiguredByUser(SslContextFactory sslFactory, SSLConfig sslConf) { + private void useOptionsConfiguredByUser(SslContextFactory.Server sslFactory, SSLConfig sslConf) { logger.info("Using configured SSL settings for web server"); sslFactory.setKeyStorePath(sslConf.getKeyStorePath()); sslFactory.setKeyStorePassword(sslConf.getKeyStorePassword()); @@ -159,7 +159,7 @@ private void setIfPresent(String optKey, Function optGet, Consume } - private void useAutoGeneratedSelfSignedCertificate(SslContextFactory sslContextFactory) throws Exception { + private void useAutoGeneratedSelfSignedCertificate(SslContextFactory.Server sslContextFactory) throws Exception { logger.info("Using generated self-signed SSL settings for web server"); final SecureRandom random = new SecureRandom(); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperTestUtil.java b/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperTestUtil.java index 7256a6082df..c83ae5a41c6 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperTestUtil.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperTestUtil.java @@ -25,7 +25,7 @@ public class ZookeeperTestUtil { - private static final String LOGIN_CONF_RESOURCE_PATHNAME = "login.conf"; + private static final String LOGIN_CONF_RESOURCE_PATHNAME = "login.jaasconf"; /** * Sets zookeeper server and client SASL test config properties. diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/TestOutputBatchSize.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/TestOutputBatchSize.java index d7b6170386a..d1562ee0715 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/TestOutputBatchSize.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/TestOutputBatchSize.java @@ -132,7 +132,7 @@ public void testProjectMap() throws Exception { Long[] baseLineValues = {(5l + 100l)}; // a + c.amount for (int i = 0; i < numRows; i++) { - opTestBuilder.baselineValues(baseLineValues); + opTestBuilder.baselineValues((Object[]) baseLineValues); } opTestBuilder.go(); } @@ -208,7 +208,7 @@ public void testProjectVariableWidthFunctions() throws Exception { String[] baseLineValues = {operationResult}; //operation(a, a) for (int i = 0; i < numRows; i++) { - opTestBuilder.baselineValues(baseLineValues); + opTestBuilder.baselineValues((Object[]) baseLineValues); } opTestBuilder.go(); } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/ssl/SslContextFactoryConfiguratorTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/ssl/SslContextFactoryConfiguratorTest.java index f7dbdd7ec21..fdc37bc3473 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/ssl/SslContextFactoryConfiguratorTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/ssl/SslContextFactoryConfiguratorTest.java @@ -62,7 +62,7 @@ public static void setUpClass() throws Exception { @Test public void configureNewSslContextFactory() throws Exception { - SslContextFactory sslContextFactory = sslContextFactoryConfigurator.configureNewSslContextFactory(); + SslContextFactory.Server sslContextFactory = sslContextFactoryConfigurator.configureNewSslContextFactory(); assertEquals(30, sslContextFactory.getSslSessionTimeout()); assertTrue(sslContextFactory.getWantClientAuth()); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java index 7619ea013ac..ceda39eaeeb 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java @@ -111,7 +111,7 @@ public void showTablesFromDb() throws Exception { .unOrdered() .baselineColumns("TABLE_SCHEMA", "TABLE_NAME"); for (String[] expectedRow : expected) { - t1.baselineValues(expectedRow); + t1.baselineValues((Object[]) expectedRow); } t1.go(); @@ -120,7 +120,7 @@ public void showTablesFromDb() throws Exception { .unOrdered() .baselineColumns("TABLE_SCHEMA", "TABLE_NAME"); for (String[] expectedRow : expected) { - t2.baselineValues(expectedRow); + t2.baselineValues((Object[]) expectedRow); } t2.go(); } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonReader.java index f1509787555..cdb57809696 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonReader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonReader.java @@ -139,8 +139,8 @@ private void doSchemaChangeValidate() throws Exception { .sqlQuery("select b from dfs.`vector/complex/writer/schemaChange/`") .unOrdered() .baselineColumns("b") - .baselineValues(null) - .baselineValues(null) + .baselineValues((Object[]) null) + .baselineValues((Object[]) null) .baselineValues(mapOf()) .baselineValues(mapOf("x", 1L, "y", 2L)) .build() diff --git a/exec/java-exec/src/test/resources/login.conf b/exec/java-exec/src/test/resources/login.jaasconf similarity index 99% rename from exec/java-exec/src/test/resources/login.conf rename to exec/java-exec/src/test/resources/login.jaasconf index 091612028f3..56a10c4fd41 100644 --- a/exec/java-exec/src/test/resources/login.conf +++ b/exec/java-exec/src/test/resources/login.jaasconf @@ -22,4 +22,4 @@ hadoop_simple { org.apache.hadoop.security.login.GenericOSLoginModule required; org.apache.hadoop.security.login.HadoopLoginModule required; -}; \ No newline at end of file +}; diff --git a/exec/java-exec/src/test/resources/vault/read-vault-secrets.hcl b/exec/java-exec/src/test/resources/vault/read-vault-secrets.hcl index 62bbdd3b415..098bafba42a 100644 --- a/exec/java-exec/src/test/resources/vault/read-vault-secrets.hcl +++ b/exec/java-exec/src/test/resources/vault/read-vault-secrets.hcl @@ -1,3 +1,4 @@ +// // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -13,6 +14,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +// path "secret/*" { diff --git a/exec/jdbc-all/pom.xml b/exec/jdbc-all/pom.xml index e689853e33a..d66822b0fc8 100644 --- a/exec/jdbc-all/pom.xml +++ b/exec/jdbc-all/pom.xml @@ -33,7 +33,7 @@ "package.namespace.prefix" equals to "oadd.". It can be overridden if necessary within any profile --> oadd. - 48000000 + 46500000 @@ -176,7 +176,7 @@ jetty-util - org.mortbay.jetty + org.eclipse.jetty freemarker @@ -413,7 +413,7 @@ org.tukaani:* org.apache.velocity:* net.hydromatic:linq4j - org.mortbay.jetty:* + org.eclipse.jetty:* org.slf4j:jul-to-slf4j org.hamcrest:hamcrest-core org.mockito:mockito-core @@ -870,7 +870,7 @@ org.apache.velocity:* net.hydromatic:linq4j org.codehaus.janino:* - org.mortbay.jetty:* + org.eclipse.jetty:* org.slf4j:jul-to-slf4j org.slf4j:log4j-over-slf4j org.hamcrest:hamcrest-core diff --git a/exec/jdbc/pom.xml b/exec/jdbc/pom.xml index a00fc9331df..6f02d45fb0a 100644 --- a/exec/jdbc/pom.xml +++ b/exec/jdbc/pom.xml @@ -42,7 +42,7 @@ org.apache.drill drill-logical ${project.version} - + org.apache.drill.exec drill-java-exec @@ -66,7 +66,6 @@ foodmart-data-json 0.4 - sqlline sqlline @@ -137,8 +136,12 @@ commons-logging - org.mortbay.jetty - servlet-api-2.5 + org.eclipse.jetty + jetty-server + + + org.eclipse.jetty + jetty-servlet diff --git a/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java b/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java index 186c9f86dfb..b233ab96474 100644 --- a/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java +++ b/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java @@ -22,7 +22,7 @@ import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.TransferPair; import org.apache.drill.exec.vector.complex.BaseRepeatedValueVector; -import org.mortbay.jetty.servlet.Holder; +import org.eclipse.jetty.servlet.Holder; <@pp.dropOutputFile /> <#list vv.types as type> diff --git a/pom.xml b/pom.xml index 93d524e94a8..5f25bef7f8b 100644 --- a/pom.xml +++ b/pom.xml @@ -63,7 +63,7 @@ 6.1.0-mapr 3.0-mapr-1808 1.0.0 - 3.0.0 + 3.0.5 2.7 4.4 @@ -80,56 +80,56 @@ --> 3.1.3 3.2.4 - 2.4.9 + 2.5.5-hadoop3 1.0 2.3.30 3.29.2-GA 0.6.6 0.9.10 - 1.11.1 - 4.2.10 - 9.4.44.v20210927 - 2.34 + 1.11.2 + 4.2.19 + 9.4.51.v20230217 + 2.40 2.0.1.Final - 9.2 + 9.5 2500 4500 true true apache/drill - 4.8-1 + 4.9.3 3.8.4 3.6.3 3.9.0 - 1.6 + 1.7 1.10.0 3.16.3 - 1.7.1 + 1.8.0 2.6 - 2.10.14 + 2.12.5 3.0.0 - 3.0.0-M8 - 5.8.0 - 1.21 + 3.1.2 + 5.13.0 + 1.23.0 4.0.3 4.1.73.Final - 4.5.13 - 0.14.0 + 4.5.14 + 0.18.1 10.14.2.0 1.4 - 2.0 + 2.1 3.10 - 1.17.3 + 1.18.3 1.4.2 1.14 2.12.2 1.10 1.9.4 - 5.8 + 5.10.0 7.19.2 - 2.19.0 - 0.20 + 2.20.0 + 0.25 0.12.1 2.8.3 4.3.3 @@ -225,7 +225,6 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.4.1 @@ -659,6 +658,7 @@ org.apache.maven.plugins maven-jar-plugin + 3.3.0 @@ -667,146 +667,157 @@ + + org.apache.maven.plugins + maven-resources-plugin + 3.3.1 + + + pl.project13.maven + git-commit-id-plugin + 4.0.5 + com.mycila license-maven-plugin - 3.0 + 4.2 true
${maven.multiModuleProjectDirectory}/header
- **/clientlib/y2038/*.c - **/clientlib/y2038/*.h - **/resources/parquet/**/* - **/.asf.yaml - **/*.woff2 - **/*.ks - **/*.pcap - **/*.sas7bdat - **/*.sav - **/*.log1 - **/*.log2 - **/*.h5 - **/*.sqllog - **/*.sqllog2 - **/*.syslog - **/*.xls - **/*.xlsx - **/*.mdb **/*.accdb - **/*.syslog1 - **/*.ssdlog - **/*.ltsv - **/*.log - **/*.css - **/*.js - **/*.map - **/*.md - **/*.eps - **/*.json - **/*.seq - **/*.parquet + **/*.access_log + **/.asf.yaml + **/*.autotools + **/*.avi **/*.avro - **/*.sql - **/git.properties - **/*.csv - **/*.csvh - **/*.pdf - **/*.csvh-test - **/*.tsv - **/*.txt - **/*.ssv **/.buildpath - **/target/** - **/*.iml - **/.idea/** - **/*.project - **/TAGS **/*.checkstyle **/.classpath - **/.settings/** - **/*.patch - **/*.pb.cc - **/*.pb.h **/client/*build*/** + **/clientlib/y2038/*.c + **/clientlib/y2038/*.h **/client/tags **/cmake_install.cmake - **/ssl/*.csr - **/ssl/*.pem - **/ssl/*.p12 - **/*.tbl - **/*.httpd - **/*.autotools + **/control **/*.cproject + **/.*.crc + **/*.css + **/*.csv + **/*.csvh + **/*.csvh-test + **/*.dbf + + dependency-reduced-pom.xml **/*.drill - **/LICENSE - **/NOTICE - KEYS - header **/.drill.parquet_metadata + **/*.eps + **/git.properties + **/*.h5 + header + **/*.httpd + **/.idea/** + **/*.iml + **/*.js + **/*.json + KEYS + **/*.ks + **/LICENSE + **/*.log1 + **/*.log1 + **/*.log2 + **/*.log2 + **/*.log + **/*.ltsv + **/*.ltsv + **/*.map + **/*.mdb + **/*.md **/*.mov **/*.mp4 - **/*.avi - **/*.pcx - **/*.webp + **/NOTICE + **/*.parquet + **/*.patch + **/*.pb.cc + **/*.pb.h + **/*.pcap **/*.pcapng - **/*.psd **/*.pcx - **/*.log1 - **/*.log2 - **/*.sqllog - **/*.sqllog2 - **/*.ltsv - **/*.wav - **/control - - dependency-reduced-pom.xml + **/*.pcx + **/*.pdf **/*.prj + **/*.project + **/*.psd + **/resources/parquet/**/* + **/*.sas7bdat + **/*.sav + **/*.seq + **/.settings/** **/*.shp - **/*.dbf - **/*.access_log + **/*.sql + **/*.sqllog2 + **/*.sqllog2 + **/*.sqllog + **/*.sqllog + **/*.ssdlog + **/ssl/*.csr + **/ssl/*.p12 + **/ssl/*.pem + **/*.ssv + **/*.syslog1 + **/*.syslog + **/TAGS + **/target/** + **/*.tbl + **/*.tsv + **/*.txt + **/*.wav + **/*.webp + **/*.woff2 + **/*.xls + **/*.xlsx - SLASHSTAR_STYLE + SLASHSTAR_STYLE + SCRIPT_STYLE + SCRIPT_STYLE + SCRIPT_STYLE + SCRIPT_STYLE + SCRIPT_STYLE + SCRIPT_STYLE SLASHSTAR_STYLE + SLASHSTAR_STYLE + SCRIPT_STYLE + SCRIPT_STYLE + SCRIPT_STYLE + SCRIPT_STYLE + SCRIPT_STYLE + SCRIPT_STYLE + SCRIPT_STYLE + SCRIPT_STYLE + SLASHSTAR_STYLE + DOUBLESLASH_STYLE SLASHSTAR_STYLE SLASHSTAR_STYLE SLASHSTAR_STYLE - SLASHSTAR_STYLE - SLASHSTAR_STYLE - SLASHSTAR_STYLE - SLASHSTAR_STYLE + SLASHSTAR_STYLE + SLASHSTAR_STYLE + SLASHSTAR_STYLE SLASHSTAR_STYLE - SLASHSTAR_STYLE - SCRIPT_STYLE - SLASHSTAR_STYLE + SLASHSTAR_STYLE + XML_STYLE + SCRIPT_STYLE + DOUBLESLASH_STYLE + SCRIPT_STYLE SCRIPT_STYLE SCRIPT_STYLE - SCRIPT_STYLE + SLASHSTAR_STYLE SCRIPT_STYLE - SCRIPT_STYLE - SCRIPT_STYLE - SCRIPT_STYLE + SLASHSTAR_STYLE SCRIPT_STYLE - SCRIPT_STYLE - SCRIPT_STYLE - - SLASHSTAR_STYLE - SCRIPT_STYLE - SCRIPT_STYLE - DOUBLESLASH_STYLE - SCRIPT_STYLE SCRIPT_STYLE - SLASHSTAR_STYLE - XML_STYLE - SLASHSTAR_STYLE - SCRIPT_STYLE - SCRIPT_STYLE - SCRIPT_STYLE - SCRIPT_STYLE - SCRIPT_STYLE - SCRIPT_STYLE + SLASHSTAR_STYLE @@ -820,21 +831,21 @@ org.apache.maven.plugins maven-compiler-plugin - 3.8.1 + 3.11.0 maven-enforcer-plugin - 3.0.0-M2 + 3.3.0 org.codehaus.mojo build-helper-maven-plugin - 3.0.0 + 3.4.0 org.codehaus.mojo exec-maven-plugin - 1.6.0 + 3.1.0 process-classes @@ -915,7 +926,7 @@ org.codehaus.mojo xml-maven-plugin - 1.0 + 1.1.0