From cdeff0feb65e74b46ba03b85320cd60f5b8d7bd1 Mon Sep 17 00:00:00 2001 From: Balazs Meszaros Date: Wed, 3 Jul 2019 14:51:03 +0200 Subject: [PATCH 1/6] HBASE-15666 shaded dependencies for hbase-testing-util Added new artifact hbase-shaded-testing-util. It wraps a whole hbase-server with its testing dependencies. Users should use only the following dependency in pom: org.apache.hbase hbase-shaded-testing-util ${hbase.version} test Added hbase-shaded-testing-util-tester maven module which ensures that hbase-shaded-testing-util works with a shaded client. --- .../main/resources/supplemental-models.xml | 13 + .../ensure-jars-have-correct-contents.sh | 2 + .../hbase-shaded-testing-util-tester/pom.xml | 62 ++ .../shaded/TestShadedHBaseTestingUtility.java | 61 ++ .../hbase-shaded-testing-util/pom.xml | 150 +++++ .../org/eclipse/jetty/webapp/webdefault.xml | 534 ++++++++++++++++++ .../ensure-jars-have-correct-contents.sh | 2 + hbase-shaded/pom.xml | 23 +- pom.xml | 1 + 9 files changed, 829 insertions(+), 19 deletions(-) create mode 100644 hbase-shaded/hbase-shaded-testing-util-tester/pom.xml create mode 100644 hbase-shaded/hbase-shaded-testing-util-tester/src/test/java/org/apache/hbase/shaded/TestShadedHBaseTestingUtility.java create mode 100644 hbase-shaded/hbase-shaded-testing-util/pom.xml create mode 100644 hbase-shaded/hbase-shaded-testing-util/src/main/resources/org/apache/hadoop/hbase/shaded/org/eclipse/jetty/webapp/webdefault.xml diff --git a/hbase-resource-bundle/src/main/resources/supplemental-models.xml b/hbase-resource-bundle/src/main/resources/supplemental-models.xml index f3a6afcfcc6a..81dbff1c6d13 100644 --- a/hbase-resource-bundle/src/main/resources/supplemental-models.xml +++ b/hbase-resource-bundle/src/main/resources/supplemental-models.xml @@ -332,6 +332,19 @@ under the License. + + + javax.servlet.jsp + jsp-api + + + CDDL 1.1 + https://glassfish.java.net/public/CDDL+GPL_1_1.html + repo + + + + javax.servlet.jsp diff --git a/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh b/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh index eff1d2030225..50bcbfc40e8b 100644 --- a/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh +++ b/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh @@ -73,6 +73,8 @@ allowed_expr+="|^org/apache/hbase/" allowed_expr+="|^META-INF/" # * the folding tables from jcodings allowed_expr+="|^tables/" +# * contents of hbase-webapps +allowed_expr+="|^hbase-webapps/" # * HBase's default configuration files, which have the form # "_module_-default.xml" allowed_expr+="|^hbase-default.xml$" diff --git a/hbase-shaded/hbase-shaded-testing-util-tester/pom.xml b/hbase-shaded/hbase-shaded-testing-util-tester/pom.xml new file mode 100644 index 000000000000..6e6a258518ec --- /dev/null +++ b/hbase-shaded/hbase-shaded-testing-util-tester/pom.xml @@ -0,0 +1,62 @@ + + + 4.0.0 + + + org.apache.hbase + hbase-build-configuration + 3.0.0-SNAPSHOT + ../../hbase-build-configuration + + + hbase-shaded-testing-util-tester + Apache HBase - Shaded - Testing Util Tester + Ensures that hbase-shaded-testing-util works with hbase-shaded-client. + + + + junit + junit + test + + + + org.slf4j + slf4j-log4j12 + + + + + org.apache.hbase + hbase-shaded-client + ${project.version} + + + org.apache.hbase + hbase-shaded-testing-util + ${project.version} + test + + + + diff --git a/hbase-shaded/hbase-shaded-testing-util-tester/src/test/java/org/apache/hbase/shaded/TestShadedHBaseTestingUtility.java b/hbase-shaded/hbase-shaded-testing-util-tester/src/test/java/org/apache/hbase/shaded/TestShadedHBaseTestingUtility.java new file mode 100644 index 000000000000..13653aa92a7a --- /dev/null +++ b/hbase-shaded/hbase-shaded-testing-util-tester/src/test/java/org/apache/hbase/shaded/TestShadedHBaseTestingUtility.java @@ -0,0 +1,61 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hbase.shaded; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertEquals; + +@Category({ ClientTests.class, MediumTests.class }) +public class TestShadedHBaseTestingUtility { + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestShadedHBaseTestingUtility.class); + + @BeforeClass + public static void setUp() throws Exception { + TEST_UTIL.startMiniCluster(); + } + + @AfterClass + public static void tearDown() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testCreateTable() throws Exception { + TableName tableName = TableName.valueOf("test"); + + TEST_UTIL.createTable(tableName, "family"); + + int rows = TEST_UTIL.countRows(tableName); + assertEquals(0, rows); + } +} diff --git a/hbase-shaded/hbase-shaded-testing-util/pom.xml b/hbase-shaded/hbase-shaded-testing-util/pom.xml new file mode 100644 index 000000000000..9848c16a7379 --- /dev/null +++ b/hbase-shaded/hbase-shaded-testing-util/pom.xml @@ -0,0 +1,150 @@ + + + 4.0.0 + + + hbase-shaded + org.apache.hbase + 3.0.0-SNAPSHOT + .. + + + hbase-shaded-testing-util + Apache HBase - Shaded - Testing Util + + + + + org.apache.hadoop + hadoop-common + test-jar + compile + + + org.apache.hadoop + hadoop-hdfs + test-jar + compile + + + org.apache.hbase + hbase-common + test-jar + compile + + + org.apache.hbase + hbase-server + test-jar + compile + + + org.apache.hbase + hbase-zookeeper + test-jar + compile + + + org.apache.hbase + hbase-hadoop-compat + test-jar + compile + + + org.apache.hbase + hbase-hadoop2-compat + test-jar + compile + + + + org.apache.hbase + hbase-testing-util + ${project.version} + compile + + + + + + + org.apache.maven.plugins + maven-site-plugin + + true + + + + + maven-assembly-plugin + + true + + + + org.apache.maven.plugins + maven-shade-plugin + + + aggregate-into-a-jar-with-relocated-third-parties + + + + + javax.annotation:javax.annotation-api + javax.activation:javax.activation-api + + + org.apache.hbase:hbase-resource-bundle + org.slf4j:* + com.google.code.findbugs:* + com.github.stephenc.findbugs:* + org.apache.htrace:* + org.apache.yetus:* + log4j:* + commons-logging:* + + + + + + + + org.apache.rat + apache-rat-plugin + + + dependency-reduced-pom.xml + **/webdefault.xml + + + + + + + diff --git a/hbase-shaded/hbase-shaded-testing-util/src/main/resources/org/apache/hadoop/hbase/shaded/org/eclipse/jetty/webapp/webdefault.xml b/hbase-shaded/hbase-shaded-testing-util/src/main/resources/org/apache/hadoop/hbase/shaded/org/eclipse/jetty/webapp/webdefault.xml new file mode 100644 index 000000000000..bd23f3e924a5 --- /dev/null +++ b/hbase-shaded/hbase-shaded-testing-util/src/main/resources/org/apache/hadoop/hbase/shaded/org/eclipse/jetty/webapp/webdefault.xml @@ -0,0 +1,534 @@ + + + + + + + + + + + + + + + + + + + + + + + Default web.xml file. + This file is applied to a Web application before it's own WEB_INF/web.xml file + + + + + + + + org.apache.hadoop.hbase.shaded.org.eclipse.jetty.servlet.listener.ELContextCleaner + + + + + + + + org.apache.hadoop.hbase.shaded.org.eclipse.jetty.servlet.listener.IntrospectorCleaner + + + + + + + + + + + + + + + + + default + org.apache.hadoop.hbase.shaded.org.eclipse.jetty.servlet.DefaultServlet + + aliases + false + + + acceptRanges + true + + + dirAllowed + true + + + welcomeServlets + false + + + redirectWelcome + false + + + maxCacheSize + 256000000 + + + maxCachedFileSize + 200000000 + + + maxCachedFiles + 2048 + + + gzip + false + + + etags + false + + + useFileMappedBuffer + true + + + + 0 + + + + default + / + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + jsp + org.apache.hadoop.hbase.shaded.org.eclipse.jetty.jsp.JettyJspServlet + + logVerbosityLevel + DEBUG + + + fork + false + + + xpoweredBy + false + + + compilerTargetVM + 1.7 + + + compilerSourceVM + 1.7 + + + 0 + + + + jsp + *.jsp + *.jspf + *.jspx + *.xsp + *.JSP + *.JSPF + *.JSPX + *.XSP + + + + + + + + 30 + + + + + + + + + + + + + + + index.html + index.htm + index.jsp + + + + + + + + ar + ISO-8859-6 + + + be + ISO-8859-5 + + + bg + ISO-8859-5 + + + ca + ISO-8859-1 + + + cs + ISO-8859-2 + + + da + ISO-8859-1 + + + de + ISO-8859-1 + + + el + ISO-8859-7 + + + en + ISO-8859-1 + + + es + ISO-8859-1 + + + et + ISO-8859-1 + + + fi + ISO-8859-1 + + + fr + ISO-8859-1 + + + hr + ISO-8859-2 + + + hu + ISO-8859-2 + + + is + ISO-8859-1 + + + it + ISO-8859-1 + + + iw + ISO-8859-8 + + + ja + Shift_JIS + + + ko + EUC-KR + + + lt + ISO-8859-2 + + + lv + ISO-8859-2 + + + mk + ISO-8859-5 + + + nl + ISO-8859-1 + + + no + ISO-8859-1 + + + pl + ISO-8859-2 + + + pt + ISO-8859-1 + + + ro + ISO-8859-2 + + + ru + ISO-8859-5 + + + sh + ISO-8859-5 + + + sk + ISO-8859-2 + + + sl + ISO-8859-2 + + + sq + ISO-8859-2 + + + sr + ISO-8859-5 + + + sv + ISO-8859-1 + + + tr + ISO-8859-9 + + + uk + ISO-8859-5 + + + zh + GB2312 + + + zh_TW + Big5 + + + + + + + + + Disable TRACE + / + TRACE + + + + + + Enable everything but TRACE + / + TRACE + + + + + diff --git a/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh index eff1d2030225..50bcbfc40e8b 100644 --- a/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh +++ b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh @@ -73,6 +73,8 @@ allowed_expr+="|^org/apache/hbase/" allowed_expr+="|^META-INF/" # * the folding tables from jcodings allowed_expr+="|^tables/" +# * contents of hbase-webapps +allowed_expr+="|^hbase-webapps/" # * HBase's default configuration files, which have the form # "_module_-default.xml" allowed_expr+="|^hbase-default.xml$" diff --git a/hbase-shaded/pom.xml b/hbase-shaded/pom.xml index cdaeee076d92..c7465ad3aa78 100644 --- a/hbase-shaded/pom.xml +++ b/hbase-shaded/pom.xml @@ -42,6 +42,8 @@ hbase-shaded-client-byo-hadoop hbase-shaded-client hbase-shaded-mapreduce + hbase-shaded-testing-util + hbase-shaded-testing-util-tester hbase-shaded-check-invariants hbase-shaded-with-hadoop-check-invariants @@ -443,15 +445,6 @@ ${shaded.prefix}.net - - - junit - ${shaded.prefix}.junit - - - org.junit - ${shaded.prefix}.org.junit - overview.html @@ -471,6 +463,8 @@ false ${project.name} + + @@ -501,15 +495,6 @@ org/apache/commons/collections/*.class - - - org.apache.hbase:hbase-server - - hbase-webapps/* - hbase-webapps/**/* - **/*_jsp.class - - org.apache.hadoop:hadoop-yarn-common diff --git a/pom.xml b/pom.xml index 65f3466fe002..f518f720ef7b 100755 --- a/pom.xml +++ b/pom.xml @@ -2257,6 +2257,7 @@ junit junit + test From b2912992fb5dfb3602c39aae24eb5598a0a0ccb3 Mon Sep 17 00:00:00 2001 From: Balazs Meszaros Date: Thu, 11 Jul 2019 10:39:07 +0200 Subject: [PATCH 2/6] license header fix, test enhancements --- NOTICE.txt | 3 +++ .../shaded/TestShadedHBaseTestingUtility.java | 19 +++++++++++++---- .../hbase-shaded-testing-util/pom.xml | 10 --------- .../org/eclipse/jetty/webapp/webdefault.xml | 21 +++++++++++++++++++ 4 files changed, 39 insertions(+), 14 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index a6a3d0a09c9d..02d0501c3a54 100755 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -45,3 +45,6 @@ This product includes portions of the Guava project v14 and v21, specifically Copyright (C) 2007 The Guava Authors Licensed under the Apache License, Version 2.0 +-- +This product includes portions of Jetty project, specially +'hbase-shaded-hbase-shaded-testing-util/src/main/resources/org/apache/hadoop/hbase/shaded/org/mortbay/jetty/webapp/webdefault.xml' diff --git a/hbase-shaded/hbase-shaded-testing-util-tester/src/test/java/org/apache/hbase/shaded/TestShadedHBaseTestingUtility.java b/hbase-shaded/hbase-shaded-testing-util-tester/src/test/java/org/apache/hbase/shaded/TestShadedHBaseTestingUtility.java index 13653aa92a7a..b18352eb45c1 100644 --- a/hbase-shaded/hbase-shaded-testing-util-tester/src/test/java/org/apache/hbase/shaded/TestShadedHBaseTestingUtility.java +++ b/hbase-shaded/hbase-shaded-testing-util-tester/src/test/java/org/apache/hbase/shaded/TestShadedHBaseTestingUtility.java @@ -18,19 +18,22 @@ */ package org.apache.hbase.shaded; +import static org.junit.Assert.assertEquals; + import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; -import static org.junit.Assert.assertEquals; - @Category({ ClientTests.class, MediumTests.class }) public class TestShadedHBaseTestingUtility { private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); @@ -53,9 +56,17 @@ public static void tearDown() throws Exception { public void testCreateTable() throws Exception { TableName tableName = TableName.valueOf("test"); - TEST_UTIL.createTable(tableName, "family"); + Table table = TEST_UTIL.createTable(tableName, "cf"); + + Put put1 = new Put(Bytes.toBytes("r1")); + put1.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("c"), Bytes.toBytes(1)); + table.put(put1); + + Put put2 = new Put(Bytes.toBytes("r2")); + put2.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("c"), Bytes.toBytes(2)); + table.put(put2); int rows = TEST_UTIL.countRows(tableName); - assertEquals(0, rows); + assertEquals(2, rows); } } diff --git a/hbase-shaded/hbase-shaded-testing-util/pom.xml b/hbase-shaded/hbase-shaded-testing-util/pom.xml index 9848c16a7379..0707ea3b542f 100644 --- a/hbase-shaded/hbase-shaded-testing-util/pom.xml +++ b/hbase-shaded/hbase-shaded-testing-util/pom.xml @@ -134,16 +134,6 @@ - - org.apache.rat - apache-rat-plugin - - - dependency-reduced-pom.xml - **/webdefault.xml - - - diff --git a/hbase-shaded/hbase-shaded-testing-util/src/main/resources/org/apache/hadoop/hbase/shaded/org/eclipse/jetty/webapp/webdefault.xml b/hbase-shaded/hbase-shaded-testing-util/src/main/resources/org/apache/hadoop/hbase/shaded/org/eclipse/jetty/webapp/webdefault.xml index bd23f3e924a5..e31d7f46f5f6 100644 --- a/hbase-shaded/hbase-shaded-testing-util/src/main/resources/org/apache/hadoop/hbase/shaded/org/eclipse/jetty/webapp/webdefault.xml +++ b/hbase-shaded/hbase-shaded-testing-util/src/main/resources/org/apache/hadoop/hbase/shaded/org/eclipse/jetty/webapp/webdefault.xml @@ -1,4 +1,25 @@ + + + Date: Thu, 11 Jul 2019 17:25:39 +0200 Subject: [PATCH 3/6] license fixes --- .../shaded/org/eclipse/jetty/webapp/webdefault.xml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/hbase-shaded/hbase-shaded-testing-util/src/main/resources/org/apache/hadoop/hbase/shaded/org/eclipse/jetty/webapp/webdefault.xml b/hbase-shaded/hbase-shaded-testing-util/src/main/resources/org/apache/hadoop/hbase/shaded/org/eclipse/jetty/webapp/webdefault.xml index e31d7f46f5f6..127dd95aee07 100644 --- a/hbase-shaded/hbase-shaded-testing-util/src/main/resources/org/apache/hadoop/hbase/shaded/org/eclipse/jetty/webapp/webdefault.xml +++ b/hbase-shaded/hbase-shaded-testing-util/src/main/resources/org/apache/hadoop/hbase/shaded/org/eclipse/jetty/webapp/webdefault.xml @@ -3,13 +3,9 @@ org.apache.hbase hbase-shaded-client