Merge remote-tracking branch 'upstream/master' into used-segments

.travis.yml

@@ -114,6 +114,18 @@ jobs:
         https://maven.apache.org/plugins/maven-dependency-plugin/analyze-mojo.html
         "
 
+    - name: "security vulnerabilities"
+      install: skip
+      script: ${MVN} dependency-check:check
+      after_failure: |-
+        echo "FAILURE EXPLANATION:
+
+        The OWASP dependency check has found security vulnerabilities. Please use a newer version
+        of the dependency that does not have vulenerabilities. If the analysis has false positives,
+        they can be suppressed by adding entries to owasp-dependency-check-suppressions.xml (for more
+        information, see https://jeremylong.github.io/DependencyCheck/general/suppression.html).
+        "
+
     - &package
       name: "(openjdk8) packaging check"
       install: skip

LICENSE

@@ -314,3 +314,14 @@ SOURCE/JAVA-CORE
     This product uses a smear function adapted from MurmurHash3, written by Austin Appleby who has placed
      MurmurHash3 in the public domain (https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp).
       * processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/Groupers.java
+
+
+Creative Commons Attribution-ShareAlike 3.0 Unported License
+================================
+
+SOURCE/JAVA-CORE:
+    This product contains example data collected from the Wikipedia edit stream API (this content is NOT part of the source
+     code and is only used for example and tutorial purposes), copyright Wikipedia editors and contributors for the text
+     content which is available under Creative Commons Attribution-ShareAlike 3.0 Unported License. For details, see
+     licenses/src/CC-BY-SA.txt.
+      * examples/quickstart/tutorial/wikiticker-2015-09-12-sampled.json.gz

README.md

@@ -17,14 +17,26 @@
   ~ under the License.
   -->
 
-[![Slack](https://img.shields.io/badge/slack-%23druid-72eff8?logo=slack)](https://druid.apache.org/community/join-slack)
+[![Slack](https://img.shields.io/badge/slack-%23druid-72eff8?logo=slack)](https://s.apache.org/slack-invite)
 [![Build Status](https://travis-ci.org/apache/incubator-druid.svg?branch=master)](https://travis-ci.org/apache/incubator-druid)
 [![Language grade: Java](https://img.shields.io/lgtm/grade/java/g/apache/incubator-druid.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/apache/incubator-druid/context:java)
 [![Coverage Status](https://img.shields.io/codecov/c/gh/apache/incubator-druid)](https://codecov.io/gh/apache/incubator-druid)
 [![Docker](https://img.shields.io/badge/container-docker-blue.svg)](https://hub.docker.com/r/apache/incubator-druid)
 <!--- Following badges are disabled until they can be fixed: -->
 <!--- [![Inspections Status](https://img.shields.io/teamcity/http/teamcity.jetbrains.com/s/OpenSourceProjects_Druid_Inspections.svg?label=TeamCity%20inspections)](https://teamcity.jetbrains.com/viewType.html?buildTypeId=OpenSourceProjects_Druid_Inspections) -->
 
+---
+
+[Website](https://druid.apache.org/) |
+[Documentation](https://druid.apache.org/docs/latest/design/) |
+[Developer Mailing List](https://lists.apache.org/[email protected]) |
+[User Mailing List](https://groups.google.com/forum/#!forum/druid-user) |
+[Slack](https://s.apache.org/slack-invite) |
+[Twitter](https://twitter.com/druidio) |
+[Download](https://druid.apache.org/downloads.html)
+
+---
+
 ## Apache Druid (incubating)
 
 Apache Druid (incubating) is a high performance real-time analytics database.

benchmarks/pom.xml

@@ -122,10 +122,6 @@
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-math3</artifactId>
     </dependency>
-    <dependency>
-      <groupId>commons-io</groupId>
-      <artifactId>commons-io</artifactId>
-    </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-annotations</artifactId>
@@ -155,8 +151,9 @@
       <artifactId>jsr305</artifactId>
     </dependency>
     <dependency>
-      <groupId>com.yahoo.datasketches</groupId>
-      <artifactId>sketches-core</artifactId>
+      <groupId>org.apache.datasketches</groupId>
+      <artifactId>datasketches-java</artifactId>
+      <version>1.1.0-incubating</version>
     </dependency>
     <dependency>
       <groupId>junit</groupId>
@@ -219,6 +216,13 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <groupId>org.owasp</groupId>
+        <artifactId>dependency-check-maven</artifactId>
+        <configuration>
+          <skip>true</skip>
+        </configuration>
+      </plugin>
     </plugins>
     <pluginManagement>
       <plugins>

benchmarks/src/main/java/org/apache/druid/benchmark/DataSketchesHllBenchmark.java

@@ -19,7 +19,7 @@
 
 package org.apache.druid.benchmark;
 
-import com.yahoo.sketches.hll.HllSketch;
+import org.apache.datasketches.hll.HllSketch;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.query.aggregation.BufferAggregator;

benchmarks/src/main/java/org/apache/druid/benchmark/FilterPartitionBenchmark.java

@@ -22,14 +22,13 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Predicate;
 import com.google.common.collect.ImmutableList;
-import com.google.common.io.Files;
-import org.apache.commons.io.FileUtils;
 import org.apache.druid.benchmark.datagen.BenchmarkDataGenerator;
 import org.apache.druid.benchmark.datagen.BenchmarkSchemaInfo;
 import org.apache.druid.benchmark.datagen.BenchmarkSchemas;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.data.input.InputRow;
 import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.java.util.common.FileUtils;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.guava.Sequences;
@@ -173,7 +172,7 @@ public void setup() throws IOException
       incIndex.add(row);
     }
 
-    tmpDir = Files.createTempDir();
+    tmpDir = FileUtils.createTempDir();
     log.info("Using temp dir: " + tmpDir.getAbsolutePath());
 
     indexFile = INDEX_MERGER_V9.persist(

benchmarks/src/main/java/org/apache/druid/benchmark/FilteredAggregatorBenchmark.java

@@ -21,15 +21,14 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.io.Files;
-import org.apache.commons.io.FileUtils;
 import org.apache.druid.benchmark.datagen.BenchmarkDataGenerator;
 import org.apache.druid.benchmark.datagen.BenchmarkSchemaInfo;
 import org.apache.druid.benchmark.datagen.BenchmarkSchemas;
 import org.apache.druid.benchmark.query.QueryBenchmarkUtil;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.data.input.InputRow;
 import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.java.util.common.FileUtils;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.logger.Logger;
@@ -186,7 +185,7 @@ public void setup() throws IOException
       inputRows.add(row);
     }
 
-    tmpDir = Files.createTempDir();
+    tmpDir = FileUtils.createTempDir();
     log.info("Using temp dir: " + tmpDir.getAbsolutePath());
 
     indexFile = INDEX_MERGER_V9.persist(

benchmarks/src/main/java/org/apache/druid/benchmark/GenericIndexedBenchmark.java

@@ -19,9 +19,9 @@
 
 package org.apache.druid.benchmark;
 
-import com.google.common.io.Files;
 import com.google.common.primitives.Ints;
 import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.java.util.common.FileUtils;
 import org.apache.druid.java.util.common.io.smoosh.FileSmoosher;
 import org.apache.druid.java.util.common.io.smoosh.SmooshedFileMapper;
 import org.apache.druid.segment.data.GenericIndexed;
@@ -126,7 +126,7 @@ public void createGenericIndexed() throws IOException
       element.putInt(0, i);
       genericIndexedWriter.write(element.array());
     }
-    smooshDir = Files.createTempDir();
+    smooshDir = FileUtils.createTempDir();
     file = File.createTempFile("genericIndexedBenchmark", "meta");
 
     try (FileChannel fileChannel =

benchmarks/src/main/java/org/apache/druid/benchmark/GroupByTypeInterfaceBenchmark.java

@@ -23,8 +23,6 @@
 import com.fasterxml.jackson.dataformat.smile.SmileFactory;
 import com.google.common.base.Supplier;
 import com.google.common.base.Suppliers;
-import com.google.common.io.Files;
-import org.apache.commons.io.FileUtils;
 import org.apache.druid.benchmark.datagen.BenchmarkDataGenerator;
 import org.apache.druid.benchmark.datagen.BenchmarkSchemaInfo;
 import org.apache.druid.benchmark.datagen.BenchmarkSchemas;
@@ -36,6 +34,7 @@
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.data.input.InputRow;
 import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.java.util.common.FileUtils;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.granularity.Granularity;
 import org.apache.druid.java.util.common.guava.Sequence;
@@ -291,7 +290,7 @@ public void setup() throws IOException
         rowsPerSegment
     );
 
-    tmpDir = Files.createTempDir();
+    tmpDir = FileUtils.createTempDir();
     log.info("Using temp dir: %s", tmpDir.getAbsolutePath());
 
     // queryableIndexes   -> numSegments worth of on-disk segments

benchmarks/src/main/java/org/apache/druid/benchmark/TopNTypeInterfaceBenchmark.java

@@ -20,7 +20,6 @@
 package org.apache.druid.benchmark;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.io.Files;
 import org.apache.druid.benchmark.datagen.BenchmarkDataGenerator;
 import org.apache.druid.benchmark.datagen.BenchmarkSchemaInfo;
 import org.apache.druid.benchmark.datagen.BenchmarkSchemas;
@@ -29,6 +28,7 @@
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.data.input.InputRow;
 import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.java.util.common.FileUtils;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.logger.Logger;
@@ -277,7 +277,7 @@ public void setup() throws IOException
       incIndexes.add(incIndex);
     }
 
-    File tmpFile = Files.createTempDir();
+    File tmpFile = FileUtils.createTempDir();
     log.info("Using temp dir: " + tmpFile.getAbsolutePath());
     tmpFile.deleteOnExit();
 

benchmarks/src/main/java/org/apache/druid/benchmark/datagen/SegmentGenerator.java

@@ -21,8 +21,6 @@
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.hash.Hashing;
-import com.google.common.io.Files;
-import org.apache.commons.io.FileUtils;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.data.input.InputRow;
 import org.apache.druid.data.input.impl.DimensionSchema;
@@ -31,6 +29,7 @@
 import org.apache.druid.data.input.impl.FloatDimensionSchema;
 import org.apache.druid.data.input.impl.LongDimensionSchema;
 import org.apache.druid.data.input.impl.StringDimensionSchema;
+import org.apache.druid.java.util.common.FileUtils;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.granularity.Granularity;
@@ -95,7 +94,7 @@ public SegmentGenerator(@Nullable final File cacheDir)
         log.warn("No cache directory provided; benchmark data caching is disabled. "
                  + "Set the 'druid.benchmark.cacheDir' property or 'DRUID_BENCHMARK_CACHE_DIR' environment variable "
                  + "to use caching.");
-        this.cacheDir = Files.createTempDir();
+        this.cacheDir = FileUtils.createTempDir();
         this.cleanupCacheDir = true;
       }
     }

benchmarks/src/main/java/org/apache/druid/benchmark/indexing/IndexMergeBenchmark.java

@@ -21,14 +21,13 @@
 
 import com.fasterxml.jackson.databind.InjectableValues;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.io.Files;
-import org.apache.commons.io.FileUtils;
 import org.apache.druid.benchmark.datagen.BenchmarkDataGenerator;
 import org.apache.druid.benchmark.datagen.BenchmarkSchemaInfo;
 import org.apache.druid.benchmark.datagen.BenchmarkSchemas;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.data.input.InputRow;
 import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.java.util.common.FileUtils;
 import org.apache.druid.java.util.common.logger.Logger;
 import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesSerde;
@@ -141,7 +140,7 @@ public void setup() throws IOException
         incIndex.add(row);
       }
 
-      tmpDir = Files.createTempDir();
+      tmpDir = FileUtils.createTempDir();
       log.info("Using temp dir: " + tmpDir.getAbsolutePath());
 
       File indexFile = INDEX_MERGER_V9.persist(

benchmarks/src/main/java/org/apache/druid/benchmark/indexing/IndexPersistBenchmark.java

@@ -20,14 +20,13 @@
 package org.apache.druid.benchmark.indexing;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.io.Files;
-import org.apache.commons.io.FileUtils;
 import org.apache.druid.benchmark.datagen.BenchmarkDataGenerator;
 import org.apache.druid.benchmark.datagen.BenchmarkSchemaInfo;
 import org.apache.druid.benchmark.datagen.BenchmarkSchemas;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.data.input.InputRow;
 import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.java.util.common.FileUtils;
 import org.apache.druid.java.util.common.logger.Logger;
 import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesSerde;
 import org.apache.druid.segment.IndexIO;
@@ -169,7 +168,7 @@ private IncrementalIndex makeIncIndex()
   @OutputTimeUnit(TimeUnit.MICROSECONDS)
   public void persistV9(Blackhole blackhole) throws Exception
   {
-    File tmpDir = Files.createTempDir();
+    File tmpDir = FileUtils.createTempDir();
     log.info("Using temp dir: " + tmpDir.getAbsolutePath());
     try {
       File indexFile = INDEX_MERGER_V9.persist(

benchmarks/src/main/java/org/apache/druid/benchmark/query/GroupByBenchmark.java

@@ -25,8 +25,6 @@
 import com.google.common.base.Suppliers;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.io.Files;
-import org.apache.commons.io.FileUtils;
 import org.apache.druid.benchmark.datagen.BenchmarkDataGenerator;
 import org.apache.druid.benchmark.datagen.BenchmarkSchemaInfo;
 import org.apache.druid.benchmark.datagen.BenchmarkSchemas;
@@ -37,6 +35,7 @@
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.data.input.InputRow;
 import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.java.util.common.FileUtils;
 import org.apache.druid.java.util.common.concurrent.Execs;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.granularity.Granularity;
@@ -419,7 +418,7 @@ public void setup() throws IOException
         rowsPerSegment
     );
 
-    tmpDir = Files.createTempDir();
+    tmpDir = FileUtils.createTempDir();
     log.info("Using temp dir: %s", tmpDir.getAbsolutePath());
 
     // queryableIndexes   -> numSegments worth of on-disk segments

benchmarks/src/main/java/org/apache/druid/benchmark/query/ScanBenchmark.java

@@ -22,15 +22,14 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.io.Files;
-import org.apache.commons.io.FileUtils;
 import org.apache.druid.benchmark.datagen.BenchmarkDataGenerator;
 import org.apache.druid.benchmark.datagen.BenchmarkSchemaInfo;
 import org.apache.druid.benchmark.datagen.BenchmarkSchemas;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.data.input.InputRow;
 import org.apache.druid.data.input.Row;
 import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.java.util.common.FileUtils;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.concurrent.Execs;
 import org.apache.druid.java.util.common.guava.Sequence;
@@ -285,7 +284,7 @@ public void setup() throws IOException
       incIndexes.add(incIndex);
     }
 
-    tmpDir = Files.createTempDir();
+    tmpDir = FileUtils.createTempDir();
     log.info("Using temp dir: " + tmpDir.getAbsolutePath());
 
     qIndexes = new ArrayList<>();

benchmarks/src/main/java/org/apache/druid/benchmark/query/SearchBenchmark.java

@@ -24,15 +24,14 @@
 import com.google.common.base.Suppliers;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
-import com.google.common.io.Files;
-import org.apache.commons.io.FileUtils;
 import org.apache.druid.benchmark.datagen.BenchmarkDataGenerator;
 import org.apache.druid.benchmark.datagen.BenchmarkSchemaInfo;
 import org.apache.druid.benchmark.datagen.BenchmarkSchemas;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.data.input.InputRow;
 import org.apache.druid.data.input.Row;
 import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.java.util.common.FileUtils;
 import org.apache.druid.java.util.common.concurrent.Execs;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.guava.Sequence;
@@ -355,7 +354,7 @@ public void setup() throws IOException
       incIndexes.add(incIndex);
     }
 
-    tmpDir = Files.createTempDir();
+    tmpDir = FileUtils.createTempDir();
     log.info("Using temp dir: " + tmpDir.getAbsolutePath());
 
     qIndexes = new ArrayList<>();