Further optimize memory for Travis jobs

.travis.yml

@@ -3,7 +3,7 @@ language: java
 # On 12-12-2017, Travis updated their trusty image, which caused integration tests to fail.
 # The group: config instructs Travis to use the previous trusty image.
 # Please see https://github.com/druid-io/druid/pull/5155 for more information.
-sudo: required
+sudo: false
 dist: trusty
 group: deprecated-2017Q4
 
@@ -17,71 +17,76 @@ cache:
 matrix:
   include:
       # strict compilation
-    - sudo: false
-      env:
+    - env:
         - NAME="strict compilation"
       install: true
       # Strict compilation requires more than 2 GB
-      script: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn clean -Pstrict -pl '!benchmarks' compile test-compile -B --fail-at-end
+      script: MAVEN_OPTS='-Xmx3000m' mvn clean -Pstrict -pl '!benchmarks' compile test-compile -B --fail-at-end
 
       # processing module test
-    - sudo: false
-      env:
+    - env:
         - NAME="processing module test"
-      install: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn install -q -ff -DskipTests -B
-      before_script:
-        - unset _JAVA_OPTIONS
-      script: echo "MAVEN_OPTS='-Xmx512m'" > ~/.mavenrc && mvn test -B -Pparallel-test -Dmaven.fork.count=2 -pl processing
+      install: MAVEN_OPTS='-Xmx3000m' mvn install -q -ff -DskipTests -B
+      before_script: unset _JAVA_OPTIONS
+      script:
+        # Set MAVEN_OPTS for Surefire launcher
+        - MAVEN_OPTS='-Xmx512m' mvn test -B -pl processing
+        - sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 || exit 0"
+        - free -m
 
       # processing module tests with SQL Compatibility enabled
-    - sudo: false
-      env:
+    - env:
         - NAME="processing module test with SQL Compatibility"
-      install: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn install -q -ff -DskipTests -B
-      before_script:
-        - unset _JAVA_OPTIONS
-      script: echo "MAVEN_OPTS='-Xmx512m'" > ~/.mavenrc && mvn test -B -Pparallel-test -Dmaven.fork.count=2 -Ddruid.generic.useDefaultValueForNull=false -pl processing
+      install: MAVEN_OPTS='-Xmx3000m' mvn install -q -ff -DskipTests -B
+      before_script: unset _JAVA_OPTIONS
+      script:
+        # Set MAVEN_OPTS for Surefire launcher
+        - MAVEN_OPTS='-Xmx512m' mvn test -B -Ddruid.generic.useDefaultValueForNull=false -pl processing
+        - sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 || exit 0"
+        - free -m
 
       # server module test
-    - sudo: false
-      env:
+    - env:
         - NAME="server module test"
-      install: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn install -q -ff -DskipTests -B
-      before_script:
-        - unset _JAVA_OPTIONS
-      # Server module test is run without the parallel-test option because it's memory sensitive and often fails with that option.
-      script: echo "MAVEN_OPTS='-Xmx512m'" > ~/.mavenrc && mvn test -B -pl server
+      install: MAVEN_OPTS='-Xmx3000m' mvn install -q -ff -DskipTests -B
+      before_script: unset _JAVA_OPTIONS
+      script:
+        # Set MAVEN_OPTS for Surefire launcher
+        - MAVEN_OPTS='-Xmx512m' mvn test -B -pl server
 
       # server module test with SQL Compatibility enabled
-    - sudo: false
-      env:
+    - env:
         - NAME="server module test with SQL Compatibility enabled"
-      install: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn install -q -ff -DskipTests -B
-      before_script:
-        - unset _JAVA_OPTIONS
-      # Server module test is run without the parallel-test option because it's memory sensitive and often fails with that option.
-      script: echo "MAVEN_OPTS='-Xmx512m'" > ~/.mavenrc && mvn test -B -pl server -Ddruid.generic.useDefaultValueForNull=false
+      install: MAVEN_OPTS='-Xmx3000m' mvn install -q -ff -DskipTests -B
+      before_script: unset _JAVA_OPTIONS
+      script:
+        # Set MAVEN_OPTS for Surefire launcher
+        - MAVEN_OPTS='-Xmx512m' mvn test -B -pl server -Ddruid.generic.useDefaultValueForNull=false
 
 
       # other modules test
-    - sudo: false
-      env:
+    - env:
         - NAME="other modules test"
         - AWS_REGION=us-east-1 # set a aws region for unit tests
-      install: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn install -q -ff -DskipTests -B
-      before_script:
-        - unset _JAVA_OPTIONS
-      script: echo "MAVEN_OPTS='-Xmx512m'" > ~/.mavenrc && mvn test -B -Pparallel-test -Dmaven.fork.count=2 -pl '!processing,!server'
+      install: MAVEN_OPTS='-Xmx3000m' mvn install -q -ff -DskipTests -B
+      before_script: unset _JAVA_OPTIONS
+      script:
+        # Set MAVEN_OPTS for Surefire launcher
+        - MAVEN_OPTS='-Xmx512m' mvn test -B -pl '!processing,!server'
+        - sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 || exit 0"
+        - free -m
 
       # other modules test with SQL Compatibility enabled
-    - sudo: false
-      env:
+    - env:
         - NAME="other modules test with SQL Compatibility"
         - AWS_REGION=us-east-1 # set a aws region for unit tests
-      install: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn install -q -ff -DskipTests -B
-      before_script:
-        - unset _JAVA_OPTIONS
-      script: echo "MAVEN_OPTS='-Xmx512m'" > ~/.mavenrc && mvn test -B -Pparallel-test -Dmaven.fork.count=2 -Ddruid.generic.useDefaultValueForNull=false -pl '!processing,!server'
+      install: MAVEN_OPTS='-Xmx3000m' mvn install -q -ff -DskipTests -B
+      before_script: unset _JAVA_OPTIONS
+      script:
+        # Set MAVEN_OPTS for Surefire launcher
+        - MAVEN_OPTS='-Xmx512m' mvn test -B -Ddruid.generic.useDefaultValueForNull=false -pl '!processing,!server'
+        - sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 || exit 0"
+        - free -m
 
       # run integration tests
     - sudo: required

benchmarks/src/main/java/io/druid/benchmark/query/SqlBenchmark.java

@@ -26,8 +26,10 @@
 import io.druid.benchmark.datagen.SegmentGenerator;
 import io.druid.data.input.Row;
 import io.druid.java.util.common.Intervals;
+import io.druid.java.util.common.Pair;
 import io.druid.java.util.common.granularity.Granularities;
 import io.druid.java.util.common.guava.Sequence;
+import io.druid.java.util.common.io.Closer;
 import io.druid.java.util.common.logger.Logger;
 import io.druid.query.QueryPlus;
 import io.druid.query.QueryRunnerFactoryConglomerate;
@@ -85,6 +87,7 @@ public class SqlBenchmark
   private PlannerFactory plannerFactory;
   private GroupByQuery groupByQuery;
   private String sqlQuery;
+  private Closer resourceCloser;
 
   @Setup(Level.Trial)
   public void setup()
@@ -104,13 +107,15 @@ public void setup()
     this.segmentGenerator = new SegmentGenerator();
 
     final QueryableIndex index = segmentGenerator.generate(dataSegment, schemaInfo, Granularities.NONE, rowsPerSegment);
-    final QueryRunnerFactoryConglomerate conglomerate = CalciteTests.queryRunnerFactoryConglomerate();
+    final Pair<QueryRunnerFactoryConglomerate, Closer> conglomerateCloserPair = CalciteTests
+        .createQueryRunnerFactoryConglomerate();
+    final QueryRunnerFactoryConglomerate conglomerate = conglomerateCloserPair.lhs;
     final PlannerConfig plannerConfig = new PlannerConfig();
 
     this.walker = new SpecificSegmentsQuerySegmentWalker(conglomerate).add(dataSegment, index);
     plannerFactory = new PlannerFactory(
-        CalciteTests.createMockSchema(walker, plannerConfig),
-        CalciteTests.createMockQueryLifecycleFactory(walker),
+        CalciteTests.createMockSchema(conglomerate, walker, plannerConfig),
+        CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate),
         CalciteTests.createOperatorTable(),
         CalciteTests.createExprMacroTable(),
         plannerConfig,
@@ -147,6 +152,10 @@ public void tearDown() throws Exception
       segmentGenerator = null;
     }
 
+    if (resourceCloser != null) {
+      resourceCloser.close();
+    }
+
     if (tmpDir != null) {
       FileUtils.deleteDirectory(tmpDir);
     }

common/src/main/java/io/druid/collections/DefaultBlockingPool.java

@@ -41,7 +41,9 @@ public class DefaultBlockingPool<T> implements BlockingPool<T>
 {
   private static final TimeUnit TIME_UNIT = TimeUnit.MILLISECONDS;
 
-  private final ArrayDeque<T> objects;
+  @VisibleForTesting
+  final ArrayDeque<T> objects;
+
   private final ReentrantLock lock;
   private final Condition notEnough;
   private final int maxSize;

common/src/main/java/io/druid/collections/StupidPool.java

@@ -39,9 +39,6 @@ public class StupidPool<T> implements NonBlockingPool<T>
 {
   private static final Logger log = new Logger(StupidPool.class);
 
-  private final String name;
-  private final Supplier<T> generator;
-
   /**
    * StupidPool Implementation Note
    * It is assumed that StupidPools are never reclaimed by the GC, either stored in static fields or global singleton
@@ -50,13 +47,20 @@ public class StupidPool<T> implements NonBlockingPool<T>
    * and registered in the global lifecycle), in this close() method all {@link ObjectResourceHolder}s should be drained
    * from the {@code objects} queue, and notifier.disable() called for them.
    */
-  private final Queue<ObjectResourceHolder> objects = new ConcurrentLinkedQueue<>();
+  @VisibleForTesting
+  final Queue<ObjectResourceHolder> objects = new ConcurrentLinkedQueue<>();
+
   /**
    * {@link ConcurrentLinkedQueue}'s size() is O(n) queue traversal apparently for the sake of being 100%
    * wait-free, that is not required by {@code StupidPool}. In {@code poolSize} we account the queue size
    * ourselves, to avoid traversal of {@link #objects} in {@link #tryReturnToPool}.
    */
-  private final AtomicLong poolSize = new AtomicLong(0);
+  @VisibleForTesting
+  final AtomicLong poolSize = new AtomicLong(0);
+
+  private final String name;
+  private final Supplier<T> generator;
+
   private final AtomicLong leakedObjectsCounter = new AtomicLong(0);
 
   //note that this is just the max entries in the cache, pool can still create as many buffers as needed.

common/src/test/java/io/druid/collections/BlockingPoolTest.java

@@ -43,8 +43,8 @@ public class BlockingPoolTest
 {
   private ExecutorService service;
 
-  private DefaultBlockingPool<Integer> pool;
-  private BlockingPool<Integer> emptyPool;
+  private CloseableDefaultBlockingPool<Integer> pool;
+  private CloseableDefaultBlockingPool<Integer> emptyPool;
 
   @Rule
   public ExpectedException expectedException = ExpectedException.none();
@@ -53,13 +53,15 @@ public class BlockingPoolTest
   public void setup()
   {
     service = Execs.multiThreaded(2, "blocking-pool-test");
-    pool = new DefaultBlockingPool<>(Suppliers.ofInstance(1), 10);
-    emptyPool = new DefaultBlockingPool<>(Suppliers.ofInstance(1), 0);
+    pool = new CloseableDefaultBlockingPool<>(Suppliers.ofInstance(1), 10);
+    emptyPool = new CloseableDefaultBlockingPool<>(Suppliers.ofInstance(1), 0);
   }
 
   @After
   public void teardown()
   {
+    pool.close();
+    emptyPool.close();
     service.shutdownNow();
   }
 

common/src/test/java/io/druid/collections/CloseableDefaultBlockingPool.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package io.druid.collections;
+
+import com.google.common.base.Supplier;
+
+import java.io.Closeable;
+
+public class CloseableDefaultBlockingPool<T> extends DefaultBlockingPool<T> implements Closeable
+{
+  public CloseableDefaultBlockingPool(Supplier<T> generator, int limit)
+  {
+    super(generator, limit);
+  }
+
+  @Override
+  public void close()
+  {
+    objects.clear();
+  }
+}

common/src/test/java/io/druid/collections/CloseableStupidPool.java

@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package io.druid.collections;
+
+import com.google.common.base.Supplier;
+
+import java.io.Closeable;
+
+public class CloseableStupidPool<T> extends StupidPool<T> implements Closeable
+{
+  public CloseableStupidPool(String name, Supplier<T> generator)
+  {
+    super(name, generator);
+  }
+
+  public CloseableStupidPool(String name, Supplier<T> generator, int initCount, int objectsCacheMaxCount)
+  {
+    super(name, generator, initCount, objectsCacheMaxCount);
+  }
+
+  @Override
+  public void close()
+  {
+    objects.clear();
+    poolSize.set(0);
+  }
+}

common/src/test/java/io/druid/collections/StupidPoolTest.java

@@ -31,7 +31,7 @@
 public class StupidPoolTest
 {
   private Supplier<String> generator;
-  private StupidPool<String> poolOfString;
+  private CloseableStupidPool<String> poolOfString;
   private ResourceHolder<String> resourceHolderObj;
   private String defaultString = new String("test");
 
@@ -41,7 +41,7 @@ public void setUp()
     generator = EasyMock.createMock(Supplier.class);
     EasyMock.expect(generator.get()).andReturn(defaultString).anyTimes();
     EasyMock.replay(generator);
-    poolOfString = new StupidPool<>("poolOfString", generator);
+    poolOfString = new CloseableStupidPool<>("poolOfString", generator);
     resourceHolderObj = poolOfString.take();
   }
 
@@ -51,6 +51,7 @@ public void tearDown()
     if (resourceHolderObj != null) {
       resourceHolderObj.close();
     }
+    poolOfString.close();
   }
 
   @Test

extensions-contrib/distinctcount/pom.xml

@@ -43,6 +43,13 @@
         </dependency>
 
         <!-- Tests -->
+        <dependency>
+            <groupId>io.druid</groupId>
+            <artifactId>druid-common</artifactId>
+            <version>${project.parent.version}</version>
+            <scope>test</scope>
+            <type>test-jar</type>
+        </dependency>
         <dependency>
             <groupId>io.druid</groupId>
             <artifactId>druid-processing</artifactId>