Limit the number of bytes that can be allocated to process requests.

This should prevent costly requests from killing the whole cluster.

Close #6050

src/main/java/org/elasticsearch/common/util/AbstractArray.java

@@ -19,21 +19,26 @@
 
 package org.elasticsearch.common.util;
 
-import org.elasticsearch.common.lease.Releasable;
 
-abstract class AbstractArray implements Releasable {
+abstract class AbstractArray implements BigArray {
 
+    private final BigArrays bigArrays;
     public final boolean clearOnResize;
     private boolean released = false;
 
-    AbstractArray(boolean clearOnResize) {
+    AbstractArray(BigArrays bigArrays, boolean clearOnResize) {
+        this.bigArrays = bigArrays;
         this.clearOnResize = clearOnResize;
     }
 
     @Override
-    public void close() {
+    public final void close() {
+        bigArrays.ramBytesUsed.addAndGet(-sizeInBytes());
         assert !released : "double release";
         released = true;
+        doClose();
     }
 
+    protected abstract void doClose();
+
 }

src/main/java/org/elasticsearch/common/util/AbstractBigArray.java

@@ -32,16 +32,18 @@
 /** Common implementation for array lists that slice data into fixed-size blocks. */
 abstract class AbstractBigArray extends AbstractArray {
 
+    private static final long EMPTY_SIZE = RamUsageEstimator.shallowSizeOfInstance(AbstractBigArray.class) + RamUsageEstimator.NUM_BYTES_OBJECT_REF + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
+
     private final PageCacheRecycler recycler;
     private Recycler.V<?>[] cache;
 
     private final int pageShift;
     private final int pageMask;
     protected long size;
 
-    protected AbstractBigArray(int pageSize, PageCacheRecycler recycler, boolean clearOnResize) {
-        super(clearOnResize);
-        this.recycler = recycler;
+    protected AbstractBigArray(int pageSize, BigArrays bigArrays, boolean clearOnResize) {
+        super(bigArrays, clearOnResize);
+        this.recycler = bigArrays.recycler;
         Preconditions.checkArgument(pageSize >= 128, "pageSize must be >= 128");
         Preconditions.checkArgument((pageSize & (pageSize - 1)) == 0, "pageSize must be a power of two");
         this.pageShift = Integer.numberOfTrailingZeros(pageSize);
@@ -76,6 +78,8 @@ public final long size() {
         return size;
     }
 
+    public abstract void resize(long newSize);
+
     protected abstract int numBytesPerElement();
 
     public final long sizeInBytes() {
@@ -161,8 +165,7 @@ protected final void releasePage(int page) {
     }
 
     @Override
-    public final void close() {
-        super.close();
+    protected final void doClose() {
         if (recycler != null) {
             Releasables.close(cache);
             cache = null;

src/main/java/org/elasticsearch/common/util/BigArray.java

@@ -27,4 +27,9 @@ public interface BigArray extends Releasable {
     /** Return the length of this array. */
     public long size();
 
+    /**
+     * Return an estimated memory usage of this instance.
+     */
+    public long sizeInBytes();
+
 }