diff --git a/benchmarks/jmh/src/jmh/java/com/linecorp/armeria/client/EventLoopStateBenchmark.java b/benchmarks/jmh/src/jmh/java/com/linecorp/armeria/client/EventLoopStateBenchmark.java new file mode 100644 index 00000000000..7468971acc6 --- /dev/null +++ b/benchmarks/jmh/src/jmh/java/com/linecorp/armeria/client/EventLoopStateBenchmark.java @@ -0,0 +1,95 @@ +/* + * Copyright 2023 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.armeria.client; + +import static com.google.common.collect.ImmutableList.toImmutableList; + +import java.util.List; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Streams; + +import com.linecorp.armeria.common.util.EventLoopGroups; + +import io.netty.channel.EventLoop; +import io.netty.channel.EventLoopGroup; + +@State(Scope.Benchmark) +public class EventLoopStateBenchmark { + private AbstractEventLoopState state; + private AbstractEventLoopEntry[] acquired; + private EventLoopGroup eventLoopGroup; + + @Param({ "32", "64", "128", "256" }) + private int maxNumEventLoops; + @Param({ "true", "false" }) + private boolean arrayBased; + + @Setup + public void setUp() { + acquired = new AbstractEventLoopEntry[maxNumEventLoops]; + + eventLoopGroup = EventLoopGroups.newEventLoopGroup(maxNumEventLoops); + final DefaultEventLoopScheduler scheduler = new DefaultEventLoopScheduler( + eventLoopGroup, maxNumEventLoops, maxNumEventLoops, ImmutableList.of()); + final List eventLoops = Streams.stream(eventLoopGroup) + .map(EventLoop.class::cast) + .collect(toImmutableList()); + if (arrayBased) { + state = new ArrayBasedEventLoopState(eventLoops, maxNumEventLoops, scheduler); + } else { + state = new HeapBasedEventLoopState(eventLoops, maxNumEventLoops, scheduler); + } + + // Acquire as many as the number of eventLoops so that the active request of all states are + // greater than 1. + for (int i = 0; i < maxNumEventLoops; i++) { + state.acquire(); + } + } + + @TearDown + public void tearDown() { + eventLoopGroup.shutdownGracefully(); + } + + @Benchmark + public void lastInFirstOut() { + for (int i = 0; i < maxNumEventLoops; ++i) { + acquired[i] = state.acquire(); + } + for (int i = maxNumEventLoops - 1; i >= 0; --i) { + acquired[i].release(); + } + } + + @Benchmark + public void firstInFirstOut() { + for (int i = 0; i < maxNumEventLoops; ++i) { + acquired[i] = state.acquire(); + } + for (int i = 0; i < maxNumEventLoops; ++i) { + acquired[i].release(); + } + } +} diff --git a/core/src/main/java/com/linecorp/armeria/client/AbstractEventLoopState.java b/core/src/main/java/com/linecorp/armeria/client/AbstractEventLoopState.java index ac0b3c5e75a..593f3466d48 100644 --- a/core/src/main/java/com/linecorp/armeria/client/AbstractEventLoopState.java +++ b/core/src/main/java/com/linecorp/armeria/client/AbstractEventLoopState.java @@ -31,9 +31,9 @@ static AbstractEventLoopState of(List eventLoops, int maxNumEventLoop DefaultEventLoopScheduler scheduler) { if (maxNumEventLoops == 1) { return new OneEventLoopState(eventLoops, scheduler); + } else if (maxNumEventLoops <= 128) { + return new ArrayBasedEventLoopState(eventLoops, maxNumEventLoops, scheduler); } - // TODO(minwoox) Introduce array based state which is used when the maxNumEventLoops is greater than 1 - // and less than N for the performance. return new HeapBasedEventLoopState(eventLoops, maxNumEventLoops, scheduler); } @@ -80,7 +80,7 @@ protected final void unlock() { abstract void release(AbstractEventLoopEntry e); @VisibleForTesting - abstract List entries(); + abstract AbstractEventLoopEntry[] entries(); abstract int allActiveRequests(); } diff --git a/core/src/main/java/com/linecorp/armeria/client/ArrayBasedEventLoopState.java b/core/src/main/java/com/linecorp/armeria/client/ArrayBasedEventLoopState.java new file mode 100644 index 00000000000..232601d8652 --- /dev/null +++ b/core/src/main/java/com/linecorp/armeria/client/ArrayBasedEventLoopState.java @@ -0,0 +1,151 @@ +/* + * Copyright 2023 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.client; + +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; + +import io.netty.channel.EventLoop; + +final class ArrayBasedEventLoopState extends AbstractEventLoopState { + + private final AbstractEventLoopEntry[] entries; + private final int maxNumEventLoops; + private int allActiveRequests; + + ArrayBasedEventLoopState(List eventLoops, int maxNumEventLoops, + DefaultEventLoopScheduler scheduler) { + super(eventLoops, scheduler); + this.maxNumEventLoops = maxNumEventLoops; + entries = new AbstractEventLoopEntry[maxNumEventLoops]; + if (eventLoops.size() == maxNumEventLoops) { + init(0); + } else { + init(scheduler().acquisitionStartIndex(maxNumEventLoops)); + } + } + + private void init(final int acquisitionStartIndex) { + final int initialEventLoopOffset = ThreadLocalRandom.current().nextInt(maxNumEventLoops); + final int eventLoopSize = eventLoops().size(); + for (int i = 0; i < maxNumEventLoops; ++i) { + final int nextIndex = (acquisitionStartIndex + (initialEventLoopOffset + i) % maxNumEventLoops) % + eventLoopSize; + entries[i] = new Entry(this, eventLoops().get(nextIndex), i); + } + } + + private AbstractEventLoopEntry targetEntry() { + int minActiveRequest = Integer.MAX_VALUE; + int targetIndex = 0; + for (int i = 0; i < maxNumEventLoops; ++i) { + final AbstractEventLoopEntry e = entries[i]; + final int activeRequests = e.activeRequests(); + if (activeRequests == 0) { + return e; + } + if (minActiveRequest > activeRequests) { + minActiveRequest = activeRequests; + targetIndex = i; + } + } + return entries[targetIndex]; + } + + @Override + AbstractEventLoopEntry acquire() { + lock(); + try { + final AbstractEventLoopEntry e = targetEntry(); + e.incrementActiveRequests(); + allActiveRequests++; + return e; + } finally { + unlock(); + } + } + + @Override + void release(AbstractEventLoopEntry e) { + lock(); + try { + e.decrementActiveRequests(); + if (--allActiveRequests == 0) { + setLastActivityTimeNanos(); + } + } finally { + unlock(); + } + } + + @Override + AbstractEventLoopEntry[] entries() { + return entries; + } + + @Override + int allActiveRequests() { + return allActiveRequests; + } + + private static final class Entry extends AbstractEventLoopEntry { + + private final int id; + + private int activeRequests; + + Entry(AbstractEventLoopState parent, EventLoop eventLoop, int id) { + super(parent, eventLoop); + this.id = id; + } + + @Override + int activeRequests() { + return activeRequests; + } + + @Override + void incrementActiveRequests() { + activeRequests++; + } + + @Override + void decrementActiveRequests() { + activeRequests--; + } + + @Override + int id() { + return id; + } + + @Override + int index() { + throw new UnsupportedOperationException(); + } + + @Override + void setIndex(int index) { + throw new UnsupportedOperationException(); + } + + @Override + public String toString() { + return "(" + id + ", " + activeRequests() + ')'; + } + } +} diff --git a/core/src/main/java/com/linecorp/armeria/client/DefaultEventLoopScheduler.java b/core/src/main/java/com/linecorp/armeria/client/DefaultEventLoopScheduler.java index 21e4cfee07a..be96031c1d3 100644 --- a/core/src/main/java/com/linecorp/armeria/client/DefaultEventLoopScheduler.java +++ b/core/src/main/java/com/linecorp/armeria/client/DefaultEventLoopScheduler.java @@ -130,9 +130,9 @@ public ReleasableHolder acquire(SessionProtocol sessionProtocol, } @VisibleForTesting - List entries(SessionProtocol sessionProtocol, - EndpointGroup endpointGroup, - @Nullable Endpoint endpoint) { + AbstractEventLoopEntry[] entries(SessionProtocol sessionProtocol, + EndpointGroup endpointGroup, + @Nullable Endpoint endpoint) { return state(sessionProtocol, endpointGroup, endpoint).entries(); } diff --git a/core/src/main/java/com/linecorp/armeria/client/HeapBasedEventLoopState.java b/core/src/main/java/com/linecorp/armeria/client/HeapBasedEventLoopState.java index c5c72e3b406..2c182a78207 100644 --- a/core/src/main/java/com/linecorp/armeria/client/HeapBasedEventLoopState.java +++ b/core/src/main/java/com/linecorp/armeria/client/HeapBasedEventLoopState.java @@ -16,7 +16,6 @@ package com.linecorp.armeria.client; -import java.util.ArrayList; import java.util.List; import java.util.concurrent.ThreadLocalRandom; @@ -33,7 +32,8 @@ final class HeapBasedEventLoopState extends AbstractEventLoopState { *
  • {@link AbstractEventLoopEntry#id()} (lower is better)
    • * */ - private final List entries = new ArrayList<>(); + private final AbstractEventLoopEntry[] entries; + private int entriesSize; private final int maxNumEventLoops; private int acquisitionStartIndex = -1; @@ -44,6 +44,7 @@ final class HeapBasedEventLoopState extends AbstractEventLoopState { DefaultEventLoopScheduler scheduler) { super(eventLoops, scheduler); this.maxNumEventLoops = maxNumEventLoops; + entries = new AbstractEventLoopEntry[maxNumEventLoops]; if (eventLoops.size() == maxNumEventLoops) { // We use all event loops so initialize early. init(0); @@ -57,10 +58,10 @@ private void init(int acquisitionStartIndex) { } private boolean addUnusedEventLoop() { - if (entries.size() < maxNumEventLoops) { + if (entriesSize < maxNumEventLoops) { final int nextIndex = (acquisitionStartIndex + nextUnusedEventLoopOffset) % eventLoops().size(); - push(new Entry(this, eventLoops().get(nextIndex), entries.size())); + push(new Entry(this, eventLoops().get(nextIndex), entriesSize)); nextUnusedEventLoopOffset = (nextUnusedEventLoopOffset + 1) % maxNumEventLoops; return true; } @@ -68,7 +69,7 @@ private boolean addUnusedEventLoop() { } @Override - List entries() { + AbstractEventLoopEntry[] entries() { return entries; } @@ -84,11 +85,11 @@ AbstractEventLoopEntry acquire() { if (acquisitionStartIndex == -1) { init(scheduler().acquisitionStartIndex(maxNumEventLoops)); } - AbstractEventLoopEntry e = entries.get(0); + AbstractEventLoopEntry e = entries[0]; if (e.activeRequests() > 0) { // All event loops are handling connections; try to add an unused event loop. if (addUnusedEventLoop()) { - e = entries.get(0); + e = entries[0]; assert e.activeRequests() == 0; } } @@ -119,8 +120,8 @@ void release(AbstractEventLoopEntry e) { // Heap implementation, modified from the public domain code at https://stackoverflow.com/a/714873 private void push(Entry e) { - entries.add(e); - bubbleUp(entries.size() - 1); + entries[entriesSize++] = e; + bubbleUp(entriesSize - 1); } private void bubbleDown() { @@ -129,10 +130,10 @@ private void bubbleDown() { final int oldBest = best; final int left = left(best); - if (left < entries.size()) { + if (left < entriesSize) { final int right = right(best); if (isBetter(left, best)) { - if (right < entries.size()) { + if (right < entriesSize) { if (isBetter(right, left)) { // Left leaf is better but right leaf is even better. best = right; @@ -144,7 +145,7 @@ private void bubbleDown() { // Left leaf is better and there's no right leaf. best = left; } - } else if (right < entries.size()) { + } else if (right < entriesSize) { if (isBetter(right, best)) { // Left leaf is not better but right leaf is better. best = right; @@ -181,8 +182,8 @@ private void bubbleUp(int i) { * Returns {@code true} if the entry at {@code a} is a better choice than the entry at {@code b}. */ private boolean isBetter(int a, int b) { - final AbstractEventLoopEntry entryA = entries.get(a); - final AbstractEventLoopEntry entryB = entries.get(b); + final AbstractEventLoopEntry entryA = entries[a]; + final AbstractEventLoopEntry entryB = entries[b]; if (entryA.activeRequests() < entryB.activeRequests()) { return true; } @@ -206,10 +207,10 @@ private static int right(int i) { } private void swap(int i, int j) { - final AbstractEventLoopEntry entryI = entries.get(i); - final AbstractEventLoopEntry entryJ = entries.get(j); - entries.set(i, entryJ); - entries.set(j, entryI); + final AbstractEventLoopEntry entryI = entries[i]; + final AbstractEventLoopEntry entryJ = entries[j]; + entries[i] = entryJ; + entries[j] = entryI; // Swap the index as well. entryJ.setIndex(i); diff --git a/core/src/main/java/com/linecorp/armeria/client/OneEventLoopState.java b/core/src/main/java/com/linecorp/armeria/client/OneEventLoopState.java index b8c7170f58c..b2fe3dfaf36 100644 --- a/core/src/main/java/com/linecorp/armeria/client/OneEventLoopState.java +++ b/core/src/main/java/com/linecorp/armeria/client/OneEventLoopState.java @@ -16,29 +16,26 @@ package com.linecorp.armeria.client; -import java.util.ArrayList; import java.util.List; import io.netty.channel.EventLoop; final class OneEventLoopState extends AbstractEventLoopState { - private final List entry = new ArrayList<>(); + private final AbstractEventLoopEntry[] entry = new AbstractEventLoopEntry[1]; private int allActiveRequests; OneEventLoopState(List eventLoops, DefaultEventLoopScheduler scheduler) { super(eventLoops, scheduler); + entry[0] = new Entry(this, eventLoops().get(scheduler().acquisitionStartIndex(1))); } @Override AbstractEventLoopEntry acquire() { lock(); try { - if (entry.isEmpty()) { - entry.add(new Entry(this, eventLoops().get(scheduler().acquisitionStartIndex(1)))); - } - final AbstractEventLoopEntry e = entry.get(0); + final AbstractEventLoopEntry e = entry[0]; allActiveRequests++; return e; } finally { @@ -59,7 +56,7 @@ void release(AbstractEventLoopEntry e) { } @Override - List entries() { + AbstractEventLoopEntry[] entries() { return entry; } diff --git a/core/src/test/java/com/linecorp/armeria/client/DefaultEventLoopSchedulerTest.java b/core/src/test/java/com/linecorp/armeria/client/DefaultEventLoopSchedulerTest.java index ebae3cbbcd3..5db9c933cd5 100644 --- a/core/src/test/java/com/linecorp/armeria/client/DefaultEventLoopSchedulerTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/DefaultEventLoopSchedulerTest.java @@ -271,16 +271,19 @@ void stressTest() { // Release all acquired entries to make sure activeRequests are all 0. acquiredEntries.forEach(AbstractEventLoopEntry::release); - final List entries = s.entries(SessionProtocol.HTTP, endpoint, endpoint); + final AbstractEventLoopEntry[] entries = s.entries(SessionProtocol.HTTP, endpoint, endpoint); for (AbstractEventLoopEntry e : entries) { - assertThat(e.activeRequests()).withFailMessage("All entries must have 0 activeRequests.").isZero(); + if (e != null) { + assertThat(e.activeRequests()).withFailMessage( + "All entries must have 0 activeRequests.").isZero(); + } } - assertThat(entries.get(0).id()).isZero(); + assertThat(entries[0].id()).isZero(); } private static void stressTest(DefaultEventLoopScheduler s, List acquiredEntries, double acquireRatio) { - final List entries = s.entries(SessionProtocol.HTTP, endpoint, endpoint); + final AbstractEventLoopEntry[] entries = s.entries(SessionProtocol.HTTP, endpoint, endpoint); final Random random = ThreadLocalRandom.current(); final int acquireRatioAsInt = (int) (Integer.MAX_VALUE * acquireRatio); @@ -295,6 +298,9 @@ private static void stressTest(DefaultEventLoopScheduler s, List entries1 = s.entries(SessionProtocol.H1C, endpointA, endpointA); - assertThat(entries1).hasSize(0); - acquireTenEntries(s, SessionProtocol.H1C, endpointA, endpointA); - assertThat(entries1).hasSize(1); + final AbstractEventLoopEntry[] entries1 = s.entries(SessionProtocol.H1C, endpointA, endpointA); + assertThat(removeNullOrInactiveElements(entries1)).hasSize(1); + } + + private static List removeNullElements(AbstractEventLoopEntry[] entries) { + return Arrays.stream(entries) + .filter(Objects::nonNull) + .collect(toImmutableList()); + } + + private static List removeNullOrInactiveElements(AbstractEventLoopEntry[] entries) { + return removeNullElements(entries).stream() + .filter(x -> { + try { + return x.activeRequests() > 0; + } catch (UnsupportedOperationException e) { + return true; + } + }) + .collect(toImmutableList()); } @Test @@ -71,32 +90,32 @@ void singleEndpointMaxNumEventLoops() { private static void checkMaxNumEventLoops(DefaultEventLoopScheduler s, Endpoint preDefined, Endpoint undefined) { - final List entries1 = s.entries(SessionProtocol.H1C, preDefined, preDefined); - assertThat(entries1).hasSize(0); + final AbstractEventLoopEntry[] entries1 = s.entries(SessionProtocol.H1C, preDefined, preDefined); + assertThat(removeNullOrInactiveElements(entries1)).hasSize(0); acquireTenEntries(s, SessionProtocol.H1C, preDefined, preDefined); - assertThat(entries1).hasSize(3); + assertThat(removeNullOrInactiveElements(entries1)).hasSize(3); - final List entries2 = s.entries(SessionProtocol.H2, undefined, undefined); - assertThat(entries2).hasSize(0); + final AbstractEventLoopEntry[] entries2 = s.entries(SessionProtocol.H2, undefined, undefined); + assertThat(removeNullOrInactiveElements(entries2)).hasSize(0); acquireTenEntries(s, SessionProtocol.H2, undefined, undefined); - assertThat(entries2).hasSize(4); + assertThat(removeNullOrInactiveElements(entries2)).hasSize(4); - final List entries3 = s.entries(SessionProtocol.H2C, undefined, undefined); - assertThat(entries2).isNotSameAs(entries3); - assertThat(entries3).hasSize(0); + final AbstractEventLoopEntry[] entries3 = s.entries(SessionProtocol.H2C, undefined, undefined); + assertThat(removeNullOrInactiveElements(entries2)).isNotEqualTo(removeNullOrInactiveElements(entries3)); + assertThat(removeNullOrInactiveElements(entries3)).hasSize(0); acquireTenEntries(s, SessionProtocol.H2C, undefined, undefined); - assertThat(entries3).hasSize(4); + assertThat(removeNullOrInactiveElements(entries3)).hasSize(4); - final List entries4 = s.entries(SessionProtocol.H1, undefined, undefined); - assertThat(entries4).hasSize(0); + final AbstractEventLoopEntry[] entries4 = s.entries(SessionProtocol.H1, undefined, undefined); + assertThat(removeNullOrInactiveElements(entries4)).hasSize(0); acquireTenEntries(s, SessionProtocol.H1, undefined, undefined); - assertThat(entries4).hasSize(5); + assertThat(removeNullOrInactiveElements(entries4)).hasSize(5); - final List entries5 = s.entries(SessionProtocol.H1C, undefined, undefined); - assertThat(entries4).isNotSameAs(entries5); - assertThat(entries5).hasSize(0); + final AbstractEventLoopEntry[] entries5 = s.entries(SessionProtocol.H1C, undefined, undefined); + assertThat(removeNullOrInactiveElements(entries4)).isNotSameAs(entries5); + assertThat(removeNullOrInactiveElements(entries5)).hasSize(0); acquireTenEntries(s, SessionProtocol.H1C, undefined, undefined); - assertThat(entries5).hasSize(5); + assertThat(removeNullOrInactiveElements(entries5)).hasSize(5); } @Test @@ -130,65 +149,65 @@ void maxNumEventLoopsFunction() { }); final DefaultEventLoopScheduler s = new DefaultEventLoopScheduler(group, 7, 7, maxNumEventLoopsFunctions); - final List entries1 = s.entries(SessionProtocol.H1C, endpointA, endpointA); - assertThat(entries1).hasSize(0); + final AbstractEventLoopEntry[] entries1 = s.entries(SessionProtocol.H1C, endpointA, endpointA); + assertThat(removeNullOrInactiveElements(entries1)).hasSize(0); acquireTenEntries(s, SessionProtocol.H1C, endpointA, endpointA); - assertThat(entries1).hasSize(2); + assertThat(removeNullOrInactiveElements(entries1)).hasSize(2); - final List entries2 = s.entries(SessionProtocol.H1C, endpointA80, endpointA80); - assertThat(entries2).hasSize(2); + final AbstractEventLoopEntry[] entries2 = s.entries(SessionProtocol.H1C, endpointA80, endpointA80); + assertThat(removeNullOrInactiveElements(entries2)).hasSize(2); - final List entries3 = + final AbstractEventLoopEntry[] entries3 = s.entries(SessionProtocol.H1C, endpointA443, endpointA443); - assertThat(entries3).hasSize(0); - acquireTenEntries(s, SessionProtocol.H1C, endpointA443, endpointA443); - assertThat(entries3).hasSize(1); // Fallback to "a.com" + assertThat(removeNullOrInactiveElements(entries3)).hasSize(1); // Fallback to "a.com" - final List entries4 = + final AbstractEventLoopEntry[] entries4 = s.entries(SessionProtocol.H1C, endpointA8443, endpointA8443); - assertThat(entries4).hasSize(0); + assertThat(removeNullOrInactiveElements(entries4)).hasSize(0); acquireTenEntries(s, SessionProtocol.H1C, endpointA8443, endpointA8443); - assertThat(entries4).hasSize(3); // Matched to Endpoint.of("a.com", 36462) + assertThat(removeNullOrInactiveElements(entries4)).hasSize(3); // Matched to Endpoint.of("a.com", 36462) // Clear text SessionProtocols. - final List bComClearText = + final AbstractEventLoopEntry[] bComClearText = s.entries(SessionProtocol.H1C, endpointB80, endpointB80); - assertThat(bComClearText).hasSize(0); + assertThat(removeNullOrInactiveElements(bComClearText)).hasSize(0); acquireTenEntries(s, SessionProtocol.H1C, endpointB, endpointB); - assertThat(bComClearText).hasSize(4); // Fallback to "b.com:80" + assertThat(removeNullOrInactiveElements(bComClearText)).hasSize(4); // Fallback to "b.com:80" - final List entries5 = s.entries(SessionProtocol.H1C, endpointB, endpointB); - assertThat(bComClearText).isSameAs(entries5); + final AbstractEventLoopEntry[] entries5 = s.entries(SessionProtocol.H1C, endpointB, endpointB); + assertThat(removeNullOrInactiveElements(bComClearText)) + .isEqualTo(removeNullOrInactiveElements(entries5)); - final List entries6 = s.entries(SessionProtocol.H2C, endpointB, endpointB); + final AbstractEventLoopEntry[] entries6 = s.entries(SessionProtocol.H2C, endpointB, endpointB); acquireTenEntries(s, SessionProtocol.H2C, endpointB, endpointB); - assertThat(bComClearText).hasSize(4); - final List entries7 = s.entries(SessionProtocol.HTTP, endpointB, endpointB); - assertThat(entries6).isSameAs(entries7); + assertThat(removeNullOrInactiveElements(bComClearText)).hasSize(4); + final AbstractEventLoopEntry[] entries7 = s.entries(SessionProtocol.HTTP, endpointB, endpointB); + assertThat(removeNullOrInactiveElements(entries6)).isEqualTo(removeNullOrInactiveElements(entries7)); // TLS SessionProtocols. - final List bComTls = s.entries(SessionProtocol.H1, endpointB443, endpointB443); - assertThat(bComTls).hasSize(0); + final AbstractEventLoopEntry[] bComTls = s.entries(SessionProtocol.H1, endpointB443, endpointB443); + assertThat(removeNullOrInactiveElements(bComTls)).hasSize(0); acquireTenEntries(s, SessionProtocol.H1, endpointB, endpointB); - assertThat(bComTls).hasSize(5); // Fallback to "b.com:433" + assertThat(removeNullOrInactiveElements(bComTls)).hasSize(5); // Fallback to "b.com:433" - final List entries8 = s.entries(SessionProtocol.H1, endpointB, endpointB); - assertThat(bComTls).isSameAs(entries8); + final AbstractEventLoopEntry[] entries8 = s.entries(SessionProtocol.H1, endpointB, endpointB); + assertThat(removeNullOrInactiveElements(bComTls)).isEqualTo(removeNullOrInactiveElements(entries8)); - final List entries9 = s.entries(SessionProtocol.H2, endpointB, endpointB); + final AbstractEventLoopEntry[] entries9 = s.entries(SessionProtocol.H2, endpointB, endpointB); acquireTenEntries(s, SessionProtocol.H2, endpointB, endpointB); - assertThat(entries9).hasSize(5); - final List entries10 = s.entries(SessionProtocol.HTTPS, endpointB, endpointB); - assertThat(entries9).isSameAs(entries10); + assertThat(removeNullOrInactiveElements(entries9)).hasSize(5); + final AbstractEventLoopEntry[] entries10 = s.entries(SessionProtocol.HTTPS, endpointB, endpointB); + assertThat(removeNullOrInactiveElements(entries9)).isEqualTo(removeNullOrInactiveElements(entries10)); - final List entries11 = + final AbstractEventLoopEntry[] entries11 = s.entries(SessionProtocol.H1, endpointB8443, endpointB8443); - assertThat(entries11).hasSize( - 1); // One entry is pushed when eventLoops.size() == maxNumEventLoops + // All entries pushed early on ArrayBasedEventLoopState + // assertThat(removeNullElements(entries11)) + // .hasSize(1); // One entry is pushed when eventLoops.size() == maxNumEventLoops acquireTenEntries(s, SessionProtocol.H1, endpointB8443, endpointB8443); - assertThat(entries11).hasSize(7); // No match + assertThat(removeNullOrInactiveElements(entries11)).hasSize(7); // No match } private static void acquireTenEntries(DefaultEventLoopScheduler s,