diff --git a/acceptance-tests/src/acceptance-test/java/tech/pegasys/teku/test/acceptance/ValidatorLivenessAcceptanceTest.java b/acceptance-tests/src/acceptance-test/java/tech/pegasys/teku/test/acceptance/ValidatorLivenessAcceptanceTest.java new file mode 100644 index 00000000000..ac94d06f096 --- /dev/null +++ b/acceptance-tests/src/acceptance-test/java/tech/pegasys/teku/test/acceptance/ValidatorLivenessAcceptanceTest.java @@ -0,0 +1,79 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.test.acceptance; + +import static tech.pegasys.teku.test.acceptance.dsl.ValidatorLivenessExpectation.expectLive; +import static tech.pegasys.teku.test.acceptance.dsl.ValidatorLivenessExpectation.expectNotLive; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import tech.pegasys.teku.infrastructure.time.SystemTimeProvider; +import tech.pegasys.teku.test.acceptance.dsl.AcceptanceTestBase; +import tech.pegasys.teku.test.acceptance.dsl.TekuNode; + +public class ValidatorLivenessAcceptanceTest extends AcceptanceTestBase { + + private static final int NODE_VALIDATORS = 2; + private static final int TOTAL_VALIDATORS = NODE_VALIDATORS * 2; + + private final SystemTimeProvider timeProvider = new SystemTimeProvider(); + private TekuNode primaryNode; + private TekuNode secondaryNode; + + @BeforeEach + public void setup() { + final int genesisTime = timeProvider.getTimeInSeconds().plus(5).intValue(); + primaryNode = + createTekuNode( + config -> configureNode(config, genesisTime).withInteropValidators(0, NODE_VALIDATORS)); + secondaryNode = + createTekuNode( + config -> + configureNode(config, genesisTime) + .withInteropValidators(NODE_VALIDATORS, NODE_VALIDATORS) + .withPeers(primaryNode)); + } + + /* + * Primary and Secondary node, each with half of the validators + * - Primary is online at genesis, so during epoch 1 all validators should be performing duties. + * - no validator keys from the secondary will be seen as active in epoch 1. + * - Secondary is online at epoch 2, so by epoch 3 should all be performing duties. + * - by epoch 4, all validators should be seen as performing duties in epoch 3 + */ + @Test + void shouldTrackValidatorLivenessOverEpochs() throws Exception { + primaryNode.start(); + + primaryNode.waitForBlockAtOrAfterSlot(8); + secondaryNode.start(); + primaryNode.checkValidatorLiveness( + 1, + TOTAL_VALIDATORS, + expectLive(0, NODE_VALIDATORS), + expectNotLive(NODE_VALIDATORS, NODE_VALIDATORS)); + + primaryNode.waitForBlockAtOrAfterSlot(16); + primaryNode.checkValidatorLiveness(3, TOTAL_VALIDATORS, expectLive(0, TOTAL_VALIDATORS)); + secondaryNode.checkValidatorLiveness(3, TOTAL_VALIDATORS, expectLive(0, TOTAL_VALIDATORS)); + } + + private TekuNode.Config configureNode(final TekuNode.Config node, final int genesisTime) { + return node.withNetwork("swift") + .withGenesisTime(genesisTime) + .withValidatorLivenessTracking() + .withInteropNumberOfValidators(TOTAL_VALIDATORS) + .withRealNetwork(); + } +} diff --git a/acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/SimpleHttpClient.java b/acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/SimpleHttpClient.java index 1260864debf..dbcc0802c67 100644 --- a/acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/SimpleHttpClient.java +++ b/acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/SimpleHttpClient.java @@ -19,11 +19,14 @@ import java.net.URI; import okhttp3.OkHttpClient; import okhttp3.Request; +import okhttp3.RequestBody; import okhttp3.Response; import okhttp3.ResponseBody; public class SimpleHttpClient { private final OkHttpClient httpClient = new OkHttpClient(); + private static final okhttp3.MediaType JSON = + okhttp3.MediaType.parse("application/json; charset=utf-8"); public String get(final URI baseUrl, final String path) throws IOException { final Response response = @@ -35,4 +38,18 @@ public String get(final URI baseUrl, final String path) throws IOException { assertThat(body).isNotNull(); return body.string(); } + + public String post(final URI baseUrl, final String path, final String jsonBody) + throws IOException { + final RequestBody requestBody = RequestBody.create(jsonBody, JSON); + final Response response = + httpClient + .newCall( + new Request.Builder().url(baseUrl.resolve(path).toURL()).post(requestBody).build()) + .execute(); + assertThat(response.isSuccessful()).isTrue(); + final ResponseBody responseBody = response.body(); + assertThat(responseBody).isNotNull(); + return responseBody.string(); + } } diff --git a/acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuNode.java b/acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuNode.java index dd2e49fbd8c..8a859b9c570 100644 --- a/acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuNode.java +++ b/acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuNode.java @@ -50,12 +50,16 @@ import org.testcontainers.containers.Network; import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; import org.testcontainers.utility.MountableFile; +import tech.pegasys.teku.api.request.v1.validator.ValidatorLivenessRequest; import tech.pegasys.teku.api.response.v1.EventType; +import tech.pegasys.teku.api.response.v1.HeadEvent; import tech.pegasys.teku.api.response.v1.beacon.FinalityCheckpointsResponse; import tech.pegasys.teku.api.response.v1.beacon.GetBlockRootResponse; import tech.pegasys.teku.api.response.v1.beacon.GetGenesisResponse; import tech.pegasys.teku.api.response.v1.beacon.GetStateFinalityCheckpointsResponse; import tech.pegasys.teku.api.response.v1.debug.GetStateResponse; +import tech.pegasys.teku.api.response.v1.validator.PostValidatorLivenessResponse; +import tech.pegasys.teku.api.response.v1.validator.ValidatorLivenessAtEpoch; import tech.pegasys.teku.api.response.v2.beacon.GetBlockResponseV2; import tech.pegasys.teku.api.schema.BeaconState; import tech.pegasys.teku.api.schema.SignedBeaconBlock; @@ -169,6 +173,69 @@ public void waitForGenesis() { waitFor(this::fetchGenesisTime); } + public void waitForBlockAtOrAfterSlot(final long slot) { + if (maybeEventStreamListener.isEmpty()) { + startEventListener(List.of(EventType.head)); + } + waitFor( + () -> + assertThat( + getSlotsFromHeadEvents().stream() + .filter(v -> v.isGreaterThanOrEqualTo(slot)) + .count()) + .isGreaterThan(0)); + } + + private List getSlotsFromHeadEvents() { + return maybeEventStreamListener.get().getMessages().stream() + .filter(packedMessage -> packedMessage.getEvent().equals(EventType.head.name())) + .map(this::getSlotFromHeadEvent) + .flatMap(Optional::stream) + .collect(toList()); + } + + private Optional getSlotFromHeadEvent( + final Eth2EventHandler.PackedMessage packedMessage) { + try { + return Optional.of( + jsonProvider.jsonToObject(packedMessage.getMessageEvent().getData(), HeadEvent.class) + .slot); + } catch (JsonProcessingException e) { + LOG.error("Failed to process head event", e); + return Optional.empty(); + } + } + + public void checkValidatorLiveness( + final int epoch, final int totalValidatorCount, ValidatorLivenessExpectation... args) + throws IOException { + final List validators = new ArrayList<>(); + for (UInt64 i = UInt64.ZERO; i.isLessThan(totalValidatorCount); i = i.increment()) { + validators.add(i); + } + final Map data = + getValidatorLivenessAtEpoch(UInt64.valueOf(epoch), validators); + for (ValidatorLivenessExpectation expectation : args) { + expectation.verify(data); + } + } + + private Map getValidatorLivenessAtEpoch( + final UInt64 epoch, List validators) throws IOException { + + final ValidatorLivenessRequest request = new ValidatorLivenessRequest(epoch, validators); + final String response = + httpClient.post( + getRestApiUrl(), "/eth/v1/validator/liveness", jsonProvider.objectToJSON(request)); + final PostValidatorLivenessResponse livenessResponse = + jsonProvider.jsonToObject(response, PostValidatorLivenessResponse.class); + final Map output = new HashMap<>(); + for (ValidatorLivenessAtEpoch entry : livenessResponse.data) { + output.put(entry.index, entry.isLive); + } + return output; + } + public void waitForGenesisTime(final UInt64 expectedGenesisTime) { waitFor(() -> assertThat(fetchGenesisTime()).isEqualTo(expectedGenesisTime)); } @@ -546,6 +613,11 @@ public Config withInteropValidators(final int startIndex, final int validatorCou return this; } + public Config withValidatorLivenessTracking() { + configMap.put("Xbeacon-liveness-tracking-enabled", true); + return this; + } + public Config withInteropNumberOfValidators(final int validatorCount) { configMap.put("Xinterop-number-of-validators", validatorCount); return this; diff --git a/acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/ValidatorLivenessExpectation.java b/acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/ValidatorLivenessExpectation.java new file mode 100644 index 00000000000..830a9f3cb37 --- /dev/null +++ b/acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/ValidatorLivenessExpectation.java @@ -0,0 +1,52 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.test.acceptance.dsl; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +import java.util.Map; +import tech.pegasys.teku.infrastructure.unsigned.UInt64; + +public class ValidatorLivenessExpectation { + private final int start; + private final int count; + private final boolean isLive; + + private ValidatorLivenessExpectation(final int start, final int count, final boolean isLive) { + this.start = start; + this.count = count; + this.isLive = isLive; + } + + public void verify(final Map liveness) { + StringBuilder result = new StringBuilder(); + for (int i = start; i < start + count; i++) { + if (liveness.get(UInt64.valueOf(i)) != isLive) { + result.append( + String.format( + "Validator %s was expected to be %s, but was not.\n", + i, isLive ? "live" : "offline")); + } + } + assertThat(result.toString()).isEmpty(); + } + + public static ValidatorLivenessExpectation expectLive(final int start, final int count) { + return new ValidatorLivenessExpectation(start, count, true); + } + + public static ValidatorLivenessExpectation expectNotLive(final int start, final int count) { + return new ValidatorLivenessExpectation(start, count, false); + } +}