Acceptance test for liveness endpoint

acceptance-tests/src/acceptance-test/java/tech/pegasys/teku/test/acceptance/ValidatorLivenessAcceptanceTest.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2021 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.test.acceptance;
+
+import static tech.pegasys.teku.test.acceptance.dsl.ValidatorLivenessExpectation.expectLive;
+import static tech.pegasys.teku.test.acceptance.dsl.ValidatorLivenessExpectation.expectNotLive;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import tech.pegasys.teku.infrastructure.time.SystemTimeProvider;
+import tech.pegasys.teku.test.acceptance.dsl.AcceptanceTestBase;
+import tech.pegasys.teku.test.acceptance.dsl.TekuNode;
+
+public class ValidatorLivenessAcceptanceTest extends AcceptanceTestBase {
+
+  private static final int NODE_VALIDATORS = 2;
+  private static final int TOTAL_VALIDATORS = NODE_VALIDATORS * 2;
+
+  private final SystemTimeProvider timeProvider = new SystemTimeProvider();
+  private TekuNode primaryNode;
+  private TekuNode secondaryNode;
+
+  @BeforeEach
+  public void setup() {
+    final int genesisTime = timeProvider.getTimeInSeconds().plus(5).intValue();
+    primaryNode =
+        createTekuNode(
+            config -> configureNode(config, genesisTime).withInteropValidators(0, NODE_VALIDATORS));
+    secondaryNode =
+        createTekuNode(
+            config ->
+                configureNode(config, genesisTime)
+                    .withInteropValidators(NODE_VALIDATORS, NODE_VALIDATORS)
+                    .withPeers(primaryNode));
+  }
+
+  /*
+   * Primary and Secondary node, each with half of the validators
+   *  - Primary is online at genesis, so during epoch 1 all validators should be performing duties.
+   *  - no validator keys from the secondary will be seen as active in epoch 1.
+   *  - Secondary is online at epoch 2, so by epoch 3 should all be performing duties.
+   *  - by epoch 4, all validators should be seen as performing duties in epoch 3
+   */
+  @Test
+  void shouldTrackValidatorLivenessOverEpochs() throws Exception {
+    primaryNode.start();
+
+    primaryNode.waitForBlockAtOrAfterSlot(8);
+    secondaryNode.start();
+    primaryNode.checkValidatorLiveness(
+        1,
+        TOTAL_VALIDATORS,
+        expectLive(0, NODE_VALIDATORS),
+        expectNotLive(NODE_VALIDATORS, NODE_VALIDATORS));
+
+    primaryNode.waitForBlockAtOrAfterSlot(16);
+    primaryNode.checkValidatorLiveness(3, TOTAL_VALIDATORS, expectLive(0, TOTAL_VALIDATORS));
+    secondaryNode.checkValidatorLiveness(3, TOTAL_VALIDATORS, expectLive(0, TOTAL_VALIDATORS));
+  }
+
+  private TekuNode.Config configureNode(final TekuNode.Config node, final int genesisTime) {
+    return node.withNetwork("swift")
+        .withGenesisTime(genesisTime)
+        .withValidatorLivenessTracking()
+        .withInteropNumberOfValidators(TOTAL_VALIDATORS)
+        .withRealNetwork();
+  }
+}

acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/SimpleHttpClient.java

@@ -19,11 +19,14 @@
 import java.net.URI;
 import okhttp3.OkHttpClient;
 import okhttp3.Request;
+import okhttp3.RequestBody;
 import okhttp3.Response;
 import okhttp3.ResponseBody;
 
 public class SimpleHttpClient {
   private final OkHttpClient httpClient = new OkHttpClient();
+  private static final okhttp3.MediaType JSON =
+      okhttp3.MediaType.parse("application/json; charset=utf-8");
 
   public String get(final URI baseUrl, final String path) throws IOException {
     final Response response =
@@ -35,4 +38,18 @@ public String get(final URI baseUrl, final String path) throws IOException {
     assertThat(body).isNotNull();
     return body.string();
   }
+
+  public String post(final URI baseUrl, final String path, final String jsonBody)
+      throws IOException {
+    final RequestBody requestBody = RequestBody.create(jsonBody, JSON);
+    final Response response =
+        httpClient
+            .newCall(
+                new Request.Builder().url(baseUrl.resolve(path).toURL()).post(requestBody).build())
+            .execute();
+    assertThat(response.isSuccessful()).isTrue();
+    final ResponseBody responseBody = response.body();
+    assertThat(responseBody).isNotNull();
+    return responseBody.string();
+  }
 }

acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuNode.java

@@ -50,12 +50,16 @@
 import org.testcontainers.containers.Network;
 import org.testcontainers.containers.wait.strategy.HttpWaitStrategy;
 import org.testcontainers.utility.MountableFile;
+import tech.pegasys.teku.api.request.v1.validator.ValidatorLivenessRequest;
 import tech.pegasys.teku.api.response.v1.EventType;
+import tech.pegasys.teku.api.response.v1.HeadEvent;
 import tech.pegasys.teku.api.response.v1.beacon.FinalityCheckpointsResponse;
 import tech.pegasys.teku.api.response.v1.beacon.GetBlockRootResponse;
 import tech.pegasys.teku.api.response.v1.beacon.GetGenesisResponse;
 import tech.pegasys.teku.api.response.v1.beacon.GetStateFinalityCheckpointsResponse;
 import tech.pegasys.teku.api.response.v1.debug.GetStateResponse;
+import tech.pegasys.teku.api.response.v1.validator.PostValidatorLivenessResponse;
+import tech.pegasys.teku.api.response.v1.validator.ValidatorLivenessAtEpoch;
 import tech.pegasys.teku.api.response.v2.beacon.GetBlockResponseV2;
 import tech.pegasys.teku.api.schema.BeaconState;
 import tech.pegasys.teku.api.schema.SignedBeaconBlock;
@@ -169,6 +173,69 @@ public void waitForGenesis() {
     waitFor(this::fetchGenesisTime);
   }
 
+  public void waitForBlockAtOrAfterSlot(final long slot) {
+    if (maybeEventStreamListener.isEmpty()) {
+      startEventListener(List.of(EventType.head));
+    }
+    waitFor(
+        () ->
+            assertThat(
+                    getSlotsFromHeadEvents().stream()
+                        .filter(v -> v.isGreaterThanOrEqualTo(slot))
+                        .count())
+                .isGreaterThan(0));
+  }
+
+  private List<UInt64> getSlotsFromHeadEvents() {
+    return maybeEventStreamListener.get().getMessages().stream()
+        .filter(packedMessage -> packedMessage.getEvent().equals(EventType.head.name()))
+        .map(this::getSlotFromHeadEvent)
+        .flatMap(Optional::stream)
+        .collect(toList());
+  }
+
+  private Optional<UInt64> getSlotFromHeadEvent(
+      final Eth2EventHandler.PackedMessage packedMessage) {
+    try {
+      return Optional.of(
+          jsonProvider.jsonToObject(packedMessage.getMessageEvent().getData(), HeadEvent.class)
+              .slot);
+    } catch (JsonProcessingException e) {
+      LOG.error("Failed to process head event", e);
+      return Optional.empty();
+    }
+  }
+
+  public void checkValidatorLiveness(
+      final int epoch, final int totalValidatorCount, ValidatorLivenessExpectation... args)
+      throws IOException {
+    final List<UInt64> validators = new ArrayList<>();
+    for (UInt64 i = UInt64.ZERO; i.isLessThan(totalValidatorCount); i = i.increment()) {
+      validators.add(i);
+    }
+    final Map<UInt64, Boolean> data =
+        getValidatorLivenessAtEpoch(UInt64.valueOf(epoch), validators);
+    for (ValidatorLivenessExpectation expectation : args) {
+      expectation.verify(data);
+    }
+  }
+
+  private Map<UInt64, Boolean> getValidatorLivenessAtEpoch(
+      final UInt64 epoch, List<UInt64> validators) throws IOException {
+
+    final ValidatorLivenessRequest request = new ValidatorLivenessRequest(epoch, validators);
+    final String response =
+        httpClient.post(
+            getRestApiUrl(), "/eth/v1/validator/liveness", jsonProvider.objectToJSON(request));
+    final PostValidatorLivenessResponse livenessResponse =
+        jsonProvider.jsonToObject(response, PostValidatorLivenessResponse.class);
+    final Map<UInt64, Boolean> output = new HashMap<>();
+    for (ValidatorLivenessAtEpoch entry : livenessResponse.data) {
+      output.put(entry.index, entry.isLive);
+    }
+    return output;
+  }
+
   public void waitForGenesisTime(final UInt64 expectedGenesisTime) {
     waitFor(() -> assertThat(fetchGenesisTime()).isEqualTo(expectedGenesisTime));
   }
@@ -546,6 +613,11 @@ public Config withInteropValidators(final int startIndex, final int validatorCou
       return this;
     }
 
+    public Config withValidatorLivenessTracking() {
+      configMap.put("Xbeacon-liveness-tracking-enabled", true);
+      return this;
+    }
+
     public Config withInteropNumberOfValidators(final int validatorCount) {
       configMap.put("Xinterop-number-of-validators", validatorCount);
       return this;

acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/ValidatorLivenessExpectation.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2021 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.test.acceptance.dsl;
+
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
+import java.util.Map;
+import tech.pegasys.teku.infrastructure.unsigned.UInt64;
+
+public class ValidatorLivenessExpectation {
+  private final int start;
+  private final int count;
+  private final boolean isLive;
+
+  private ValidatorLivenessExpectation(final int start, final int count, final boolean isLive) {
+    this.start = start;
+    this.count = count;
+    this.isLive = isLive;
+  }
+
+  public void verify(final Map<UInt64, Boolean> liveness) {
+    StringBuilder result = new StringBuilder();
+    for (int i = start; i < start + count; i++) {
+      if (liveness.get(UInt64.valueOf(i)) != isLive) {
+        result.append(
+            String.format(
+                "Validator %s was expected to be %s, but was not.\n",
+                i, isLive ? "live" : "offline"));
+      }
+    }
+    assertThat(result.toString()).isEmpty();
+  }
+
+  public static ValidatorLivenessExpectation expectLive(final int start, final int count) {
+    return new ValidatorLivenessExpectation(start, count, true);
+  }
+
+  public static ValidatorLivenessExpectation expectNotLive(final int start, final int count) {
+    return new ValidatorLivenessExpectation(start, count, false);
+  }
+}