Proper liveness probe

tillerinobot-irc/pom.xml

@@ -71,6 +71,11 @@
 			<artifactId>rabbitmq</artifactId>
 			<scope>test</scope>
 		</dependency>
+		<dependency>
+			<groupId>io.rest-assured</groupId>
+			<artifactId>rest-assured</artifactId>
+			<scope>test</scope>
+		</dependency>
 	</dependencies>
 
 	<build>

tillerinobot-irc/src/main/java/org/tillerino/ppaddict/chat/irc/BotRunnerImpl.java

@@ -42,7 +42,7 @@
 
 @Slf4j
 @Singleton
-public class BotRunnerImpl implements GameChatClient, Runnable {
+class BotRunnerImpl implements GameChatClient, Runnable {
 	public static final int DEFAULT_MESSAGE_DELAY = 250;
 	@SuppressFBWarnings(value = "MS", justification = "We're modifying this in tests")
 	public static int MESSAGE_DELAY = DEFAULT_MESSAGE_DELAY;

tillerinobot-irc/src/main/java/org/tillerino/ppaddict/chat/irc/IrcHooks.java

@@ -41,7 +41,7 @@
  */
 @SuppressWarnings("rawtypes")
 @Slf4j
-public class IrcHooks extends CoreHooks {
+class IrcHooks extends CoreHooks {
 	private final GameChatEventConsumer downStream;
 	private final GameChatClientMetrics botInfo;
 	private final IrcWriter queue;

tillerinobot-irc/src/main/java/org/tillerino/ppaddict/chat/irc/IrcWriter.java

@@ -20,7 +20,7 @@
 
 @Slf4j
 @RequiredArgsConstructor
-public class IrcWriter implements GameChatWriter {
+class IrcWriter implements GameChatWriter {
 	private final Pinger pinger;
 
 	private final AtomicReference<CloseableBot> bot = new AtomicReference<>();

tillerinobot-irc/src/main/java/org/tillerino/ppaddict/chat/irc/Main.java

@@ -10,10 +10,8 @@
 import java.util.concurrent.TimeoutException;
 import java.util.function.Function;
 
-import javax.annotation.CheckForNull;
-
-import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import org.tillerino.ppaddict.chat.GameChatClient;
+import org.tillerino.ppaddict.chat.GameChatClientMetrics;
 import org.tillerino.ppaddict.chat.GameChatWriter;
 import org.tillerino.ppaddict.rabbit.RabbitMqConfiguration;
 import org.tillerino.ppaddict.rabbit.RabbitRpc;
@@ -23,7 +21,9 @@
 import com.rabbitmq.client.Connection;
 import com.rabbitmq.client.ConnectionFactory;
 
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import io.undertow.Undertow;
+import io.undertow.server.HttpHandler;
 import io.undertow.util.Headers;
 import lombok.extern.slf4j.Slf4j;
 
@@ -35,32 +35,41 @@ public static void main(String[] args) throws Exception {
 		Bot bot = createBotRunner();
 		Undertow httpServer = Undertow.builder()
 			.addHttpListener(8080, "0.0.0.0")
-			.setHandler(exchange -> {
-				exchange.getResponseHeaders().put(Headers.CONTENT_TYPE, "text/plain");
-				exchange.getResponseSender().send("Hello World");
-			})
+			.setHandler(probes(bot))
 			.setIoThreads(1)
 			.setWorkerThreads(1)
 			.build();
 		httpServer.start();
-		if (bot != null) {
-			ExecutorService exec = Executors.newCachedThreadPool();
-			exec.submit(RabbitRpc.handleRemoteCalls(bot.conn(), GameChatClient.class, bot.runner(), new GameChatClient.Error.Unknown())::mainloop);
-			exec.submit(RabbitRpc.handleRemoteCalls(bot.conn(), GameChatWriter.class, bot.runner().getWriter(), new GameChatWriter.Error.Unknown())::mainloop);
-			exec.submit(bot.runner());
-		} else {
-			log.info("IRC server not configured. Only starting HTTP.");
-		}
+		ExecutorService exec = Executors.newCachedThreadPool();
+		exec.submit(RabbitRpc.handleRemoteCalls(bot.conn(), GameChatClient.class, bot.runner(), new GameChatClient.Error.Unknown())::mainloop);
+		exec.submit(RabbitRpc.handleRemoteCalls(bot.conn(), GameChatWriter.class, bot.runner().getWriter(), new GameChatWriter.Error.Unknown())::mainloop);
+		exec.submit(bot.runner());
 	}
 
-	private static @CheckForNull Bot createBotRunner() throws IOException, TimeoutException {
-		String server = null;
-		try {
-			server = env("TILLERINOBOT_IRC_SERVER", identity(), null);
-		} catch (NoSuchElementException e) {
-			return null;
-		}
+	private static HttpHandler probes(Bot bot) {
+		return exchange -> {
+			exchange.getResponseHeaders().put(Headers.CONTENT_TYPE, "text/plain");
+			// we don't have a service, so we don't need a readiness probe
+			if (exchange.getRequestPath().equals("/live")) {
+				if (bot.isLive()) {
+					exchange.setStatusCode(200);
+					exchange.getResponseSender().send("live");
+				} else {
+					exchange.setStatusCode(503);
+					exchange.getResponseSender().send("not live");
+				}
+			} else if (exchange.getRequestPath().equals("/hello")) {
+				exchange.setStatusCode(200);
+				exchange.getResponseSender().send("yes?");
+			} else {
+				exchange.setStatusCode(404);
+				exchange.getResponseSender().send("not found");
+			}
+		};
+	}
 
+	private static Bot createBotRunner() throws IOException, TimeoutException {
+		String server = env("TILLERINOBOT_IRC_SERVER", identity(), null);
 		int port = env("TILLERINOBOT_IRC_PORT", Integer::valueOf, null);
 		String nickname = env("TILLERINOBOT_IRC_NICKNAME", identity(), null);
 		String password = env("TILLERINOBOT_IRC_PASSWORD", identity(), null);
@@ -94,7 +103,15 @@ static <T> T env(String name, Function<String, T> parser, T defaultValue) {
 		return optional.orElseThrow(() -> new NoSuchElementException("Need to configure environment variable " + name));
 	}
 
-	record Bot(Connection conn, BotRunnerImpl runner) { }
+	record Bot(Connection conn, BotRunnerImpl runner) {
+		public boolean isLive() {
+			return conn.isOpen()
+				&& runner.getMetrics().map(GameChatClientMetrics::isConnected).unwrapOr(false)
+				// The bot is considered live if it has received any event in the last 10 minutes.
+				// This is a last resort if the connection somehow gets stuck.
+				&& runner.getMetrics().map(metrics -> metrics.getLastInteraction() > Clock.system().currentTimeMillis() - 10 * 60 * 1000).unwrapOr(false);
+		}
+	}
 
 	record Rabbit(Connection conn, RemoteEventQueue queue) { }
 }

tillerinobot-irc/src/main/java/org/tillerino/ppaddict/chat/irc/Pinger.java

@@ -19,7 +19,7 @@
 
 @Slf4j
 @RequiredArgsConstructor
-public class Pinger {
+class Pinger {
 	volatile String pingMessage = null;
 	volatile CountDownLatch pingLatch = null;
 

tillerinobot-irc/src/test/java/org/tillerino/ppaddict/chat/irc/BotIT.java

@@ -0,0 +1,36 @@
+package org.tillerino.ppaddict.chat.irc;
+
+import org.awaitility.Awaitility;
+import org.junit.Before;
+import org.junit.Test;
+import org.tillerino.ppaddict.rabbit.RabbitMqContainer;
+
+import io.restassured.RestAssured;
+
+public class BotIT {
+	@Before
+	public void setUp() throws Exception {
+		RabbitMqContainer.RABBIT_MQ.start();
+		NgircdContainer.NGIRCD.start();
+		IrcContainer.TILLERINOBOT_IRC.start();
+		RestAssured.baseURI = "http://" + IrcContainer.TILLERINOBOT_IRC.getHost() + ":"
+				+ IrcContainer.TILLERINOBOT_IRC.getMappedPort(8080) + "/";
+		Awaitility.await().untilAsserted(() -> RestAssured.when().get("/live").then().statusCode(200));
+	}
+
+	@Test
+	public void livenessReactsToRabbit() throws Exception {
+		RabbitMqContainer.RABBIT_MQ.stop();
+		Awaitility.await().untilAsserted(() -> RestAssured.when().get("/live").then().statusCode(503));
+		RabbitMqContainer.RABBIT_MQ.start();
+		Awaitility.await().untilAsserted(() -> RestAssured.when().get("/live").then().statusCode(200));
+	}
+
+	@Test
+	public void livenessReactsToNgircd() throws Exception {
+		NgircdContainer.NGIRCD.stop();
+		Awaitility.await().untilAsserted(() -> RestAssured.when().get("/live").then().statusCode(503));
+		NgircdContainer.NGIRCD.start();
+		Awaitility.await().untilAsserted(() -> RestAssured.when().get("/live").then().statusCode(200));
+	}
+}

tillerinobot-irc/src/test/java/org/tillerino/ppaddict/chat/irc/IrcContainer.java

@@ -17,7 +17,7 @@ public class IrcContainer {
 		.withCreateContainerCmdModifier(cmd -> cmd.withName("tillerinobot-irc"))
 		.withNetwork(NETWORK)
 		.withExposedPorts(8080)
-		.waitingFor(new HttpWaitStrategy().forPort(8080).forPath("/ready"))
+		.waitingFor(new HttpWaitStrategy().forPort(8080).forPath("/live"))
 		.withEnv("TILLERINOBOT_IRC_SERVER", "irc")
 		.withEnv("TILLERINOBOT_IRC_PORT", "6667")
 		.withEnv("TILLERINOBOT_IRC_NICKNAME", "tillerinobot")

tillerinobot-model/src/main/java/org/tillerino/ppaddict/util/Result.java

@@ -65,6 +65,13 @@ default T unwrapOrElse(Function<E, T> op) {
 		else return op.apply(((Err<T, E>) this).e);
 	}
 
+	default T unwrapOr(T def) {
+		if (this instanceof Ok<T, E> ok) {
+			return ok.t;
+		}
+		else return def;
+	}
+
 	default <U> Result<U, E> map(Function<T, U> op) {
 		if (this instanceof Ok<T, E> ok) {
 			return ok(op.apply(ok.t));

tillerinobot/src/main/java/org/tillerino/ppaddict/chat/impl/ResponsePostprocessor.java

@@ -33,7 +33,7 @@
  * enables us to switch out the implementation of {@link GameChatWriter}.
  */
 @Slf4j
-@RequiredArgsConstructor(onConstructor = @__(@Inject))
+@RequiredArgsConstructor(onConstructor_ = {@Inject, @SuppressFBWarnings("EI_EXPOSE_REP2")})
 public class ResponsePostprocessor implements GameChatResponseConsumer {
 	private final Bouncer bouncer;