From 6edf61c72caabf8a8453dbab7a2f37124d8ba72b Mon Sep 17 00:00:00 2001 From: Michael Simons Date: Fri, 17 Sep 2021 16:28:10 +0200 Subject: [PATCH] Add support for Dev Services inside Neo4j module. This change adds one additional processor that conditionally (when docker is working and Neo4j is not reachable on the default address) starts up a Neo4j Test container based on the latest image. Tests have been added that will be executed when docker is enabled. The existing functional integration tests won't be affected as they use explicit configuration. --- docs/src/main/asciidoc/neo4j.adoc | 19 ++ extensions/neo4j/deployment/pom.xml | 39 ++++ .../neo4j/deployment/BoltHandshaker.java | 77 +++++++ .../DevServicesBuildTimeConfig.java | 32 +++ .../deployment/Neo4jBuildTimeConfig.java | 9 +- .../deployment/Neo4jDevServiceBuildItem.java | 6 + .../deployment/Neo4jDevServicesProcessor.java | 188 ++++++++++++++++++ .../neo4j/deployment/Neo4jDevModeTests.java | 142 +++++++++++++ .../src/test/resources/application.properties | 8 + 9 files changed, 519 insertions(+), 1 deletion(-) create mode 100644 extensions/neo4j/deployment/src/main/java/io/quarkus/neo4j/deployment/BoltHandshaker.java create mode 100644 extensions/neo4j/deployment/src/main/java/io/quarkus/neo4j/deployment/DevServicesBuildTimeConfig.java create mode 100644 extensions/neo4j/deployment/src/main/java/io/quarkus/neo4j/deployment/Neo4jDevServiceBuildItem.java create mode 100644 extensions/neo4j/deployment/src/main/java/io/quarkus/neo4j/deployment/Neo4jDevServicesProcessor.java create mode 100644 extensions/neo4j/deployment/src/test/java/io/quarkus/neo4j/deployment/Neo4jDevModeTests.java create mode 100644 extensions/neo4j/deployment/src/test/resources/application.properties diff --git a/docs/src/main/asciidoc/neo4j.adoc b/docs/src/main/asciidoc/neo4j.adoc index 2fcb8348c1496..62f63d64d8f5b 100644 --- a/docs/src/main/asciidoc/neo4j.adoc +++ b/docs/src/main/asciidoc/neo4j.adoc @@ -167,6 +167,7 @@ The Neo4j driver can be configured with standard Quarkus properties: [source,properties] .src/main/resources/application.properties ---- +# Those are the default values and are implicitly assumed quarkus.neo4j.uri = bolt://localhost:7687 quarkus.neo4j.authentication.username = neo4j quarkus.neo4j.authentication.password = secret @@ -176,6 +177,24 @@ You'll recognize the authentication here that you passed on to the docker comman Having done that, the driver is ready to use, there are however other configuration options, detailed below. +[[dev-services]] +=== Dev Services (Configuration Free Databases) + +Quarkus supports a feature called Dev Services that allows you to create various datasources without any config. +In the case of Neo4j this support applies to the single Neo4j driver instance. +Dev Services will bring up a Neo4j container if you didn't explicit add the default values or configured custom values for +any of `quarkus.neo4j.uri`, `quarkus.neo4j.authentication.username` or `quarkus.neo4j.authentication.password`. +If Neo4j seems to be reachable via the default properties, Dev Services will also step back. + +Otherwise, Quarkus will automatically start a Neo4j container when running tests or dev-mode, +and automatically configure the connection. + +When running the production version of the application, the Neo4j connection need to be configured as normal, +so if you want to include a production database config in your `application.properties` and continue to use Dev Services +we recommend that you use the `%prod.` profile to define your Neo4j settings. + +include::{generated-dir}/config/quarkus-neo4j-config-group-dev-services-build-time-config.adoc[opts=optional, leveloffset=+1] + == Using the driver === General remarks diff --git a/extensions/neo4j/deployment/pom.xml b/extensions/neo4j/deployment/pom.xml index e299429a06662..c1a26e9eab907 100644 --- a/extensions/neo4j/deployment/pom.xml +++ b/extensions/neo4j/deployment/pom.xml @@ -29,6 +29,36 @@ io.quarkus quarkus-neo4j + + org.testcontainers + neo4j + + + junit + junit + + + + + io.quarkus + quarkus-devservices-common + + + + io.quarkus + quarkus-junit5-internal + test + + + org.testcontainers + junit-jupiter + test + + + org.assertj + assertj-core + test + @@ -45,6 +75,15 @@ + + + maven-surefire-plugin + + + + + + diff --git a/extensions/neo4j/deployment/src/main/java/io/quarkus/neo4j/deployment/BoltHandshaker.java b/extensions/neo4j/deployment/src/main/java/io/quarkus/neo4j/deployment/BoltHandshaker.java new file mode 100644 index 0000000000000..687074e94cb3b --- /dev/null +++ b/extensions/neo4j/deployment/src/main/java/io/quarkus/neo4j/deployment/BoltHandshaker.java @@ -0,0 +1,77 @@ +package io.quarkus.neo4j.deployment; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.time.Duration; + +/** + * This implements the protocol version negotiation of bolt. Testing to see if in address will respond to this is a + * quick way to find out if it's a running bolt server. + *

+ * This class first appeared in https://github.com/michael-simons/junit-jupiter-causal-cluster-testcontainer-extension + * by Andrew Jefferson and Michael Simons + */ +final class BoltHandshaker { + + private static final int magicToken = 1616949271; + + // Versions message that cannot be matched because it is all zeros. + private static final byte[] versionsMessage = { + (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00, + (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00, + (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00, + (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00 + }; + + private final String address; + private final int port; + + BoltHandshaker(String address, int port) { + this.address = address; + this.port = port; + } + + private boolean doBoltHandshake(String address, int port, int timeoutMillis) { + + try (Socket socket = new Socket()) { + + // Set the socket timeout for blocking operations + socket.setSoTimeout(timeoutMillis); + + // Connects this socket to the server (also with the specified timeout value). + socket.connect(new InetSocketAddress(address, port), timeoutMillis); + + DataOutputStream dOut = new DataOutputStream(socket.getOutputStream()); + DataInputStream dIn = new DataInputStream(socket.getInputStream()); + + // Send magic token (0x6060B017) + dOut.writeInt(magicToken); + dOut.flush(); + + // Send 4 supported versions + // Except we don't support any versions and communicate that by sending all zeros + dOut.write(versionsMessage); + dOut.flush(); + + // Receive agreed version + // It should be 0 because there are no possible versions we can agree on + int response = dIn.readInt(); + assert response == 0; + + // Because we cannot agree on a version the server should close its side of the connection + // resulting in EOF (-1) on all subsequent reads. + return dIn.read() == -1; + } catch (IOException exception) { + // Return false if handshake fails + return false; + } + } + + boolean isBoltPortReachable(Duration timeout) { + int timeoutMillis = Math.toIntExact(timeout.toMillis()); + return doBoltHandshake(address, port, timeoutMillis); + } +} diff --git a/extensions/neo4j/deployment/src/main/java/io/quarkus/neo4j/deployment/DevServicesBuildTimeConfig.java b/extensions/neo4j/deployment/src/main/java/io/quarkus/neo4j/deployment/DevServicesBuildTimeConfig.java new file mode 100644 index 0000000000000..2b5cc03f60782 --- /dev/null +++ b/extensions/neo4j/deployment/src/main/java/io/quarkus/neo4j/deployment/DevServicesBuildTimeConfig.java @@ -0,0 +1,32 @@ +package io.quarkus.neo4j.deployment; + +import java.util.Map; +import java.util.Optional; + +import io.quarkus.runtime.annotations.ConfigGroup; +import io.quarkus.runtime.annotations.ConfigItem; + +@ConfigGroup +public class DevServicesBuildTimeConfig { + + /** + * If DevServices has been explicitly enabled or disabled. DevServices is generally enabled + * by default, unless there is an existing configuration present. + * When DevServices is enabled Quarkus will attempt to automatically configure and start + * a database when running in Dev or Test mode. + */ + @ConfigItem + public Optional enabled = Optional.empty(); + + /** + * The container image name to use, for container based DevServices providers. + */ + @ConfigItem(defaultValue = "neo4j:4.3") + public String imageName; + + /** + * Additional environment entries that can be added to the container before its start. + */ + @ConfigItem + public Map additionalEnv; +} diff --git a/extensions/neo4j/deployment/src/main/java/io/quarkus/neo4j/deployment/Neo4jBuildTimeConfig.java b/extensions/neo4j/deployment/src/main/java/io/quarkus/neo4j/deployment/Neo4jBuildTimeConfig.java index c968d7f59eae6..a02812ef7a9d3 100644 --- a/extensions/neo4j/deployment/src/main/java/io/quarkus/neo4j/deployment/Neo4jBuildTimeConfig.java +++ b/extensions/neo4j/deployment/src/main/java/io/quarkus/neo4j/deployment/Neo4jBuildTimeConfig.java @@ -6,9 +6,16 @@ @ConfigRoot(name = "neo4j", phase = ConfigPhase.BUILD_TIME) public class Neo4jBuildTimeConfig { + /** - * Whether or not an health check is published in case the smallrye-health extension is present. + * Whether a health check is published in case the smallrye-health extension is present. */ @ConfigItem(name = "health.enabled", defaultValue = "true") public boolean healthEnabled; + + /** + * Configuration for DevServices. DevServices allows Quarkus to automatically start a Neo4j instance in dev and test mode. + */ + @ConfigItem + public DevServicesBuildTimeConfig devservices; } diff --git a/extensions/neo4j/deployment/src/main/java/io/quarkus/neo4j/deployment/Neo4jDevServiceBuildItem.java b/extensions/neo4j/deployment/src/main/java/io/quarkus/neo4j/deployment/Neo4jDevServiceBuildItem.java new file mode 100644 index 0000000000000..a4df6e02fbb20 --- /dev/null +++ b/extensions/neo4j/deployment/src/main/java/io/quarkus/neo4j/deployment/Neo4jDevServiceBuildItem.java @@ -0,0 +1,6 @@ +package io.quarkus.neo4j.deployment; + +import io.quarkus.builder.item.SimpleBuildItem; + +public final class Neo4jDevServiceBuildItem extends SimpleBuildItem { +} diff --git a/extensions/neo4j/deployment/src/main/java/io/quarkus/neo4j/deployment/Neo4jDevServicesProcessor.java b/extensions/neo4j/deployment/src/main/java/io/quarkus/neo4j/deployment/Neo4jDevServicesProcessor.java new file mode 100644 index 0000000000000..543cdd453496a --- /dev/null +++ b/extensions/neo4j/deployment/src/main/java/io/quarkus/neo4j/deployment/Neo4jDevServicesProcessor.java @@ -0,0 +1,188 @@ +package io.quarkus.neo4j.deployment; + +import java.io.Closeable; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.function.BooleanSupplier; + +import org.jboss.logging.Logger; +import org.testcontainers.containers.Neo4jContainer; + +import io.quarkus.deployment.IsNormal; +import io.quarkus.deployment.annotations.BuildProducer; +import io.quarkus.deployment.annotations.BuildStep; +import io.quarkus.deployment.builditem.CuratedApplicationShutdownBuildItem; +import io.quarkus.deployment.builditem.DevServicesConfigResultBuildItem; +import io.quarkus.deployment.builditem.LaunchModeBuildItem; +import io.quarkus.deployment.console.ConsoleInstalledBuildItem; +import io.quarkus.deployment.console.StartupLogCompressor; +import io.quarkus.deployment.dev.devservices.GlobalDevServicesConfig; +import io.quarkus.deployment.logging.LoggingSetupBuildItem; +import io.quarkus.runtime.configuration.ConfigUtils; + +class Neo4jDevServicesProcessor { + + private static final Logger log = Logger.getLogger("io.quarkus.neo4j.deployment"); + + private static final String NEO4J_URI = "quarkus.neo4j.uri"; + private static final String NEO4J_USER_PROP = "quarkus.neo4j.authentication.username"; + private static final String NEO4J_PASSWORD_PROP = "quarkus.neo4j.authentication.password"; + + static volatile Closeable closeable; + static volatile Neo4jDevServiceConfig runningConfiguration; + static volatile boolean first = true; + + static final class IsDockerWorking implements BooleanSupplier { + + private final io.quarkus.deployment.IsDockerWorking delegate = new io.quarkus.deployment.IsDockerWorking(true); + + @Override + public boolean getAsBoolean() { + + return delegate.getAsBoolean(); + } + } + + static final class BoldHandshakeOnDefaultAddressIsPossible implements BooleanSupplier { + + private final BoltHandshaker boltHandshaker = new BoltHandshaker("localhost", 7687); + + @Override + public boolean getAsBoolean() { + + var boldIsReachable = Boolean.getBoolean("quarkus.neo4j.devservices.assumeBoltIsReachable") + || boltHandshaker.isBoltPortReachable(Duration.ofSeconds(5)); + if (boldIsReachable && log.isDebugEnabled()) { + log.info("Not starting Dev Services for Neo4j, as the default config points to a reachable address."); + } + return boldIsReachable; + } + } + + @BuildStep(onlyIfNot = { IsNormal.class, BoldHandshakeOnDefaultAddressIsPossible.class }, onlyIf = { + IsDockerWorking.class, + GlobalDevServicesConfig.Enabled.class }) + public Neo4jDevServiceBuildItem startNeo4jDevService( + LaunchModeBuildItem launchMode, + Neo4jBuildTimeConfig neo4jBuildTimeConfig, + BuildProducer devServicePropertiesProducer, + Optional consoleInstalledBuildItem, + CuratedApplicationShutdownBuildItem closeBuildItem, + LoggingSetupBuildItem loggingSetupBuildItem) { + + var configuration = new Neo4jDevServiceConfig(neo4jBuildTimeConfig.devservices); + + if (closeable != null) { + if (configuration.equals(runningConfiguration)) { + return null; + } + shutdownNeo4j(); + runningConfiguration = null; + } + + var compressor = new StartupLogCompressor( + (launchMode.isTest() ? "(test) " : "") + "Neo4j Dev Services Starting:", consoleInstalledBuildItem, + loggingSetupBuildItem); + try { + var neo4jContainer = startNeo4j(configuration, launchMode); + if (neo4jContainer != null) { + devServicePropertiesProducer.produce( + new DevServicesConfigResultBuildItem(NEO4J_URI, neo4jContainer.getBoltUrl())); + devServicePropertiesProducer.produce(new DevServicesConfigResultBuildItem(NEO4J_USER_PROP, "neo4j")); + devServicePropertiesProducer.produce(new DevServicesConfigResultBuildItem(NEO4J_PASSWORD_PROP, + neo4jContainer.getAdminPassword())); + + log.infof("Dev Services started a Neo4j container reachable at %s.", neo4jContainer.getBoltUrl()); + + closeable = neo4jContainer::close; + } + } catch (Throwable t) { + compressor.closeAndDumpCaptured(); + throw new RuntimeException(t); + } + + // Configure the watch dog + if (first) { + first = false; + Runnable closeTask = () -> { + if (closeable != null) { + shutdownNeo4j(); + log.info("Dev Services for Neo4j shut down."); + } + first = true; + closeable = null; + runningConfiguration = null; + }; + closeBuildItem.addCloseTask(closeTask, true); + } + runningConfiguration = configuration; + + return new Neo4jDevServiceBuildItem(); + } + + private Neo4jContainer startNeo4j(Neo4jDevServiceConfig configuration, LaunchModeBuildItem launchMode) { + + if (!configuration.devServicesEnabled) { + log.debug("Not starting Dev Services for Neo4j, as it has been disabled in the config."); + return null; + } + + // Check if Neo4j URL or password is set to explicitly + if (ConfigUtils.isPropertyPresent(NEO4J_URI) || ConfigUtils.isPropertyPresent(NEO4J_USER_PROP) + || ConfigUtils.isPropertyPresent(NEO4J_PASSWORD_PROP)) { + log.debug("Not starting Dev Services for Neo4j, as there is explicit configuration present."); + return null; + } + + var neo4jContainer = new Neo4jContainer<>(configuration.imageName); + configuration.additionalEnv.forEach(neo4jContainer::addEnv); + neo4jContainer.start(); + return neo4jContainer; + } + + private void shutdownNeo4j() { + if (closeable != null) { + try { + closeable.close(); + } catch (Throwable e) { + log.error("Failed to stop Neo4j container", e); + } finally { + closeable = null; + } + } + } + + private static final class Neo4jDevServiceConfig { + final boolean devServicesEnabled; + final String imageName; + final Map additionalEnv; + + Neo4jDevServiceConfig(DevServicesBuildTimeConfig devServicesConfig) { + this.devServicesEnabled = devServicesConfig.enabled.orElse(true); + this.imageName = devServicesConfig.imageName; + this.additionalEnv = new HashMap<>(devServicesConfig.additionalEnv); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Neo4jDevServiceConfig that = (Neo4jDevServiceConfig) o; + return devServicesEnabled == that.devServicesEnabled && imageName.equals(that.imageName) + && additionalEnv.equals( + that.additionalEnv); + } + + @Override + public int hashCode() { + return Objects.hash(devServicesEnabled, imageName, additionalEnv); + } + } +} diff --git a/extensions/neo4j/deployment/src/test/java/io/quarkus/neo4j/deployment/Neo4jDevModeTests.java b/extensions/neo4j/deployment/src/test/java/io/quarkus/neo4j/deployment/Neo4jDevModeTests.java new file mode 100644 index 0000000000000..b9187a541600c --- /dev/null +++ b/extensions/neo4j/deployment/src/test/java/io/quarkus/neo4j/deployment/Neo4jDevModeTests.java @@ -0,0 +1,142 @@ +package io.quarkus.neo4j.deployment; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.assertj.core.api.Assertions.assertThatNoException; + +import java.util.logging.LogRecord; + +import javax.inject.Inject; + +import org.jboss.shrinkwrap.api.ShrinkWrap; +import org.jboss.shrinkwrap.api.spec.JavaArchive; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.neo4j.driver.Driver; +import org.neo4j.driver.exceptions.ServiceUnavailableException; +import org.testcontainers.junit.jupiter.Testcontainers; + +import io.quarkus.test.QuarkusUnitTest; + +public class Neo4jDevModeTests { + + @Testcontainers(disabledWithoutDocker = true) + static class DevServicesShouldStartNeo4j { + + @RegisterExtension + static QuarkusUnitTest test = new QuarkusUnitTest() + .setArchiveProducer(() -> ShrinkWrap.create(JavaArchive.class)) + .setLogRecordPredicate(record -> true) + .withConfigurationResource("application.properties") + .assertLogRecords(records -> assertThat(records).extracting(LogRecord::getMessage) + .contains("Dev Services started a Neo4j container reachable at %s.")); + + @Inject + Driver driver; + + @Test + public void shouldBeAbleToConnect() { + + assertThatNoException().isThrownBy(() -> driver.verifyConnectivity()); + + } + } + + @Testcontainers(disabledWithoutDocker = true) + static class WorkingWithDifferentImageAndAdditionalEnv { + + @RegisterExtension + static QuarkusUnitTest test = new QuarkusUnitTest() + .setArchiveProducer(() -> ShrinkWrap.create(JavaArchive.class)) + .setLogRecordPredicate(record -> true) + .withConfigurationResource("application.properties") + .overrideConfigKey("quarkus.neo4j.devservices.image-name", "neo4j:4.3-enterprise") + .overrideConfigKey("quarkus.neo4j.devservices.additional-env.NEO4J_ACCEPT_LICENSE_AGREEMENT", "yes") + .assertLogRecords(records -> assertThat(records).extracting(LogRecord::getMessage) + .contains("Dev Services started a Neo4j container reachable at %s.")); + + @Inject + Driver driver; + + @Test + public void shouldBeAbleToConnect() { + + assertThatNoException().isThrownBy(() -> driver.verifyConnectivity()); + try (var session = driver.session()) { + var cypher = "CALL dbms.components() YIELD versions, name, edition WHERE name = 'Neo4j Kernel' RETURN edition, versions[0] as version"; + var result = session.run(cypher).single(); + assertThat(result.get("edition").asString()).isEqualToIgnoringCase("enterprise"); + } + } + } + + @Testcontainers(disabledWithoutDocker = true) + static class WithLocallyDisabledDevServicesTest { + + @RegisterExtension + static QuarkusUnitTest test = new QuarkusUnitTest() + .setArchiveProducer(() -> ShrinkWrap.create(JavaArchive.class)) + .setLogRecordPredicate(record -> true) + .withConfigurationResource("application.properties") + .overrideConfigKey("quarkus.neo4j.devservices.enabled", "false") + .assertLogRecords(records -> assertThat(records).extracting(LogRecord::getMessage) + .contains("Not starting Dev Services for Neo4j, as it has been disabled in the config.")); + + @Inject + Driver driver; + + @Test + public void shouldNotBeAbleToConnect() { + + assertThatExceptionOfType(ServiceUnavailableException.class).isThrownBy(() -> driver.verifyConnectivity()); + } + } + + @Testcontainers(disabledWithoutDocker = true) + static class WithExplicitProperty { + + @RegisterExtension + static QuarkusUnitTest test = new QuarkusUnitTest() + .setArchiveProducer(() -> ShrinkWrap.create(JavaArchive.class)) + .setLogRecordPredicate(record -> true) + .withConfigurationResource("application.properties") + .overrideConfigKey("quarkus.neo4j.uri", "bolt://localhost:7687") + .assertLogRecords(records -> assertThat(records).extracting(LogRecord::getMessage) + .contains("Not starting Dev Services for Neo4j, as there is explicit configuration present.")); + + @Inject + Driver driver; + + @Test + public void shouldNotBeAbleToConnect() { + + assertThatExceptionOfType(ServiceUnavailableException.class).isThrownBy(() -> driver.verifyConnectivity()); + } + } + + @Testcontainers(disabledWithoutDocker = true) + static class WithAlreadyReachableInstance { + + static { + // Make our check think that + System.setProperty("quarkus.neo4j.devservices.assumeBoltIsReachable", "true"); + } + + @RegisterExtension + static QuarkusUnitTest test = new QuarkusUnitTest() + .setArchiveProducer(() -> ShrinkWrap.create(JavaArchive.class)) + .setLogRecordPredicate(record -> true) + .withConfigurationResource("application.properties") + .assertLogRecords(records -> assertThat(records).extracting(LogRecord::getMessage) + .contains("Not starting Dev Services for Neo4j, as the default config points to a reachable address.")); + + @Inject + Driver driver; + + @Test + public void shouldNotBeAbleToConnect() { + + assertThatExceptionOfType(ServiceUnavailableException.class).isThrownBy(() -> driver.verifyConnectivity()); + } + } +} diff --git a/extensions/neo4j/deployment/src/test/resources/application.properties b/extensions/neo4j/deployment/src/test/resources/application.properties new file mode 100644 index 0000000000000..cf7bbd37413ba --- /dev/null +++ b/extensions/neo4j/deployment/src/test/resources/application.properties @@ -0,0 +1,8 @@ +# I much prefer just setting this but I was not able to capture +# debug log with QuarkusUnitTest and the InMemoryLogHandler without... +quarkus.log.category."io.quarkus.neo4j.deployment".level=DEBUG + +# Forcing the whole of Quarkus into logging +quarkus.log.level=DEBUG +# And than taming console again +quarkus.log.console.level=INFO