From dec7c214c5cdf5b9e6a9c91d1c2bc388b7f7972f Mon Sep 17 00:00:00 2001 From: David Roberts Date: Thu, 14 May 2020 07:12:18 +0100 Subject: [PATCH] Prevent unexpected native controller output hanging the process (#56688) In normal operation native controllers are not expected to write anything to stdout or stderr. However, if due to an error or something unexpected with the environment a native controller does write something to stdout or stderr then it will block if nothing is reading that output. This change makes the stdout and stderr of native controllers reuse the same stdout and stderr as the Elasticsearch JVM (which are by default redirected to es.stdout.log and es.stderr.log) so that if something unexpected is written to native controller output then: 1. The native controller process does not block, waiting for something to read the output 2. We can see what the output was, making it easier to debug obscure environmental problems Backport of #56491 --- .../bootstrap/SpawnerNoBootstrapTests.java | 10 +++++----- .../org/elasticsearch/bootstrap/Bootstrap.java | 2 +- .../org/elasticsearch/bootstrap/Spawner.java | 18 ++++++++++++++---- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java b/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java index 00ad1b7e9b231..b1ae584f6633c 100644 --- a/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java +++ b/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java @@ -91,7 +91,7 @@ public void testNoControllerSpawn() throws IOException { "has.native.controller", "false"); try (Spawner spawner = new Spawner()) { - spawner.spawnNativeControllers(environment); + spawner.spawnNativeControllers(environment, false); assertThat(spawner.getProcesses(), hasSize(0)); } } @@ -149,7 +149,7 @@ private void assertControllerSpawns(final Function pluginsDir "has.native.controller", "false"); Spawner spawner = new Spawner(); - spawner.spawnNativeControllers(environment); + spawner.spawnNativeControllers(environment, false); List processes = spawner.getProcesses(); @@ -196,7 +196,7 @@ public void testControllerSpawnWithIncorrectDescriptor() throws IOException { Spawner spawner = new Spawner(); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> spawner.spawnNativeControllers(environment)); + () -> spawner.spawnNativeControllers(environment, false)); assertThat( e.getMessage(), equalTo("module [test_plugin] does not have permission to fork native controller")); @@ -217,10 +217,10 @@ public void testSpawnerHandlingOfDesktopServicesStoreFiles() throws IOException final Spawner spawner = new Spawner(); if (Constants.MAC_OS_X) { // if the spawner were not skipping the Desktop Services Store files on macOS this would explode - spawner.spawnNativeControllers(environment); + spawner.spawnNativeControllers(environment, false); } else { // we do not ignore these files on non-macOS systems - final FileSystemException e = expectThrows(FileSystemException.class, () -> spawner.spawnNativeControllers(environment)); + final FileSystemException e = expectThrows(FileSystemException.class, () -> spawner.spawnNativeControllers(environment, false)); if (Constants.WINDOWS) { assertThat(e, instanceOf(NoSuchFileException.class)); } else { diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java index f44722a1148ff..8e8bf3fe30217 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java @@ -162,7 +162,7 @@ private void setup(boolean addShutdownHook, Environment environment) throws Boot Settings settings = environment.settings(); try { - spawner.spawnNativeControllers(environment); + spawner.spawnNativeControllers(environment, true); } catch (IOException e) { throw new BootstrapException(e); } diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Spawner.java b/server/src/main/java/org/elasticsearch/bootstrap/Spawner.java index f1c6c36dc5cca..6a384a2aefcd4 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Spawner.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Spawner.java @@ -55,10 +55,12 @@ public void close() throws IOException { /** * Spawns the native controllers for each module. * - * @param environment the node environment + * @param environment The node environment + * @param inheritIo Should the stdout and stderr of the spawned process inherit the + * stdout and stderr of the JVM spawning it? * @throws IOException if an I/O error occurs reading the module or spawning a native process */ - void spawnNativeControllers(final Environment environment) throws IOException { + void spawnNativeControllers(final Environment environment, final boolean inheritIo) throws IOException { if (!spawned.compareAndSet(false, true)) { throw new IllegalStateException("native controllers already spawned"); } @@ -83,7 +85,7 @@ void spawnNativeControllers(final Environment environment) throws IOException { modules.getFileName()); throw new IllegalArgumentException(message); } - final Process process = spawnNativeController(spawnPath, environment.tmpFile()); + final Process process = spawnNativeController(spawnPath, environment.tmpFile(), inheritIo); processes.add(process); } } @@ -92,7 +94,7 @@ void spawnNativeControllers(final Environment environment) throws IOException { * Attempt to spawn the controller daemon for a given module. The spawned process will remain connected to this JVM via its stdin, * stdout, and stderr streams, but the references to these streams are not available to code outside this package. */ - private Process spawnNativeController(final Path spawnPath, final Path tmpPath) throws IOException { + private Process spawnNativeController(final Path spawnPath, final Path tmpPath, final boolean inheritIo) throws IOException { final String command; if (Constants.WINDOWS) { /* @@ -114,6 +116,14 @@ private Process spawnNativeController(final Path spawnPath, final Path tmpPath) pb.environment().clear(); pb.environment().put("TMPDIR", tmpPath.toString()); + // The process _shouldn't_ write any output via its stdout or stderr, but if it does then + // it will block if nothing is reading that output. To avoid this we can inherit the + // JVM's stdout and stderr (which are redirected to files in standard installations). + if (inheritIo) { + pb.redirectOutput(ProcessBuilder.Redirect.INHERIT); + pb.redirectError(ProcessBuilder.Redirect.INHERIT); + } + // the output stream of the process object corresponds to the daemon's stdin return pb.start(); }