diff --git a/bin/logstash b/bin/logstash index 2091fd7a001..d9596d8b57e 100755 --- a/bin/logstash +++ b/bin/logstash @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Run logstash from source # # This is most useful when done from a git checkout. @@ -58,5 +58,14 @@ if [ "$1" = "-V" ] || [ "$1" = "--version" ]; then fi echo "logstash $LOGSTASH_VERSION" else - ruby_exec "${LOGSTASH_HOME}/lib/bootstrap/environment.rb" "logstash/runner.rb" "$@" + function classpath() { + echo -n "$1" + shift + while [ $# -gt 0 ] ; do + echo -n ":${1}" + shift + done + } + CLASSPATH="$(classpath ${LOGSTASH_HOME}/logstash-core/lib/jars/*.jar)" + exec "${JAVACMD}" ${JAVA_OPTS} -cp "${CLASSPATH}" org.logstash.Logstash "$@" fi diff --git a/bin/logstash.bat b/bin/logstash.bat index 055d085cdf3..a61bc3db2a8 100644 --- a/bin/logstash.bat +++ b/bin/logstash.bat @@ -45,7 +45,18 @@ if exist %LS_JVM_OPTIONS_CONFIG% ( ) set JAVA_OPTS=%LS_JAVA_OPTS% -rem jruby launcher will pickup JAVA_OPTS set above to set the JVM options before launching jruby -%JRUBY_BIN% "%LS_HOME%\lib\bootstrap\environment.rb" "logstash\runner.rb" %* +for %%i in ("%LS_HOME%\logstash-core\lib\jars\*.jar") do ( + call :concat "%%i" +) + +%JAVA% %JAVA_OPTS% -cp %CLASSPATH% org.logstash.Logstash %* endlocal + +goto :eof +:concat +IF not defined CLASSPATH ( + set CLASSPATH="%~1" +) ELSE ( + set CLASSPATH=%CLASSPATH%;"%~1" +) diff --git a/bin/logstash.lib.sh b/bin/logstash.lib.sh index c352fb9c47e..d8b88d394e4 100755 --- a/bin/logstash.lib.sh +++ b/bin/logstash.lib.sh @@ -25,6 +25,7 @@ fi LOGSTASH_HOME="$(cd `dirname $SOURCEPATH`/..; pwd)" export LOGSTASH_HOME +export LS_HOME="${LOGSTASH_HOME}" SINCEDB_DIR="${LOGSTASH_HOME}" export SINCEDB_DIR diff --git a/logstash-core/build.gradle b/logstash-core/build.gradle index abad3e33295..446747f9039 100644 --- a/logstash-core/build.gradle +++ b/logstash-core/build.gradle @@ -33,10 +33,15 @@ task javadocJar(type: Jar, dependsOn: javadoc) { extension 'jar' } +task copyRuntimeLibs(type: Copy) { + into project.file('lib/jars/') + from configurations.compile, configurations.runtime +} + // copy jar file into the gem lib dir but without the version number in filename -task copyGemjar(type: Copy, dependsOn: sourcesJar) { +task copyGemjar(type: Copy, dependsOn: [sourcesJar, copyRuntimeLibs]) { from project.jar - into project.file('lib/logstash-core/') + into project.file('lib/jars/') rename(/(.+)-${project.version}.jar/, '$1.jar') } @@ -48,22 +53,6 @@ task cleanGemjar { clean.dependsOn(cleanGemjar) jar.finalizedBy(copyGemjar) -task gemspec_jars { - def gemspecJars = file("${projectDir}/gemspec_jars.rb") - outputs.files gemspecJars - doLast { - gemspecJars.newWriter().withWriter { w -> - w << "# This file is generated by Gradle as part of the build process. It extracts the build.gradle\n" - w << "# runtime dependencies to generate this gemspec dependencies file to be eval'ed by the gemspec\n" - w << "# for the jar-dependencies requirements.\n\n" - configurations.runtime.allDependencies.each { dependency -> - w << "gem.requirements << \"jar ${dependency.group}:${dependency.name}, ${dependency.version}\"\n" - } - } - } -} - -compileJava.dependsOn gemspec_jars configurations.create('sources') configurations.create('javadoc') @@ -128,13 +117,11 @@ dependencies { compile "com.fasterxml.jackson.core:jackson-annotations:${jacksonVersion}" compile 'org.codehaus.janino:janino:3.0.7' compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${jacksonVersion}" + compile "org.jruby:jruby-complete:${jrubyVersion}" testCompile 'org.apache.logging.log4j:log4j-core:2.6.2:tests' testCompile 'org.apache.logging.log4j:log4j-api:2.6.2:tests' testCompile 'junit:junit:4.12' testCompile 'net.javacrumbs.json-unit:json-unit:1.9.0' testCompile 'org.elasticsearch:securemock:1.2' testCompile 'org.assertj:assertj-core:3.8.0' - testCompile "org.jruby:jruby-complete:${jrubyVersion}" - provided "org.jruby:jruby-core:${jrubyVersion}" } - diff --git a/logstash-core/gemspec_jars.rb b/logstash-core/gemspec_jars.rb deleted file mode 100644 index 857e8f648f2..00000000000 --- a/logstash-core/gemspec_jars.rb +++ /dev/null @@ -1,12 +0,0 @@ -# This file is generated by Gradle as part of the build process. It extracts the build.gradle -# runtime dependencies to generate this gemspec dependencies file to be eval'ed by the gemspec -# for the jar-dependencies requirements. - -gem.requirements << "jar org.apache.logging.log4j:log4j-slf4j-impl, 2.9.1" -gem.requirements << "jar org.apache.logging.log4j:log4j-api, 2.9.1" -gem.requirements << "jar org.apache.logging.log4j:log4j-core, 2.9.1" -gem.requirements << "jar com.fasterxml.jackson.core:jackson-core, 2.9.1" -gem.requirements << "jar com.fasterxml.jackson.core:jackson-databind, 2.9.1" -gem.requirements << "jar com.fasterxml.jackson.core:jackson-annotations, 2.9.1" -gem.requirements << "jar org.codehaus.janino:janino, 3.0.7" -gem.requirements << "jar com.fasterxml.jackson.dataformat:jackson-dataformat-cbor, 2.9.1" diff --git a/logstash-core/lib/logstash-core/logstash-core.rb b/logstash-core/lib/logstash-core/logstash-core.rb index fc62ca933e9..36b8e906c02 100644 --- a/logstash-core/lib/logstash-core/logstash-core.rb +++ b/logstash-core/lib/logstash-core/logstash-core.rb @@ -2,36 +2,15 @@ require "java" -module LogStash -end - -require "logstash-core_jars" - -# local dev setup -alt_classdir = File.expand_path("../../../out/production/classes", __FILE__) #IntelliJ's gradle output as of 2017.02 https://youtrack.jetbrains.com/issue/IDEA-175172 -if File.directory?(alt_classdir) - classes_dir = alt_classdir - resources_dir = File.expand_path("../../../out/production/resources", __FILE__) -else - classes_dir = File.expand_path("../../../build/classes/java/main", __FILE__) - resources_dir = File.expand_path("../../../build/resources/main", __FILE__) -end - - - -if File.directory?(classes_dir) && File.directory?(resources_dir) - # if in local dev setup, add target to classpath - $CLASSPATH << classes_dir unless $CLASSPATH.include?(classes_dir) - $CLASSPATH << resources_dir unless $CLASSPATH.include?(resources_dir) -else - # otherwise use included jar - begin - require "logstash-core/logstash-core.jar" - rescue Exception => e - raise("Error loading logstash-core/logstash-core.jar file, cause: #{e.message}") +# This block is used to load Logstash's Java libraries when using a Ruby entrypoint and +# LS_JARS_LOADED is not globally set. +# Currently this happens when using the `bin/rspec` executable to invoke specs instead of the JUnit +# wrapper. +unless $LS_JARS_LOADED + jar_path = File.join(File.dirname(File.dirname(__FILE__)), "jars") + $:.unshift jar_path + Dir.glob(jar_path + '/*.jar') do |jar| + require File.basename(jar) end + java_import org.logstash.RubyUtil end - -# Load Logstash's Java-defined RubyClasses by classloading RubyUtil which sets them up in its -# static constructor -java_import org.logstash.RubyUtil diff --git a/logstash-core/lib/logstash-core_jars.rb b/logstash-core/lib/logstash-core_jars.rb deleted file mode 100644 index 8802ba2b874..00000000000 --- a/logstash-core/lib/logstash-core_jars.rb +++ /dev/null @@ -1,28 +0,0 @@ -# this is a generated file, to avoid over-writing it just delete this comment -begin - require 'jar_dependencies' -rescue LoadError - require 'org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25.jar' - require 'com/fasterxml/jackson/core/jackson-databind/2.9.1/jackson-databind-2.9.1.jar' - require 'com/fasterxml/jackson/core/jackson-annotations/2.9.1/jackson-annotations-2.9.1.jar' - require 'org/apache/logging/log4j/log4j-api/2.9.1/log4j-api-2.9.1.jar' - require 'org/apache/logging/log4j/log4j-core/2.9.1/log4j-core-2.9.1.jar' - require 'com/fasterxml/jackson/dataformat/jackson-dataformat-cbor/2.9.1/jackson-dataformat-cbor-2.9.1.jar' - require 'org/codehaus/janino/commons-compiler/3.0.7/commons-compiler-3.0.7.jar' - require 'org/apache/logging/log4j/log4j-slf4j-impl/2.9.1/log4j-slf4j-impl-2.9.1.jar' - require 'com/fasterxml/jackson/core/jackson-core/2.9.1/jackson-core-2.9.1.jar' - require 'org/codehaus/janino/janino/3.0.7/janino-3.0.7.jar' -end - -if defined? Jars - require_jar( 'org.slf4j', 'slf4j-api', '1.7.25' ) - require_jar( 'com.fasterxml.jackson.core', 'jackson-databind', '2.9.1' ) - require_jar( 'com.fasterxml.jackson.core', 'jackson-annotations', '2.9.1' ) - require_jar( 'org.apache.logging.log4j', 'log4j-api', '2.9.1' ) - require_jar( 'org.apache.logging.log4j', 'log4j-core', '2.9.1' ) - require_jar( 'com.fasterxml.jackson.dataformat', 'jackson-dataformat-cbor', '2.9.1' ) - require_jar( 'org.codehaus.janino', 'commons-compiler', '3.0.7' ) - require_jar( 'org.apache.logging.log4j', 'log4j-slf4j-impl', '2.9.1' ) - require_jar( 'com.fasterxml.jackson.core', 'jackson-core', '2.9.1' ) - require_jar( 'org.codehaus.janino', 'janino', '3.0.7' ) -end diff --git a/logstash-core/lib/logstash/environment.rb b/logstash-core/lib/logstash/environment.rb index fe9f4f7cdf6..1b40ff0916e 100644 --- a/logstash-core/lib/logstash/environment.rb +++ b/logstash-core/lib/logstash/environment.rb @@ -1,4 +1,5 @@ # encoding: utf-8 +require "logstash-core/logstash-core" require "logstash/errors" require "logstash/java_integration" require "logstash/config/cpu_core_strategy" diff --git a/logstash-core/logstash-core.gemspec b/logstash-core/logstash-core.gemspec index 48eef59a8fa..aef15267f0b 100644 --- a/logstash-core/logstash-core.gemspec +++ b/logstash-core/logstash-core.gemspec @@ -70,13 +70,6 @@ Gem::Specification.new do |gem| gem.add_runtime_dependency "jrjackson", "~> #{ALL_VERSIONS.fetch('jrjackson')}" #(Apache 2.0 license) - gem.add_runtime_dependency "jar-dependencies" - # as of Feb 3rd 2016, the ruby-maven gem is resolved to version 3.3.3 and that version - # has an rdoc problem that causes a bundler exception. 3.3.9 is the current latest version - # which does not have this problem. - gem.add_runtime_dependency "ruby-maven", "~> 3.3.9" gem.add_runtime_dependency "elasticsearch", "~> 5.0", ">= 5.0.4" # Ruby client for ES (Apache 2.0 license) gem.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0' - - eval(File.read(File.expand_path("../gemspec_jars.rb", __FILE__))) end diff --git a/logstash-core/src/main/java/org/logstash/Logstash.java b/logstash-core/src/main/java/org/logstash/Logstash.java new file mode 100644 index 00000000000..459ca33b7e0 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/Logstash.java @@ -0,0 +1,146 @@ +package org.logstash; + +import java.io.InputStream; +import java.io.PrintStream; +import java.nio.file.Path; +import java.nio.file.Paths; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jruby.Ruby; +import org.jruby.RubyException; +import org.jruby.RubyInstanceConfig; +import org.jruby.RubyNumeric; +import org.jruby.exceptions.RaiseException; +import org.jruby.runtime.builtin.IRubyObject; + +/** + * Logstash Main Entrypoint. + */ +public final class Logstash implements Runnable, AutoCloseable { + + private static final Logger LOGGER = LogManager.getLogger(Logstash.class); + + /** + * Configuration for {@link #ruby}. + */ + private final RubyInstanceConfig config; + + /** + * JRuby Runtime Environment. + */ + private final Ruby ruby; + + /** + * Ruby Entrypoint Script. + */ + private final InputStream script; + + /** + * Main Entrypoint. + * Requires environment {@code "LS_HOME"} to be set to the Logstash root directory. + * @param args Logstash CLI Arguments + */ + public static void main(final String... args) { + final String lsHome = System.getenv("LS_HOME"); + if (lsHome == null) { + throw new IllegalStateException( + "LS_HOME environment variable must be set. This is likely a bug that should be reported." + ); + } + final Path home = Paths.get(lsHome).toAbsolutePath(); + try ( + final Logstash logstash = new Logstash(home, args, System.out, System.err, System.in) + ) { + logstash.run(); + } catch (final Throwable t) { + LOGGER.error(t); + System.exit(1); + } + System.exit(0); + } + + /** + * Ctor. + * @param home Logstash Root Directory + * @param args Commandline Arguments + * @param output Output Stream Capturing StdOut + * @param error Output Stream Capturing StdErr + * @param input Input Stream Capturing StdIn + */ + Logstash(final Path home, final String[] args, final PrintStream output, + final PrintStream error, final InputStream input) { + config = buildConfig(home, args); + config.setOutput(output); + config.setError(error); + config.setInput(input); + script = config.getScriptSource(); + ruby = Ruby.newInstance(config); + } + + @Override + public void run() { + // @todo: Refactor codebase to not rely on global constant for Ruby Runtime + if (RubyUtil.RUBY != ruby) { + throw new IllegalStateException( + "More than one JRuby Runtime detected in the current JVM!" + ); + } + try { + Thread.currentThread().setContextClassLoader(ruby.getJRubyClassLoader()); + ruby.runFromMain(script, config.displayedFileName()); + } catch (final RaiseException ex) { + final RubyException rexep = ex.getException(); + if (ruby.getSystemExit().isInstance(rexep)) { + final IRubyObject status = + rexep.callMethod(ruby.getCurrentContext(), "status"); + if (status != null && !status.isNil() && RubyNumeric.fix2int(status) != 0) { + throw new IllegalStateException(ex); + } + } + } + } + + @Override + public void close() throws Exception { + ruby.tearDown(false); + script.close(); + } + + /** + * Sets up the correct {@link RubyInstanceConfig} for a given Logstash installation and set of + * CLI arguments. + * @param home Logstash Root Path + * @param args Commandline Arguments Passed to Logstash + * @return RubyInstanceConfig + */ + private static RubyInstanceConfig buildConfig(final Path home, final String[] args) { + final String[] arguments = new String[args.length + 2]; + System.arraycopy(args, 0, arguments, 2, args.length); + arguments[0] = safePath(home, "lib", "bootstrap", "environment.rb"); + arguments[1] = safePath(home, "logstash-core", "lib", "logstash", "runner.rb"); + final RubyInstanceConfig config = new RubyInstanceConfig(); + config.processArguments(arguments); + return config; + } + + /** + * Builds the correct path for a file under the given Logstash root and defined by its sub path + * elements relative to the Logstash root. + * Ensures that the file exists and throws an exception of it's missing. + * This is done to avoid hard to interpret errors thrown by JRuby that could result from missing + * Ruby bootstrap scripts. + * @param home Logstash Root Path + * @param subs Path elements relative to {@code home} + * @return Absolute Path a File under the Logstash Root. + */ + private static String safePath(final Path home, final String... subs) { + Path resolved = home; + for (final String element : subs) { + resolved = resolved.resolve(element); + } + if (!resolved.toFile().exists()) { + throw new IllegalArgumentException(String.format("Missing: %s.", resolved)); + } + return resolved.toString(); + } +} diff --git a/logstash-core/src/main/java/org/logstash/RubyUtil.java b/logstash-core/src/main/java/org/logstash/RubyUtil.java index eec17665604..f22edf9c608 100644 --- a/logstash-core/src/main/java/org/logstash/RubyUtil.java +++ b/logstash-core/src/main/java/org/logstash/RubyUtil.java @@ -92,6 +92,7 @@ public final class RubyUtil { abstractQueue, AbstractJRubyQueue.RubyAckedMemoryQueue::new, AbstractJRubyQueue.RubyAckedMemoryQueue.class ); + RUBY.getGlobalVariables().set("$LS_JARS_LOADED", RUBY.newString("true")); } private RubyUtil() { diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index 37289c66fca..116b743728a 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -22,7 +22,6 @@ namespace "artifact" do "logstash-core/vendor/**/*", "logstash-core/versions-gem-copy.yml", "logstash-core/*.gemspec", - "logstash-core/gemspec_jars.rb", "logstash-core-plugin-api/lib/**/*", "logstash-core-plugin-api/*.gemspec",