Skip to content

Commit

Permalink
JAVA EntryPoint
Browse files Browse the repository at this point in the history
* Created `org.logstash.Logstash` as entrypoint
* Safely handle `Ruby` runtime (which sadly is still a singleton, moving away from that will require a few iterations on top of this)
* Adjusted `bat` and `sh` entry point wrappers
* Verified manually that performance is unchanged (i.e. all Java opts are still loaded properly)
* Flattened `.jar` path to make it a little less bothersome to build the `-cp` string
* Retained ability to load jars from Ruby via the global `$LS_JARS_LOADED` variable hack, to keep plugin specs that load LS as a `.gem` functional (like e.g. the ITs in LS itself)
* No need for the gem jars magic anymore, the downloading and moving into place of jars is now all handled by Gradle
  • Loading branch information
original-brownbear committed Jan 3, 2018
1 parent 19b7f77 commit 166c88c
Show file tree
Hide file tree
Showing 12 changed files with 191 additions and 104 deletions.
13 changes: 11 additions & 2 deletions bin/logstash
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/bin/sh
#!/bin/bash
# Run logstash from source
#
# This is most useful when done from a git checkout.
Expand Down Expand Up @@ -58,5 +58,14 @@ if [ "$1" = "-V" ] || [ "$1" = "--version" ]; then
fi
echo "logstash $LOGSTASH_VERSION"
else
ruby_exec "${LOGSTASH_HOME}/lib/bootstrap/environment.rb" "logstash/runner.rb" "$@"
function classpath() {
echo -n "$1"
shift
while [ $# -gt 0 ] ; do
echo -n ":${1}"
shift
done
}
CLASSPATH="$(classpath ${LOGSTASH_HOME}/logstash-core/lib/jars/*.jar)"
exec "${JAVACMD}" ${JAVA_OPTS} -cp "${CLASSPATH}" org.logstash.Logstash "$@"
fi
15 changes: 13 additions & 2 deletions bin/logstash.bat
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,18 @@ if exist %LS_JVM_OPTIONS_CONFIG% (
)
set JAVA_OPTS=%LS_JAVA_OPTS%

rem jruby launcher will pickup JAVA_OPTS set above to set the JVM options before launching jruby
%JRUBY_BIN% "%LS_HOME%\lib\bootstrap\environment.rb" "logstash\runner.rb" %*
for %%i in ("%LS_HOME%\logstash-core\lib\jars\*.jar") do (
call :concat "%%i"
)

%JAVA% %JAVA_OPTS% -cp %CLASSPATH% org.logstash.Logstash %*

endlocal

goto :eof
:concat
IF not defined CLASSPATH (
set CLASSPATH="%~1"
) ELSE (
set CLASSPATH=%CLASSPATH%;"%~1"
)
1 change: 1 addition & 0 deletions bin/logstash.lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ fi

LOGSTASH_HOME="$(cd `dirname $SOURCEPATH`/..; pwd)"
export LOGSTASH_HOME
export LS_HOME="${LOGSTASH_HOME}"
SINCEDB_DIR="${LOGSTASH_HOME}"
export SINCEDB_DIR

Expand Down
29 changes: 8 additions & 21 deletions logstash-core/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,15 @@ task javadocJar(type: Jar, dependsOn: javadoc) {
extension 'jar'
}

task copyRuntimeLibs(type: Copy) {
into project.file('lib/jars/')
from configurations.compile, configurations.runtime
}

// copy jar file into the gem lib dir but without the version number in filename
task copyGemjar(type: Copy, dependsOn: sourcesJar) {
task copyGemjar(type: Copy, dependsOn: [sourcesJar, copyRuntimeLibs]) {
from project.jar
into project.file('lib/logstash-core/')
into project.file('lib/jars/')
rename(/(.+)-${project.version}.jar/, '$1.jar')
}

Expand All @@ -48,22 +53,6 @@ task cleanGemjar {

clean.dependsOn(cleanGemjar)
jar.finalizedBy(copyGemjar)
task gemspec_jars {
def gemspecJars = file("${projectDir}/gemspec_jars.rb")
outputs.files gemspecJars
doLast {
gemspecJars.newWriter().withWriter { w ->
w << "# This file is generated by Gradle as part of the build process. It extracts the build.gradle\n"
w << "# runtime dependencies to generate this gemspec dependencies file to be eval'ed by the gemspec\n"
w << "# for the jar-dependencies requirements.\n\n"
configurations.runtime.allDependencies.each { dependency ->
w << "gem.requirements << \"jar ${dependency.group}:${dependency.name}, ${dependency.version}\"\n"
}
}
}
}

compileJava.dependsOn gemspec_jars

configurations.create('sources')
configurations.create('javadoc')
Expand Down Expand Up @@ -128,13 +117,11 @@ dependencies {
compile "com.fasterxml.jackson.core:jackson-annotations:${jacksonVersion}"
compile 'org.codehaus.janino:janino:3.0.7'
compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${jacksonVersion}"
compile "org.jruby:jruby-complete:${jrubyVersion}"
testCompile 'org.apache.logging.log4j:log4j-core:2.6.2:tests'
testCompile 'org.apache.logging.log4j:log4j-api:2.6.2:tests'
testCompile 'junit:junit:4.12'
testCompile 'net.javacrumbs.json-unit:json-unit:1.9.0'
testCompile 'org.elasticsearch:securemock:1.2'
testCompile 'org.assertj:assertj-core:3.8.0'
testCompile "org.jruby:jruby-complete:${jrubyVersion}"
provided "org.jruby:jruby-core:${jrubyVersion}"
}

12 changes: 0 additions & 12 deletions logstash-core/gemspec_jars.rb

This file was deleted.

41 changes: 10 additions & 31 deletions logstash-core/lib/logstash-core/logstash-core.rb
Original file line number Diff line number Diff line change
Expand Up @@ -2,36 +2,15 @@

require "java"

module LogStash
end

require "logstash-core_jars"

# local dev setup
alt_classdir = File.expand_path("../../../out/production/classes", __FILE__) #IntelliJ's gradle output as of 2017.02 https://youtrack.jetbrains.com/issue/IDEA-175172
if File.directory?(alt_classdir)
classes_dir = alt_classdir
resources_dir = File.expand_path("../../../out/production/resources", __FILE__)
else
classes_dir = File.expand_path("../../../build/classes/java/main", __FILE__)
resources_dir = File.expand_path("../../../build/resources/main", __FILE__)
end



if File.directory?(classes_dir) && File.directory?(resources_dir)
# if in local dev setup, add target to classpath
$CLASSPATH << classes_dir unless $CLASSPATH.include?(classes_dir)
$CLASSPATH << resources_dir unless $CLASSPATH.include?(resources_dir)
else
# otherwise use included jar
begin
require "logstash-core/logstash-core.jar"
rescue Exception => e
raise("Error loading logstash-core/logstash-core.jar file, cause: #{e.message}")
# This block is used to load Logstash's Java libraries when using a Ruby entrypoint and
# LS_JARS_LOADED is not globally set.
# Currently this happens when using the `bin/rspec` executable to invoke specs instead of the JUnit
# wrapper.
unless $LS_JARS_LOADED
jar_path = File.join(File.dirname(File.dirname(__FILE__)), "jars")
$:.unshift jar_path
Dir.glob(jar_path + '/*.jar') do |jar|
require File.basename(jar)
end
java_import org.logstash.RubyUtil
end

# Load Logstash's Java-defined RubyClasses by classloading RubyUtil which sets them up in its
# static constructor
java_import org.logstash.RubyUtil
28 changes: 0 additions & 28 deletions logstash-core/lib/logstash-core_jars.rb

This file was deleted.

1 change: 1 addition & 0 deletions logstash-core/lib/logstash/environment.rb
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
# encoding: utf-8
require "logstash-core/logstash-core"
require "logstash/errors"
require "logstash/java_integration"
require "logstash/config/cpu_core_strategy"
Expand Down
7 changes: 0 additions & 7 deletions logstash-core/logstash-core.gemspec
Original file line number Diff line number Diff line change
Expand Up @@ -70,13 +70,6 @@ Gem::Specification.new do |gem|

gem.add_runtime_dependency "jrjackson", "~> #{ALL_VERSIONS.fetch('jrjackson')}" #(Apache 2.0 license)

gem.add_runtime_dependency "jar-dependencies"
# as of Feb 3rd 2016, the ruby-maven gem is resolved to version 3.3.3 and that version
# has an rdoc problem that causes a bundler exception. 3.3.9 is the current latest version
# which does not have this problem.
gem.add_runtime_dependency "ruby-maven", "~> 3.3.9"
gem.add_runtime_dependency "elasticsearch", "~> 5.0", ">= 5.0.4" # Ruby client for ES (Apache 2.0 license)
gem.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0'

eval(File.read(File.expand_path("../gemspec_jars.rb", __FILE__)))
end
146 changes: 146 additions & 0 deletions logstash-core/src/main/java/org/logstash/Logstash.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
package org.logstash;

import java.io.InputStream;
import java.io.PrintStream;
import java.nio.file.Path;
import java.nio.file.Paths;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jruby.Ruby;
import org.jruby.RubyException;
import org.jruby.RubyInstanceConfig;
import org.jruby.RubyNumeric;
import org.jruby.exceptions.RaiseException;
import org.jruby.runtime.builtin.IRubyObject;

/**
* Logstash Main Entrypoint.
*/
public final class Logstash implements Runnable, AutoCloseable {

private static final Logger LOGGER = LogManager.getLogger(Logstash.class);

/**
* Configuration for {@link #ruby}.
*/
private final RubyInstanceConfig config;

/**
* JRuby Runtime Environment.
*/
private final Ruby ruby;

/**
* Ruby Entrypoint Script.
*/
private final InputStream script;

/**
* Main Entrypoint.
* Requires environment {@code "LS_HOME"} to be set to the Logstash root directory.
* @param args Logstash CLI Arguments
*/
public static void main(final String... args) {
final String lsHome = System.getenv("LS_HOME");
if (lsHome == null) {
throw new IllegalStateException(
"LS_HOME environment variable must be set. This is likely a bug that should be reported."
);
}
final Path home = Paths.get(lsHome).toAbsolutePath();
try (
final Logstash logstash = new Logstash(home, args, System.out, System.err, System.in)
) {
logstash.run();
} catch (final Throwable t) {
LOGGER.error(t);
System.exit(1);
}
System.exit(0);
}

/**
* Ctor.
* @param home Logstash Root Directory
* @param args Commandline Arguments
* @param output Output Stream Capturing StdOut
* @param error Output Stream Capturing StdErr
* @param input Input Stream Capturing StdIn
*/
Logstash(final Path home, final String[] args, final PrintStream output,
final PrintStream error, final InputStream input) {
config = buildConfig(home, args);
config.setOutput(output);
config.setError(error);
config.setInput(input);
script = config.getScriptSource();
ruby = Ruby.newInstance(config);
}

@Override
public void run() {
// @todo: Refactor codebase to not rely on global constant for Ruby Runtime
if (RubyUtil.RUBY != ruby) {
throw new IllegalStateException(
"More than one JRuby Runtime detected in the current JVM!"
);
}
try {
Thread.currentThread().setContextClassLoader(ruby.getJRubyClassLoader());
ruby.runFromMain(script, config.displayedFileName());
} catch (final RaiseException ex) {
final RubyException rexep = ex.getException();
if (ruby.getSystemExit().isInstance(rexep)) {
final IRubyObject status =
rexep.callMethod(ruby.getCurrentContext(), "status");
if (status != null && !status.isNil() && RubyNumeric.fix2int(status) != 0) {
throw new IllegalStateException(ex);
}
}
}
}

@Override
public void close() throws Exception {
ruby.tearDown(false);
script.close();
}

/**
* Sets up the correct {@link RubyInstanceConfig} for a given Logstash installation and set of
* CLI arguments.
* @param home Logstash Root Path
* @param args Commandline Arguments Passed to Logstash
* @return RubyInstanceConfig
*/
private static RubyInstanceConfig buildConfig(final Path home, final String[] args) {
final String[] arguments = new String[args.length + 2];
System.arraycopy(args, 0, arguments, 2, args.length);
arguments[0] = safePath(home, "lib", "bootstrap", "environment.rb");
arguments[1] = safePath(home, "logstash-core", "lib", "logstash", "runner.rb");
final RubyInstanceConfig config = new RubyInstanceConfig();
config.processArguments(arguments);
return config;
}

/**
* Builds the correct path for a file under the given Logstash root and defined by its sub path
* elements relative to the Logstash root.
* Ensures that the file exists and throws an exception of it's missing.
* This is done to avoid hard to interpret errors thrown by JRuby that could result from missing
* Ruby bootstrap scripts.
* @param home Logstash Root Path
* @param subs Path elements relative to {@code home}
* @return Absolute Path a File under the Logstash Root.
*/
private static String safePath(final Path home, final String... subs) {
Path resolved = home;
for (final String element : subs) {
resolved = resolved.resolve(element);
}
if (!resolved.toFile().exists()) {
throw new IllegalArgumentException(String.format("Missing: %s.", resolved));
}
return resolved.toString();
}
}
1 change: 1 addition & 0 deletions logstash-core/src/main/java/org/logstash/RubyUtil.java
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ public final class RubyUtil {
abstractQueue, AbstractJRubyQueue.RubyAckedMemoryQueue::new,
AbstractJRubyQueue.RubyAckedMemoryQueue.class
);
RUBY.getGlobalVariables().set("$LS_JARS_LOADED", RUBY.newString("true"));
}

private RubyUtil() {
Expand Down
1 change: 0 additions & 1 deletion rakelib/artifacts.rake
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ namespace "artifact" do
"logstash-core/vendor/**/*",
"logstash-core/versions-gem-copy.yml",
"logstash-core/*.gemspec",
"logstash-core/gemspec_jars.rb",

"logstash-core-plugin-api/lib/**/*",
"logstash-core-plugin-api/*.gemspec",
Expand Down

0 comments on commit 166c88c

Please sign in to comment.