#8442 Java Pipeline Compiler

Dockerfile

@@ -9,7 +9,7 @@ RUN /opt/logstash/gradlew wrapper
 ADD versions.yml /opt/logstash/versions.yml
 ADD LICENSE /opt/logstash/LICENSE
 ADD CONTRIBUTORS /opt/logstash/CONTRIBUTORS
-ADD Gemfile.template /opt/logstash/Gemfile
+ADD Gemfile.template /opt/logstash/Gemfile.template
 ADD Rakefile /opt/logstash/Rakefile
 ADD build.gradle /opt/logstash/build.gradle
 ADD rakelib /opt/logstash/rakelib

README.md

@@ -82,7 +82,7 @@ jruby 9.1.10.0 (2.3.3) 2017-05-25 b09c48a Java HotSpot(TM) 64-Bit Server VM 25.1
 * To run Logstash from the repo you must first bootstrap the environment:
 
 ```sh
-rake bootstrap
+./gradlew bootstrap
 ```
 
 * You can then use `bin/logstash` to start Logstash, but there are no plugins installed. To install default plugins, you can run:
@@ -126,22 +126,18 @@ Most of the unit tests in Logstash are written using [rspec](http://rspec.info/)
 
 ### Core tests
 
-1- In order to run the core tests, a small set of plugins must first be installed:
+1- To run the core tests you can use the rake task:
 
-    rake test:install-core
-
-2- To run the core tests you can use the rake task:
-
-    rake test:core
+    ./gradlew test
 
   or use the `rspec` tool to run all tests or run a specific test:
 
     bin/rspec
     bin/rspec spec/foo/bar_spec.rb
 
-3- To run the subset of tests covering the Java codebase only run:
+2- To run the subset of tests covering the Java codebase only run:
 
-    ./gradlew test
+    ./gradlew javaTests
 
 ### Plugins tests
 
@@ -163,8 +159,8 @@ Note that if a plugin is installed using the plugin manager `bin/logstash-plugin
 You can build a Logstash snapshot package as tarball or zip file
 
 ```sh
-rake artifact:tar
-rake artifact:zip
+./gradlew bootstrap && rake artifact:tar
+./gradlew bootstrap && rake artifact:zip
 ```
 
 This will create the artifact `LS_HOME/build` directory

build.gradle

@@ -17,6 +17,8 @@ import de.undercouch.gradle.tasks.download.Download
 import de.undercouch.gradle.tasks.download.Verify
 import org.yaml.snakeyaml.Yaml
 
+task bootstrap {}
+
 allprojects {
   group = 'org.logstash'
 
@@ -42,6 +44,7 @@ allprojects {
 
   //https://stackoverflow.com/questions/3963708/gradle-how-to-display-test-results-in-the-console-in-real-time
   tasks.withType(Test) {
+    dependsOn bootstrap
     testLogging {
       // set options for log level LIFECYCLE
       events "passed", "skipped", "failed", "standardOut"
@@ -89,9 +92,6 @@ if (versionMap["jruby-runtime-override"]) {
     doChecksum = true
 }
 
-// Tasks
-task bootstrap {}
-
 task downloadJRuby(type: Download) {
     description "Download JRuby artifact from this specific URL: ${jRubyURL}"
     src jRubyURL
@@ -119,4 +119,32 @@ task downloadAndInstallJRuby(dependsOn: verifyFile, type: Copy) {
 
 // If you are running a JRuby snapshot we will skip the integrity check.
 verifyFile.onlyIf { doChecksum }
-bootstrap.dependsOn downloadAndInstallJRuby
+
+task installGems(dependsOn: [downloadAndInstallJRuby], type: Exec) {
+  workingDir projectDir
+  inputs.files file("$projectDir/Gemfile.template")
+  inputs.files fileTree("$projectDir/rakelib")
+  inputs.files fileTree("$projectDir/versions.yml")
+  outputs.files file("$projectDir/Gemfile")
+  outputs.files file("$projectDir/Gemfile.lock")
+  outputs.files fileTree("$projectDir/vendor/bundle")
+  outputs.files fileTree("$projectDir/vendor/jruby")
+  commandLine './vendor/jruby/bin/jruby', "${projectDir}/vendor/jruby/bin/rake".toString(), "vendor:gems", "test:install-core"
+  standardOutput = new ByteArrayOutputStream()
+  ext.output = {
+    return standardOutput.toString()
+  }
+}
+
+project(":logstash-core") {
+  installGems.dependsOn assemble
+}
+
+clean {
+  delete "${projectDir}/vendor/"
+  delete "${projectDir}/Gemfile"
+  delete "${projectDir}/Gemfile.lock"
+  delete "${projectDir}/NOTICE.TXT"
+}
+
+bootstrap.dependsOn installGems

ci/unit_tests.sh

@@ -13,24 +13,23 @@ SELECTED_TEST_SUITE=$1
 
 if [[ $SELECTED_TEST_SUITE == $"core-fail-fast" ]]; then
   echo "Running Java and Ruby unit tests, but will fail fast"
-  echo "Running test:install-core"
-  rake test:install-core
+  echo "Running Gradle clean bootstrap"
+  ./gradlew clean bootstrap
   echo "Running test:core-fail-fast"
   rake test:core-fail-fast
 elif [[ $SELECTED_TEST_SUITE == $"java" ]]; then
   echo "Running Java unit tests"
-  echo "Running test:core-java"
-  rake test:core-java
+  ./gradlew clean javaTests
 elif [[ $SELECTED_TEST_SUITE == $"ruby" ]]; then
   echo "Running Ruby unit tests"
-  echo "Running test:install-core"
-  rake test:install-core
+  echo "Running Gradle clean bootstrap"
+  ./gradlew clean bootstrap
   echo "Running test:core-ruby"
   rake test:core-ruby
 else
   echo "Running Java and Ruby unit tests"
-  echo "Running test:install-core"
-  rake test:install-core
+  echo "Running Gradle clean bootstrap"
+  ./gradlew clean bootstrap
   echo "Running test:core"
   rake test:core
 fi

logstash-core/build.gradle

@@ -74,6 +74,10 @@ task javaTests(type: Test) {
     exclude '/org/logstash/RSpecTests.class'
 }
 
+tasks.withType(Test) {
+    systemProperty 'logstash.core.root.dir', projectDir.absolutePath
+}
+
 artifacts {
     sources(sourcesJar) {
         // Weird Gradle quirk where type will be used for the extension, but only for sources

logstash-core/lib/logstash/compiler.rb

@@ -7,9 +7,9 @@
 module LogStash; class Compiler
   include ::LogStash::Util::Loggable
 
-  def self.compile_sources(sources_with_metadata, settings)
+  def self.compile_sources(sources_with_metadata, support_escapes)
     graph_sections = sources_with_metadata.map do |swm|
-      self.compile_graph(swm, settings)
+      self.compile_graph(swm, support_escapes)
     end
 
     input_graph = Graph.combine(*graph_sections.map {|s| s[:input] }).graph
@@ -30,7 +30,7 @@ def self.compile_sources(sources_with_metadata, settings)
     PipelineIR.new(input_graph, filter_graph, output_graph, original_source)
   end
 
-  def self.compile_ast(source_with_metadata, settings)
+  def self.compile_imperative(source_with_metadata, support_escapes)
     if !source_with_metadata.is_a?(org.logstash.common.SourceWithMetadata)
       raise ArgumentError, "Expected 'org.logstash.common.SourceWithMetadata', got #{source_with_metadata.class}"
     end
@@ -42,15 +42,11 @@ def self.compile_ast(source_with_metadata, settings)
       raise ConfigurationError, grammar.failure_reason
     end
 
-    config.process_escape_sequences = settings.get_value("config.support_escapes")
+    config.process_escape_sequences = support_escapes
     config.compile(source_with_metadata)
   end
 
-  def self.compile_imperative(source_with_metadata, settings)
-    compile_ast(source_with_metadata, settings)
-  end
-
-  def self.compile_graph(source_with_metadata, settings)
-    Hash[compile_imperative(source_with_metadata, settings).map {|section,icompiled| [section, icompiled.toGraph]}]
+  def self.compile_graph(source_with_metadata, support_escapes)
+    Hash[compile_imperative(source_with_metadata, support_escapes).map {|section,icompiled| [section, icompiled.toGraph]}]
   end
 end; end

logstash-core/lib/logstash/java_pipeline.rb

@@ -29,6 +29,7 @@
 java_import org.logstash.common.SourceWithMetadata
 java_import org.logstash.common.io.DeadLetterQueueWriter
 java_import org.logstash.config.ir.CompiledPipeline
+java_import org.logstash.config.ir.ConfigCompiler
 
 module LogStash; class JavaBasePipeline
   include org.logstash.config.ir.compiler.RubyIntegration::Pipeline 
@@ -47,7 +48,9 @@ def initialize(pipeline_config, namespaced_metric = nil, agent = nil)
     @settings = pipeline_config.settings
     @config_hash = Digest::SHA1.hexdigest(@config_str)
 
-    @lir = compile_lir
+    @lir = ConfigCompiler.configToPipelineIR(
+      @config_str, @settings.get_value("config.support_escapes")
+    )
 
     # Every time #plugin is invoked this is incremented to give each plugin
     # a unique id when auto-generating plugin ids
@@ -85,13 +88,6 @@ def close_dlq_writer
     end
   end
 
-  def compile_lir
-    sources_with_metadata = [
-      SourceWithMetadata.new("str", "pipeline", 0, 0, self.config_str)
-    ]
-    LogStash::Compiler.compile_sources(sources_with_metadata, @settings)
-  end
-
   def buildOutput(name, line, column, *args)
     plugin("output", name, line, column, *args)
   end

logstash-core/lib/logstash/pipeline.rb

@@ -29,6 +29,7 @@
 java_import org.logstash.common.DeadLetterQueueFactory
 java_import org.logstash.common.SourceWithMetadata
 java_import org.logstash.common.io.DeadLetterQueueWriter
+java_import org.logstash.config.ir.ConfigCompiler
 
 module LogStash; class BasePipeline
   include LogStash::Util::Loggable
@@ -46,7 +47,9 @@ def initialize(pipeline_config, namespaced_metric = nil, agent = nil)
     @settings = pipeline_config.settings
     @config_hash = Digest::SHA1.hexdigest(@config_str)
 
-    @lir = compile_lir
+    @lir = ConfigCompiler.configToPipelineIR(
+      @config_str, @settings.get_value("config.support_escapes")
+    )
 
     # Every time #plugin is invoked this is incremented to give each plugin
     # a unique id when auto-generating plugin ids
@@ -101,10 +104,9 @@ def close_dlq_writer
   end
 
   def compile_lir
-    sources_with_metadata = [
-      SourceWithMetadata.new("str", "pipeline", 0, 0, self.config_str)
-    ]
-    LogStash::Compiler.compile_sources(sources_with_metadata, @settings)
+    org.logstash.config.ir.ConfigCompiler.configToPipelineIR(
+      self.config_str, @settings.get_value("config.support_escapes")
+    )
   end
 
   def plugin(plugin_type, name, line, column, *args)

logstash-core/spec/logstash/compiler/compiler_spec.rb

@@ -47,7 +47,7 @@ def j
         end
       end
 
-      subject(:pipeline) { described_class.compile_sources(sources_with_metadata, settings) }
+      subject(:pipeline) { described_class.compile_sources(sources_with_metadata, false) }
 
       it "should generate a hash" do
         expect(pipeline.unique_hash).to be_a(String)
@@ -100,7 +100,7 @@ def j
   describe "compiling imperative" do
     let(:source_id) { "fake_sourcefile" }
     let(:source_with_metadata) { org.logstash.common.SourceWithMetadata.new(source_protocol, source_id, 0, 0, source) }
-    subject(:compiled) { described_class.compile_imperative(source_with_metadata, settings) }
+    subject(:compiled) { described_class.compile_imperative(source_with_metadata, settings.get_value("config.support_escapes")) }
 
     context "when config.support_escapes" do
       let(:parser) { LogStashCompilerLSCLGrammarParser.new }

logstash-core/src/main/java/org/logstash/config/ir/ConfigCompiler.java

@@ -0,0 +1,69 @@
+package org.logstash.config.ir;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import org.jruby.RubyHash;
+import org.jruby.javasupport.JavaUtil;
+import org.jruby.runtime.builtin.IRubyObject;
+import org.jruby.runtime.load.LoadService;
+import org.logstash.RubyUtil;
+import org.logstash.common.IncompleteSourceWithMetadataException;
+import org.logstash.common.SourceWithMetadata;
+
+/**
+ * Java Implementation of the config compiler that is implemented by wrapping the Ruby
+ * {@code LogStash::Compiler}.
+ */
+public final class ConfigCompiler {
+
+    private ConfigCompiler() {
+        // Utility Class
+    }
+
+    /**
+     * @param config Logstash Config String
+     * @param supportEscapes The value of the setting {@code config.support_escapes}
+     * @return Compiled {@link PipelineIR}
+     * @throws IncompleteSourceWithMetadataException On Broken Configuration
+     */
+    public static PipelineIR configToPipelineIR(final String config, final boolean supportEscapes)
+        throws IncompleteSourceWithMetadataException {
+        ensureLoadpath();
+        final IRubyObject compiler = RubyUtil.RUBY.executeScript(
+            "require 'logstash/compiler'\nLogStash::Compiler",
+            ""
+        );
+        final IRubyObject code =
+            compiler.callMethod(RubyUtil.RUBY.getCurrentContext(), "compile_sources",
+                new IRubyObject[]{
+                    RubyUtil.RUBY.newArray(
+                        JavaUtil.convertJavaToRuby(
+                            RubyUtil.RUBY,
+                            new SourceWithMetadata("str", "pipeline", 0, 0, config)
+                        )
+                    ),
+                    RubyUtil.RUBY.newBoolean(supportEscapes)
+                }
+            );
+        return (PipelineIR) code.toJava(PipelineIR.class);
+    }
+
+    /**
+     * Loads the logstash-core/lib path if the load service can't find {@code logstash/compiler}.
+     */
+    private static void ensureLoadpath() {
+        final LoadService loader = RubyUtil.RUBY.getLoadService();
+        if (loader.findFileForLoad("logstash/compiler").library == null) {
+            final RubyHash environment = RubyUtil.RUBY.getENV();
+            final Path root = Paths.get(
+                System.getProperty("logstash.core.root.dir", "")
+            ).toAbsolutePath();
+            final String gems = root.getParent().resolve("vendor").resolve("bundle")
+                .resolve("jruby").resolve("2.3.0").toFile().getAbsolutePath();
+            environment.put("GEM_HOME", gems);
+            environment.put("GEM_PATH", gems);
+            loader.addPaths(root.resolve("lib").toFile().getAbsolutePath()
+            );
+        }
+    }
+}

logstash-core/src/test/java/org/logstash/config/ir/ConfigCompilerTest.java

@@ -0,0 +1,17 @@
+package org.logstash.config.ir;
+
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+public class ConfigCompilerTest {
+
+    @Test
+    public void testConfigToPipelineIR() throws Exception {
+        final PipelineIR pipelineIR =
+            ConfigCompiler.configToPipelineIR("input {stdin{}} output{stdout{}}", false);
+        assertThat(pipelineIR.getOutputPluginVertices().size(), is(1));
+        assertThat(pipelineIR.getFilterPluginVertices().size(), is(0));
+    }
+}

rakelib/artifacts.rake

@@ -191,8 +191,10 @@ namespace "artifact" do
   end
 
   task "prepare" do
+    exit 1 unless system('./gradlew assemble bootstrap :ingest-converter:assemble')
+    Rake::Task["vendor:gems"].invoke
     if ENV['SKIP_PREPARE'] != "1"
-      ["bootstrap", "plugin:install-default", "artifact:clean-bundle-config"].each {|task| Rake::Task[task].invoke }
+      ["plugin:install-default", "artifact:clean-bundle-config"].each {|task| Rake::Task[task].invoke }
     end
   end