diff --git a/logstash-core/lib/logstash/compiler.rb b/logstash-core/lib/logstash/compiler.rb index 3f4a109e079..6c568c22859 100644 --- a/logstash-core/lib/logstash/compiler.rb +++ b/logstash-core/lib/logstash/compiler.rb @@ -7,9 +7,9 @@ module LogStash; class Compiler include ::LogStash::Util::Loggable - def self.compile_sources(sources_with_metadata, settings) + def self.compile_sources(sources_with_metadata, support_escapes) graph_sections = sources_with_metadata.map do |swm| - self.compile_graph(swm, settings) + self.compile_graph(swm, support_escapes) end input_graph = Graph.combine(*graph_sections.map {|s| s[:input] }).graph @@ -30,7 +30,7 @@ def self.compile_sources(sources_with_metadata, settings) PipelineIR.new(input_graph, filter_graph, output_graph, original_source) end - def self.compile_ast(source_with_metadata, settings) + def self.compile_imperative(source_with_metadata, support_escapes) if !source_with_metadata.is_a?(org.logstash.common.SourceWithMetadata) raise ArgumentError, "Expected 'org.logstash.common.SourceWithMetadata', got #{source_with_metadata.class}" end @@ -42,15 +42,11 @@ def self.compile_ast(source_with_metadata, settings) raise ConfigurationError, grammar.failure_reason end - config.process_escape_sequences = settings.get_value("config.support_escapes") + config.process_escape_sequences = support_escapes config.compile(source_with_metadata) end - def self.compile_imperative(source_with_metadata, settings) - compile_ast(source_with_metadata, settings) - end - - def self.compile_graph(source_with_metadata, settings) - Hash[compile_imperative(source_with_metadata, settings).map {|section,icompiled| [section, icompiled.toGraph]}] + def self.compile_graph(source_with_metadata, support_escapes) + Hash[compile_imperative(source_with_metadata, support_escapes).map {|section,icompiled| [section, icompiled.toGraph]}] end end; end diff --git a/logstash-core/lib/logstash/java_pipeline.rb b/logstash-core/lib/logstash/java_pipeline.rb index 2cdd00eb9ef..8a27443e82f 100644 --- a/logstash-core/lib/logstash/java_pipeline.rb +++ b/logstash-core/lib/logstash/java_pipeline.rb @@ -47,7 +47,9 @@ def initialize(pipeline_config, namespaced_metric = nil, agent = nil) @settings = pipeline_config.settings @config_hash = Digest::SHA1.hexdigest(@config_str) - @lir = compile_lir + @lir = org.logstash.config.ir.ConfigCompiler.configToPipelineIR( + @config_str, @settings.get_value("config.support_escapes") + ) # Every time #plugin is invoked this is incremented to give each plugin # a unique id when auto-generating plugin ids @@ -85,13 +87,6 @@ def close_dlq_writer end end - def compile_lir - sources_with_metadata = [ - SourceWithMetadata.new("str", "pipeline", 0, 0, self.config_str) - ] - LogStash::Compiler.compile_sources(sources_with_metadata, @settings) - end - def buildOutput(name, line, column, *args) plugin("output", name, line, column, *args) end diff --git a/logstash-core/lib/logstash/pipeline.rb b/logstash-core/lib/logstash/pipeline.rb index 37aeaeb3fa1..d10ae138226 100644 --- a/logstash-core/lib/logstash/pipeline.rb +++ b/logstash-core/lib/logstash/pipeline.rb @@ -46,7 +46,9 @@ def initialize(pipeline_config, namespaced_metric = nil, agent = nil) @settings = pipeline_config.settings @config_hash = Digest::SHA1.hexdigest(@config_str) - @lir = compile_lir + @lir = org.logstash.config.ir.ConfigCompiler.configToPipelineIR( + @config_str, @settings.get_value("config.support_escapes") + ) # Every time #plugin is invoked this is incremented to give each plugin # a unique id when auto-generating plugin ids @@ -101,10 +103,9 @@ def close_dlq_writer end def compile_lir - sources_with_metadata = [ - SourceWithMetadata.new("str", "pipeline", 0, 0, self.config_str) - ] - LogStash::Compiler.compile_sources(sources_with_metadata, @settings) + org.logstash.config.ir.ConfigCompiler.configToPipelineIR( + self.config_str, @settings.get_value("config.support_escapes") + ) end def plugin(plugin_type, name, line, column, *args) diff --git a/logstash-core/spec/logstash/compiler/compiler_spec.rb b/logstash-core/spec/logstash/compiler/compiler_spec.rb index da018062011..16c81e85462 100644 --- a/logstash-core/spec/logstash/compiler/compiler_spec.rb +++ b/logstash-core/spec/logstash/compiler/compiler_spec.rb @@ -47,7 +47,7 @@ def j end end - subject(:pipeline) { described_class.compile_sources(sources_with_metadata, settings) } + subject(:pipeline) { described_class.compile_sources(sources_with_metadata, false) } it "should generate a hash" do expect(pipeline.unique_hash).to be_a(String) @@ -100,7 +100,7 @@ def j describe "compiling imperative" do let(:source_id) { "fake_sourcefile" } let(:source_with_metadata) { org.logstash.common.SourceWithMetadata.new(source_protocol, source_id, 0, 0, source) } - subject(:compiled) { described_class.compile_imperative(source_with_metadata, settings) } + subject(:compiled) { described_class.compile_imperative(source_with_metadata, settings.get_value("config.support_escapes")) } context "when config.support_escapes" do let(:parser) { LogStashCompilerLSCLGrammarParser.new } diff --git a/logstash-core/src/test/java/org/logstash/config/ir/ConfigCompiler.java b/logstash-core/src/test/java/org/logstash/config/ir/ConfigCompiler.java index 493630b84c8..abefb4989db 100644 --- a/logstash-core/src/test/java/org/logstash/config/ir/ConfigCompiler.java +++ b/logstash-core/src/test/java/org/logstash/config/ir/ConfigCompiler.java @@ -1,23 +1,57 @@ package org.logstash.config.ir; -import org.jruby.Ruby; +import org.jruby.javasupport.JavaUtil; import org.jruby.runtime.builtin.IRubyObject; +import org.jruby.runtime.load.LoadService; +import org.logstash.RubyUtil; +import org.logstash.common.IncompleteSourceWithMetadataException; +import org.logstash.common.SourceWithMetadata; +/** + * Java Implementation of the config compiler that is implemented by wrapping the Ruby + * {@code LogStash::Compiler}. + */ public final class ConfigCompiler { - private final Ruby ruby; - - public ConfigCompiler(final Ruby ruby) { - this.ruby = ruby; + private ConfigCompiler() { + // Utility Class } - public String configToRuby(final String config) { - final IRubyObject grammar = ruby.executeScript( - "require 'logstash/config/grammar'\ngrammar = LogStashConfigParser.new", "" + /** + * @param config Logstash Config String + * @param supportEscapes The value of the setting {@code config.support_escapes} + * @return Compiled {@link PipelineIR} + * @throws IncompleteSourceWithMetadataException On Broken Configuration + */ + public static PipelineIR configToPipelineIR(final String config, final boolean supportEscapes) + throws IncompleteSourceWithMetadataException { + ensureLoadpath(); + final IRubyObject compiler = RubyUtil.RUBY.executeScript( + "require 'logstash/compiler'\nLogStash::Compiler", + "" ); - final IRubyObject parsed = - grammar.callMethod(ruby.getCurrentContext(), "parse", ruby.newString(config)); - final IRubyObject code = parsed.callMethod(ruby.getCurrentContext(), "compile"); - return code.toString(); + final IRubyObject code = + compiler.callMethod(RubyUtil.RUBY.getCurrentContext(), "compile_sources", + new IRubyObject[]{ + RubyUtil.RUBY.newArray( + JavaUtil.convertJavaToRuby( + RubyUtil.RUBY, + new SourceWithMetadata("str", "pipeline", 0, 0, config) + ) + ), + RubyUtil.RUBY.newBoolean(supportEscapes) + } + ); + return (PipelineIR) code.toJava(PipelineIR.class); + } + + /** + * Loads the logstash-core/lib path if the load service can't find {@code logstash/compiler}. + */ + private static void ensureLoadpath() { + final LoadService loader = RubyUtil.RUBY.getLoadService(); + if (loader.findFileForLoad("logstash/compiler").library == null) { + loader.addPaths("lib"); + } } } diff --git a/logstash-core/src/test/java/org/logstash/config/ir/ConfigCompilerTest.java b/logstash-core/src/test/java/org/logstash/config/ir/ConfigCompilerTest.java index 07a9c755261..6a2f89f33d7 100644 --- a/logstash-core/src/test/java/org/logstash/config/ir/ConfigCompilerTest.java +++ b/logstash-core/src/test/java/org/logstash/config/ir/ConfigCompilerTest.java @@ -1,13 +1,17 @@ package org.logstash.config.ir; import org.junit.Test; -import org.logstash.RubyUtil; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; public class ConfigCompilerTest { @Test - public void testConfigToRuby() { - new ConfigCompiler(RubyUtil.RUBY) - .configToRuby("input {stdin{}} output{stdout{}}"); + public void testConfigToPipelineIR() throws Exception { + final PipelineIR pipelineIR = + ConfigCompiler.configToPipelineIR("input {stdin{}} output{stdout{}}", false); + assertThat(pipelineIR.getOutputPluginVertices().size(), is(1)); + assertThat(pipelineIR.getFilterPluginVertices().size(), is(0)); } }