Skip to content

Commit

Permalink
Wrap PipelineIR Compilation in Java Call
Browse files Browse the repository at this point in the history
  • Loading branch information
original-brownbear committed Oct 27, 2017
1 parent f7ca6d9 commit 761b551
Show file tree
Hide file tree
Showing 6 changed files with 71 additions and 41 deletions.
16 changes: 6 additions & 10 deletions logstash-core/lib/logstash/compiler.rb
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
module LogStash; class Compiler
include ::LogStash::Util::Loggable

def self.compile_sources(sources_with_metadata, settings)
def self.compile_sources(sources_with_metadata, support_escapes)
graph_sections = sources_with_metadata.map do |swm|
self.compile_graph(swm, settings)
self.compile_graph(swm, support_escapes)
end

input_graph = Graph.combine(*graph_sections.map {|s| s[:input] }).graph
Expand All @@ -30,7 +30,7 @@ def self.compile_sources(sources_with_metadata, settings)
PipelineIR.new(input_graph, filter_graph, output_graph, original_source)
end

def self.compile_ast(source_with_metadata, settings)
def self.compile_imperative(source_with_metadata, support_escapes)
if !source_with_metadata.is_a?(org.logstash.common.SourceWithMetadata)
raise ArgumentError, "Expected 'org.logstash.common.SourceWithMetadata', got #{source_with_metadata.class}"
end
Expand All @@ -42,15 +42,11 @@ def self.compile_ast(source_with_metadata, settings)
raise ConfigurationError, grammar.failure_reason
end

config.process_escape_sequences = settings.get_value("config.support_escapes")
config.process_escape_sequences = support_escapes
config.compile(source_with_metadata)
end

def self.compile_imperative(source_with_metadata, settings)
compile_ast(source_with_metadata, settings)
end

def self.compile_graph(source_with_metadata, settings)
Hash[compile_imperative(source_with_metadata, settings).map {|section,icompiled| [section, icompiled.toGraph]}]
def self.compile_graph(source_with_metadata, support_escapes)
Hash[compile_imperative(source_with_metadata, support_escapes).map {|section,icompiled| [section, icompiled.toGraph]}]
end
end; end
11 changes: 3 additions & 8 deletions logstash-core/lib/logstash/java_pipeline.rb
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,9 @@ def initialize(pipeline_config, namespaced_metric = nil, agent = nil)
@settings = pipeline_config.settings
@config_hash = Digest::SHA1.hexdigest(@config_str)

@lir = compile_lir
@lir = org.logstash.config.ir.ConfigCompiler.configToPipelineIR(
@config_str, @settings.get_value("config.support_escapes")
)

# Every time #plugin is invoked this is incremented to give each plugin
# a unique id when auto-generating plugin ids
Expand Down Expand Up @@ -85,13 +87,6 @@ def close_dlq_writer
end
end

def compile_lir
sources_with_metadata = [
SourceWithMetadata.new("str", "pipeline", 0, 0, self.config_str)
]
LogStash::Compiler.compile_sources(sources_with_metadata, @settings)
end

def buildOutput(name, line, column, *args)
plugin("output", name, line, column, *args)
end
Expand Down
11 changes: 6 additions & 5 deletions logstash-core/lib/logstash/pipeline.rb
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,9 @@ def initialize(pipeline_config, namespaced_metric = nil, agent = nil)
@settings = pipeline_config.settings
@config_hash = Digest::SHA1.hexdigest(@config_str)

@lir = compile_lir
@lir = org.logstash.config.ir.ConfigCompiler.configToPipelineIR(
@config_str, @settings.get_value("config.support_escapes")
)

# Every time #plugin is invoked this is incremented to give each plugin
# a unique id when auto-generating plugin ids
Expand Down Expand Up @@ -101,10 +103,9 @@ def close_dlq_writer
end

def compile_lir
sources_with_metadata = [
SourceWithMetadata.new("str", "pipeline", 0, 0, self.config_str)
]
LogStash::Compiler.compile_sources(sources_with_metadata, @settings)
org.logstash.config.ir.ConfigCompiler.configToPipelineIR(
self.config_str, @settings.get_value("config.support_escapes")
)
end

def plugin(plugin_type, name, line, column, *args)
Expand Down
4 changes: 2 additions & 2 deletions logstash-core/spec/logstash/compiler/compiler_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ def j
end
end

subject(:pipeline) { described_class.compile_sources(sources_with_metadata, settings) }
subject(:pipeline) { described_class.compile_sources(sources_with_metadata, false) }

it "should generate a hash" do
expect(pipeline.unique_hash).to be_a(String)
Expand Down Expand Up @@ -100,7 +100,7 @@ def j
describe "compiling imperative" do
let(:source_id) { "fake_sourcefile" }
let(:source_with_metadata) { org.logstash.common.SourceWithMetadata.new(source_protocol, source_id, 0, 0, source) }
subject(:compiled) { described_class.compile_imperative(source_with_metadata, settings) }
subject(:compiled) { described_class.compile_imperative(source_with_metadata, settings.get_value("config.support_escapes")) }

context "when config.support_escapes" do
let(:parser) { LogStashCompilerLSCLGrammarParser.new }
Expand Down
Original file line number Diff line number Diff line change
@@ -1,23 +1,57 @@
package org.logstash.config.ir;

import org.jruby.Ruby;
import org.jruby.javasupport.JavaUtil;
import org.jruby.runtime.builtin.IRubyObject;
import org.jruby.runtime.load.LoadService;
import org.logstash.RubyUtil;
import org.logstash.common.IncompleteSourceWithMetadataException;
import org.logstash.common.SourceWithMetadata;

/**
* Java Implementation of the config compiler that is implemented by wrapping the Ruby
* {@code LogStash::Compiler}.
*/
public final class ConfigCompiler {

private final Ruby ruby;

public ConfigCompiler(final Ruby ruby) {
this.ruby = ruby;
private ConfigCompiler() {
// Utility Class
}

public String configToRuby(final String config) {
final IRubyObject grammar = ruby.executeScript(
"require 'logstash/config/grammar'\ngrammar = LogStashConfigParser.new", ""
/**
* @param config Logstash Config String
* @param supportEscapes The value of the setting {@code config.support_escapes}
* @return Compiled {@link PipelineIR}
* @throws IncompleteSourceWithMetadataException On Broken Configuration
*/
public static PipelineIR configToPipelineIR(final String config, final boolean supportEscapes)
throws IncompleteSourceWithMetadataException {
ensureLoadpath();
final IRubyObject compiler = RubyUtil.RUBY.executeScript(
"require 'logstash/compiler'\nLogStash::Compiler",
""
);
final IRubyObject parsed =
grammar.callMethod(ruby.getCurrentContext(), "parse", ruby.newString(config));
final IRubyObject code = parsed.callMethod(ruby.getCurrentContext(), "compile");
return code.toString();
final IRubyObject code =
compiler.callMethod(RubyUtil.RUBY.getCurrentContext(), "compile_sources",
new IRubyObject[]{
RubyUtil.RUBY.newArray(
JavaUtil.convertJavaToRuby(
RubyUtil.RUBY,
new SourceWithMetadata("str", "pipeline", 0, 0, config)
)
),
RubyUtil.RUBY.newBoolean(supportEscapes)
}
);
return (PipelineIR) code.toJava(PipelineIR.class);
}

/**
* Loads the logstash-core/lib path if the load service can't find {@code logstash/compiler}.
*/
private static void ensureLoadpath() {
final LoadService loader = RubyUtil.RUBY.getLoadService();
if (loader.findFileForLoad("logstash/compiler").library == null) {
loader.addPaths("lib");
}
}
}
Original file line number Diff line number Diff line change
@@ -1,13 +1,17 @@
package org.logstash.config.ir;

import org.junit.Test;
import org.logstash.RubyUtil;

import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;

public class ConfigCompilerTest {

@Test
public void testConfigToRuby() {
new ConfigCompiler(RubyUtil.RUBY)
.configToRuby("input {stdin{}} output{stdout{}}");
public void testConfigToPipelineIR() throws Exception {
final PipelineIR pipelineIR =
ConfigCompiler.configToPipelineIR("input {stdin{}} output{stdout{}}", false);
assertThat(pipelineIR.getOutputPluginVertices().size(), is(1));
assertThat(pipelineIR.getFilterPluginVertices().size(), is(0));
}
}

0 comments on commit 761b551

Please sign in to comment.