From 7ee9fed0ae720523d19af42fbd71d6a4bce72553 Mon Sep 17 00:00:00 2001 From: Armin Date: Fri, 27 Oct 2017 11:35:06 +0200 Subject: [PATCH 1/8] #8442 Wrap Ruby PipelineIR Compiler in Java Call --- build.gradle | 16 ++++- ci/unit_tests.sh | 15 ++-- logstash-core/build.gradle | 4 ++ logstash-core/lib/logstash/compiler.rb | 16 ++--- logstash-core/lib/logstash/java_pipeline.rb | 12 ++-- logstash-core/lib/logstash/pipeline.rb | 12 ++-- .../spec/logstash/compiler/compiler_spec.rb | 4 +- .../logstash/config/ir/ConfigCompiler.java | 69 +++++++++++++++++++ .../config/ir/ConfigCompilerTest.java | 17 +++++ rakelib/artifacts.rake | 2 +- rakelib/bootstrap.rake | 1 - rakelib/compile.rake | 14 +--- rakelib/dependency.rake | 4 -- rakelib/license.rake | 2 +- rakelib/plugin.rake | 8 +-- rakelib/test.rake | 15 ++-- rakelib/vendor.rake | 7 -- rakelib/z_rubycheck.rake | 21 ------ 18 files changed, 143 insertions(+), 96 deletions(-) create mode 100644 logstash-core/src/main/java/org/logstash/config/ir/ConfigCompiler.java create mode 100644 logstash-core/src/test/java/org/logstash/config/ir/ConfigCompilerTest.java delete mode 100644 rakelib/bootstrap.rake delete mode 100644 rakelib/z_rubycheck.rake diff --git a/build.gradle b/build.gradle index 19ceb1ba64b..d4570b7f02f 100644 --- a/build.gradle +++ b/build.gradle @@ -119,4 +119,18 @@ task downloadAndInstallJRuby(dependsOn: verifyFile, type: Copy) { // If you are running a JRuby snapshot we will skip the integrity check. verifyFile.onlyIf { doChecksum } -bootstrap.dependsOn downloadAndInstallJRuby + +task installGems(dependsOn: [downloadAndInstallJRuby], type: Exec) { + workingDir projectDir + commandLine './vendor/jruby/bin/jruby', "${projectDir}/vendor/jruby/bin/rake".toString(), "vendor:gems", "test:install-core" + standardOutput = new ByteArrayOutputStream() + ext.output = { + return standardOutput.toString() + } +} + +project(":logstash-core") { + installGems.dependsOn assemble +} + +bootstrap.dependsOn installGems diff --git a/ci/unit_tests.sh b/ci/unit_tests.sh index 2a0ad094451..6dedd7d8a17 100755 --- a/ci/unit_tests.sh +++ b/ci/unit_tests.sh @@ -13,24 +13,23 @@ SELECTED_TEST_SUITE=$1 if [[ $SELECTED_TEST_SUITE == $"core-fail-fast" ]]; then echo "Running Java and Ruby unit tests, but will fail fast" - echo "Running test:install-core" - rake test:install-core + echo "Running Gradle clean bootstrap" + ./gradlew clean bootstrap echo "Running test:core-fail-fast" rake test:core-fail-fast elif [[ $SELECTED_TEST_SUITE == $"java" ]]; then echo "Running Java unit tests" - echo "Running test:core-java" - rake test:core-java + ./gradlew clean javaTests elif [[ $SELECTED_TEST_SUITE == $"ruby" ]]; then echo "Running Ruby unit tests" - echo "Running test:install-core" - rake test:install-core + echo "Running Gradle clean bootstrap" + ./gradlew clean bootstrap echo "Running test:core-ruby" rake test:core-ruby else echo "Running Java and Ruby unit tests" - echo "Running test:install-core" - rake test:install-core + echo "Running Gradle clean bootstrap" + ./gradlew clean bootstrap echo "Running test:core" rake test:core fi diff --git a/logstash-core/build.gradle b/logstash-core/build.gradle index 8584dc4b8b3..4d285cd0a9a 100644 --- a/logstash-core/build.gradle +++ b/logstash-core/build.gradle @@ -74,6 +74,10 @@ task javaTests(type: Test) { exclude '/org/logstash/RSpecTests.class' } +tasks.withType(Test) { + systemProperty 'logstash.core.root.dir', projectDir.absolutePath +} + artifacts { sources(sourcesJar) { // Weird Gradle quirk where type will be used for the extension, but only for sources diff --git a/logstash-core/lib/logstash/compiler.rb b/logstash-core/lib/logstash/compiler.rb index 3f4a109e079..6c568c22859 100644 --- a/logstash-core/lib/logstash/compiler.rb +++ b/logstash-core/lib/logstash/compiler.rb @@ -7,9 +7,9 @@ module LogStash; class Compiler include ::LogStash::Util::Loggable - def self.compile_sources(sources_with_metadata, settings) + def self.compile_sources(sources_with_metadata, support_escapes) graph_sections = sources_with_metadata.map do |swm| - self.compile_graph(swm, settings) + self.compile_graph(swm, support_escapes) end input_graph = Graph.combine(*graph_sections.map {|s| s[:input] }).graph @@ -30,7 +30,7 @@ def self.compile_sources(sources_with_metadata, settings) PipelineIR.new(input_graph, filter_graph, output_graph, original_source) end - def self.compile_ast(source_with_metadata, settings) + def self.compile_imperative(source_with_metadata, support_escapes) if !source_with_metadata.is_a?(org.logstash.common.SourceWithMetadata) raise ArgumentError, "Expected 'org.logstash.common.SourceWithMetadata', got #{source_with_metadata.class}" end @@ -42,15 +42,11 @@ def self.compile_ast(source_with_metadata, settings) raise ConfigurationError, grammar.failure_reason end - config.process_escape_sequences = settings.get_value("config.support_escapes") + config.process_escape_sequences = support_escapes config.compile(source_with_metadata) end - def self.compile_imperative(source_with_metadata, settings) - compile_ast(source_with_metadata, settings) - end - - def self.compile_graph(source_with_metadata, settings) - Hash[compile_imperative(source_with_metadata, settings).map {|section,icompiled| [section, icompiled.toGraph]}] + def self.compile_graph(source_with_metadata, support_escapes) + Hash[compile_imperative(source_with_metadata, support_escapes).map {|section,icompiled| [section, icompiled.toGraph]}] end end; end diff --git a/logstash-core/lib/logstash/java_pipeline.rb b/logstash-core/lib/logstash/java_pipeline.rb index c96ad138744..8c722624bf0 100644 --- a/logstash-core/lib/logstash/java_pipeline.rb +++ b/logstash-core/lib/logstash/java_pipeline.rb @@ -29,6 +29,7 @@ java_import org.logstash.common.SourceWithMetadata java_import org.logstash.common.io.DeadLetterQueueWriter java_import org.logstash.config.ir.CompiledPipeline +java_import org.logstash.config.ir.ConfigCompiler module LogStash; class JavaBasePipeline include org.logstash.config.ir.compiler.RubyIntegration::Pipeline @@ -47,7 +48,9 @@ def initialize(pipeline_config, namespaced_metric = nil, agent = nil) @settings = pipeline_config.settings @config_hash = Digest::SHA1.hexdigest(@config_str) - @lir = compile_lir + @lir = ConfigCompiler.configToPipelineIR( + @config_str, @settings.get_value("config.support_escapes") + ) # Every time #plugin is invoked this is incremented to give each plugin # a unique id when auto-generating plugin ids @@ -85,13 +88,6 @@ def close_dlq_writer end end - def compile_lir - sources_with_metadata = [ - SourceWithMetadata.new("str", "pipeline", 0, 0, self.config_str) - ] - LogStash::Compiler.compile_sources(sources_with_metadata, @settings) - end - def buildOutput(name, line, column, *args) plugin("output", name, line, column, *args) end diff --git a/logstash-core/lib/logstash/pipeline.rb b/logstash-core/lib/logstash/pipeline.rb index 37aeaeb3fa1..9d61065e263 100644 --- a/logstash-core/lib/logstash/pipeline.rb +++ b/logstash-core/lib/logstash/pipeline.rb @@ -29,6 +29,7 @@ java_import org.logstash.common.DeadLetterQueueFactory java_import org.logstash.common.SourceWithMetadata java_import org.logstash.common.io.DeadLetterQueueWriter +java_import org.logstash.config.ir.ConfigCompiler module LogStash; class BasePipeline include LogStash::Util::Loggable @@ -46,7 +47,9 @@ def initialize(pipeline_config, namespaced_metric = nil, agent = nil) @settings = pipeline_config.settings @config_hash = Digest::SHA1.hexdigest(@config_str) - @lir = compile_lir + @lir = ConfigCompiler.configToPipelineIR( + @config_str, @settings.get_value("config.support_escapes") + ) # Every time #plugin is invoked this is incremented to give each plugin # a unique id when auto-generating plugin ids @@ -101,10 +104,9 @@ def close_dlq_writer end def compile_lir - sources_with_metadata = [ - SourceWithMetadata.new("str", "pipeline", 0, 0, self.config_str) - ] - LogStash::Compiler.compile_sources(sources_with_metadata, @settings) + org.logstash.config.ir.ConfigCompiler.configToPipelineIR( + self.config_str, @settings.get_value("config.support_escapes") + ) end def plugin(plugin_type, name, line, column, *args) diff --git a/logstash-core/spec/logstash/compiler/compiler_spec.rb b/logstash-core/spec/logstash/compiler/compiler_spec.rb index da018062011..16c81e85462 100644 --- a/logstash-core/spec/logstash/compiler/compiler_spec.rb +++ b/logstash-core/spec/logstash/compiler/compiler_spec.rb @@ -47,7 +47,7 @@ def j end end - subject(:pipeline) { described_class.compile_sources(sources_with_metadata, settings) } + subject(:pipeline) { described_class.compile_sources(sources_with_metadata, false) } it "should generate a hash" do expect(pipeline.unique_hash).to be_a(String) @@ -100,7 +100,7 @@ def j describe "compiling imperative" do let(:source_id) { "fake_sourcefile" } let(:source_with_metadata) { org.logstash.common.SourceWithMetadata.new(source_protocol, source_id, 0, 0, source) } - subject(:compiled) { described_class.compile_imperative(source_with_metadata, settings) } + subject(:compiled) { described_class.compile_imperative(source_with_metadata, settings.get_value("config.support_escapes")) } context "when config.support_escapes" do let(:parser) { LogStashCompilerLSCLGrammarParser.new } diff --git a/logstash-core/src/main/java/org/logstash/config/ir/ConfigCompiler.java b/logstash-core/src/main/java/org/logstash/config/ir/ConfigCompiler.java new file mode 100644 index 00000000000..98b48f75de3 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/ConfigCompiler.java @@ -0,0 +1,69 @@ +package org.logstash.config.ir; + +import java.nio.file.Path; +import java.nio.file.Paths; +import org.jruby.RubyHash; +import org.jruby.javasupport.JavaUtil; +import org.jruby.runtime.builtin.IRubyObject; +import org.jruby.runtime.load.LoadService; +import org.logstash.RubyUtil; +import org.logstash.common.IncompleteSourceWithMetadataException; +import org.logstash.common.SourceWithMetadata; + +/** + * Java Implementation of the config compiler that is implemented by wrapping the Ruby + * {@code LogStash::Compiler}. + */ +public final class ConfigCompiler { + + private ConfigCompiler() { + // Utility Class + } + + /** + * @param config Logstash Config String + * @param supportEscapes The value of the setting {@code config.support_escapes} + * @return Compiled {@link PipelineIR} + * @throws IncompleteSourceWithMetadataException On Broken Configuration + */ + public static PipelineIR configToPipelineIR(final String config, final boolean supportEscapes) + throws IncompleteSourceWithMetadataException { + ensureLoadpath(); + final IRubyObject compiler = RubyUtil.RUBY.executeScript( + "require 'logstash/compiler'\nLogStash::Compiler", + "" + ); + final IRubyObject code = + compiler.callMethod(RubyUtil.RUBY.getCurrentContext(), "compile_sources", + new IRubyObject[]{ + RubyUtil.RUBY.newArray( + JavaUtil.convertJavaToRuby( + RubyUtil.RUBY, + new SourceWithMetadata("str", "pipeline", 0, 0, config) + ) + ), + RubyUtil.RUBY.newBoolean(supportEscapes) + } + ); + return (PipelineIR) code.toJava(PipelineIR.class); + } + + /** + * Loads the logstash-core/lib path if the load service can't find {@code logstash/compiler}. + */ + private static void ensureLoadpath() { + final LoadService loader = RubyUtil.RUBY.getLoadService(); + if (loader.findFileForLoad("logstash/compiler").library == null) { + final RubyHash environment = RubyUtil.RUBY.getENV(); + final Path root = Paths.get( + System.getProperty("logstash.core.root.dir", "") + ).toAbsolutePath(); + final String gems = root.getParent().resolve("vendor").resolve("bundle") + .resolve("jruby").resolve("2.3.0").toFile().getAbsolutePath(); + environment.put("GEM_HOME", gems); + environment.put("GEM_PATH", gems); + loader.addPaths(root.resolve("lib").toFile().getAbsolutePath() + ); + } + } +} diff --git a/logstash-core/src/test/java/org/logstash/config/ir/ConfigCompilerTest.java b/logstash-core/src/test/java/org/logstash/config/ir/ConfigCompilerTest.java new file mode 100644 index 00000000000..6a2f89f33d7 --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/config/ir/ConfigCompilerTest.java @@ -0,0 +1,17 @@ +package org.logstash.config.ir; + +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +public class ConfigCompilerTest { + + @Test + public void testConfigToPipelineIR() throws Exception { + final PipelineIR pipelineIR = + ConfigCompiler.configToPipelineIR("input {stdin{}} output{stdout{}}", false); + assertThat(pipelineIR.getOutputPluginVertices().size(), is(1)); + assertThat(pipelineIR.getFilterPluginVertices().size(), is(0)); + } +} diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index 37289c66fca..0625b5a83fc 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -192,7 +192,7 @@ namespace "artifact" do task "prepare" do if ENV['SKIP_PREPARE'] != "1" - ["bootstrap", "plugin:install-default", "artifact:clean-bundle-config"].each {|task| Rake::Task[task].invoke } + ["plugin:install-default", "artifact:clean-bundle-config"].each {|task| Rake::Task[task].invoke } end end diff --git a/rakelib/bootstrap.rake b/rakelib/bootstrap.rake deleted file mode 100644 index 7d9c2e6880b..00000000000 --- a/rakelib/bootstrap.rake +++ /dev/null @@ -1 +0,0 @@ -task "bootstrap" => [ "vendor:all", "compile:all" ] diff --git a/rakelib/compile.rake b/rakelib/compile.rake index 99c7e320efb..1e0354efc5c 100644 --- a/rakelib/compile.rake +++ b/rakelib/compile.rake @@ -10,19 +10,7 @@ namespace "compile" do desc "Compile the config grammar" task "grammar" => "logstash-core/lib/logstash/config/grammar.rb" - - def safe_system(*args) - if !system(*args) - status = $? - raise "Got exit status #{status.exitstatus} attempting to execute #{args.inspect}!" - end - end - - task "logstash-core-java" do - puts("Building logstash-core using gradle") - safe_system("./gradlew", "assemble") - end desc "Build everything" - task "all" => ["grammar", "logstash-core-java"] + task "all" => ["grammar"] end diff --git a/rakelib/dependency.rake b/rakelib/dependency.rake index 323e76ff275..1f3b753bac2 100644 --- a/rakelib/dependency.rake +++ b/rakelib/dependency.rake @@ -4,10 +4,6 @@ namespace "dependency" do Rake::Task["gem:require"].invoke("bundler", "~> 1.9.4") end - task "rbx-stdlib" do - Rake::Task["gem:require"].invoke("rubysl", ">= 0") - end # task rbx-stdlib - task "archive-tar-minitar" do Rake::Task["gem:require"].invoke("minitar", "0.5.4") end # task archive-minitar diff --git a/rakelib/license.rake b/rakelib/license.rake index d790444cdb7..8b2b88c97c0 100644 --- a/rakelib/license.rake +++ b/rakelib/license.rake @@ -14,7 +14,7 @@ namespace "license" do NOTICE_FILE_PATH = File.join(LogStash::Environment::LOGSTASH_HOME, "NOTICE.TXT") desc "Generate a license/notice file for default plugin dependencies" - task "generate-notice-file" => ["bootstrap", "plugin:install-default"] do + task "generate-notice-file" => ["plugin:install-default"] do puts("[license:generate-notice-file] Generating notice file for default plugin dependencies") generate_notice_file end diff --git a/rakelib/plugin.rake b/rakelib/plugin.rake index 57693bda709..2b25d2a9f2a 100644 --- a/rakelib/plugin.rake +++ b/rakelib/plugin.rake @@ -8,7 +8,7 @@ namespace "plugin" do raise(RuntimeError, $!.to_s) unless $?.success? end - task "install-development-dependencies" => "bootstrap" do + task "install-development-dependencies" do puts("[plugin:install-development-dependencies] Installing development dependencies of all installed plugins") install_plugins("--development", "--preserve") @@ -23,21 +23,21 @@ namespace "plugin" do task.reenable # Allow this task to be run again end # task "install" - task "install-default" => "bootstrap" do + task "install-default" do puts("[plugin:install-default] Installing default plugins") install_plugins("--no-verify", "--preserve", *LogStash::RakeLib::DEFAULT_PLUGINS) task.reenable # Allow this task to be run again end - task "install-core" => "bootstrap" do + task "install-core" do puts("[plugin:install-core] Installing core plugins") install_plugins("--no-verify", "--preserve", *LogStash::RakeLib::CORE_SPECS_PLUGINS) task.reenable # Allow this task to be run again end - task "install-jar-dependencies" => "bootstrap" do + task "install-jar-dependencies" do puts("[plugin:install-jar-dependencies] Installing jar_dependencies plugins for testing") install_plugins("--no-verify", "--preserve", *LogStash::RakeLib::TEST_JAR_DEPENDENCIES_PLUGINS) diff --git a/rakelib/test.rake b/rakelib/test.rake index 58cff40ad1f..f0b4923c84e 100644 --- a/rakelib/test.rake +++ b/rakelib/test.rake @@ -6,11 +6,6 @@ require 'pathname' namespace "test" do - desc "run the java unit tests" - task "core-java" do - exit(1) unless system('./gradlew clean javaTests') - end - desc "run the ruby unit tests" task "core-ruby" do exit 1 unless system(*default_spec_command) @@ -39,7 +34,7 @@ namespace "test" do end desc "run all installed plugins specs" - task "plugins" => "bootstrap" do + task "plugins" do plugins_to_exclude = ENV.fetch("EXCLUDE_PLUGIN", "").split(",") # grab all spec files using the live plugins gem specs. this allows correctly also running the specs # of a local plugin dir added using the Gemfile :path option. before this, any local plugin spec would @@ -59,16 +54,16 @@ namespace "test" do end desc "install core plugins and dev dependencies" - task "install-core" => ["bootstrap", "plugin:install-core", "plugin:install-development-dependencies"] + task "install-core" => ["plugin:install-core", "plugin:install-development-dependencies"] desc "install default plugins and dev dependencies" - task "install-default" => ["bootstrap", "plugin:install-default", "plugin:install-development-dependencies"] + task "install-default" => ["plugin:install-default", "plugin:install-development-dependencies"] desc "install vendor plugins and dev dependencies" - task "install-vendor-plugins" => ["bootstrap", "plugin:install-vendor", "plugin:install-development-dependencies"] + task "install-vendor-plugins" => ["plugin:install-vendor", "plugin:install-development-dependencies"] desc "install jar dependencies and dev dependencies" - task "install-jar-dependencies-plugins" => ["bootstrap", "plugin:install-jar-dependencies", "plugin:install-development-dependencies"] + task "install-jar-dependencies-plugins" => ["plugin:install-jar-dependencies", "plugin:install-development-dependencies"] end task "test" => [ "test:core" ] diff --git a/rakelib/vendor.rake b/rakelib/vendor.rake index e846ec3e2e7..88becaaac1a 100644 --- a/rakelib/vendor.rake +++ b/rakelib/vendor.rake @@ -3,12 +3,6 @@ namespace "vendor" do return File.join("vendor", *args) end - task "jruby" do |task, args| - system('./gradlew bootstrap') - end # jruby - - task "all" => "jruby" - task "system_gem", :jruby_bin, :name, :version do |task, args| jruby_bin = args[:jruby_bin] name = args[:name] @@ -39,7 +33,6 @@ namespace "vendor" do puts(output) raise(exception) if exception end # task gems - task "all" => "gems" desc "Clean the vendored files" task :clean do diff --git a/rakelib/z_rubycheck.rake b/rakelib/z_rubycheck.rake deleted file mode 100644 index 2ed3223f50a..00000000000 --- a/rakelib/z_rubycheck.rake +++ /dev/null @@ -1,21 +0,0 @@ -if ENV['USE_RUBY'] != '1' - if RUBY_ENGINE != "jruby" or Gem.ruby !~ /vendor\/jruby\/bin\/jruby/ - puts "Restarting myself under Vendored JRuby (currently #{RUBY_ENGINE} #{RUBY_VERSION})" if ENV['DEBUG'] - - # Make sure we have JRuby, then rerun ourselves under jruby. - Rake::Task["vendor:jruby"].invoke - jruby = File.join("vendor", "jruby", "bin", "jruby") - rake = File.join("vendor", "jruby", "bin", "rake") - - # if required at this point system gems can be installed using the system_gem task, for example: - # Rake::Task["vendor:system_gem"].invoke(jruby, "ffi", "1.9.6") - - # Ignore Environment JAVA_OPTS - ENV["JAVA_OPTS"] = "" - exec(jruby, "-J-Xmx1g", "-S", rake, *ARGV) - end -end - -def discover_rake() - Dir.glob('vendor', 'bundle', 'rake') -end From 0c85312447f6ce34425a7bd2603fb2bda101411d Mon Sep 17 00:00:00 2001 From: Armin Date: Fri, 27 Oct 2017 19:48:28 +0200 Subject: [PATCH 2/8] bck --- build.gradle | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index d4570b7f02f..a68724620b5 100644 --- a/build.gradle +++ b/build.gradle @@ -17,6 +17,8 @@ import de.undercouch.gradle.tasks.download.Download import de.undercouch.gradle.tasks.download.Verify import org.yaml.snakeyaml.Yaml +task bootstrap {} + allprojects { group = 'org.logstash' @@ -42,6 +44,7 @@ allprojects { //https://stackoverflow.com/questions/3963708/gradle-how-to-display-test-results-in-the-console-in-real-time tasks.withType(Test) { + dependsOn bootstrap testLogging { // set options for log level LIFECYCLE events "passed", "skipped", "failed", "standardOut" @@ -89,9 +92,6 @@ if (versionMap["jruby-runtime-override"]) { doChecksum = true } -// Tasks -task bootstrap {} - task downloadJRuby(type: Download) { description "Download JRuby artifact from this specific URL: ${jRubyURL}" src jRubyURL @@ -133,4 +133,9 @@ project(":logstash-core") { installGems.dependsOn assemble } +clean { + delete "${projectDir}/vendor/" + delete "${projectDir}/Gemfile" +} + bootstrap.dependsOn installGems From 7189f1cd2185ef466d518b0d67a3db712dc83a73 Mon Sep 17 00:00:00 2001 From: Armin Date: Fri, 27 Oct 2017 19:50:38 +0200 Subject: [PATCH 3/8] bck --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index a68724620b5..1ccd8bf9cac 100644 --- a/build.gradle +++ b/build.gradle @@ -136,6 +136,7 @@ project(":logstash-core") { clean { delete "${projectDir}/vendor/" delete "${projectDir}/Gemfile" + delete "${projectDir}/Gemfile.lock" } bootstrap.dependsOn installGems From ccdcc8a26be09e0f6c5f22d88149a99cf49893b1 Mon Sep 17 00:00:00 2001 From: Armin Date: Fri, 27 Oct 2017 20:31:44 +0200 Subject: [PATCH 4/8] bck --- rakelib/artifacts.rake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index 0625b5a83fc..a8a9e8e781c 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -191,6 +191,8 @@ namespace "artifact" do end task "prepare" do + exit 1 unless system('./gradlew assemble bootstrap :ingest-converter:assemble') + Rake::Task["vendor:gems"].invoke if ENV['SKIP_PREPARE'] != "1" ["plugin:install-default", "artifact:clean-bundle-config"].each {|task| Rake::Task[task].invoke } end From dbd98ce20cc32b405229d387e7ce293616a87675 Mon Sep 17 00:00:00 2001 From: Armin Date: Fri, 27 Oct 2017 23:31:52 +0200 Subject: [PATCH 5/8] bck --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bac1168c8bb..e75f3535ec5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN /opt/logstash/gradlew wrapper ADD versions.yml /opt/logstash/versions.yml ADD LICENSE /opt/logstash/LICENSE ADD CONTRIBUTORS /opt/logstash/CONTRIBUTORS -ADD Gemfile.template /opt/logstash/Gemfile +ADD Gemfile.template /opt/logstash/Gemfile.template ADD Rakefile /opt/logstash/Rakefile ADD build.gradle /opt/logstash/build.gradle ADD rakelib /opt/logstash/rakelib From c1d8ff388e597dcbb28fae56991ebe16cc8a691b Mon Sep 17 00:00:00 2001 From: Armin Date: Fri, 27 Oct 2017 23:49:37 +0200 Subject: [PATCH 6/8] bck --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index 1ccd8bf9cac..516349e236d 100644 --- a/build.gradle +++ b/build.gradle @@ -137,6 +137,7 @@ clean { delete "${projectDir}/vendor/" delete "${projectDir}/Gemfile" delete "${projectDir}/Gemfile.lock" + delete "${projectDir}/NOTICE.TXT" } bootstrap.dependsOn installGems From 6406405765f420f03808e1b0f4b3e4d877cc541f Mon Sep 17 00:00:00 2001 From: Armin Date: Fri, 27 Oct 2017 23:58:19 +0200 Subject: [PATCH 7/8] bck --- build.gradle | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/build.gradle b/build.gradle index 516349e236d..b62fba07e84 100644 --- a/build.gradle +++ b/build.gradle @@ -122,6 +122,13 @@ verifyFile.onlyIf { doChecksum } task installGems(dependsOn: [downloadAndInstallJRuby], type: Exec) { workingDir projectDir + inputs.files file("$projectDir/Gemfile.template") + inputs.files fileTree("$projectDir/rakelib") + inputs.files fileTree("$projectDir/versions.yml") + outputs.files file("$projectDir/Gemfile") + outputs.files file("$projectDir/Gemfile.lock") + outputs.files fileTree("$projectDir/vendor/bundle") + outputs.files fileTree("$projectDir/vendor/jruby") commandLine './vendor/jruby/bin/jruby', "${projectDir}/vendor/jruby/bin/rake".toString(), "vendor:gems", "test:install-core" standardOutput = new ByteArrayOutputStream() ext.output = { From 7a1cf35c57f06821f65c7e2aa0fb27e1f36c6673 Mon Sep 17 00:00:00 2001 From: Armin Date: Tue, 31 Oct 2017 15:40:04 +0100 Subject: [PATCH 8/8] updated docs --- README.md | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 0862700934c..7fcdfbe6da5 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ jruby 9.1.10.0 (2.3.3) 2017-05-25 b09c48a Java HotSpot(TM) 64-Bit Server VM 25.1 * To run Logstash from the repo you must first bootstrap the environment: ```sh -rake bootstrap +./gradlew bootstrap ``` * You can then use `bin/logstash` to start Logstash, but there are no plugins installed. To install default plugins, you can run: @@ -126,22 +126,18 @@ Most of the unit tests in Logstash are written using [rspec](http://rspec.info/) ### Core tests -1- In order to run the core tests, a small set of plugins must first be installed: +1- To run the core tests you can use the rake task: - rake test:install-core - -2- To run the core tests you can use the rake task: - - rake test:core + ./gradlew test or use the `rspec` tool to run all tests or run a specific test: bin/rspec bin/rspec spec/foo/bar_spec.rb -3- To run the subset of tests covering the Java codebase only run: +2- To run the subset of tests covering the Java codebase only run: - ./gradlew test + ./gradlew javaTests ### Plugins tests @@ -163,8 +159,8 @@ Note that if a plugin is installed using the plugin manager `bin/logstash-plugin You can build a Logstash snapshot package as tarball or zip file ```sh -rake artifact:tar -rake artifact:zip +./gradlew bootstrap && rake artifact:tar +./gradlew bootstrap && rake artifact:zip ``` This will create the artifact `LS_HOME/build` directory