From 6f8d8ca91de1d63ca61ee3db8fbf7e73667f1624 Mon Sep 17 00:00:00 2001 From: Pere Urbon-Bayes Date: Wed, 26 Aug 2015 18:52:15 +0200 Subject: [PATCH 01/52] add basic lookup test for the plugin Fixes #7 --- spec/inputs/unix_spec.rb | 16 +++++++++++++++- spec/spec_helper.rb | 3 +++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 spec/spec_helper.rb diff --git a/spec/inputs/unix_spec.rb b/spec/inputs/unix_spec.rb index 6bf5945..dc75e63 100644 --- a/spec/inputs/unix_spec.rb +++ b/spec/inputs/unix_spec.rb @@ -1 +1,15 @@ -require "logstash/devutils/rspec/spec_helper" +# encoding: utf-8 +require_relative "../spec_helper" +require "stud/temporary" +require "tempfile" + +describe LogStash::Inputs::Unix do + + let(:tempfile) { Tempfile.new("/tmp/foo") } + + it "should register without errors" do + plugin = LogStash::Plugin.lookup("input", "unix").new({ "path" => tempfile.path, "force_unlink" => true }) + expect { plugin.register }.to_not raise_error + end + +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 0000000..365dbbe --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,3 @@ +# encoding: utf-8 +require "logstash/devutils/rspec/spec_helper" +require 'logstash/inputs/unix' From 6281ca30b062fbc3806d73e793723b7257d86c5e Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 11 Sep 2015 10:34:53 +0100 Subject: [PATCH 02/52] remove git ls-files from gemspec --- logstash-input-unix.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash-input-unix.gemspec b/logstash-input-unix.gemspec index 76d76ca..7fa4d26 100644 --- a/logstash-input-unix.gemspec +++ b/logstash-input-unix.gemspec @@ -11,7 +11,7 @@ Gem::Specification.new do |s| s.require_paths = ["lib"] # Files - s.files = `git ls-files`.split($\)+::Dir.glob('vendor/*') + s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT'] # Tests s.test_files = s.files.grep(%r{^(test|spec|features)/}) From 65ef9b418ea805df924056f0a00578792ccfb8e6 Mon Sep 17 00:00:00 2001 From: Pere Urbon-Bayes Date: Mon, 14 Sep 2015 16:57:24 +0200 Subject: [PATCH 03/52] Make usage of the new stop api Make sure the client socket can be closed when the underneath socket has not more data and readpartial blocks Fixes #10 --- lib/logstash/inputs/unix.rb | 63 ++++++++++++------------------------- spec/inputs/unix_spec.rb | 36 +++++++++++++++++++++ spec/spec_helper.rb | 30 ++++++++++++++++++ 3 files changed, 86 insertions(+), 43 deletions(-) diff --git a/lib/logstash/inputs/unix.rb b/lib/logstash/inputs/unix.rb index 05ff793..c9bb55b 100644 --- a/lib/logstash/inputs/unix.rb +++ b/lib/logstash/inputs/unix.rb @@ -68,7 +68,7 @@ def register def handle_socket(socket, output_queue) begin hostname = Socket.gethostname - loop do + while !stop? buf = nil # NOTE(petef): the timeout only hits after the line is read # or socket dies @@ -86,13 +86,11 @@ def handle_socket(socket, output_queue) event["path"] = @path output_queue << event end - end # loop do + end rescue => e - @logger.debug("Closing connection", :path => @path, - :exception => e, :backtrace => e.backtrace) + @logger.debug("Closing connection", :path => @path, :exception => e, :backtrace => e.backtrace) rescue Timeout::Error - @logger.debug("Closing connection after read timeout", - :path => @path) + @logger.debug("Closing connection after read timeout", :path => @path) end # begin ensure @@ -100,7 +98,7 @@ def handle_socket(socket, output_queue) socket.close rescue IOError #pass - end # begin + end end private @@ -111,52 +109,31 @@ def server? public def run(output_queue) if server? - @thread = Thread.current @client_threads = [] - loop do + while !stop? # Start a new thread for each connection. - begin - @client_threads << Thread.start(@server_socket.accept) do |s| - # TODO(sissel): put this block in its own method. - - @logger.debug("Accepted connection", - :server => "#{@path}") - begin - handle_socket(s, output_queue) - rescue Interrupted - s.close rescue nil - end - end # Thread.start - rescue IOError, Interrupted - if @interrupted - # Intended shutdown, get out of the loop - @server_socket.close - @client_threads.each do |thread| - thread.raise(IOError.new) - end - break - else - # Else it was a genuine IOError caused by something else, so propagate it up.. - raise - end + @client_threads << Thread.start(@server_socket.accept) do |s| + @logger.debug("Accepted connection", :server => "#{@path}") + handle_socket(s, output_queue) end - end # loop + end else - loop do - client_socket = UNIXSocket.new(@path) - client_socket.instance_eval { class << self; include ::LogStash::Util::SocketPeer end } + while !stop? + @client_socket = UNIXSocket.new(@path) + @client_socket.instance_eval { class << self; include ::LogStash::Util::SocketPeer end } @logger.debug("Opened connection", :client => @path) - handle_socket(client_socket, output_queue) - end # loop + handle_socket(@client_socket, output_queue) + end end end # def run public - def teardown + def stop if server? File.unlink(@path) - @interrupted = true - @thread.raise(Interrupted.new) + @server_socket.close + else + @client_socket.close end - end # def teardown + end # def stop end # class LogStash::Inputs::Unix diff --git a/spec/inputs/unix_spec.rb b/spec/inputs/unix_spec.rb index dc75e63..32be8ba 100644 --- a/spec/inputs/unix_spec.rb +++ b/spec/inputs/unix_spec.rb @@ -12,4 +12,40 @@ expect { plugin.register }.to_not raise_error end + describe "when interrupting the plugin" do + + context "#server" do + it_behaves_like "an interruptible input plugin" do + let(:config) { { "path" => tempfile.path, "force_unlink" => true } } + end + end + + context "#client" do + let(:tempfile) { "/tmp/sock#{rand(65532)}" } + let(:config) { { "path" => tempfile, "mode" => "client" } } + let(:unix_socket) { UnixSocketHelper.new.new_socket(tempfile) } + let(:run_forever) { true } + + before(:each) do + unix_socket.loop(run_forever) + end + + after(:each) do + unix_socket.close + end + + context "when the unix socket has data to be read" do + it_behaves_like "an interruptible input plugin" do + let(:run_forever) { true } + end + end + + context "when the unix socket has no data to be read" do + it_behaves_like "an interruptible input plugin" do + let(:run_forever) { false } + end + end + end + + end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 365dbbe..4e01beb 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,3 +1,33 @@ # encoding: utf-8 require "logstash/devutils/rspec/spec_helper" require 'logstash/inputs/unix' + +class UnixSocketHelper + + attr_reader :path + + def initialize + @socket = nil + end + + def new_socket(path) + @path = path + File.unlink if File.exists?(path) && File.socket?(path) + @socket = UNIXServer.new(path) + self + end + + def loop(forever=false) + @thread = Thread.new do + s = @socket.accept + s.puts "hi" while forever + end + self + end + + def close + @thread.kill + @socket.close + File.unlink(path) + end +end From 6508ef19ef3c1161e56a07bb6897d16ccc9507ab Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Tue, 22 Sep 2015 09:41:07 -0400 Subject: [PATCH 04/52] Update dependencies to logstash-core 2.0 and udate the major version --- logstash-input-unix.gemspec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/logstash-input-unix.gemspec b/logstash-input-unix.gemspec index 7fa4d26..a4c4c46 100644 --- a/logstash-input-unix.gemspec +++ b/logstash-input-unix.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-unix' - s.version = '1.0.0' + s.version = '2.0.0' s.licenses = ['Apache License (2.0)'] s.summary = "Read events over a UNIX socket." s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" @@ -20,7 +20,7 @@ Gem::Specification.new do |s| s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" } # Gem dependencies - s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0' + s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 3.0.0" s.add_runtime_dependency 'logstash-codec-line' s.add_development_dependency 'logstash-devutils' From 0c64f31acd96f522eabbe78a741f36a6b7df91a9 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Tue, 22 Sep 2015 13:14:38 -0400 Subject: [PATCH 05/52] Declare a dependency on logstash-core snapshot instead of the release version --- logstash-input-unix.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash-input-unix.gemspec b/logstash-input-unix.gemspec index a4c4c46..8b6ae66 100644 --- a/logstash-input-unix.gemspec +++ b/logstash-input-unix.gemspec @@ -20,7 +20,7 @@ Gem::Specification.new do |s| s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" } # Gem dependencies - s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 3.0.0" + s.add_runtime_dependency "logstash-core", "~> 2.0.0.snapshot" s.add_runtime_dependency 'logstash-codec-line' s.add_development_dependency 'logstash-devutils' From 1667ee4ef735f82035788161568aefd7327339aa Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Wed, 7 Oct 2015 11:11:23 -0400 Subject: [PATCH 06/52] update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e69de29..53045b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +## 2.0.0 + - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, + instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895 + - Dependency on logstash-core update to 2.0 + From 9cf9a7640afcca001d7107b1a514d27e4af99b54 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Wed, 7 Oct 2015 11:16:14 -0400 Subject: [PATCH 07/52] minor version bump --- logstash-input-unix.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash-input-unix.gemspec b/logstash-input-unix.gemspec index 8b6ae66..c8029e5 100644 --- a/logstash-input-unix.gemspec +++ b/logstash-input-unix.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-unix' - s.version = '2.0.0' + s.version = '2.0.1' s.licenses = ['Apache License (2.0)'] s.summary = "Read events over a UNIX socket." s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" From 7abea465d0e33f024d856f0abead01cf05ebde43 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 8 Oct 2015 09:39:11 -0400 Subject: [PATCH 08/52] dependency logstash-core >= 2.0.0.snapshot < 3.0.0 --- logstash-input-unix.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash-input-unix.gemspec b/logstash-input-unix.gemspec index c8029e5..ba6dfce 100644 --- a/logstash-input-unix.gemspec +++ b/logstash-input-unix.gemspec @@ -20,7 +20,7 @@ Gem::Specification.new do |s| s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" } # Gem dependencies - s.add_runtime_dependency "logstash-core", "~> 2.0.0.snapshot" + s.add_runtime_dependency "logstash-core", ">= 2.0.0.snapshot", "< 3.0.0" s.add_runtime_dependency 'logstash-codec-line' s.add_development_dependency 'logstash-devutils' From 5981b148909ed6da5a4b8365bb466fe65d69d0d2 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Wed, 7 Oct 2015 17:57:13 +0100 Subject: [PATCH 09/52] ensure plugin works with abort_on_exception --- CHANGELOG.md | 3 +++ lib/logstash/inputs/unix.rb | 4 ++++ logstash-input-unix.gemspec | 2 +- spec/spec_helper.rb | 8 ++++++-- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53045b4..f7f5484 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 2.0.2 + - Make plugin and spec work when Thread.abort_on_exception is true + ## 2.0.0 - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895 diff --git a/lib/logstash/inputs/unix.rb b/lib/logstash/inputs/unix.rb index c9bb55b..a5a2676 100644 --- a/lib/logstash/inputs/unix.rb +++ b/lib/logstash/inputs/unix.rb @@ -125,6 +125,10 @@ def run(output_queue) handle_socket(@client_socket, output_queue) end end + rescue IOError + # if stop is called during @server_socket.accept + # the thread running `run` will raise an IOError + # We catch IOError here and do nothing, just let the method terminate end # def run public diff --git a/logstash-input-unix.gemspec b/logstash-input-unix.gemspec index ba6dfce..16e1d7b 100644 --- a/logstash-input-unix.gemspec +++ b/logstash-input-unix.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-unix' - s.version = '2.0.1' + s.version = '2.0.2' s.licenses = ['Apache License (2.0)'] s.summary = "Read events over a UNIX socket." s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 4e01beb..1e0f20d 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -19,8 +19,12 @@ def new_socket(path) def loop(forever=false) @thread = Thread.new do - s = @socket.accept - s.puts "hi" while forever + begin + s = @socket.accept + s.puts "hi" while forever + rescue Errno::EPIPE + # ... + end end self end From f4e886dc2ea2c3215b413996f3fa54cd02424299 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Wed, 14 Oct 2015 14:11:28 +0100 Subject: [PATCH 10/52] ensure spec_helper handles the plugin shutdown gracefully Fixes #14 --- logstash-input-unix.gemspec | 2 +- spec/spec_helper.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/logstash-input-unix.gemspec b/logstash-input-unix.gemspec index 16e1d7b..a2fbb07 100644 --- a/logstash-input-unix.gemspec +++ b/logstash-input-unix.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-unix' - s.version = '2.0.2' + s.version = '2.0.3' s.licenses = ['Apache License (2.0)'] s.summary = "Read events over a UNIX socket." s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 1e0f20d..f2c640f 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -22,7 +22,7 @@ def loop(forever=false) begin s = @socket.accept s.puts "hi" while forever - rescue Errno::EPIPE + rescue Errno::EPIPE, Errno::ECONNRESET # ... end end From 8cb4051b56aca4bbecb2ec35f7f6c25cae9437f4 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Wed, 14 Oct 2015 14:02:29 -0400 Subject: [PATCH 11/52] dependency logstash-core >= 2.0.0.beta2 < 3.0.0 --- logstash-input-unix.gemspec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/logstash-input-unix.gemspec b/logstash-input-unix.gemspec index a2fbb07..781eb31 100644 --- a/logstash-input-unix.gemspec +++ b/logstash-input-unix.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-unix' - s.version = '2.0.3' + s.version = '2.0.4' s.licenses = ['Apache License (2.0)'] s.summary = "Read events over a UNIX socket." s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" @@ -20,7 +20,7 @@ Gem::Specification.new do |s| s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" } # Gem dependencies - s.add_runtime_dependency "logstash-core", ">= 2.0.0.snapshot", "< 3.0.0" + s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0" s.add_runtime_dependency 'logstash-codec-line' s.add_development_dependency 'logstash-devutils' From e3373505a694e2ce14e2e42c0b75fb59c8eab790 Mon Sep 17 00:00:00 2001 From: Pere Urbon-Bayes Date: Thu, 22 Oct 2015 16:08:06 +0200 Subject: [PATCH 12/52] Add a CI badge to show the build status --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 3a070f9..0e036fb 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # Logstash Plugin +[![Build +Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-unix-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-unix-unit/) + This is a plugin for [Logstash](https://github.com/elastic/logstash). It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way. From 82f0d60e10f30d8bffcff093e3b2f4587e3a7432 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Fri, 18 Mar 2016 14:23:56 -0400 Subject: [PATCH 13/52] dependency logstash-core > 2.0.0 < 6.0.0.alpha1 --- CHANGELOG.md | 2 ++ logstash-input-unix.gemspec | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7f5484..a647a27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +# 2.0.5 + - New dependency requirements for logstash-core for the 5.0 release ## 2.0.2 - Make plugin and spec work when Thread.abort_on_exception is true diff --git a/logstash-input-unix.gemspec b/logstash-input-unix.gemspec index 781eb31..6c42cfe 100644 --- a/logstash-input-unix.gemspec +++ b/logstash-input-unix.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-unix' - s.version = '2.0.4' + s.version = '2.0.5' s.licenses = ['Apache License (2.0)'] s.summary = "Read events over a UNIX socket." s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" @@ -20,7 +20,7 @@ Gem::Specification.new do |s| s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" } # Gem dependencies - s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0" + s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 6.0.0.alpha1" s.add_runtime_dependency 'logstash-codec-line' s.add_development_dependency 'logstash-devutils' From ac6e29e0aaa6cdec9ad113e1ce9c819dd709a97d Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Wed, 23 Mar 2016 21:36:44 -0400 Subject: [PATCH 14/52] dependency logstash-core-plugin-api ~> 1.0 --- CHANGELOG.md | 2 ++ logstash-input-unix.gemspec | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a647a27..c613871 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +# 2.0.6 + - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash # 2.0.5 - New dependency requirements for logstash-core for the 5.0 release ## 2.0.2 diff --git a/logstash-input-unix.gemspec b/logstash-input-unix.gemspec index 6c42cfe..5b1a82f 100644 --- a/logstash-input-unix.gemspec +++ b/logstash-input-unix.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-unix' - s.version = '2.0.5' + s.version = '2.0.6' s.licenses = ['Apache License (2.0)'] s.summary = "Read events over a UNIX socket." s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" @@ -20,7 +20,7 @@ Gem::Specification.new do |s| s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" } # Gem dependencies - s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 6.0.0.alpha1" + s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0" s.add_runtime_dependency 'logstash-codec-line' s.add_development_dependency 'logstash-devutils' From 169e2216220c648b5870619cf9c153fe7ec17bca Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Fri, 8 Apr 2016 14:38:17 -0400 Subject: [PATCH 15/52] adding .travis.yml --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..350c4eb --- /dev/null +++ b/.travis.yml @@ -0,0 +1,7 @@ +sudo: false +language: ruby +cache: bundler +rvm: + - jruby-1.7.23 +script: + - bundle exec rspec spec From 2aeb7545165b3dfa58ce99dfbe459dfac50035a7 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Fri, 8 Apr 2016 15:20:41 -0400 Subject: [PATCH 16/52] adding travis badge --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 0e036fb..0419ccd 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # Logstash Plugin -[![Build -Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-unix-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-unix-unit/) +[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-input-unix.svg)](https://travis-ci.org/logstash-plugins/logstash-input-unix) This is a plugin for [Logstash](https://github.com/elastic/logstash). From e800416a6e5843c3096eb4da2e2b324f91dccd40 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Wed, 20 Apr 2016 09:42:09 -0400 Subject: [PATCH 17/52] License 2016 and new plugin installing command --- .github/CONTRIBUTING.md | 65 ++++++++++++++++++++++++++++++++ .github/ISSUE_TEMPLATE.md | 9 +++++ .github/PULL_REQUEST_TEMPLATE.md | 1 + LICENSE | 2 +- README.md | 12 +++++- logstash-input-unix.gemspec | 2 +- 6 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 .github/CONTRIBUTING.md create mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 0000000..d325328 --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,65 @@ +# Contributing to Logstash + +All contributions are welcome: ideas, patches, documentation, bug reports, +complaints, etc! + +Programming is not a required skill, and there are many ways to help out! +It is more important to us that you are able to contribute. + +That said, some basic guidelines, which you are free to ignore :) + +## Want to learn? + +Want to lurk about and see what others are doing with Logstash? + +* The irc channel (#logstash on irc.freenode.org) is a good place for this +* The [forum](https://discuss.elastic.co/c/logstash) is also + great for learning from others. + +## Got Questions? + +Have a problem you want Logstash to solve for you? + +* You can ask a question in the [forum](https://discuss.elastic.co/c/logstash) +* Alternately, you are welcome to join the IRC channel #logstash on +irc.freenode.org and ask for help there! + +## Have an Idea or Feature Request? + +* File a ticket on [GitHub](https://github.com/elastic/logstash/issues). Please remember that GitHub is used only for issues and feature requests. If you have a general question, the [forum](https://discuss.elastic.co/c/logstash) or IRC would be the best place to ask. + +## Something Not Working? Found a Bug? + +If you think you found a bug, it probably is a bug. + +* If it is a general Logstash or a pipeline issue, file it in [Logstash GitHub](https://github.com/elasticsearch/logstash/issues) +* If it is specific to a plugin, please file it in the respective repository under [logstash-plugins](https://github.com/logstash-plugins) +* or ask the [forum](https://discuss.elastic.co/c/logstash). + +# Contributing Documentation and Code Changes + +If you have a bugfix or new feature that you would like to contribute to +logstash, and you think it will take more than a few minutes to produce the fix +(ie; write code), it is worth discussing the change with the Logstash users and developers first! You can reach us via [GitHub](https://github.com/elastic/logstash/issues), the [forum](https://discuss.elastic.co/c/logstash), or via IRC (#logstash on freenode irc) +Please note that Pull Requests without tests will not be merged. If you would like to contribute but do not have experience with writing tests, please ping us on IRC/forum or create a PR and ask our help. + +## Contributing to plugins + +Check our [documentation](https://www.elastic.co/guide/en/logstash/current/contributing-to-logstash.html) on how to contribute to plugins or write your own! It is super easy! + +## Contribution Steps + +1. Test your changes! [Run](https://github.com/elastic/logstash#testing) the test suite +2. Please make sure you have signed our [Contributor License + Agreement](https://www.elastic.co/contributor-agreement/). We are not + asking you to assign copyright to us, but to give us the right to distribute + your code without restriction. We ask this of all contributors in order to + assure our users of the origin and continuing existence of the code. You + only need to sign the CLA once. +3. Send a pull request! Push your changes to your fork of the repository and + [submit a pull + request](https://help.github.com/articles/using-pull-requests). In the pull + request, describe what your changes do and mention any bugs/issues related + to the pull request. + + diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..c3cc91d --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,9 @@ +Please post all product and debugging questions on our [forum](https://discuss.elastic.co/c/logstash). Your questions will reach our wider community members there, and if we confirm that there is a bug, then we can open a new issue here. + +For all general issues, please provide the following details for fast resolution: + +- Version: +- Operating System: +- Config File (if you have sensitive info, please remove it): +- Sample Data: +- Steps to Reproduce: diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..a153827 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1 @@ +Thanks for contributing to Logstash! If you haven't already signed our CLA, here's a handy link: https://www.elastic.co/contributor-agreement/ diff --git a/LICENSE b/LICENSE index 8026afd..43976b7 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2012–2015 Elasticsearch +Copyright (c) 2012–2016 Elasticsearch Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index 0419ccd..ed712f8 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,12 @@ gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome" ``` - Install plugin ```sh +# Logstash 2.3 and higher +bin/logstah-plugin install --no-verify + +# Prior to Logstash 2.3 bin/plugin install --no-verify + ``` - Run Logstash with your plugin ```sh @@ -73,7 +78,12 @@ gem build logstash-filter-awesome.gemspec ``` - Install the plugin from the Logstash home ```sh -bin/plugin install /your/local/plugin/logstash-filter-awesome.gem +# Logstash 2.3 and higher +bin/logstah-plugin install --no-verify + +# Prior to Logstash 2.3 +bin/plugin install --no-verify + ``` - Start Logstash and proceed to test the plugin diff --git a/logstash-input-unix.gemspec b/logstash-input-unix.gemspec index 5b1a82f..399dc2c 100644 --- a/logstash-input-unix.gemspec +++ b/logstash-input-unix.gemspec @@ -4,7 +4,7 @@ Gem::Specification.new do |s| s.version = '2.0.6' s.licenses = ['Apache License (2.0)'] s.summary = "Read events over a UNIX socket." - s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" + s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" s.authors = ["Elastic"] s.email = 'info@elastic.co' s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html" From 3b9fbef91e85750e692b42da5944f1969af9f654 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Wed, 20 Apr 2016 13:59:15 -0400 Subject: [PATCH 18/52] Fix a typo in the command --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ed712f8..1cf9d90 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome" - Install plugin ```sh # Logstash 2.3 and higher -bin/logstah-plugin install --no-verify +bin/logstash-plugin install --no-verify # Prior to Logstash 2.3 bin/plugin install --no-verify @@ -79,7 +79,7 @@ gem build logstash-filter-awesome.gemspec - Install the plugin from the Logstash home ```sh # Logstash 2.3 and higher -bin/logstah-plugin install --no-verify +bin/logstash-plugin install --no-verify # Prior to Logstash 2.3 bin/plugin install --no-verify From e58fedae9c55c1eecb063f68783c336b137205b9 Mon Sep 17 00:00:00 2001 From: Suyog Rao Date: Tue, 3 May 2016 11:54:58 -0700 Subject: [PATCH 19/52] Update with new Event APIs --- .travis.yml | 10 +++++++--- Gemfile | 7 ++++++- lib/logstash/inputs/unix.rb | 4 ++-- logstash-input-unix.gemspec | 2 +- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 350c4eb..6b858e5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,11 @@ +--- sudo: false language: ruby cache: bundler rvm: - - jruby-1.7.23 -script: - - bundle exec rspec spec +- jruby-1.7.25 +script: +- bundle exec rspec spec +jdk: oraclejdk8 +before_install: +- git clone -b feature/event_interface https://github.com/elastic/logstash diff --git a/Gemfile b/Gemfile index d926697..b2d50bb 100644 --- a/Gemfile +++ b/Gemfile @@ -1,2 +1,7 @@ source 'https://rubygems.org/' -gemspec \ No newline at end of file +gemspec +gem "logstash-core", :path => "./logstash/logstash-core" +gem "logstash-core-plugin-api", :path => "./logstash/logstash-core-plugin-api" +gem "logstash-core-event-java", :path => "./logstash/logstash-core-event-java" +gem "logstash-devutils", :github => "elastic/logstash-devutils", :branch => "feature/plugin-api-2_0" +gem "logstash-codec-line", :github => "logstash-plugins/logstash-codec-line", :branch => "feature/plugin-api-2_0" diff --git a/lib/logstash/inputs/unix.rb b/lib/logstash/inputs/unix.rb index a5a2676..7d7e8be 100644 --- a/lib/logstash/inputs/unix.rb +++ b/lib/logstash/inputs/unix.rb @@ -82,8 +82,8 @@ def handle_socket(socket, output_queue) end @codec.decode(buf) do |event| decorate(event) - event["host"] = hostname - event["path"] = @path + event.set("host", hostname) + event.set("path", @path) output_queue << event end end diff --git a/logstash-input-unix.gemspec b/logstash-input-unix.gemspec index 399dc2c..1f1be81 100644 --- a/logstash-input-unix.gemspec +++ b/logstash-input-unix.gemspec @@ -20,7 +20,7 @@ Gem::Specification.new do |s| s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" } # Gem dependencies - s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0" + s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0" s.add_runtime_dependency 'logstash-codec-line' s.add_development_dependency 'logstash-devutils' From 6012ffd995625324501e06ddab2421f6ce68d407 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Fri, 6 May 2016 14:53:05 -0400 Subject: [PATCH 20/52] Mass update for 2.0 plugin api changes --- .travis.yml | 3 +-- CHANGELOG.md | 2 ++ Gemfile | 7 ++----- logstash-input-unix.gemspec | 2 +- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6b858e5..5c63af6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,5 +7,4 @@ rvm: script: - bundle exec rspec spec jdk: oraclejdk8 -before_install: -- git clone -b feature/event_interface https://github.com/elastic/logstash +before_install: [] diff --git a/CHANGELOG.md b/CHANGELOG.md index c613871..1e5fcee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## 3.0.0 + - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141 # 2.0.6 - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash # 2.0.5 diff --git a/Gemfile b/Gemfile index b2d50bb..2b03d18 100644 --- a/Gemfile +++ b/Gemfile @@ -1,7 +1,4 @@ source 'https://rubygems.org/' + +# Specify your gem's dependencies in logstash-mass_effect.gemspec gemspec -gem "logstash-core", :path => "./logstash/logstash-core" -gem "logstash-core-plugin-api", :path => "./logstash/logstash-core-plugin-api" -gem "logstash-core-event-java", :path => "./logstash/logstash-core-event-java" -gem "logstash-devutils", :github => "elastic/logstash-devutils", :branch => "feature/plugin-api-2_0" -gem "logstash-codec-line", :github => "logstash-plugins/logstash-codec-line", :branch => "feature/plugin-api-2_0" diff --git a/logstash-input-unix.gemspec b/logstash-input-unix.gemspec index 1f1be81..6632840 100644 --- a/logstash-input-unix.gemspec +++ b/logstash-input-unix.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-unix' - s.version = '2.0.6' + s.version = '3.0.0' s.licenses = ['Apache License (2.0)'] s.summary = "Read events over a UNIX socket." s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 45abb4269a97d487cbbd6e4241114d0e65d89516 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Fri, 6 May 2016 16:39:16 -0400 Subject: [PATCH 21/52] Make sure all the gems were published under jruby --- CHANGELOG.md | 2 ++ logstash-input-unix.gemspec | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e5fcee..3659d9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## 3.0.1 + - Republish all the gems under jruby. ## 3.0.0 - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141 # 2.0.6 diff --git a/logstash-input-unix.gemspec b/logstash-input-unix.gemspec index 6632840..d080f5a 100644 --- a/logstash-input-unix.gemspec +++ b/logstash-input-unix.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-unix' - s.version = '3.0.0' + s.version = '3.0.1' s.licenses = ['Apache License (2.0)'] s.summary = "Read events over a UNIX socket." s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From acdb515ca23bf7a0ff60d8c135b0ab78d46a6fb2 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Thu, 14 Jul 2016 12:25:22 +0100 Subject: [PATCH 22/52] dependency logstash-core-plugin-api >= 1.60 <= 2.99 --- CHANGELOG.md | 3 +++ logstash-input-unix.gemspec | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3659d9e..5681198 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.0.2 + - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99 + ## 3.0.1 - Republish all the gems under jruby. ## 3.0.0 diff --git a/logstash-input-unix.gemspec b/logstash-input-unix.gemspec index d080f5a..383888c 100644 --- a/logstash-input-unix.gemspec +++ b/logstash-input-unix.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-unix' - s.version = '3.0.1' + s.version = '3.0.2' s.licenses = ['Apache License (2.0)'] s.summary = "Read events over a UNIX socket." s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -20,7 +20,7 @@ Gem::Specification.new do |s| s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" } # Gem dependencies - s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0" + s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" s.add_runtime_dependency 'logstash-codec-line' s.add_development_dependency 'logstash-devutils' From 961213b0c2a000c99d0fb5af2e8741d0ddd48ff9 Mon Sep 17 00:00:00 2001 From: Lucas Bremgartner Date: Tue, 24 Jan 2017 11:03:32 +0100 Subject: [PATCH 23/52] Preserve values provided in `add_field` for `host` and `path`. See https://github.com/elastic/logstash/issues/6581 --- lib/logstash/inputs/unix.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/logstash/inputs/unix.rb b/lib/logstash/inputs/unix.rb index 7d7e8be..4e62fb4 100644 --- a/lib/logstash/inputs/unix.rb +++ b/lib/logstash/inputs/unix.rb @@ -82,8 +82,8 @@ def handle_socket(socket, output_queue) end @codec.decode(buf) do |event| decorate(event) - event.set("host", hostname) - event.set("path", @path) + event.set("host", hostname) if !event.include?("host") + event.set("path", @path) if !event.include?("path") output_queue << event end end From 15511a1da41cf5866ff869e00be484ca34b91a4e Mon Sep 17 00:00:00 2001 From: Lucas Bremgartner Date: Sat, 11 Feb 2017 01:44:35 +0100 Subject: [PATCH 24/52] Version bump (#21) cleanup negation with unless --- CHANGELOG.md | 3 +++ lib/logstash/inputs/unix.rb | 4 ++-- logstash-input-unix.gemspec | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5681198..63abff9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.0.3 + - Preserve values provided in `add_field` for `host` and `path`. + ## 3.0.2 - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99 diff --git a/lib/logstash/inputs/unix.rb b/lib/logstash/inputs/unix.rb index 4e62fb4..729ed99 100644 --- a/lib/logstash/inputs/unix.rb +++ b/lib/logstash/inputs/unix.rb @@ -82,8 +82,8 @@ def handle_socket(socket, output_queue) end @codec.decode(buf) do |event| decorate(event) - event.set("host", hostname) if !event.include?("host") - event.set("path", @path) if !event.include?("path") + event.set("host", hostname) unless event.include?("host") + event.set("path", @path) unless event.include?("path") output_queue << event end end diff --git a/logstash-input-unix.gemspec b/logstash-input-unix.gemspec index 383888c..82fa4a8 100644 --- a/logstash-input-unix.gemspec +++ b/logstash-input-unix.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-unix' - s.version = '3.0.2' + s.version = '3.0.3' s.licenses = ['Apache License (2.0)'] s.summary = "Read events over a UNIX socket." s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 1719ad5e2a6a838acfda55c42abec7e50c913331 Mon Sep 17 00:00:00 2001 From: Lucas Bremgartner Date: Sat, 11 Feb 2017 01:45:16 +0100 Subject: [PATCH 25/52] Catch IOError in stop (#18) If socket with @mode == client was closed by the client, an other call to @client_socket.close will raise an IOError. We catch this IOError, log a warning and let logstash stop without error. --- lib/logstash/inputs/unix.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/logstash/inputs/unix.rb b/lib/logstash/inputs/unix.rb index 729ed99..c061158 100644 --- a/lib/logstash/inputs/unix.rb +++ b/lib/logstash/inputs/unix.rb @@ -139,5 +139,9 @@ def stop else @client_socket.close end + rescue IOError + # if socket with @mode == client was closed by the client, an other call to @client_socket.close + # will raise an IOError. We catch IOError here and do nothing, just let logstash terminate + @logger.warn("Cloud not close socket while Logstash is shutting down. Socket already closed by the other party?", :path => @path) end # def stop end # class LogStash::Inputs::Unix From 73acd8c31aa3135eca201ff13864b12d2aded2eb Mon Sep 17 00:00:00 2001 From: Lucas Bremgartner Date: Tue, 14 Feb 2017 09:14:52 +0100 Subject: [PATCH 26/52] Wait for client socket to appear (#19) * Wait for client socket to appear More resilent in case of errors (handle error conditions). * Configurable socket not present retry interval if `mode` is `client` --- lib/logstash/inputs/unix.rb | 28 ++++++++++++++++++++++------ spec/inputs/unix_spec.rb | 13 +++++++++++++ 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/lib/logstash/inputs/unix.rb b/lib/logstash/inputs/unix.rb index c061158..efe9d02 100644 --- a/lib/logstash/inputs/unix.rb +++ b/lib/logstash/inputs/unix.rb @@ -32,6 +32,12 @@ class Interrupted < StandardError; end # `client` connects to a server. config :mode, :validate => ["server", "client"], :default => "server" + # Amount of time in seconds to wait if the socket file is not present, before retrying. + # Only positive values are allowed. + # + # This setting is only used if `mode` is `client`. + config :socket_not_present_retry_interval_seconds, :validate => :number, :required => true, :default => 5 + def initialize(*args) super(*args) end # def initialize @@ -61,6 +67,11 @@ def register :path => @path) raise end + else # client + if @socket_not_present_retry_interval_seconds < 0 + @logger.warn("Value #{@socket_not_present_retry_interval_seconds} for socket_not_present_retry_interval_seconds is not valid, using default value of 5 instead") + @socket_not_present_retry_interval_seconds = 5 + end end end # def register @@ -119,10 +130,15 @@ def run(output_queue) end else while !stop? - @client_socket = UNIXSocket.new(@path) - @client_socket.instance_eval { class << self; include ::LogStash::Util::SocketPeer end } - @logger.debug("Opened connection", :client => @path) - handle_socket(@client_socket, output_queue) + if File.socket?(@path) then + @client_socket = UNIXSocket.new(@path) + @client_socket.instance_eval { class << self; include ::LogStash::Util::SocketPeer end } + @logger.debug("Opened connection", :client => @path) + handle_socket(@client_socket, output_queue) + else + @logger.warn("Socket not present, wait for #{@subscription_retry_interval_seconds} seconds for socket to appear", :client => @path) + sleep @socket_not_present_retry_interval_seconds + end end end rescue IOError @@ -135,9 +151,9 @@ def run(output_queue) def stop if server? File.unlink(@path) - @server_socket.close + @server_socket.close unless @server_socket.nil? else - @client_socket.close + @client_socket.close unless @client_socket.nil? end rescue IOError # if socket with @mode == client was closed by the client, an other call to @client_socket.close diff --git a/spec/inputs/unix_spec.rb b/spec/inputs/unix_spec.rb index 32be8ba..a84dd20 100644 --- a/spec/inputs/unix_spec.rb +++ b/spec/inputs/unix_spec.rb @@ -12,6 +12,19 @@ expect { plugin.register }.to_not raise_error end + describe "when mode is client" do + + let(:mode) { "client" } + + context "if socket_not_present_retry_interval_seconds is out of bounds" do + it "should fallback to default value" do + plugin = LogStash::Plugin.lookup("input", "unix").new({ "path" => tempfile.path, "force_unlink" => true, "mode" => mode, "socket_not_present_retry_interval_seconds" => -1 }) + plugin.register + expect(plugin.instance_variable_get(:@socket_not_present_retry_interval_seconds)).to be 5 + end + end + end + describe "when interrupting the plugin" do context "#server" do From 30ed69b250d46d5f48fa493cbdbdcd162a45368c Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Fri, 28 Apr 2017 21:08:32 +0000 Subject: [PATCH 27/52] Initial doc move --- docs/index.asciidoc | 102 ++++++++++++++++++++++++++++++++++++ logstash-input-unix.gemspec | 2 +- 2 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 docs/index.asciidoc diff --git a/docs/index.asciidoc b/docs/index.asciidoc new file mode 100644 index 0000000..db9fc96 --- /dev/null +++ b/docs/index.asciidoc @@ -0,0 +1,102 @@ +:plugin: unix +:type: input + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: %VERSION% +:release_date: %RELEASE_DATE% +:changelog_url: %CHANGELOG_URL% +:include_path: ../../../logstash/docs/include +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="plugins-{type}-{plugin}"] + +=== Unix + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read events over a UNIX socket. + +Like `stdin` and `file` inputs, each event is assumed to be one line of text. + +Can either accept connections from clients or connect to a server, +depending on `mode`. + +[id="plugins-{type}s-{plugin}-options"] +==== Unix Input Configuration Options + +This plugin supports the following configuration options plus the <> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <> |<>|No +| <> |<>|No +| <> |<>, one of `["server", "client"]`|No +| <> |<>|Yes +| <> |<>|Yes +|======================================================================= + +Also see <> for a list of options supported by all +input plugins. + +  + +[id="plugins-{type}s-{plugin}-data_timeout"] +===== `data_timeout` + + * Value type is <> + * Default value is `-1` + +The 'read' timeout in seconds. If a particular connection is idle for +more than this timeout period, we will assume it is dead and close it. + +If you never want to timeout, use -1. + +[id="plugins-{type}s-{plugin}-force_unlink"] +===== `force_unlink` + + * Value type is <> + * Default value is `false` + +Remove socket file in case of EADDRINUSE failure + +[id="plugins-{type}s-{plugin}-mode"] +===== `mode` + + * Value can be any of: `server`, `client` + * Default value is `"server"` + +Mode to operate in. `server` listens for client connections, +`client` connects to a server. + +[id="plugins-{type}s-{plugin}-path"] +===== `path` + + * This is a required setting. + * Value type is <> + * There is no default value for this setting. + +When mode is `server`, the path to listen on. +When mode is `client`, the path to connect to. + +[id="plugins-{type}s-{plugin}-socket_not_present_retry_interval_seconds"] +===== `socket_not_present_retry_interval_seconds` + + * This is a required setting. + * Value type is <> + * Default value is `5` + +Amount of time in seconds to wait if the socket file is not present, before retrying. +Only positive values are allowed. + +This setting is only used if `mode` is `client`. + + + +include::{include_path}/{type}.asciidoc[] diff --git a/logstash-input-unix.gemspec b/logstash-input-unix.gemspec index 82fa4a8..8b90a60 100644 --- a/logstash-input-unix.gemspec +++ b/logstash-input-unix.gemspec @@ -11,7 +11,7 @@ Gem::Specification.new do |s| s.require_paths = ["lib"] # Files - s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT'] + s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"] # Tests s.test_files = s.files.grep(%r{^(test|spec|features)/}) From 327a6fa6aaef4092cceeb3500fa7f293a4ad03f4 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Fri, 26 May 2017 13:01:05 -0400 Subject: [PATCH 28/52] new build system for jruby9k --- .travis.yml | 19 +++++++++++++++---- Gemfile | 8 +++++++- ci/build.sh | 21 +++++++++++++++++++++ ci/setup.sh | 26 ++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 5 deletions(-) create mode 100755 ci/build.sh create mode 100755 ci/setup.sh diff --git a/.travis.yml b/.travis.yml index 5c63af6..f274087 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,9 +2,20 @@ sudo: false language: ruby cache: bundler +env: rvm: -- jruby-1.7.25 -script: -- bundle exec rspec spec + - jruby-1.7.25 +matrix: + include: + - rvm: jruby-1.7.25 + env: LOGSTASH_BRANCH=master + - rvm: jruby-1.7.25 + env: LOGSTASH_BRANCH=5.x + - rvm: jruby-9.1.9.0 + env: LOGSTASH_BRANCH=feature/9000 + allow_failures: + - rvm: jruby-9.1.9.0 + fast_finish: true +install: true +script: ci/build.sh jdk: oraclejdk8 -before_install: [] diff --git a/Gemfile b/Gemfile index 2b03d18..93e5e5d 100644 --- a/Gemfile +++ b/Gemfile @@ -1,4 +1,10 @@ source 'https://rubygems.org/' -# Specify your gem's dependencies in logstash-mass_effect.gemspec gemspec + +logstash_path = "../../logstash" + +if Dir.exist?(logstash_path) && ENV["LOGSTASH_SOURCE"] == 1 + gem 'logstash-core', :path => "#{logstash_path}/logstash-core" + gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api" +end diff --git a/ci/build.sh b/ci/build.sh new file mode 100755 index 0000000..076e908 --- /dev/null +++ b/ci/build.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# version: 1 +######################################################## +# +# AUTOMATICALLY GENERATED! DO NOT EDIT +# +######################################################## +set -e + +echo "Starting build process in: `pwd`" +./ci/setup.sh + +if [[ -f "ci/run.sh" ]]; then + echo "Running custom build script in: `pwd`/ci/run.sh" + ./ci/run.sh +else + echo "Running default build scripts in: `pwd`/ci/build.sh" + bundle install + bundle exec rake vendor + bundle exec rspec spec +fi diff --git a/ci/setup.sh b/ci/setup.sh new file mode 100755 index 0000000..835fa43 --- /dev/null +++ b/ci/setup.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# version: 1 +######################################################## +# +# AUTOMATICALLY GENERATED! DO NOT EDIT +# +######################################################## +set -e +if [ "$LOGSTASH_BRANCH" ]; then + echo "Building plugin using Logstash source" + BASE_DIR=`pwd` + echo "Checking out branch: $LOGSTASH_BRANCH" + git clone -b $LOGSTASH_BRANCH https://github.com/elastic/logstash.git ../../logstash --depth 1 + printf "Checked out Logstash revision: %s\n" "$(git -C ../../logstash rev-parse HEAD)" + cd ../../logstash + echo "Building plugins with Logstash version:" + cat versions.yml + echo "---" + # We need to build the jars for that specific version + echo "Running gradle assemble in: `pwd`" + ./gradlew assemble + cd $BASE_DIR + export LOGSTASH_SOURCE=1 +else + echo "Building plugin using released gems on rubygems" +fi From 8d36c1c37999eaa349f9936517cb2fe453291428 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Wed, 31 May 2017 16:44:59 -0400 Subject: [PATCH 29/52] Adjusting the build scripts to correctly load the logstash source and allow people to override it --- Gemfile | 5 +++-- ci/build.sh | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Gemfile b/Gemfile index 93e5e5d..32cc6fb 100644 --- a/Gemfile +++ b/Gemfile @@ -2,9 +2,10 @@ source 'https://rubygems.org/' gemspec -logstash_path = "../../logstash" +logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash" +use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1" -if Dir.exist?(logstash_path) && ENV["LOGSTASH_SOURCE"] == 1 +if Dir.exist?(logstash_path) && use_logstash_source gem 'logstash-core', :path => "#{logstash_path}/logstash-core" gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api" end diff --git a/ci/build.sh b/ci/build.sh index 076e908..06caffd 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -8,11 +8,11 @@ set -e echo "Starting build process in: `pwd`" -./ci/setup.sh +source ./ci/setup.sh if [[ -f "ci/run.sh" ]]; then echo "Running custom build script in: `pwd`/ci/run.sh" - ./ci/run.sh + source ./ci/run.sh else echo "Running default build scripts in: `pwd`/ci/build.sh" bundle install From 2fca1fff52c5c8b357c0f8e494131f34c63089ad Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Tue, 13 Jun 2017 09:10:38 -0400 Subject: [PATCH 30/52] update .travis.yml for jruby9k jobs --- .travis.yml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index f274087..59c937e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,19 +2,15 @@ sudo: false language: ruby cache: bundler -env: +env: rvm: - - jruby-1.7.25 +- jruby-1.7.25 matrix: include: - - rvm: jruby-1.7.25 - env: LOGSTASH_BRANCH=master - - rvm: jruby-1.7.25 - env: LOGSTASH_BRANCH=5.x - - rvm: jruby-9.1.9.0 - env: LOGSTASH_BRANCH=feature/9000 - allow_failures: - - rvm: jruby-9.1.9.0 + - rvm: jruby-9.1.10.0 + env: LOGSTASH_BRANCH=master + - rvm: jruby-1.7.25 + env: LOGSTASH_BRANCH=5.x fast_finish: true install: true script: ci/build.sh From 827940c5b03f33d897d1388b3f05ddde24e3e04d Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 15 Jun 2017 20:00:58 -0400 Subject: [PATCH 31/52] update plugin header for better search results --- docs/index.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index db9fc96..f456384 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -14,7 +14,7 @@ END - GENERATED VARIABLES, DO NOT EDIT! [id="plugins-{type}-{plugin}"] -=== Unix +=== Unix input plugin include::{include_path}/plugin_header.asciidoc[] From dc4842b9fc31b6e3c989f3599579ea8740ee7a00 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 22 Jun 2017 22:30:36 -0400 Subject: [PATCH 32/52] [skip ci] Updating the plugin doc --- docs/index.asciidoc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index f456384..f027931 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -7,7 +7,7 @@ START - GENERATED VARIABLES, DO NOT EDIT! :version: %VERSION% :release_date: %RELEASE_DATE% :changelog_url: %CHANGELOG_URL% -:include_path: ../../../logstash/docs/include +:include_path: ../../../../logstash/docs/include /////////////////////////////////////////// END - GENERATED VARIABLES, DO NOT EDIT! /////////////////////////////////////////// @@ -30,7 +30,7 @@ depending on `mode`. [id="plugins-{type}s-{plugin}-options"] ==== Unix Input Configuration Options -This plugin supports the following configuration options plus the <> described later. +This plugin supports the following configuration options plus the <> described later. [cols="<,<,<",options="header",] |======================================================================= @@ -42,7 +42,7 @@ This plugin supports the following configuration options plus the <> |<>|Yes |======================================================================= -Also see <> for a list of options supported by all +Also see <> for a list of options supported by all input plugins.   @@ -99,4 +99,5 @@ This setting is only used if `mode` is `client`. -include::{include_path}/{type}.asciidoc[] +[id="plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] \ No newline at end of file From 874da9ba8ebe8782a18bba583b4a6bf3143aea2c Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 22 Jun 2017 22:54:11 -0400 Subject: [PATCH 33/52] bump patch level for doc generation --- logstash-input-unix.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash-input-unix.gemspec b/logstash-input-unix.gemspec index 8b90a60..3a56654 100644 --- a/logstash-input-unix.gemspec +++ b/logstash-input-unix.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-unix' - s.version = '3.0.3' + s.version = '3.0.4' s.licenses = ['Apache License (2.0)'] s.summary = "Read events over a UNIX socket." s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 9e2130aa217d5bc7f2a3e62cdb8a29b0c8ca775f Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Mon, 26 Jun 2017 21:24:13 -0400 Subject: [PATCH 34/52] [skip ci] Updating the plugin id in the doc to match the index in the docbook --- docs/index.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index f027931..49687c1 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -12,7 +12,7 @@ START - GENERATED VARIABLES, DO NOT EDIT! END - GENERATED VARIABLES, DO NOT EDIT! /////////////////////////////////////////// -[id="plugins-{type}-{plugin}"] +[id="plugins-{type}s-{plugin}"] === Unix input plugin From a30a64d610ab1bde6771d3bbb45f2bc08a07b5a9 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Wed, 26 Jul 2017 12:38:09 +0100 Subject: [PATCH 35/52] on travis test against 5.6 and 6.0 logstash-core --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 59c937e..7af01f7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,9 +9,11 @@ matrix: include: - rvm: jruby-9.1.10.0 env: LOGSTASH_BRANCH=master + - rvm: jruby-9.1.10.0 + env: LOGSTASH_BRANCH=6.x - rvm: jruby-1.7.25 - env: LOGSTASH_BRANCH=5.x + env: LOGSTASH_BRANCH=5.6 fast_finish: true install: true script: ci/build.sh -jdk: oraclejdk8 +jdk: oraclejdk8 \ No newline at end of file From f31385066d8f869cec2c046be22959c7dcb00817 Mon Sep 17 00:00:00 2001 From: Jordan Sissel Date: Tue, 15 Aug 2017 10:01:03 -0700 Subject: [PATCH 36/52] Version bump For https://github.com/elastic/logstash/issues/7993 [ci skip] --- CHANGELOG.md | 3 +++ logstash-input-unix.gemspec | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63abff9..067e38a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.0.5 + - Fix some documentation issues + ## 3.0.3 - Preserve values provided in `add_field` for `host` and `path`. diff --git a/logstash-input-unix.gemspec b/logstash-input-unix.gemspec index 3a56654..5b6198b 100644 --- a/logstash-input-unix.gemspec +++ b/logstash-input-unix.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-unix' - s.version = '3.0.4' + s.version = '3.0.5' s.licenses = ['Apache License (2.0)'] s.summary = "Read events over a UNIX socket." s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From dc1944778f10f46741594a67f25fbaff730cc0cc Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Fri, 27 Oct 2017 17:24:57 -0500 Subject: [PATCH 37/52] Travis - add 6.0 build, remove default JRuby 1.7 build, bump RVM versions --- .travis.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7af01f7..1458a3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,18 +2,17 @@ sudo: false language: ruby cache: bundler -env: -rvm: -- jruby-1.7.25 matrix: include: - - rvm: jruby-9.1.10.0 + - rvm: jruby-9.1.13.0 env: LOGSTASH_BRANCH=master - - rvm: jruby-9.1.10.0 + - rvm: jruby-9.1.13.0 env: LOGSTASH_BRANCH=6.x - - rvm: jruby-1.7.25 + - rvm: jruby-9.1.13.0 + env: LOGSTASH_BRANCH=6.0 + - rvm: jruby-1.7.27 env: LOGSTASH_BRANCH=5.6 fast_finish: true install: true script: ci/build.sh -jdk: oraclejdk8 \ No newline at end of file +jdk: oraclejdk8 From a5b08420a377f8fdaf0ec20f423df828f347c3dc Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 7 Nov 2017 11:18:31 +0000 Subject: [PATCH 38/52] [skip ci] update gemspec summary --- CHANGELOG.md | 3 +++ logstash-input-unix.gemspec | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 067e38a..91c73a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.0.6 + - Update gemspec summary + ## 3.0.5 - Fix some documentation issues diff --git a/logstash-input-unix.gemspec b/logstash-input-unix.gemspec index 5b6198b..733895c 100644 --- a/logstash-input-unix.gemspec +++ b/logstash-input-unix.gemspec @@ -1,9 +1,9 @@ Gem::Specification.new do |s| s.name = 'logstash-input-unix' - s.version = '3.0.5' + s.version = '3.0.6' s.licenses = ['Apache License (2.0)'] - s.summary = "Read events over a UNIX socket." + s.summary = "Reads events over a UNIX socket" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" s.authors = ["Elastic"] s.email = 'info@elastic.co' From 41cb63a35cc1a3c180f2b78f733ca69274dfb482 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Mon, 8 Jan 2018 21:30:56 +0000 Subject: [PATCH 39/52] [skip ci] update license to 2018 --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 43976b7..2162c9b 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2012–2016 Elasticsearch +Copyright (c) 2012-2018 Elasticsearch Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From 63ac5314f0647a256e1931fe3530b7ae63a591cb Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 6 Apr 2018 23:50:48 +0100 Subject: [PATCH 40/52] set default_codec doc attribute --- CHANGELOG.md | 3 +++ docs/index.asciidoc | 5 ++++- logstash-input-unix.gemspec | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91c73a4..5ba0181 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.0.7 + - Docs: Set the default_codec doc attribute. + ## 3.0.6 - Update gemspec summary diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 49687c1..0624741 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -1,5 +1,6 @@ :plugin: unix :type: input +:default_codec: line /////////////////////////////////////////// START - GENERATED VARIABLES, DO NOT EDIT! @@ -100,4 +101,6 @@ This setting is only used if `mode` is `client`. [id="plugins-{type}s-{plugin}-common-options"] -include::{include_path}/{type}.asciidoc[] \ No newline at end of file +include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/logstash-input-unix.gemspec b/logstash-input-unix.gemspec index 733895c..ec54cb2 100644 --- a/logstash-input-unix.gemspec +++ b/logstash-input-unix.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-unix' - s.version = '3.0.6' + s.version = '3.0.7' s.licenses = ['Apache License (2.0)'] s.summary = "Reads events over a UNIX socket" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 579d36a489d1a11af8ed69cd28e6a9dfdae543d0 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Thu, 3 Jan 2019 15:26:13 -0500 Subject: [PATCH 41/52] pin bundler version to < 2 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 1458a3b..ea27df6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,3 +16,4 @@ matrix: install: true script: ci/build.sh jdk: oraclejdk8 +before_install: gem install bundler -v '< 2' From f406ec0d30bdf69e37f6c576cac1e97e1ca00c07 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Fri, 4 Jan 2019 11:27:39 -0500 Subject: [PATCH 42/52] [skip ci] Travis: update LOGSTASH_BRANCH from 6.[0..4] to 6.5 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ea27df6..746f145 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ matrix: - rvm: jruby-9.1.13.0 env: LOGSTASH_BRANCH=6.x - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=6.0 + env: LOGSTASH_BRANCH=6.5 - rvm: jruby-1.7.27 env: LOGSTASH_BRANCH=5.6 fast_finish: true From f16c3bd39854f815d428e18b52fd61a567f642d0 Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Wed, 13 Feb 2019 00:32:31 +0000 Subject: [PATCH 43/52] update matrix to include current targets [ci skip] --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 746f145..dc96273 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,9 +7,11 @@ matrix: - rvm: jruby-9.1.13.0 env: LOGSTASH_BRANCH=master - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=6.x + env: LOGSTASH_BRANCH=7.0 - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=6.5 + env: LOGSTASH_BRANCH=6.7 + - rvm: jruby-9.1.13.0 + env: LOGSTASH_BRANCH=6.6 - rvm: jruby-1.7.27 env: LOGSTASH_BRANCH=5.6 fast_finish: true From 407d56e4d2358f79e5f044ca5116ee20287d8acf Mon Sep 17 00:00:00 2001 From: Colin Surprenant Date: Tue, 6 Aug 2019 14:22:26 -0400 Subject: [PATCH 44/52] [skip ci] Travis: switch to openjdk8 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index dc96273..3b61247 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,5 +17,5 @@ matrix: fast_finish: true install: true script: ci/build.sh -jdk: oraclejdk8 +jdk: openjdk8 before_install: gem install bundler -v '< 2' From 7f6f36540d48c2250d04a92cf03e6fd2c849d856 Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Mon, 9 Mar 2020 11:22:03 +0100 Subject: [PATCH 45/52] Test: adjust for devutils 2.0 compat --- spec/inputs/unix_spec.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/inputs/unix_spec.rb b/spec/inputs/unix_spec.rb index a84dd20..d871897 100644 --- a/spec/inputs/unix_spec.rb +++ b/spec/inputs/unix_spec.rb @@ -1,5 +1,6 @@ # encoding: utf-8 require_relative "../spec_helper" +require "logstash/devutils/rspec/shared_examples" require "stud/temporary" require "tempfile" From 3face379edb86a9a53de67fe2bc943691b77877d Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 17 Mar 2020 15:41:56 +0000 Subject: [PATCH 46/52] [skip ci] updated apache license --- LICENSE | 209 +++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 199 insertions(+), 10 deletions(-) diff --git a/LICENSE b/LICENSE index 2162c9b..a80a3fd 100644 --- a/LICENSE +++ b/LICENSE @@ -1,13 +1,202 @@ -Copyright (c) 2012-2018 Elasticsearch -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ - http://www.apache.org/licenses/LICENSE-2.0 + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2020 Elastic and contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From 5635d065d01881ec1f9608f7154b70e09556e862 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 17 Mar 2020 22:42:26 +0000 Subject: [PATCH 47/52] move testing to centralized travis configuration --- .travis.yml | 23 ++--------------------- ci/build.sh | 21 --------------------- ci/setup.sh | 26 -------------------------- 3 files changed, 2 insertions(+), 68 deletions(-) delete mode 100755 ci/build.sh delete mode 100755 ci/setup.sh diff --git a/.travis.yml b/.travis.yml index 3b61247..a50fc73 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,21 +1,2 @@ ---- -sudo: false -language: ruby -cache: bundler -matrix: - include: - - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=master - - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=7.0 - - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=6.7 - - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=6.6 - - rvm: jruby-1.7.27 - env: LOGSTASH_BRANCH=5.6 - fast_finish: true -install: true -script: ci/build.sh -jdk: openjdk8 -before_install: gem install bundler -v '< 2' +import: +- logstash-plugins/.ci:travis/travis.yml@1.x \ No newline at end of file diff --git a/ci/build.sh b/ci/build.sh deleted file mode 100755 index 06caffd..0000000 --- a/ci/build.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# version: 1 -######################################################## -# -# AUTOMATICALLY GENERATED! DO NOT EDIT -# -######################################################## -set -e - -echo "Starting build process in: `pwd`" -source ./ci/setup.sh - -if [[ -f "ci/run.sh" ]]; then - echo "Running custom build script in: `pwd`/ci/run.sh" - source ./ci/run.sh -else - echo "Running default build scripts in: `pwd`/ci/build.sh" - bundle install - bundle exec rake vendor - bundle exec rspec spec -fi diff --git a/ci/setup.sh b/ci/setup.sh deleted file mode 100755 index 835fa43..0000000 --- a/ci/setup.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# version: 1 -######################################################## -# -# AUTOMATICALLY GENERATED! DO NOT EDIT -# -######################################################## -set -e -if [ "$LOGSTASH_BRANCH" ]; then - echo "Building plugin using Logstash source" - BASE_DIR=`pwd` - echo "Checking out branch: $LOGSTASH_BRANCH" - git clone -b $LOGSTASH_BRANCH https://github.com/elastic/logstash.git ../../logstash --depth 1 - printf "Checked out Logstash revision: %s\n" "$(git -C ../../logstash rev-parse HEAD)" - cd ../../logstash - echo "Building plugins with Logstash version:" - cat versions.yml - echo "---" - # We need to build the jars for that specific version - echo "Running gradle assemble in: `pwd`" - ./gradlew assemble - cd $BASE_DIR - export LOGSTASH_SOURCE=1 -else - echo "Building plugin using released gems on rubygems" -fi From d60cb06a6b4d9581903df1f52edb4743962d1502 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Mon, 4 Jan 2021 12:49:23 +0000 Subject: [PATCH 48/52] [skip ci] update travis ci badge from .org to .com --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1cf9d90..9ed1041 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Logstash Plugin -[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-input-unix.svg)](https://travis-ci.org/logstash-plugins/logstash-input-unix) +[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-input-unix.svg)](https://travis-ci.com/logstash-plugins/logstash-input-unix) This is a plugin for [Logstash](https://github.com/elastic/logstash). From 6f8d7f789237d6f7692fdca84bec58fadf16eb0b Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Mon, 22 Nov 2021 08:59:28 +0100 Subject: [PATCH 49/52] Feat: adjust fields for ECS compatibility (#28) Co-authored-by: Karen Metts <35154725+karenzone@users.noreply.github.com> --- CHANGELOG.md | 3 ++ docs/index.asciidoc | 56 ++++++++++++++++++++++++ lib/logstash/inputs/unix.rb | 49 +++++++++++++-------- logstash-input-unix.gemspec | 5 ++- spec/inputs/unix_spec.rb | 86 +++++++++++++++++++++++++------------ spec/spec_helper.rb | 9 ++-- 6 files changed, 155 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ba0181..faf5e27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.1.0 + - Feat: adjust fields for ECS compatibility [#28](https://github.com/logstash-plugins/logstash-input-unix/pull/28) + ## 3.0.7 - Docs: Set the default_codec doc attribute. diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 0624741..f63de43 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -28,6 +28,23 @@ Like `stdin` and `file` inputs, each event is assumed to be one line of text. Can either accept connections from clients or connect to a server, depending on `mode`. +[id="plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin adds extra fields about the event's source. +Configure the <> option if you want +to ensure that these fields are compatible with {ecs-ref}[ECS]. + +These fields are added after the event has been decoded by the appropriate codec, +and will not overwrite existing values. + +|======== +| ECS Disabled | ECS v1 , v8 | Description + +| `host` | `[host][name]` | The name of the {ls} host that processed the event +| `path` | `[file][path]` | The socket path configured in the plugin +|======== + [id="plugins-{type}s-{plugin}-options"] ==== Unix Input Configuration Options @@ -37,6 +54,7 @@ This plugin supports the following configuration options plus the <> |<>|No +| <> |<>|No | <> |<>|No | <> |<>, one of `["server", "client"]`|No | <> |<>|Yes @@ -59,6 +77,44 @@ more than this timeout period, we will assume it is dead and close it. If you never want to timeout, use -1. +[id="plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is <> + * Supported values are: + ** `disabled`: uses backwards compatible field names, such as `[host]` + ** `v1`, `v8`: uses fields that are compatible with ECS, such as `[host][name]` + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. +See <> for detailed information. + + +**Sample output: ECS enabled** +[source,ruby] +----- +{ + "@timestamp" => 2021-11-16T13:20:06.308Z, + "file" => { + "path" => "/tmp/sock41299" + }, + "host" => { + "name" => "deus-ex-machina" + }, + "message" => "foo" +} +----- + +**Sample output: ECS disabled** +[source,ruby] +----- +{ + "@timestamp" => 2021-11-16T13:20:06.308Z, + "path" => "/tmp/sock41299", + "host" => "deus-ex-machina", + "message" => "foo" +} +----- + [id="plugins-{type}s-{plugin}-force_unlink"] ===== `force_unlink` diff --git a/lib/logstash/inputs/unix.rb b/lib/logstash/inputs/unix.rb index efe9d02..0baadb2 100644 --- a/lib/logstash/inputs/unix.rb +++ b/lib/logstash/inputs/unix.rb @@ -3,6 +3,8 @@ require "logstash/namespace" require "logstash/util/socket_peer" +require 'logstash/plugin_mixins/ecs_compatibility_support' + # Read events over a UNIX socket. # # Like `stdin` and `file` inputs, each event is assumed to be one line of text. @@ -10,7 +12,11 @@ # Can either accept connections from clients or connect to a server, # depending on `mode`. class LogStash::Inputs::Unix < LogStash::Inputs::Base + + include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1) + class Interrupted < StandardError; end + config_name "unix" default :codec, "line" @@ -38,8 +44,11 @@ class Interrupted < StandardError; end # This setting is only used if `mode` is `client`. config :socket_not_present_retry_interval_seconds, :validate => :number, :required => true, :default => 5 - def initialize(*args) - super(*args) + def initialize(*params) + super + + @host_name_field = ecs_select[disabled: 'host', v1: '[host][name]'] + @file_path_field = ecs_select[disabled: 'path', v1: '[file][path]'] end # def initialize public @@ -48,7 +57,7 @@ def register require "timeout" if server? - @logger.info("Starting unix input listener", :address => "#{@path}", :force_unlink => "#{@force_unlink}") + @logger.info("Starting unix input listener", :address => @path, :force_unlink => @force_unlink) begin @server_socket = UNIXServer.new(@path) rescue Errno::EADDRINUSE, IOError @@ -58,18 +67,16 @@ def register @server_socket = UNIXServer.new(@path) return rescue Errno::EADDRINUSE, IOError - @logger.error("!!!Could not start UNIX server: Address in use", - :path => @path) + @logger.error("Could not start UNIX server: address in use", :path => @path) raise end end - @logger.error("Could not start UNIX server: Address in use", - :path => @path) + @logger.error("Could not start UNIX server: address in use", :path => @path) raise end else # client - if @socket_not_present_retry_interval_seconds < 0 - @logger.warn("Value #{@socket_not_present_retry_interval_seconds} for socket_not_present_retry_interval_seconds is not valid, using default value of 5 instead") + if socket_not_present_retry_interval_seconds < 0 + @logger.warn("Value #{socket_not_present_retry_interval_seconds} for socket_not_present_retry_interval_seconds is not valid, using default value of 5 instead") @socket_not_present_retry_interval_seconds = 5 end end @@ -93,16 +100,20 @@ def handle_socket(socket, output_queue) end @codec.decode(buf) do |event| decorate(event) - event.set("host", hostname) unless event.include?("host") - event.set("path", @path) unless event.include?("path") + event.set(@host_name_field, hostname) unless event.include?(@host_name_field) + event.set(@file_path_field, @path) unless event.include?(@file_path_field) output_queue << event end end - rescue => e - @logger.debug("Closing connection", :path => @path, :exception => e, :backtrace => e.backtrace) rescue Timeout::Error - @logger.debug("Closing connection after read timeout", :path => @path) - end # begin + @logger.info("Closing connection after read timeout", :path => @path) + rescue => e + if @logger.debug? + @logger.debug("Closing connection", :path => @path, :exception => e, :backtrace => e.backtrace) + else + @logger.info("Closing connection", :path => @path, :exception => e) + end + end ensure begin @@ -124,7 +135,7 @@ def run(output_queue) while !stop? # Start a new thread for each connection. @client_threads << Thread.start(@server_socket.accept) do |s| - @logger.debug("Accepted connection", :server => "#{@path}") + @logger.debug("Accepted connection", :server => @path) handle_socket(s, output_queue) end end @@ -136,8 +147,8 @@ def run(output_queue) @logger.debug("Opened connection", :client => @path) handle_socket(@client_socket, output_queue) else - @logger.warn("Socket not present, wait for #{@subscription_retry_interval_seconds} seconds for socket to appear", :client => @path) - sleep @socket_not_present_retry_interval_seconds + @logger.warn("Socket not present, wait for #{socket_not_present_retry_interval_seconds} seconds for socket to appear", :client => @path) + sleep socket_not_present_retry_interval_seconds end end end @@ -158,6 +169,6 @@ def stop rescue IOError # if socket with @mode == client was closed by the client, an other call to @client_socket.close # will raise an IOError. We catch IOError here and do nothing, just let logstash terminate - @logger.warn("Cloud not close socket while Logstash is shutting down. Socket already closed by the other party?", :path => @path) + @logger.warn("Could not close socket while Logstash is shutting down. Socket already closed by the other party?", :path => @path) end # def stop end # class LogStash::Inputs::Unix diff --git a/logstash-input-unix.gemspec b/logstash-input-unix.gemspec index ec54cb2..666765b 100644 --- a/logstash-input-unix.gemspec +++ b/logstash-input-unix.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-unix' - s.version = '3.0.7' + s.version = '3.1.0' s.licenses = ['Apache License (2.0)'] s.summary = "Reads events over a UNIX socket" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -21,8 +21,9 @@ Gem::Specification.new do |s| # Gem dependencies s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" - + s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.3' s.add_runtime_dependency 'logstash-codec-line' + s.add_development_dependency 'logstash-devutils' end diff --git a/spec/inputs/unix_spec.rb b/spec/inputs/unix_spec.rb index d871897..a1a499c 100644 --- a/spec/inputs/unix_spec.rb +++ b/spec/inputs/unix_spec.rb @@ -1,65 +1,95 @@ # encoding: utf-8 require_relative "../spec_helper" require "logstash/devutils/rspec/shared_examples" +require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper' require "stud/temporary" require "tempfile" describe LogStash::Inputs::Unix do - let(:tempfile) { Tempfile.new("/tmp/foo") } + let(:config) { { 'path' => tempfile.path, 'socket_not_present_retry_interval_seconds' => 1, 'force_unlink' => true } } + let(:tempfile) { Tempfile.new("unix-input-test") } + + subject(:input) { described_class.new(config) } it "should register without errors" do - plugin = LogStash::Plugin.lookup("input", "unix").new({ "path" => tempfile.path, "force_unlink" => true }) - expect { plugin.register }.to_not raise_error + expect { subject.register }.to_not raise_error end describe "when mode is client" do - let(:mode) { "client" } + let(:config) { super().merge("mode" => 'client', "socket_not_present_retry_interval_seconds" => -1) } context "if socket_not_present_retry_interval_seconds is out of bounds" do it "should fallback to default value" do - plugin = LogStash::Plugin.lookup("input", "unix").new({ "path" => tempfile.path, "force_unlink" => true, "mode" => mode, "socket_not_present_retry_interval_seconds" => -1 }) - plugin.register - expect(plugin.instance_variable_get(:@socket_not_present_retry_interval_seconds)).to be 5 + subject.register + expect( subject.socket_not_present_retry_interval_seconds ).to eql 5 end end end - describe "when interrupting the plugin" do + context "#server" do + it_behaves_like "an interruptible input plugin" do + let(:config) { super().merge "mode" => 'server' } + end + end - context "#server" do - it_behaves_like "an interruptible input plugin" do - let(:config) { { "path" => tempfile.path, "force_unlink" => true } } - end + context "#client", :ecs_compatibility_support do + let(:temp_path) { "/tmp/sock#{rand(65532)}" } + let(:config) { super().merge "path" => temp_path, "mode" => "client" } + let(:unix_socket) { UnixSocketHelper.new('foo').new_socket(temp_path) } + let(:run_forever) { true } + + before(:each) do + unix_socket.loop(run_forever) end - context "#client" do - let(:tempfile) { "/tmp/sock#{rand(65532)}" } - let(:config) { { "path" => tempfile, "mode" => "client" } } - let(:unix_socket) { UnixSocketHelper.new.new_socket(tempfile) } - let(:run_forever) { true } + after(:each) do + unix_socket.close + end - before(:each) do - unix_socket.loop(run_forever) + context "when the unix socket has data to be read" do + it_behaves_like "an interruptible input plugin" do + let(:run_forever) { true } end + end - after(:each) do - unix_socket.close + context "when the unix socket has no data to be read" do + it_behaves_like "an interruptible input plugin" do + let(:run_forever) { false } end + end + + ecs_compatibility_matrix(:disabled, :v1, :v8) do |ecs_select| + + let(:config) { super().merge 'ecs_compatibility' => ecs_compatibility } - context "when the unix socket has data to be read" do - it_behaves_like "an interruptible input plugin" do - let(:run_forever) { true } + let(:queue) { java.util.Vector.new } + + it 'generates events with host, path and message set' do + subject.register + Thread.new(subject, queue) { |subject, queue| subject.run(queue) } + try(10) do + expect( queue.size ).to_not eql 0 end - end + subject.do_stop # stop the plugin - context "when the unix socket has no data to be read" do - it_behaves_like "an interruptible input plugin" do - let(:run_forever) { false } + event = queue.first + + if ecs_select.active_mode == :disabled + expect( event.get('host') ).to be_a String + expect( event.get('path') ).to eql temp_path + else + expect( event.get('[host][name]') ).to be_a String + expect( event.get('[file][path]') ).to eql temp_path + expect( event.include?('path') ).to be false end + + expect( event.get('message') ).to eql 'foo' end + end end + end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index f2c640f..f4e20c0 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -6,8 +6,9 @@ class UnixSocketHelper attr_reader :path - def initialize + def initialize(line = 'hi!') @socket = nil + @line = line end def new_socket(path) @@ -21,9 +22,9 @@ def loop(forever=false) @thread = Thread.new do begin s = @socket.accept - s.puts "hi" while forever - rescue Errno::EPIPE, Errno::ECONNRESET - # ... + s.puts @line while forever + rescue Errno::EPIPE, Errno::ECONNRESET => e + warn e.inspect if $VERBOSE end end self From cadcdcf9e225873294dcc4f13c930b501cc802f0 Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Thu, 23 Dec 2021 14:19:00 +0100 Subject: [PATCH 50/52] Fix: unable to stop plugin (on older versions of LS) (#29) --- CHANGELOG.md | 4 ++++ lib/logstash/inputs/unix.rb | 31 +++++++++++++------------------ logstash-input-unix.gemspec | 2 +- spec/inputs/unix_spec.rb | 25 +++++++++++++++++++++++-- spec/spec_helper.rb | 3 +-- 5 files changed, 42 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index faf5e27..c6028cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 3.1.1 + - Fix: unable to stop plugin (on LS 6.x) [#29](https://github.com/logstash-plugins/logstash-input-unix/pull/29) + - Refactor: plugin internals got reviewed for `data_timeout => ...` to work reliably + ## 3.1.0 - Feat: adjust fields for ECS compatibility [#28](https://github.com/logstash-plugins/logstash-input-unix/pull/28) diff --git a/lib/logstash/inputs/unix.rb b/lib/logstash/inputs/unix.rb index 0baadb2..ebaf26e 100644 --- a/lib/logstash/inputs/unix.rb +++ b/lib/logstash/inputs/unix.rb @@ -15,8 +15,6 @@ class LogStash::Inputs::Unix < LogStash::Inputs::Base include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1) - class Interrupted < StandardError; end - config_name "unix" default :codec, "line" @@ -54,7 +52,6 @@ def initialize(*params) public def register require "socket" - require "timeout" if server? @logger.info("Starting unix input listener", :address => @path, :force_unlink => @force_unlink) @@ -87,26 +84,25 @@ def handle_socket(socket, output_queue) begin hostname = Socket.gethostname while !stop? - buf = nil - # NOTE(petef): the timeout only hits after the line is read - # or socket dies - # TODO(sissel): Why do we have a timeout here? What's the point? - if @data_timeout == -1 - buf = socket.readpartial(16384) - else - Timeout::timeout(@data_timeout) do - buf = socket.readpartial(16384) + data = socket.read_nonblock(16384, exception: false) + + if data == :wait_readable + if @data_timeout == -1 || IO.select([socket], nil, nil, @data_timeout) + next # retry socket read + else + # socket not ready after @data_timeout seconds + @logger.info("Closing connection after read timeout", :path => @path) + return end end - @codec.decode(buf) do |event| + + @codec.decode(data) do |event| decorate(event) event.set(@host_name_field, hostname) unless event.include?(@host_name_field) event.set(@file_path_field, @path) unless event.include?(@file_path_field) output_queue << event end end - rescue Timeout::Error - @logger.info("Closing connection after read timeout", :path => @path) rescue => e if @logger.debug? @logger.debug("Closing connection", :path => @path, :exception => e, :backtrace => e.backtrace) @@ -114,7 +110,6 @@ def handle_socket(socket, output_queue) @logger.info("Closing connection", :path => @path, :exception => e) end end - ensure begin socket.close @@ -141,9 +136,9 @@ def run(output_queue) end else while !stop? - if File.socket?(@path) then + if File.socket?(@path) @client_socket = UNIXSocket.new(@path) - @client_socket.instance_eval { class << self; include ::LogStash::Util::SocketPeer end } + @client_socket.extend ::LogStash::Util::SocketPeer @logger.debug("Opened connection", :client => @path) handle_socket(@client_socket, output_queue) else diff --git a/logstash-input-unix.gemspec b/logstash-input-unix.gemspec index 666765b..68b2678 100644 --- a/logstash-input-unix.gemspec +++ b/logstash-input-unix.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-unix' - s.version = '3.1.0' + s.version = '3.1.1' s.licenses = ['Apache License (2.0)'] s.summary = "Reads events over a UNIX socket" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/inputs/unix_spec.rb b/spec/inputs/unix_spec.rb index a1a499c..a56d30a 100644 --- a/spec/inputs/unix_spec.rb +++ b/spec/inputs/unix_spec.rb @@ -55,8 +55,29 @@ end context "when the unix socket has no data to be read" do - it_behaves_like "an interruptible input plugin" do - let(:run_forever) { false } + + let(:run_forever) { false } + + it_behaves_like "an interruptible input plugin" + + context 'with timeout' do + + let(:config) { super().merge "data_timeout" => 1.0 } + + let(:queue) { SizedQueue.new(10) } + before(:each) { subject.register } + after(:each) { subject.do_stop } + + it "closes socket after timeout" do + plugin_thread = Thread.new(subject, queue) { |subject, queue| subject.run(queue) } + sleep 0.5 + client_socket = subject.instance_variable_get :@client_socket + expect( client_socket.closed? ).to be false + sleep 1.0 # allow timeout to kick in + expect( client_socket.closed? ).to be true + expect( plugin_thread ).to be_alive + end + end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index f4e20c0..40942e6 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -12,7 +12,7 @@ def initialize(line = 'hi!') end def new_socket(path) - @path = path + @path = path File.unlink if File.exists?(path) && File.socket?(path) @socket = UNIXServer.new(path) self @@ -31,7 +31,6 @@ def loop(forever=false) end def close - @thread.kill @socket.close File.unlink(path) end From 8495cd71b31935cc7ba0e3cd306c2931cad44e23 Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Mon, 3 Oct 2022 17:15:22 +0000 Subject: [PATCH 51/52] data_timeout: decouple blocking duration from data timeout Introduces `Unix#io_interruptable_readpartial`, which emulates `IO#readpartial` while observing both the plugin's stop condition and an optional timeout. Internally, this helper limits its blocking calls to 10s or less whether or not the plugin is configured with a `data_timeout`, allowing it to periodically observe the plugin's stop condition to exit gracefully in a reasonable amount of time when the socket is not receiving writes. This fixes a regression introduced in v3.1.1 of this plugin in which the default configuration of `data_timeout => -1` (timeout disabled) consumed excess CPU attempting non-blocking reads of the IO when it had no readable bytes. --- CHANGELOG.md | 3 +++ lib/logstash/inputs/unix.rb | 48 +++++++++++++++++++++++++++++-------- logstash-input-unix.gemspec | 2 +- 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6028cd..c8fc433 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.1.2 + - Fix: eliminate high CPU usage when data timeout is disabled and no data is available on the socket + ## 3.1.1 - Fix: unable to stop plugin (on LS 6.x) [#29](https://github.com/logstash-plugins/logstash-input-unix/pull/29) - Refactor: plugin internals got reviewed for `data_timeout => ...` to work reliably diff --git a/lib/logstash/inputs/unix.rb b/lib/logstash/inputs/unix.rb index ebaf26e..2863f27 100644 --- a/lib/logstash/inputs/unix.rb +++ b/lib/logstash/inputs/unix.rb @@ -84,16 +84,15 @@ def handle_socket(socket, output_queue) begin hostname = Socket.gethostname while !stop? - data = socket.read_nonblock(16384, exception: false) - - if data == :wait_readable - if @data_timeout == -1 || IO.select([socket], nil, nil, @data_timeout) - next # retry socket read - else - # socket not ready after @data_timeout seconds - @logger.info("Closing connection after read timeout", :path => @path) - return - end + data = io_interruptable_readpartial(socket, 16384, @data_timeout) + + if data == :data_timeout + # socket not ready after @data_timeout seconds + @logger.info("Closing connection after read timeout", :path => @path) + return + elsif data == :stopping + @logger.trace("Shutdown in progress", :path => @path) + next # let next loop handle graceful stop end @codec.decode(data) do |event| @@ -118,6 +117,35 @@ def handle_socket(socket, output_queue) end end + ## + # Emulates `IO#readpartial` with a timeout and our plugin's stop-condition, + # limiting blocking calls to windows of 10s or less to ensure it can be interrupted. + # + # @param readable_io [IO] the IO to read from + # @param maxlen [Integer] the max bytes to be read + # @param timeout [Number] the maximum number of seconds to , or -1 to disable timeouts + # + # @return [:data_timeout] if timeout was reached before bytes were available + # @return [:stopping] if plugin stop-condition was detected before bytes were available + # @return [String] a non-empty string if bytes became available before the timeout was reached + def io_interruptable_readpartial(readable_io, maxlen, timeout) + + data_timeout_deadline = timeout < 0 ? nil : Time.now + timeout + maximum_blocking_seconds = timeout < 0 || timeout > 10 ? 10 : timeout + + loop do + return :stopping if stop? + result = readable_io.read_nonblock(maxlen, exception: false) + + return result if result.kind_of?(String) + raise EOFError if result.nil? + + return :data_timeout if (data_timeout_deadline && data_timeout_deadline < Time.now) + IO.select([readable_io], nil, nil, maximum_blocking_seconds) + end + end + private :io_interruptable_readpartial + private def server? @mode == "server" diff --git a/logstash-input-unix.gemspec b/logstash-input-unix.gemspec index 68b2678..bfed81e 100644 --- a/logstash-input-unix.gemspec +++ b/logstash-input-unix.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-unix' - s.version = '3.1.1' + s.version = '3.1.2' s.licenses = ['Apache License (2.0)'] s.summary = "Reads events over a UNIX socket" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From dcd33f24f216a85a93066bcfb4665c72c85a296f Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Mon, 3 Oct 2022 10:25:47 -0700 Subject: [PATCH 52/52] Update CHANGELOG.md Add link to PR --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8fc433..4072926 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ ## 3.1.2 - - Fix: eliminate high CPU usage when data timeout is disabled and no data is available on the socket + - Fix: eliminate high CPU usage when data timeout is disabled and no data is available on the socket [#30](https://github.com/logstash-plugins/logstash-input-unix/pull/30) ## 3.1.1 - Fix: unable to stop plugin (on LS 6.x) [#29](https://github.com/logstash-plugins/logstash-input-unix/pull/29)