From b02f696190555fdf05421e86272eb981bf893243 Mon Sep 17 00:00:00 2001 From: Philippe Weber Date: Thu, 9 Jun 2016 10:13:14 +0200 Subject: [PATCH 01/34] Update to new event API (#10) * update to new event API * update travis.yml --- .travis.yml | 3 +- CHANGELOG.md | 13 ++-- lib/logstash/inputs/github.rb | 59 ++++++++++------- logstash-input-github.gemspec | 6 +- spec/fixtures/event_create.json | 113 ++++++++++++++++++++++++++++++++ spec/inputs/github_spec.rb | 46 +++++++++++++ 6 files changed, 208 insertions(+), 32 deletions(-) create mode 100644 spec/fixtures/event_create.json diff --git a/.travis.yml b/.travis.yml index 350c4eb..3a5a76d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ sudo: false language: ruby cache: bundler rvm: - - jruby-1.7.23 + - jruby-1.7.25 +jdk: oraclejdk8 script: - bundle exec rspec spec diff --git a/CHANGELOG.md b/CHANGELOG.md index a17f38d..5d1c557 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,12 @@ -# 2.0.5 - - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash -# 2.0.4 - - New dependency requirements for logstash-core for the 5.0 release +## 3.0.0 + - breaking: Updated plugin to use new Java Event APIs + +## 2.0.5 + - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash + +## 2.0.4 + - New dependency requirements for logstash-core for the 5.0 release + ## 2.0.0 - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895 diff --git a/lib/logstash/inputs/github.rb b/lib/logstash/inputs/github.rb index 4902926..612c098 100644 --- a/lib/logstash/inputs/github.rb +++ b/lib/logstash/inputs/github.rb @@ -17,9 +17,9 @@ class LogStash::Inputs::GitHub < LogStash::Inputs::Base # Your GitHub Secret Token for the webhook config :secret_token, :validate => :string, :required => false - # If Secret is defined, we drop the events that don't match. - # Otherwise, we'll just add a invalid tag - config :drop_invalid, :validate => :boolean + # If Secret is defined, we drop the events that don't match. + # Otherwise, we'll just add an invalid tag + config :drop_invalid, :validate => :boolean, :default => false def register require "ftw" @@ -29,27 +29,13 @@ def register def run(output_queue) @server = FTW::WebServer.new(@ip, @port) do |request, response| body = request.read_body - begin - event = LogStash::Event.new(JSON.parse(body)) - rescue JSON::ParserError => e - @logger.info("JSON parse failure. Falling back to plain-text", :error => e, :data => body) - event = LogStash::Event.new("message" => body, "tags" => "_invalidjson") - end - event['headers'] = request.headers.to_hash - if defined? @secret_token and event['headers']['x-hub-signature'] - event['hash'] = 'sha1=' + OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('sha1'), @secret_token, body) - if not Rack::Utils.secure_compare(event['hash'], event['headers']['x-hub-signature']) - if not @drop_invalid - event['tags'] = "_Invalid_Github_Message" - else - @logger.info("Dropping invalid Github message") - drop = true - end - end - end - if not drop - decorate(event) - output_queue << event + event = build_event_from_request(body, request.headers_.to_hash) + valid_event = verify_signature(event,body) + if !valid_event && @drop_invalid + @logger.info("Dropping invalid Github message") + else + decorate(event) + output_queue << event end response.status = 200 response.body = "Accepted!" @@ -57,6 +43,31 @@ def run(output_queue) @server.run end # def run + def build_event_from_request(body, headers) + begin + event = LogStash::Event.new(JSON.parse(body)) + rescue JSON::ParserError => e + @logger.info("JSON parse failure. Falling back to plain-text", :error => e, :data => body) + event = LogStash::Event.new("message" => body, "tags" => "_invalidjson") + end + event.set('headers', headers) + return event + end + + def verify_signature(event,body) + is_valid = true + sign_header = event.get("[headers][x-hub-signature]") + if @secret_token && sign_header + hash = 'sha1=' + OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('sha1'), @secret_token, body) + event.set("hash", hash) + if not Rack::Utils.secure_compare(hash, sign_header) + event.tag("_Invalid_Github_Message") + is_valid = false + end + end + return is_valid + end + def close @server.stop end # def close diff --git a/logstash-input-github.gemspec b/logstash-input-github.gemspec index f53fe47..670e443 100644 --- a/logstash-input-github.gemspec +++ b/logstash-input-github.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-github' - s.version = '2.0.5' + s.version = '3.0.0' s.licenses = ['Apache License (2.0)'] s.summary = "Accept events from github webhooks." s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -20,12 +20,12 @@ Gem::Specification.new do |s| s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" } # Gem dependencies - s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0" + s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0" s.add_runtime_dependency 'addressable' s.add_runtime_dependency 'logstash-codec-plain' s.add_runtime_dependency 'ftw', '~> 0.0.42' - s.add_development_dependency 'logstash-devutils', '~> 0' + s.add_development_dependency 'logstash-devutils' end diff --git a/spec/fixtures/event_create.json b/spec/fixtures/event_create.json new file mode 100644 index 0000000..2d48b54 --- /dev/null +++ b/spec/fixtures/event_create.json @@ -0,0 +1,113 @@ +{ + "ref": "0.0.1", + "ref_type": "tag", + "master_branch": "master", + "description": "", + "pusher_type": "user", + "repository": { + "id": 35129377, + "name": "public-repo", + "full_name": "baxterthehacker/public-repo", + "owner": { + "login": "baxterthehacker", + "id": 6752317, + "avatar_url": "https://avatars.githubusercontent.com/u/6752317?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/baxterthehacker", + "html_url": "https://github.com/baxterthehacker", + "followers_url": "https://api.github.com/users/baxterthehacker/followers", + "following_url": "https://api.github.com/users/baxterthehacker/following%7B/other_user%7D", + "gists_url": "https://api.github.com/users/baxterthehacker/gists%7B/gist_id%7D", + "starred_url": "https://api.github.com/users/baxterthehacker/starred%7B/owner%7D%7B/repo%7D", + "subscriptions_url": "https://api.github.com/users/baxterthehacker/subscriptions", + "organizations_url": "https://api.github.com/users/baxterthehacker/orgs", + "repos_url": "https://api.github.com/users/baxterthehacker/repos", + "events_url": "https://api.github.com/users/baxterthehacker/events%7B/privacy%7D", + "received_events_url": "https://api.github.com/users/baxterthehacker/received_events", + "type": "User", + "site_admin": false + }, + "private": false, + "html_url": "https://github.com/baxterthehacker/public-repo", + "description": "", + "fork": false, + "url": "https://api.github.com/repos/baxterthehacker/public-repo", + "forks_url": "https://api.github.com/repos/baxterthehacker/public-repo/forks", + "keys_url": "https://api.github.com/repos/baxterthehacker/public-repo/keys%7B/key_id%7D", + "collaborators_url": "https://api.github.com/repos/baxterthehacker/public-repo/collaborators%7B/collaborator%7D", + "teams_url": "https://api.github.com/repos/baxterthehacker/public-repo/teams", + "hooks_url": "https://api.github.com/repos/baxterthehacker/public-repo/hooks", + "issue_events_url": "https://api.github.com/repos/baxterthehacker/public-repo/issues/events%7B/number%7D", + "events_url": "https://api.github.com/repos/baxterthehacker/public-repo/events", + "assignees_url": "https://api.github.com/repos/baxterthehacker/public-repo/assignees%7B/user%7D", + "branches_url": "https://api.github.com/repos/baxterthehacker/public-repo/branches%7B/branch%7D", + "tags_url": "https://api.github.com/repos/baxterthehacker/public-repo/tags", + "blobs_url": "https://api.github.com/repos/baxterthehacker/public-repo/git/blobs%7B/sha%7D", + "git_tags_url": "https://api.github.com/repos/baxterthehacker/public-repo/git/tags%7B/sha%7D", + "git_refs_url": "https://api.github.com/repos/baxterthehacker/public-repo/git/refs%7B/sha%7D", + "trees_url": "https://api.github.com/repos/baxterthehacker/public-repo/git/trees%7B/sha%7D", + "statuses_url": "https://api.github.com/repos/baxterthehacker/public-repo/statuses/%7Bsha%7D", + "languages_url": "https://api.github.com/repos/baxterthehacker/public-repo/languages", + "stargazers_url": "https://api.github.com/repos/baxterthehacker/public-repo/stargazers", + "contributors_url": "https://api.github.com/repos/baxterthehacker/public-repo/contributors", + "subscribers_url": "https://api.github.com/repos/baxterthehacker/public-repo/subscribers", + "subscription_url": "https://api.github.com/repos/baxterthehacker/public-repo/subscription", + "commits_url": "https://api.github.com/repos/baxterthehacker/public-repo/commits%7B/sha%7D", + "git_commits_url": "https://api.github.com/repos/baxterthehacker/public-repo/git/commits%7B/sha%7D", + "comments_url": "https://api.github.com/repos/baxterthehacker/public-repo/comments%7B/number%7D", + "issue_comment_url": "https://api.github.com/repos/baxterthehacker/public-repo/issues/comments%7B/number%7D", + "contents_url": "https://api.github.com/repos/baxterthehacker/public-repo/contents/%7B+path%7D", + "compare_url": "https://api.github.com/repos/baxterthehacker/public-repo/compare/%7Bbase%7D...%7Bhead%7D", + "merges_url": "https://api.github.com/repos/baxterthehacker/public-repo/merges", + "archive_url": "https://api.github.com/repos/baxterthehacker/public-repo/%7Barchive_format%7D%7B/ref%7D", + "downloads_url": "https://api.github.com/repos/baxterthehacker/public-repo/downloads", + "issues_url": "https://api.github.com/repos/baxterthehacker/public-repo/issues%7B/number%7D", + "pulls_url": "https://api.github.com/repos/baxterthehacker/public-repo/pulls%7B/number%7D", + "milestones_url": "https://api.github.com/repos/baxterthehacker/public-repo/milestones%7B/number%7D", + "notifications_url": "https://api.github.com/repos/baxterthehacker/public-repo/notifications%7B?since,all,participating}", + "labels_url": "https://api.github.com/repos/baxterthehacker/public-repo/labels%7B/name%7D", + "releases_url": "https://api.github.com/repos/baxterthehacker/public-repo/releases%7B/id%7D", + "created_at": "2015-05-05T23:40:12Z", + "updated_at": "2015-05-05T23:40:30Z", + "pushed_at": "2015-05-05T23:40:38Z", + "git_url": "git://github.com/baxterthehacker/public-repo.git", + "ssh_url": "git@github.com:baxterthehacker/public-repo.git", + "clone_url": "https://github.com/baxterthehacker/public-repo.git", + "svn_url": "https://github.com/baxterthehacker/public-repo", + "homepage": null, + "size": 0, + "stargazers_count": 0, + "watchers_count": 0, + "language": null, + "has_issues": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": true, + "forks_count": 0, + "mirror_url": null, + "open_issues_count": 2, + "forks": 0, + "open_issues": 2, + "watchers": 0, + "default_branch": "master" + }, + "sender": { + "login": "baxterthehacker", + "id": 6752317, + "avatar_url": "https://avatars.githubusercontent.com/u/6752317?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/baxterthehacker", + "html_url": "https://github.com/baxterthehacker", + "followers_url": "https://api.github.com/users/baxterthehacker/followers", + "following_url": "https://api.github.com/users/baxterthehacker/following%7B/other_user%7D", + "gists_url": "https://api.github.com/users/baxterthehacker/gists%7B/gist_id%7D", + "starred_url": "https://api.github.com/users/baxterthehacker/starred%7B/owner%7D%7B/repo%7D", + "subscriptions_url": "https://api.github.com/users/baxterthehacker/subscriptions", + "organizations_url": "https://api.github.com/users/baxterthehacker/orgs", + "repos_url": "https://api.github.com/users/baxterthehacker/repos", + "events_url": "https://api.github.com/users/baxterthehacker/events%7B/privacy%7D", + "received_events_url": "https://api.github.com/users/baxterthehacker/received_events", + "type": "User", + "site_admin": false + } +} \ No newline at end of file diff --git a/spec/inputs/github_spec.rb b/spec/inputs/github_spec.rb index 34392f1..c9bb69b 100644 --- a/spec/inputs/github_spec.rb +++ b/spec/inputs/github_spec.rb @@ -9,4 +9,50 @@ it "register without errors" do expect { plugin.register }.to_not raise_error end + + describe "building Logstash event from webhook" do + let(:body) {IO.read("spec/fixtures/event_create.json")} + let(:headers) { {"fake_header" => "fake_value"} } + let(:event) {plugin.build_event_from_request(body,headers)} + + it "initialize event from webhook body" do + JSON.parse(body).each do |k,v| + expect(event.get(k)).to eq(v) + end + end + + it "copy webhook http headers to event[headers]" do + expect(event.get('headers')).to eq (headers) + end + end + + describe "verify webhook signature" do + let(:plugin) { LogStash::Plugin.lookup("input", "github").new( {"port" => 9999, "secret_token" => "my_secret"} ) } + let(:body) {IO.read("spec/fixtures/event_create.json")} + let(:headers) { {"x-hub-signature" => "hash"} } + let(:event) {plugin.build_event_from_request(body,headers)} + let(:hash) { "sha1=43b113fc453c47f1cd4d5b4ded2985581c00a715" } + + it "accept event without signature" do + event.set('headers',{}) + expect(plugin.verify_signature(event,body)).to eq(true) + expect(event.get("hash")).to be_nil + expect(event.get("tags")).to be_nil + end + + it "reject event with invalid signature" do + event.set('headers',{"x-hub-signature" => "invalid"}) + expect(plugin.verify_signature(event,body)).to eq(false) + expect(event.get("hash")).to eq(hash) + expect(event.get("tags")).to eq(["_Invalid_Github_Message"]) + end + + it "accept event with valid signature" do + event.set('headers', {"x-hub-signature" => hash}) + expect(plugin.verify_signature(event,body)).to eq(true) + expect(event.get("hash")).to eq(hash) + expect(event.get("tags")).to be_nil + end + + end end From d099b342588b316d2b68b886b88ae3de6a35f8c6 Mon Sep 17 00:00:00 2001 From: Alex Lance Date: Tue, 12 Jul 2016 15:26:38 +1000 Subject: [PATCH 02/34] Add missing dependency for rack, fixes issue #11 --- lib/logstash/inputs/github.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/logstash/inputs/github.rb b/lib/logstash/inputs/github.rb index 612c098..4b95211 100644 --- a/lib/logstash/inputs/github.rb +++ b/lib/logstash/inputs/github.rb @@ -3,6 +3,7 @@ require "logstash/namespace" require "socket" require "json" +require "rack" # Read events from github webhooks class LogStash::Inputs::GitHub < LogStash::Inputs::Base From 25febd295093618a26aaa54fb690339e3bd08c60 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Thu, 14 Jul 2016 12:25:21 +0100 Subject: [PATCH 03/34] dependency logstash-core-plugin-api >= 1.60 <= 2.99 --- CHANGELOG.md | 3 +++ logstash-input-github.gemspec | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d1c557..96c45c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.0.1 + - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99 + ## 3.0.0 - breaking: Updated plugin to use new Java Event APIs diff --git a/logstash-input-github.gemspec b/logstash-input-github.gemspec index 670e443..9759807 100644 --- a/logstash-input-github.gemspec +++ b/logstash-input-github.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-github' - s.version = '3.0.0' + s.version = '3.0.1' s.licenses = ['Apache License (2.0)'] s.summary = "Accept events from github webhooks." s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -20,7 +20,7 @@ Gem::Specification.new do |s| s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" } # Gem dependencies - s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0" + s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" s.add_runtime_dependency 'addressable' s.add_runtime_dependency 'logstash-codec-plain' From eebc761b55511d35913d9875edd7ebd9c0ade880 Mon Sep 17 00:00:00 2001 From: "Code Hugger (Matthew Jones)" Date: Thu, 10 Nov 2016 18:08:53 -0700 Subject: [PATCH 04/34] Fixing NoMethodError: undefined method `headers_' I'm getting this stack trace with logstash 5.0.0, this change fixes it. [2016-11-10T20:02:17,433][FATAL][logstash.runner ] An unexpected error occurred! {:error=>#>, :backtrace=>["/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-input-github-3.0.1/lib/logstash/inputs/github.rb:32:in `run'", "org/jruby/RubyProc.java:281:in `call'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/ftw-0.0.44/lib/ftw/webserver.rb:77:in `handle_request'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/ftw-0.0.44/lib/ftw/webserver.rb:59:in `handle_connection'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/ftw-0.0.44/lib/ftw/webserver.rb:29:in `run'"]} --- lib/logstash/inputs/github.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logstash/inputs/github.rb b/lib/logstash/inputs/github.rb index 612c098..b7e5eae 100644 --- a/lib/logstash/inputs/github.rb +++ b/lib/logstash/inputs/github.rb @@ -29,7 +29,7 @@ def register def run(output_queue) @server = FTW::WebServer.new(@ip, @port) do |request, response| body = request.read_body - event = build_event_from_request(body, request.headers_.to_hash) + event = build_event_from_request(body, request.headers.to_hash) valid_event = verify_signature(event,body) if !valid_event && @drop_invalid @logger.info("Dropping invalid Github message") From 891c2b2456c0052f6376c2cc2a60bdab4f792d79 Mon Sep 17 00:00:00 2001 From: Suyog Rao Date: Wed, 5 Apr 2017 07:38:41 -0700 Subject: [PATCH 05/34] Bump to 3.0.2 --- logstash-input-github.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash-input-github.gemspec b/logstash-input-github.gemspec index 9759807..461cfdf 100644 --- a/logstash-input-github.gemspec +++ b/logstash-input-github.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-github' - s.version = '3.0.1' + s.version = '3.0.2' s.licenses = ['Apache License (2.0)'] s.summary = "Accept events from github webhooks." s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From d4f598c1172a4461289048b156f4e89f76ead3ce Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Fri, 28 Apr 2017 21:09:25 +0000 Subject: [PATCH 06/34] Initial doc move --- docs/index.asciidoc | 80 +++++++++++++++++++++++++++++++++++ logstash-input-github.gemspec | 2 +- 2 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 docs/index.asciidoc diff --git a/docs/index.asciidoc b/docs/index.asciidoc new file mode 100644 index 0000000..690d580 --- /dev/null +++ b/docs/index.asciidoc @@ -0,0 +1,80 @@ +:plugin: github +:type: input + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: %VERSION% +:release_date: %RELEASE_DATE% +:changelog_url: %CHANGELOG_URL% +:include_path: ../../../logstash/docs/include +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="plugins-{type}-{plugin}"] + +=== Github + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Read events from github webhooks + +[id="plugins-{type}s-{plugin}-options"] +==== Github Input Configuration Options + +This plugin supports the following configuration options plus the <> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <> |<>|No +| <> |<>|No +| <> |<>|Yes +| <> |<>|No +|======================================================================= + +Also see <> for a list of options supported by all +input plugins. + +  + +[id="plugins-{type}s-{plugin}-drop_invalid"] +===== `drop_invalid` + + * Value type is <> + * Default value is `false` + +If Secret is defined, we drop the events that don't match. +Otherwise, we'll just add an invalid tag + +[id="plugins-{type}s-{plugin}-ip"] +===== `ip` + + * Value type is <> + * Default value is `"0.0.0.0"` + +The ip to listen on + +[id="plugins-{type}s-{plugin}-port"] +===== `port` + + * This is a required setting. + * Value type is <> + * There is no default value for this setting. + +The port to listen on + +[id="plugins-{type}s-{plugin}-secret_token"] +===== `secret_token` + + * Value type is <> + * There is no default value for this setting. + +Your GitHub Secret Token for the webhook + + + +include::{include_path}/{type}.asciidoc[] diff --git a/logstash-input-github.gemspec b/logstash-input-github.gemspec index 461cfdf..a1a54ec 100644 --- a/logstash-input-github.gemspec +++ b/logstash-input-github.gemspec @@ -11,7 +11,7 @@ Gem::Specification.new do |s| s.require_paths = ["lib"] # Files - s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT'] + s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"] # Tests s.test_files = s.files.grep(%r{^(test|spec|features)/}) From 73135c98324b7d8c0eaf81862e7b8b34e9565191 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Fri, 26 May 2017 13:01:03 -0400 Subject: [PATCH 07/34] new build system for jruby9k --- .travis.yml | 17 +++++++++++++++-- Gemfile | 10 +++++++++- ci/build.sh | 21 +++++++++++++++++++++ ci/setup.sh | 26 ++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 3 deletions(-) create mode 100755 ci/build.sh create mode 100755 ci/setup.sh diff --git a/.travis.yml b/.travis.yml index 3a5a76d..f274087 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,21 @@ +--- sudo: false language: ruby cache: bundler +env: rvm: - jruby-1.7.25 +matrix: + include: + - rvm: jruby-1.7.25 + env: LOGSTASH_BRANCH=master + - rvm: jruby-1.7.25 + env: LOGSTASH_BRANCH=5.x + - rvm: jruby-9.1.9.0 + env: LOGSTASH_BRANCH=feature/9000 + allow_failures: + - rvm: jruby-9.1.9.0 + fast_finish: true +install: true +script: ci/build.sh jdk: oraclejdk8 -script: - - bundle exec rspec spec diff --git a/Gemfile b/Gemfile index d926697..93e5e5d 100644 --- a/Gemfile +++ b/Gemfile @@ -1,2 +1,10 @@ source 'https://rubygems.org/' -gemspec \ No newline at end of file + +gemspec + +logstash_path = "../../logstash" + +if Dir.exist?(logstash_path) && ENV["LOGSTASH_SOURCE"] == 1 + gem 'logstash-core', :path => "#{logstash_path}/logstash-core" + gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api" +end diff --git a/ci/build.sh b/ci/build.sh new file mode 100755 index 0000000..076e908 --- /dev/null +++ b/ci/build.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# version: 1 +######################################################## +# +# AUTOMATICALLY GENERATED! DO NOT EDIT +# +######################################################## +set -e + +echo "Starting build process in: `pwd`" +./ci/setup.sh + +if [[ -f "ci/run.sh" ]]; then + echo "Running custom build script in: `pwd`/ci/run.sh" + ./ci/run.sh +else + echo "Running default build scripts in: `pwd`/ci/build.sh" + bundle install + bundle exec rake vendor + bundle exec rspec spec +fi diff --git a/ci/setup.sh b/ci/setup.sh new file mode 100755 index 0000000..835fa43 --- /dev/null +++ b/ci/setup.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# version: 1 +######################################################## +# +# AUTOMATICALLY GENERATED! DO NOT EDIT +# +######################################################## +set -e +if [ "$LOGSTASH_BRANCH" ]; then + echo "Building plugin using Logstash source" + BASE_DIR=`pwd` + echo "Checking out branch: $LOGSTASH_BRANCH" + git clone -b $LOGSTASH_BRANCH https://github.com/elastic/logstash.git ../../logstash --depth 1 + printf "Checked out Logstash revision: %s\n" "$(git -C ../../logstash rev-parse HEAD)" + cd ../../logstash + echo "Building plugins with Logstash version:" + cat versions.yml + echo "---" + # We need to build the jars for that specific version + echo "Running gradle assemble in: `pwd`" + ./gradlew assemble + cd $BASE_DIR + export LOGSTASH_SOURCE=1 +else + echo "Building plugin using released gems on rubygems" +fi From 6e6cea9029d202b23c795e322d55d3b41806cfa2 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Wed, 31 May 2017 16:44:57 -0400 Subject: [PATCH 08/34] Adjusting the build scripts to correctly load the logstash source and allow people to override it --- Gemfile | 5 +++-- ci/build.sh | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Gemfile b/Gemfile index 93e5e5d..32cc6fb 100644 --- a/Gemfile +++ b/Gemfile @@ -2,9 +2,10 @@ source 'https://rubygems.org/' gemspec -logstash_path = "../../logstash" +logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash" +use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1" -if Dir.exist?(logstash_path) && ENV["LOGSTASH_SOURCE"] == 1 +if Dir.exist?(logstash_path) && use_logstash_source gem 'logstash-core', :path => "#{logstash_path}/logstash-core" gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api" end diff --git a/ci/build.sh b/ci/build.sh index 076e908..06caffd 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -8,11 +8,11 @@ set -e echo "Starting build process in: `pwd`" -./ci/setup.sh +source ./ci/setup.sh if [[ -f "ci/run.sh" ]]; then echo "Running custom build script in: `pwd`/ci/run.sh" - ./ci/run.sh + source ./ci/run.sh else echo "Running default build scripts in: `pwd`/ci/build.sh" bundle install From ecd7985db9cc7645c63f433acaa37550422a05d4 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Tue, 13 Jun 2017 09:10:37 -0400 Subject: [PATCH 09/34] update .travis.yml for jruby9k jobs --- .travis.yml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index f274087..59c937e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,19 +2,15 @@ sudo: false language: ruby cache: bundler -env: +env: rvm: - - jruby-1.7.25 +- jruby-1.7.25 matrix: include: - - rvm: jruby-1.7.25 - env: LOGSTASH_BRANCH=master - - rvm: jruby-1.7.25 - env: LOGSTASH_BRANCH=5.x - - rvm: jruby-9.1.9.0 - env: LOGSTASH_BRANCH=feature/9000 - allow_failures: - - rvm: jruby-9.1.9.0 + - rvm: jruby-9.1.10.0 + env: LOGSTASH_BRANCH=master + - rvm: jruby-1.7.25 + env: LOGSTASH_BRANCH=5.x fast_finish: true install: true script: ci/build.sh From c1f1ae60c9cedf72c2a842d4c0e0177818cfadfe Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 15 Jun 2017 20:00:56 -0400 Subject: [PATCH 10/34] update plugin header for better search results --- docs/index.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 690d580..4855e00 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -14,7 +14,7 @@ END - GENERATED VARIABLES, DO NOT EDIT! [id="plugins-{type}-{plugin}"] -=== Github +=== Github input plugin include::{include_path}/plugin_header.asciidoc[] From eaeca11ed93210f0daa88298e99df85a3a2c9cd1 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 22 Jun 2017 22:30:32 -0400 Subject: [PATCH 11/34] [skip ci] Updating the plugin doc --- docs/index.asciidoc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 4855e00..1e9998d 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -7,7 +7,7 @@ START - GENERATED VARIABLES, DO NOT EDIT! :version: %VERSION% :release_date: %RELEASE_DATE% :changelog_url: %CHANGELOG_URL% -:include_path: ../../../logstash/docs/include +:include_path: ../../../../logstash/docs/include /////////////////////////////////////////// END - GENERATED VARIABLES, DO NOT EDIT! /////////////////////////////////////////// @@ -25,7 +25,7 @@ Read events from github webhooks [id="plugins-{type}s-{plugin}-options"] ==== Github Input Configuration Options -This plugin supports the following configuration options plus the <> described later. +This plugin supports the following configuration options plus the <> described later. [cols="<,<,<",options="header",] |======================================================================= @@ -36,7 +36,7 @@ This plugin supports the following configuration options plus the <> |<>|No |======================================================================= -Also see <> for a list of options supported by all +Also see <> for a list of options supported by all input plugins.   @@ -77,4 +77,5 @@ Your GitHub Secret Token for the webhook -include::{include_path}/{type}.asciidoc[] +[id="plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] \ No newline at end of file From dc3c5af1000b4c9f6576f1d832123bba052af5f8 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 22 Jun 2017 22:54:09 -0400 Subject: [PATCH 12/34] bump patch level for doc generation --- logstash-input-github.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash-input-github.gemspec b/logstash-input-github.gemspec index a1a54ec..c44e90c 100644 --- a/logstash-input-github.gemspec +++ b/logstash-input-github.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-github' - s.version = '3.0.2' + s.version = '3.0.3' s.licenses = ['Apache License (2.0)'] s.summary = "Accept events from github webhooks." s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From b396c9f0580c7cc635d6d7b3de24dac01963418e Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Mon, 26 Jun 2017 21:24:11 -0400 Subject: [PATCH 13/34] [skip ci] Updating the plugin id in the doc to match the index in the docbook --- docs/index.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 1e9998d..25bbfdb 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -12,7 +12,7 @@ START - GENERATED VARIABLES, DO NOT EDIT! END - GENERATED VARIABLES, DO NOT EDIT! /////////////////////////////////////////// -[id="plugins-{type}-{plugin}"] +[id="plugins-{type}s-{plugin}"] === Github input plugin From 41e52beb56b4b8d43136d1f3af64498e15e02264 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Wed, 26 Jul 2017 12:38:07 +0100 Subject: [PATCH 14/34] on travis test against 5.6 and 6.0 logstash-core --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 59c937e..7af01f7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,9 +9,11 @@ matrix: include: - rvm: jruby-9.1.10.0 env: LOGSTASH_BRANCH=master + - rvm: jruby-9.1.10.0 + env: LOGSTASH_BRANCH=6.x - rvm: jruby-1.7.25 - env: LOGSTASH_BRANCH=5.x + env: LOGSTASH_BRANCH=5.6 fast_finish: true install: true script: ci/build.sh -jdk: oraclejdk8 +jdk: oraclejdk8 \ No newline at end of file From df5996fb8aa1f0b99558d269032276cf9ae330ae Mon Sep 17 00:00:00 2001 From: Jordan Sissel Date: Tue, 15 Aug 2017 10:01:02 -0700 Subject: [PATCH 15/34] Version bump For https://github.com/elastic/logstash/issues/7993 [ci skip] --- CHANGELOG.md | 3 +++ logstash-input-github.gemspec | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96c45c7..447f36e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.0.4 + - Fix some documentation issues + ## 3.0.1 - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99 diff --git a/logstash-input-github.gemspec b/logstash-input-github.gemspec index c44e90c..c1a7ea4 100644 --- a/logstash-input-github.gemspec +++ b/logstash-input-github.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-github' - s.version = '3.0.3' + s.version = '3.0.4' s.licenses = ['Apache License (2.0)'] s.summary = "Accept events from github webhooks." s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 5c4b9e2c716183b2795be11c81ab3681571a853d Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Fri, 27 Oct 2017 17:24:58 -0500 Subject: [PATCH 16/34] Travis - add 6.0 build, remove default JRuby 1.7 build, bump RVM versions --- .travis.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7af01f7..1458a3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,18 +2,17 @@ sudo: false language: ruby cache: bundler -env: -rvm: -- jruby-1.7.25 matrix: include: - - rvm: jruby-9.1.10.0 + - rvm: jruby-9.1.13.0 env: LOGSTASH_BRANCH=master - - rvm: jruby-9.1.10.0 + - rvm: jruby-9.1.13.0 env: LOGSTASH_BRANCH=6.x - - rvm: jruby-1.7.25 + - rvm: jruby-9.1.13.0 + env: LOGSTASH_BRANCH=6.0 + - rvm: jruby-1.7.27 env: LOGSTASH_BRANCH=5.6 fast_finish: true install: true script: ci/build.sh -jdk: oraclejdk8 \ No newline at end of file +jdk: oraclejdk8 From 50d9f4196a46ed16fa1b20c950422448952401e0 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 7 Nov 2017 11:18:31 +0000 Subject: [PATCH 17/34] [skip ci] update gemspec summary --- CHANGELOG.md | 3 +++ logstash-input-github.gemspec | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 447f36e..4df1d74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.0.5 + - Update gemspec summary + ## 3.0.4 - Fix some documentation issues diff --git a/logstash-input-github.gemspec b/logstash-input-github.gemspec index c1a7ea4..bc6b7c9 100644 --- a/logstash-input-github.gemspec +++ b/logstash-input-github.gemspec @@ -1,9 +1,9 @@ Gem::Specification.new do |s| s.name = 'logstash-input-github' - s.version = '3.0.4' + s.version = '3.0.5' s.licenses = ['Apache License (2.0)'] - s.summary = "Accept events from github webhooks." + s.summary = "Reads events from a GitHub webhook" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" s.authors = ["Elastic"] s.email = 'jason.kendall@elastic.co' From 17c8a2eec6152e8d2fbb1dd5fcb06d53ca667402 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Mon, 8 Jan 2018 21:30:57 +0000 Subject: [PATCH 18/34] [skip ci] update license to 2018 --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 43976b7..2162c9b 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2012–2016 Elasticsearch +Copyright (c) 2012-2018 Elasticsearch Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From 0b26beb3a06662f0eaf03f101f6bafe50a20797d Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Fri, 16 Feb 2018 22:25:39 +0000 Subject: [PATCH 19/34] Improve malformed-input handling, restarting, and crash-recovery FTW v0.48 was recently shipped, fixing a crash that could happen when the webserver was sent an `HTTP/0.9` request (what is this, 1991!?), and fixing a scenario where the webserver would fail to return when stopped. Also properly overrides `LogStash::Inputs::Base#stop`, so we can ensure our server gets stopped when the plugin is stopped or reloaded. --- CHANGELOG.md | 5 ++++ lib/logstash/inputs/github.rb | 13 +++++++--- logstash-input-github.gemspec | 4 +-- spec/inputs/github_spec.rb | 47 +++++++++++++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4df1d74..5ed7d1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## 3.0.6 + - Improve malformed-input handling by using updated FTW + - Improve webserver crash recovery + - Properly support plugin stopping & reloading + ## 3.0.5 - Update gemspec summary diff --git a/lib/logstash/inputs/github.rb b/lib/logstash/inputs/github.rb index 48b6665..aaa38e4 100644 --- a/lib/logstash/inputs/github.rb +++ b/lib/logstash/inputs/github.rb @@ -42,6 +42,13 @@ def run(output_queue) response.body = "Accepted!" end @server.run + rescue Exception => original_exception + # If our server crashes, it may not have cleaned up after itself; + # since `FTW::WebServer#stop` is idempotent, make one last attempt + # before propagating the original exception. + @server && @server.stop rescue logger.error("Error while stopping FTW::WebServer", exception: $!.message, backtrace: $!.backtrace) + + raise original_exception end # def run def build_event_from_request(body, headers) @@ -69,8 +76,8 @@ def verify_signature(event,body) return is_valid end - def close - @server.stop - end # def close + def stop + @server && @server.stop + end # def stop end # class LogStash::Inputs::Github diff --git a/logstash-input-github.gemspec b/logstash-input-github.gemspec index bc6b7c9..a62fb92 100644 --- a/logstash-input-github.gemspec +++ b/logstash-input-github.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-github' - s.version = '3.0.5' + s.version = '3.0.6' s.licenses = ['Apache License (2.0)'] s.summary = "Reads events from a GitHub webhook" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -24,7 +24,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'addressable' s.add_runtime_dependency 'logstash-codec-plain' - s.add_runtime_dependency 'ftw', '~> 0.0.42' + s.add_runtime_dependency 'ftw', '~> 0.0.48' s.add_development_dependency 'logstash-devutils' end diff --git a/spec/inputs/github_spec.rb b/spec/inputs/github_spec.rb index c9bb69b..042678d 100644 --- a/spec/inputs/github_spec.rb +++ b/spec/inputs/github_spec.rb @@ -55,4 +55,51 @@ end end + + describe 'graceful shutdown' do + context 'when underlying webserver crashes' do + + # Stubbing out our FTW::WebServer allows us to force it to raise an exception when we try to run it. + let(:mock_webserver_class) { double('FTW::WebServer::class').as_null_object } + let(:mock_webserver) { double('FTW::WebServer').as_null_object } + before(:each) do + stub_const('FTW::WebServer', mock_webserver_class) + allow(mock_webserver_class).to receive(:new).and_return(mock_webserver) + expect(mock_webserver).to receive(:run).and_raise('testing: intentional uncaught exception') + end + + it 'makes an attempt to stop the webserver' do + expect(mock_webserver).to receive(:stop) + + plugin.run([]) rescue nil + end + + it 'propagates the original exception' do + expect do + plugin.run([]) + end.to raise_exception('testing: intentional uncaught exception') + end + + context 'and an attempt to stop the webserver also crashes' do + let(:mock_logger) { double('Logger').as_null_object } + before(:each) do + allow(plugin).to receive(:logger).and_return(mock_logger) + allow(mock_webserver).to receive(:stop).and_raise('yo dawg') + end + + it 'logs helpfully' do + expect(mock_logger).to receive(:error).with("Error while stopping FTW::WebServer", + exception: 'yo dawg', backtrace: instance_of(Array)) + + plugin.run([]) rescue nil + end + + it 'propagates the original exception' do + expect do + plugin.run([]) + end.to raise_exception('testing: intentional uncaught exception') + end + end + end + end end From 169ecfb1380004400c2c0464f768ac50a2f71886 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 6 Apr 2018 23:37:04 +0100 Subject: [PATCH 20/34] set default_codec doc attribute --- CHANGELOG.md | 3 +++ docs/index.asciidoc | 5 ++++- logstash-input-github.gemspec | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ed7d1a..06eaa3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.0.7 + - Docs: Set the default_codec doc attribute. + ## 3.0.6 - Improve malformed-input handling by using updated FTW - Improve webserver crash recovery diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 25bbfdb..7d77c2e 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -1,5 +1,6 @@ :plugin: github :type: input +:default_codec: plain /////////////////////////////////////////// START - GENERATED VARIABLES, DO NOT EDIT! @@ -78,4 +79,6 @@ Your GitHub Secret Token for the webhook [id="plugins-{type}s-{plugin}-common-options"] -include::{include_path}/{type}.asciidoc[] \ No newline at end of file +include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/logstash-input-github.gemspec b/logstash-input-github.gemspec index a62fb92..84da538 100644 --- a/logstash-input-github.gemspec +++ b/logstash-input-github.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-github' - s.version = '3.0.6' + s.version = '3.0.7' s.licenses = ['Apache License (2.0)'] s.summary = "Reads events from a GitHub webhook" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 65f5bcc85b284bd25c6a62ae9a3d1e2861648bd4 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Thu, 3 Jan 2019 15:26:10 -0500 Subject: [PATCH 21/34] pin bundler version to < 2 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 1458a3b..ea27df6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,3 +16,4 @@ matrix: install: true script: ci/build.sh jdk: oraclejdk8 +before_install: gem install bundler -v '< 2' From b1faf160e6dd152ad4da98e33e8851257d5d376f Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Fri, 4 Jan 2019 11:27:36 -0500 Subject: [PATCH 22/34] [skip ci] Travis: update LOGSTASH_BRANCH from 6.[0..4] to 6.5 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ea27df6..746f145 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ matrix: - rvm: jruby-9.1.13.0 env: LOGSTASH_BRANCH=6.x - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=6.0 + env: LOGSTASH_BRANCH=6.5 - rvm: jruby-1.7.27 env: LOGSTASH_BRANCH=5.6 fast_finish: true From c38c5b515b2b552291475ea123a3c9a21b4d6cb8 Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Wed, 13 Feb 2019 00:33:53 +0000 Subject: [PATCH 23/34] update matrix to include current targets [ci skip] --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 746f145..dc96273 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,9 +7,11 @@ matrix: - rvm: jruby-9.1.13.0 env: LOGSTASH_BRANCH=master - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=6.x + env: LOGSTASH_BRANCH=7.0 - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=6.5 + env: LOGSTASH_BRANCH=6.7 + - rvm: jruby-9.1.13.0 + env: LOGSTASH_BRANCH=6.6 - rvm: jruby-1.7.27 env: LOGSTASH_BRANCH=5.6 fast_finish: true From 197fbb5eed007ecc6e60aeb5d8372bd819b42bc9 Mon Sep 17 00:00:00 2001 From: Ron Toland Date: Mon, 17 Jun 2019 11:20:10 -0700 Subject: [PATCH 24/34] Reject reqs missing x-hub-signature header if secret_token defined --- CHANGELOG.md | 4 +++- lib/logstash/inputs/github.rb | 3 +++ logstash-input-github.gemspec | 3 +-- spec/inputs/github_spec.rb | 38 +++++++++++++++++++++++++++++++---- 4 files changed, 41 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06eaa3f..64f8da0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.0.8 + - Require x-hub-signature header if secret_token defined + ## 3.0.7 - Docs: Set the default_codec doc attribute. @@ -28,4 +31,3 @@ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895 - Dependency on logstash-core update to 2.0 - diff --git a/lib/logstash/inputs/github.rb b/lib/logstash/inputs/github.rb index aaa38e4..bf26d75 100644 --- a/lib/logstash/inputs/github.rb +++ b/lib/logstash/inputs/github.rb @@ -72,6 +72,9 @@ def verify_signature(event,body) event.tag("_Invalid_Github_Message") is_valid = false end + elsif @secret_token && !sign_header + event.tag("_Invalid_Github_Message") + is_valid = false end return is_valid end diff --git a/logstash-input-github.gemspec b/logstash-input-github.gemspec index 84da538..d54275a 100644 --- a/logstash-input-github.gemspec +++ b/logstash-input-github.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-github' - s.version = '3.0.7' + s.version = '3.0.8' s.licenses = ['Apache License (2.0)'] s.summary = "Reads events from a GitHub webhook" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -28,4 +28,3 @@ Gem::Specification.new do |s| s.add_development_dependency 'logstash-devutils' end - diff --git a/spec/inputs/github_spec.rb b/spec/inputs/github_spec.rb index 042678d..08f1198 100644 --- a/spec/inputs/github_spec.rb +++ b/spec/inputs/github_spec.rb @@ -26,18 +26,18 @@ end end - describe "verify webhook signature" do + describe "verify webhook signature if token provided" do let(:plugin) { LogStash::Plugin.lookup("input", "github").new( {"port" => 9999, "secret_token" => "my_secret"} ) } let(:body) {IO.read("spec/fixtures/event_create.json")} let(:headers) { {"x-hub-signature" => "hash"} } let(:event) {plugin.build_event_from_request(body,headers)} let(:hash) { "sha1=43b113fc453c47f1cd4d5b4ded2985581c00a715" } - it "accept event without signature" do + it "reject event without signature" do event.set('headers',{}) - expect(plugin.verify_signature(event,body)).to eq(true) + expect(plugin.verify_signature(event,body)).to eq(false) expect(event.get("hash")).to be_nil - expect(event.get("tags")).to be_nil + expect(event.get("tags")).to eq(["_Invalid_Github_Message"]) end it "reject event with invalid signature" do @@ -56,6 +56,36 @@ end + describe "don't validate webhook if token missing" do + let(:plugin) { LogStash::Plugin.lookup("input", "github").new( {"port" => 9999} ) } + let(:body) {IO.read("spec/fixtures/event_create.json")} + let(:headers) { {"x-hub-signature" => "hash"} } + let(:event) {plugin.build_event_from_request(body,headers)} + let(:hash) { "sha1=43b113fc453c47f1cd4d5b4ded2985581c00a715" } + + it "accept event without signature" do + event.set('headers',{}) + expect(plugin.verify_signature(event,body)).to eq(true) + expect(event.get("hash")).to be_nil + expect(event.get("tags")).to be_nil + end + + it "accept event with invalid signature" do + event.set('headers',{"x-hub-signature" => "invalid"}) + expect(plugin.verify_signature(event,body)).to eq(true) + expect(event.get("hash")).to be_nil + expect(event.get("tags")).to be_nil + end + + it "accept event with valid signature" do + event.set('headers', {"x-hub-signature" => hash}) + expect(plugin.verify_signature(event,body)).to eq(true) + expect(event.get("hash")).to be_nil + expect(event.get("tags")).to be_nil + end + + end + describe 'graceful shutdown' do context 'when underlying webserver crashes' do From 7d54a107ade093bf596d7c523717a36ae46557fb Mon Sep 17 00:00:00 2001 From: Ron Toland Date: Wed, 26 Jun 2019 16:13:31 -0700 Subject: [PATCH 25/34] refactor verify_signature method (with thanks to yaauie) --- lib/logstash/inputs/github.rb | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/lib/logstash/inputs/github.rb b/lib/logstash/inputs/github.rb index bf26d75..0a03ef7 100644 --- a/lib/logstash/inputs/github.rb +++ b/lib/logstash/inputs/github.rb @@ -63,20 +63,18 @@ def build_event_from_request(body, headers) end def verify_signature(event,body) - is_valid = true + # skip validation if we have no secret token + return true unless @secret_token + sign_header = event.get("[headers][x-hub-signature]") - if @secret_token && sign_header + if sign_header hash = 'sha1=' + OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('sha1'), @secret_token, body) event.set("hash", hash) - if not Rack::Utils.secure_compare(hash, sign_header) - event.tag("_Invalid_Github_Message") - is_valid = false - end - elsif @secret_token && !sign_header - event.tag("_Invalid_Github_Message") - is_valid = false + return true if Rack::Utils.secure_compare(hash, sign_header) end - return is_valid + + event.tag("_Invalid_Github_Message") + return false end def stop From c4db8fbecc195f3a795c730be223cc02e2d553b3 Mon Sep 17 00:00:00 2001 From: Colin Surprenant Date: Tue, 6 Aug 2019 14:22:22 -0400 Subject: [PATCH 26/34] [skip ci] Travis: switch to openjdk8 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index dc96273..3b61247 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,5 +17,5 @@ matrix: fast_finish: true install: true script: ci/build.sh -jdk: oraclejdk8 +jdk: openjdk8 before_install: gem install bundler -v '< 2' From deab476b33a8b8a53786f35142905dc8172af370 Mon Sep 17 00:00:00 2001 From: Ron Toland Date: Fri, 2 Aug 2019 10:47:48 -0700 Subject: [PATCH 27/34] update ftw gem dep to 0.0.49 allows newest versions of plugin to be installed on logstash 7.x --- CHANGELOG.md | 3 +++ logstash-input-github.gemspec | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64f8da0..2a326e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.0.9 + - Bump ftw dependency to 0.0.49, for compatibility with Logstash 7.x + ## 3.0.8 - Require x-hub-signature header if secret_token defined diff --git a/logstash-input-github.gemspec b/logstash-input-github.gemspec index d54275a..0e6aab9 100644 --- a/logstash-input-github.gemspec +++ b/logstash-input-github.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-github' - s.version = '3.0.8' + s.version = '3.0.9' s.licenses = ['Apache License (2.0)'] s.summary = "Reads events from a GitHub webhook" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -24,7 +24,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'addressable' s.add_runtime_dependency 'logstash-codec-plain' - s.add_runtime_dependency 'ftw', '~> 0.0.48' + s.add_runtime_dependency 'ftw', '~> 0.0.49' s.add_development_dependency 'logstash-devutils' end From 5f364de1e440c04082bb41c423410e1104421f7e Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 17 Mar 2020 15:41:54 +0000 Subject: [PATCH 28/34] [skip ci] updated apache license --- LICENSE | 209 +++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 199 insertions(+), 10 deletions(-) diff --git a/LICENSE b/LICENSE index 2162c9b..a80a3fd 100644 --- a/LICENSE +++ b/LICENSE @@ -1,13 +1,202 @@ -Copyright (c) 2012-2018 Elasticsearch -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ - http://www.apache.org/licenses/LICENSE-2.0 + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2020 Elastic and contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From e7b0f37986203d29028b55247018e8670b8d9153 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 17 Mar 2020 21:15:17 +0000 Subject: [PATCH 29/34] move testing to centralized travis configuration --- .travis.yml | 23 ++--------------------- ci/build.sh | 21 --------------------- ci/setup.sh | 26 -------------------------- 3 files changed, 2 insertions(+), 68 deletions(-) delete mode 100755 ci/build.sh delete mode 100755 ci/setup.sh diff --git a/.travis.yml b/.travis.yml index 3b61247..a50fc73 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,21 +1,2 @@ ---- -sudo: false -language: ruby -cache: bundler -matrix: - include: - - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=master - - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=7.0 - - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=6.7 - - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=6.6 - - rvm: jruby-1.7.27 - env: LOGSTASH_BRANCH=5.6 - fast_finish: true -install: true -script: ci/build.sh -jdk: openjdk8 -before_install: gem install bundler -v '< 2' +import: +- logstash-plugins/.ci:travis/travis.yml@1.x \ No newline at end of file diff --git a/ci/build.sh b/ci/build.sh deleted file mode 100755 index 06caffd..0000000 --- a/ci/build.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# version: 1 -######################################################## -# -# AUTOMATICALLY GENERATED! DO NOT EDIT -# -######################################################## -set -e - -echo "Starting build process in: `pwd`" -source ./ci/setup.sh - -if [[ -f "ci/run.sh" ]]; then - echo "Running custom build script in: `pwd`/ci/run.sh" - source ./ci/run.sh -else - echo "Running default build scripts in: `pwd`/ci/build.sh" - bundle install - bundle exec rake vendor - bundle exec rspec spec -fi diff --git a/ci/setup.sh b/ci/setup.sh deleted file mode 100755 index 835fa43..0000000 --- a/ci/setup.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# version: 1 -######################################################## -# -# AUTOMATICALLY GENERATED! DO NOT EDIT -# -######################################################## -set -e -if [ "$LOGSTASH_BRANCH" ]; then - echo "Building plugin using Logstash source" - BASE_DIR=`pwd` - echo "Checking out branch: $LOGSTASH_BRANCH" - git clone -b $LOGSTASH_BRANCH https://github.com/elastic/logstash.git ../../logstash --depth 1 - printf "Checked out Logstash revision: %s\n" "$(git -C ../../logstash rev-parse HEAD)" - cd ../../logstash - echo "Building plugins with Logstash version:" - cat versions.yml - echo "---" - # We need to build the jars for that specific version - echo "Running gradle assemble in: `pwd`" - ./gradlew assemble - cd $BASE_DIR - export LOGSTASH_SOURCE=1 -else - echo "Building plugin using released gems on rubygems" -fi From 8c979b104ce4c86f856c2ca675ad10a94803c1f7 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Mon, 4 Jan 2021 12:49:20 +0000 Subject: [PATCH 30/34] [skip ci] update travis ci badge from .org to .com --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 86a1c11..62c9cb2 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Logstash Plugin -[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-input-github.svg)](https://travis-ci.org/logstash-plugins/logstash-input-github) +[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-input-github.svg)](https://travis-ci.com/logstash-plugins/logstash-input-github) This is a plugin for [Logstash](https://github.com/elastic/logstash). From 0910a86e0f5e29e67de86bc6bed45708be5d1989 Mon Sep 17 00:00:00 2001 From: Edmo Vamerlatti Costa <11836452+edmocosta@users.noreply.github.com> Date: Thu, 22 Dec 2022 16:53:43 +0100 Subject: [PATCH 31/34] Fixed crashing when request body is not a JSON object (#24) - Fixed crashing when the request body is not a JSON object - Pin http_parser.rb gem version to ~> 0.6.0 --- CHANGELOG.md | 4 ++++ lib/logstash/inputs/github.rb | 11 ++++++++++- logstash-input-github.gemspec | 3 ++- spec/inputs/github_spec.rb | 21 +++++++++++++++++++++ 4 files changed, 37 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a326e3..d9bd064 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 3.0.10 + - Changed the transitive dependency `http_parser.rb` (ftw) version to `~-> 0.6.0` as newer versions are published without the java support. + - Fixed crashing when the request body payload is not a JSON object. [#24](https://github.com/logstash-plugins/logstash-input-github/pull/24) + ## 3.0.9 - Bump ftw dependency to 0.0.49, for compatibility with Logstash 7.x diff --git a/lib/logstash/inputs/github.rb b/lib/logstash/inputs/github.rb index 0a03ef7..8a6ba67 100644 --- a/lib/logstash/inputs/github.rb +++ b/lib/logstash/inputs/github.rb @@ -53,7 +53,16 @@ def run(output_queue) def build_event_from_request(body, headers) begin - event = LogStash::Event.new(JSON.parse(body)) + data = JSON.parse(body) + # The JSON specification defines single values as valid JSONs, it can be a string in double quotes, + # a number, true or false or null. When the body is parsed, those values are transformed into its + # corresponding types. When those types aren't a Hash (aka object), it breaks the LogStash::Event + # contract and crashes. + if data.is_a?(::Hash) + event = LogStash::Event.new(data) + else + event = LogStash::Event.new("message" => body, "tags" => "_invalidjsonobject") + end rescue JSON::ParserError => e @logger.info("JSON parse failure. Falling back to plain-text", :error => e, :data => body) event = LogStash::Event.new("message" => body, "tags" => "_invalidjson") diff --git a/logstash-input-github.gemspec b/logstash-input-github.gemspec index 0e6aab9..98cde90 100644 --- a/logstash-input-github.gemspec +++ b/logstash-input-github.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-github' - s.version = '3.0.9' + s.version = '3.0.10' s.licenses = ['Apache License (2.0)'] s.summary = "Reads events from a GitHub webhook" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -24,6 +24,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'addressable' s.add_runtime_dependency 'logstash-codec-plain' + s.add_runtime_dependency 'http_parser.rb', '~> 0.6.0' s.add_runtime_dependency 'ftw', '~> 0.0.49' s.add_development_dependency 'logstash-devutils' diff --git a/spec/inputs/github_spec.rb b/spec/inputs/github_spec.rb index 08f1198..6dc22e3 100644 --- a/spec/inputs/github_spec.rb +++ b/spec/inputs/github_spec.rb @@ -86,6 +86,27 @@ end + describe "verify event builder" do + let(:plugin) { LogStash::Plugin.lookup("input", "github").new( {"port" => 9999} ) } + let(:body) {"{}"} + let(:event) {plugin.build_event_from_request(body, {})} + + context 'when request body is a minimal JSON value' do + let(:body) {"123"} + it 'should add the body string into the message field and tag' do + expect(event.get("message")).to eq("123") + expect(event.get("tags")).to eq("_invalidjsonobject") + end + end + + context 'when request body is a JSON object' do + let(:body) {'{"action": "create"}'} + it 'should parse the body' do + expect(event.get("action")).to eq("create") + end + end + end + describe 'graceful shutdown' do context 'when underlying webserver crashes' do From 84e01ea461469815887413bbf723ad9768863936 Mon Sep 17 00:00:00 2001 From: Mashhur Date: Mon, 5 Dec 2022 13:46:30 +0500 Subject: [PATCH 32/34] Change config secret_token type to `Password`. This change ensures to protect the secret_token from leaks in debug logs. Fixes: #22 Pull-request: #23 --- CHANGELOG.md | 5 ++++- docs/index.asciidoc | 4 ++-- lib/logstash/inputs/github.rb | 4 ++-- spec/inputs/github_spec.rb | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9bd064..08ab890 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ +## 3.0.11 + - Change `secret_token` config type to `password` for better protection from leaks in debug logs [#23](https://github.com/logstash-plugins/logstash-input-github/pull/23) + ## 3.0.10 - Changed the transitive dependency `http_parser.rb` (ftw) version to `~-> 0.6.0` as newer versions are published without the java support. - - Fixed crashing when the request body payload is not a JSON object. [#24](https://github.com/logstash-plugins/logstash-input-github/pull/24) + - Fixed crashing when the request body payload is not a JSON object. [#24](https://github.com/logstash-plugins/logstash-input-github/pull/24) ## 3.0.9 - Bump ftw dependency to 0.0.49, for compatibility with Logstash 7.x diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 7d77c2e..eb333e9 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -34,7 +34,7 @@ This plugin supports the following configuration options plus the <> |<>|No | <> |<>|No | <> |<>|Yes -| <> |<>|No +| <> |<>|No |======================================================================= Also see <> for a list of options supported by all @@ -71,7 +71,7 @@ The port to listen on [id="plugins-{type}s-{plugin}-secret_token"] ===== `secret_token` - * Value type is <> + * Value type is <> * There is no default value for this setting. Your GitHub Secret Token for the webhook diff --git a/lib/logstash/inputs/github.rb b/lib/logstash/inputs/github.rb index 8a6ba67..c337d42 100644 --- a/lib/logstash/inputs/github.rb +++ b/lib/logstash/inputs/github.rb @@ -16,7 +16,7 @@ class LogStash::Inputs::GitHub < LogStash::Inputs::Base config :port, :validate => :number, :required => true # Your GitHub Secret Token for the webhook - config :secret_token, :validate => :string, :required => false + config :secret_token, :validate => :password, :required => false # If Secret is defined, we drop the events that don't match. # Otherwise, we'll just add an invalid tag @@ -77,7 +77,7 @@ def verify_signature(event,body) sign_header = event.get("[headers][x-hub-signature]") if sign_header - hash = 'sha1=' + OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('sha1'), @secret_token, body) + hash = 'sha1=' + OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('sha1'), @secret_token.value, body) event.set("hash", hash) return true if Rack::Utils.secure_compare(hash, sign_header) end diff --git a/spec/inputs/github_spec.rb b/spec/inputs/github_spec.rb index 6dc22e3..f158cbe 100644 --- a/spec/inputs/github_spec.rb +++ b/spec/inputs/github_spec.rb @@ -27,7 +27,7 @@ end describe "verify webhook signature if token provided" do - let(:plugin) { LogStash::Plugin.lookup("input", "github").new( {"port" => 9999, "secret_token" => "my_secret"} ) } + let(:plugin) { LogStash::Plugin.lookup("input", "github").new( {"port" => 9999, "secret_token" => ::LogStash::Util::Password.new("my_secret")} ) } let(:body) {IO.read("spec/fixtures/event_create.json")} let(:headers) { {"x-hub-signature" => "hash"} } let(:event) {plugin.build_event_from_request(body,headers)} From 28a64b9162fd45840f9ff9d476657dd4bdca12f3 Mon Sep 17 00:00:00 2001 From: Mashhur Date: Tue, 23 May 2023 14:43:34 -0700 Subject: [PATCH 33/34] Rebase and version upgraded in gemspec. --- logstash-input-github.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash-input-github.gemspec b/logstash-input-github.gemspec index 98cde90..e3dec90 100644 --- a/logstash-input-github.gemspec +++ b/logstash-input-github.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-github' - s.version = '3.0.10' + s.version = '3.0.11' s.licenses = ['Apache License (2.0)'] s.summary = "Reads events from a GitHub webhook" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From cbb19555d65a3a22584891fae3f4abd9c7d6e78b Mon Sep 17 00:00:00 2001 From: Mashhur Date: Sat, 27 May 2023 12:47:36 -0700 Subject: [PATCH 34/34] Add unit test case that secrets will not be shown in debug logs. --- spec/inputs/github_spec.rb | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/spec/inputs/github_spec.rb b/spec/inputs/github_spec.rb index f158cbe..11ad53d 100644 --- a/spec/inputs/github_spec.rb +++ b/spec/inputs/github_spec.rb @@ -153,4 +153,15 @@ end end end + + describe "debugging `secret_token`" do + let(:plugin) { LogStash::Plugin.lookup("input", "github").new( {"port" => 9999, "secret_token" => ::LogStash::Util::Password.new("my_secret")} ) } + + it "should not show origin value" do + expect(plugin.logger).to receive(:debug).with('') + + plugin.register + plugin.logger.send(:debug, plugin.secret_token.to_s) + end + end end