From f1d1fa6054c9483d2c821701f2e9f9c4b9994a75 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 3 May 2016 11:53:11 -0700 Subject: [PATCH 01/40] update to new event api --- .travis.yml | 12 ++++++++---- Gemfile | 9 ++++++++- logstash-output-redis.gemspec | 2 +- spec/integration/outputs/redis_spec.rb | 4 ++-- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2220b24..0b6635a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,13 @@ +--- sudo: false language: ruby cache: bundler services: - - redis-server +- redis-server rvm: - - jruby-1.7.23 -script: - - bundle exec rspec spec && bundle exec rspec spec --tag integration +- jruby-1.7.25 +script: +- bundle exec rspec spec && bundle exec rspec spec --tag integration +jdk: oraclejdk8 +before_install: +- git clone -b feature/event_interface https://github.com/elastic/logstash diff --git a/Gemfile b/Gemfile index d926697..53827bd 100644 --- a/Gemfile +++ b/Gemfile @@ -1,2 +1,9 @@ source 'https://rubygems.org/' -gemspec \ No newline at end of file +gemspec +gem "logstash-core", :path => "./logstash/logstash-core" +gem "logstash-core-plugin-api", :path => "./logstash/logstash-core-plugin-api" +gem "logstash-core-event-java", :path => "./logstash/logstash-core-event-java" +gem "logstash-devutils", :github => "elastic/logstash-devutils", :branch => "feature/plugin-api-2_0" +gem "logstash-input-generator", :github => "logstash-plugins/logstash-input-generator", :branch => "feature/plugin-api-2_0" +gem "logstash-codec-json", :github => "logstash-plugins/logstash-codec-json", :branch => "feature/plugin-api-2_0" +gem "logstash-codec-plain", :github => "logstash-plugins/logstash-codec-plain", :branch => "feature/plugin-api-2_0" diff --git a/logstash-output-redis.gemspec b/logstash-output-redis.gemspec index fd1ba01..fb0d8ba 100644 --- a/logstash-output-redis.gemspec +++ b/logstash-output-redis.gemspec @@ -20,7 +20,7 @@ Gem::Specification.new do |s| s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" } # Gem dependencies - s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0" + s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0" s.add_runtime_dependency 'redis' s.add_runtime_dependency 'stud' diff --git a/spec/integration/outputs/redis_spec.rb b/spec/integration/outputs/redis_spec.rb index e934e64..4d531f7 100644 --- a/spec/integration/outputs/redis_spec.rb +++ b/spec/integration/outputs/redis_spec.rb @@ -42,8 +42,8 @@ event_count.times do |value| id, element = redis.blpop(key, 0) event = LogStash::Event.new(LogStash::Json.load(element)) - insist { event["sequence"] } == value - insist { event["message"] } == message + insist { event.get("sequence") } == value + insist { event.get("message") } == message end # The list should now be empty From 30653aeffe53f2224974912a9891129fefe2c1be Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Fri, 6 May 2016 14:53:05 -0400 Subject: [PATCH 02/40] Mass update for 2.0 plugin api changes --- .travis.yml | 3 +-- CHANGELOG.md | 2 ++ Gemfile | 9 ++------- logstash-output-redis.gemspec | 2 +- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0b6635a..cf65f5c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,5 +9,4 @@ rvm: script: - bundle exec rspec spec && bundle exec rspec spec --tag integration jdk: oraclejdk8 -before_install: -- git clone -b feature/event_interface https://github.com/elastic/logstash +before_install: [] diff --git a/CHANGELOG.md b/CHANGELOG.md index b53b225..e96a229 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## 3.0.0 + - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141 # 2.0.5 - Fix LocalJumpError exception see https://github.com/logstash-plugins/logstash-output-redis/pull/27 # 2.0.4 diff --git a/Gemfile b/Gemfile index 53827bd..2b03d18 100644 --- a/Gemfile +++ b/Gemfile @@ -1,9 +1,4 @@ source 'https://rubygems.org/' + +# Specify your gem's dependencies in logstash-mass_effect.gemspec gemspec -gem "logstash-core", :path => "./logstash/logstash-core" -gem "logstash-core-plugin-api", :path => "./logstash/logstash-core-plugin-api" -gem "logstash-core-event-java", :path => "./logstash/logstash-core-event-java" -gem "logstash-devutils", :github => "elastic/logstash-devutils", :branch => "feature/plugin-api-2_0" -gem "logstash-input-generator", :github => "logstash-plugins/logstash-input-generator", :branch => "feature/plugin-api-2_0" -gem "logstash-codec-json", :github => "logstash-plugins/logstash-codec-json", :branch => "feature/plugin-api-2_0" -gem "logstash-codec-plain", :github => "logstash-plugins/logstash-codec-plain", :branch => "feature/plugin-api-2_0" diff --git a/logstash-output-redis.gemspec b/logstash-output-redis.gemspec index fb0d8ba..7eb36dc 100644 --- a/logstash-output-redis.gemspec +++ b/logstash-output-redis.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-output-redis' - s.version = '2.0.5' + s.version = '3.0.0' s.licenses = ['Apache License (2.0)'] s.summary = "This output will send events to a Redis queue using RPUSH" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From ceb72bcb1a636fd7153e9b5e24405572c4afc95a Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Fri, 6 May 2016 16:39:17 -0400 Subject: [PATCH 03/40] Make sure all the gems were published under jruby --- CHANGELOG.md | 2 ++ logstash-output-redis.gemspec | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e96a229..ff1a64b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## 3.0.1 + - Republish all the gems under jruby. ## 3.0.0 - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141 # 2.0.5 diff --git a/logstash-output-redis.gemspec b/logstash-output-redis.gemspec index 7eb36dc..332dfa1 100644 --- a/logstash-output-redis.gemspec +++ b/logstash-output-redis.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-output-redis' - s.version = '3.0.0' + s.version = '3.0.1' s.licenses = ['Apache License (2.0)'] s.summary = "This output will send events to a Redis queue using RPUSH" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 5e36d241bd14fb53068e3fcf2deadbab1243dc2d Mon Sep 17 00:00:00 2001 From: markwalkom Date: Sun, 8 May 2016 09:55:05 +1000 Subject: [PATCH 04/40] Expand hosts array documentation Clarified that the hosts directive can take an array, and how that works. This was raised [here](https://discuss.elastic.co/t/question-on-logstash-output-to-redis/49374). Fixes #32 --- lib/logstash/outputs/redis.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/logstash/outputs/redis.rb b/lib/logstash/outputs/redis.rb index 78cf6d6..daeb5bd 100644 --- a/lib/logstash/outputs/redis.rb +++ b/lib/logstash/outputs/redis.rb @@ -26,6 +26,8 @@ class LogStash::Outputs::Redis < LogStash::Outputs::Base # The hostname(s) of your Redis server(s). Ports may be specified on any # hostname, which will override the global port config. + # If the hosts list is an array, Logstash will pick one random host to connect to, + # if that host is disconnected it will then pick another. # # For example: # [source,ruby] From a581dea12965b24901492510b2ec4d7faf1a4595 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Thu, 14 Jul 2016 12:25:23 +0100 Subject: [PATCH 05/40] dependency logstash-core-plugin-api >= 1.60 <= 2.99 --- CHANGELOG.md | 3 +++ logstash-output-redis.gemspec | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff1a64b..abd9f88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.0.2 + - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99 + ## 3.0.1 - Republish all the gems under jruby. ## 3.0.0 diff --git a/logstash-output-redis.gemspec b/logstash-output-redis.gemspec index 332dfa1..2fc2902 100644 --- a/logstash-output-redis.gemspec +++ b/logstash-output-redis.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-output-redis' - s.version = '3.0.1' + s.version = '3.0.2' s.licenses = ['Apache License (2.0)'] s.summary = "This output will send events to a Redis queue using RPUSH" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -20,7 +20,7 @@ Gem::Specification.new do |s| s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" } # Gem dependencies - s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0" + s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" s.add_runtime_dependency 'redis' s.add_runtime_dependency 'stud' From 24635f635dce97257b2c1e3484275aacbd024ecc Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 13 Oct 2016 14:51:23 -0400 Subject: [PATCH 06/40] fix debug logging arguments to match proper signature v3.0.3 (#42) * fix debug logging arguments to match proper signature * bump version --- CHANGELOG.md | 3 ++- lib/logstash/outputs/redis.rb | 2 +- logstash-output-redis.gemspec | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abd9f88..86d4a0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ +## 3.0.3 + - Fix logging method signature for #debug ## 3.0.2 - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99 - ## 3.0.1 - Republish all the gems under jruby. ## 3.0.0 diff --git a/lib/logstash/outputs/redis.rb b/lib/logstash/outputs/redis.rb index daeb5bd..aa289a8 100644 --- a/lib/logstash/outputs/redis.rb +++ b/lib/logstash/outputs/redis.rb @@ -213,7 +213,7 @@ def connect :timeout => @timeout, :db => @db } - @logger.debug(params) + @logger.debug("connection params", params) if @password params[:password] = @password.value diff --git a/logstash-output-redis.gemspec b/logstash-output-redis.gemspec index 2fc2902..cda826a 100644 --- a/logstash-output-redis.gemspec +++ b/logstash-output-redis.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-output-redis' - s.version = '3.0.2' + s.version = '3.0.3' s.licenses = ['Apache License (2.0)'] s.summary = "This output will send events to a Redis queue using RPUSH" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 9bf2bfc9dbefcbec7201472d8f01d6a718e5d257 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Fri, 28 Apr 2017 21:14:00 +0000 Subject: [PATCH 07/40] Initial doc move --- docs/index.asciidoc | 220 ++++++++++++++++++++++++++++++++++ logstash-output-redis.gemspec | 2 +- 2 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 docs/index.asciidoc diff --git a/docs/index.asciidoc b/docs/index.asciidoc new file mode 100644 index 0000000..9d6d5bc --- /dev/null +++ b/docs/index.asciidoc @@ -0,0 +1,220 @@ +:plugin: redis +:type: output + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: %VERSION% +:release_date: %RELEASE_DATE% +:changelog_url: %CHANGELOG_URL% +:include_path: ../../../logstash/docs/include +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="plugins-{type}-{plugin}"] + +=== Redis + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This output will send events to a Redis queue using RPUSH. +The RPUSH command is supported in Redis v0.0.7+. Using +PUBLISH to a channel requires at least v1.3.8+. +While you may be able to make these Redis versions work, +the best performance and stability will be found in more +recent stable versions. Versions 2.6.0+ are recommended. + +For more information, see http://redis.io/[the Redis homepage] + + +[id="plugins-{type}s-{plugin}-options"] +==== Redis Output Configuration Options + +This plugin supports the following configuration options plus the <> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>, one of `["list", "channel"]`|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +|======================================================================= + +Also see <> for a list of options supported by all +output plugins. + +  + +[id="plugins-{type}s-{plugin}-batch"] +===== `batch` + + * Value type is <> + * Default value is `false` + +Set to true if you want Redis to batch up values and send 1 RPUSH command +instead of one command per value to push on the list. Note that this only +works with `data_type="list"` mode right now. + +If true, we send an RPUSH every "batch_events" events or +"batch_timeout" seconds (whichever comes first). +Only supported for `data_type` is "list". + +[id="plugins-{type}s-{plugin}-batch_events"] +===== `batch_events` + + * Value type is <> + * Default value is `50` + +If batch is set to true, the number of events we queue up for an RPUSH. + +[id="plugins-{type}s-{plugin}-batch_timeout"] +===== `batch_timeout` + + * Value type is <> + * Default value is `5` + +If batch is set to true, the maximum amount of time between RPUSH commands +when there are pending events to flush. + +[id="plugins-{type}s-{plugin}-congestion_interval"] +===== `congestion_interval` + + * Value type is <> + * Default value is `1` + +How often to check for congestion. Default is one second. +Zero means to check on every event. + +[id="plugins-{type}s-{plugin}-congestion_threshold"] +===== `congestion_threshold` + + * Value type is <> + * Default value is `0` + +In case Redis `data_type` is `list` and has more than `@congestion_threshold` items, +block until someone consumes them and reduces congestion, otherwise if there are +no consumers Redis will run out of memory, unless it was configured with OOM protection. +But even with OOM protection, a single Redis list can block all other users of Redis, +until Redis CPU consumption reaches the max allowed RAM size. +A default value of 0 means that this limit is disabled. +Only supported for `list` Redis `data_type`. + +[id="plugins-{type}s-{plugin}-data_type"] +===== `data_type` + + * Value can be any of: `list`, `channel` + * There is no default value for this setting. + +Either list or channel. If `redis_type` is list, then we will set +RPUSH to key. If `redis_type` is channel, then we will PUBLISH to `key`. + +[id="plugins-{type}s-{plugin}-db"] +===== `db` + + * Value type is <> + * Default value is `0` + +The Redis database number. + +[id="plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is <> + * Default value is `["127.0.0.1"]` + +The hostname(s) of your Redis server(s). Ports may be specified on any +hostname, which will override the global port config. +If the hosts list is an array, Logstash will pick one random host to connect to, +if that host is disconnected it will then pick another. + +For example: +[source,ruby] + "127.0.0.1" + ["127.0.0.1", "127.0.0.2"] + ["127.0.0.1:6380", "127.0.0.1"] + +[id="plugins-{type}s-{plugin}-key"] +===== `key` + + * Value type is <> + * There is no default value for this setting. + +The name of a Redis list or channel. Dynamic names are +valid here, for example `logstash-%{type}`. + +[id="plugins-{type}s-{plugin}-name"] +===== `name` (DEPRECATED) + + * DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions. + * Value type is <> + * Default value is `"default"` + +Name is used for logging in case there are multiple instances. + +[id="plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is <> + * There is no default value for this setting. + +Password to authenticate with. There is no authentication by default. + +[id="plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is <> + * Default value is `6379` + +The default port to connect on. Can be overridden on any hostname. + +[id="plugins-{type}s-{plugin}-queue"] +===== `queue` (DEPRECATED) + + * DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions. + * Value type is <> + * There is no default value for this setting. + +The name of the Redis queue (we'll use RPUSH on this). Dynamic names are +valid here, for example `logstash-%{type}` + +[id="plugins-{type}s-{plugin}-reconnect_interval"] +===== `reconnect_interval` + + * Value type is <> + * Default value is `1` + +Interval for reconnecting to failed Redis connections + +[id="plugins-{type}s-{plugin}-shuffle_hosts"] +===== `shuffle_hosts` + + * Value type is <> + * Default value is `true` + +Shuffle the host list during Logstash startup. + +[id="plugins-{type}s-{plugin}-timeout"] +===== `timeout` + + * Value type is <> + * Default value is `5` + +Redis initial connection timeout in seconds. + + + +include::{include_path}/{type}.asciidoc[] diff --git a/logstash-output-redis.gemspec b/logstash-output-redis.gemspec index cda826a..bb3d1b9 100644 --- a/logstash-output-redis.gemspec +++ b/logstash-output-redis.gemspec @@ -11,7 +11,7 @@ Gem::Specification.new do |s| s.require_paths = ["lib"] # Files - s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT'] + s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"] # Tests s.test_files = s.files.grep(%r{^(test|spec|features)/}) From d40cf67dd7bc5bb8311091f749fbe01db5b6cff8 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Fri, 26 May 2017 13:01:07 -0400 Subject: [PATCH 08/40] new build system for jruby9k --- .travis.yml | 23 +++++++++++++++++------ Gemfile | 8 +++++++- ci/build.sh | 21 +++++++++++++++++++++ ci/run.sh | 3 +++ ci/setup.sh | 26 ++++++++++++++++++++++++++ 5 files changed, 74 insertions(+), 7 deletions(-) create mode 100755 ci/build.sh create mode 100755 ci/run.sh create mode 100755 ci/setup.sh diff --git a/.travis.yml b/.travis.yml index cf65f5c..38f1e9d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,11 +2,22 @@ sudo: false language: ruby cache: bundler -services: -- redis-server +env: rvm: -- jruby-1.7.25 -script: -- bundle exec rspec spec && bundle exec rspec spec --tag integration + - jruby-1.7.25 +services: + - redis-server +matrix: + include: + - rvm: jruby-1.7.25 + env: LOGSTASH_BRANCH=master + - rvm: jruby-1.7.25 + env: LOGSTASH_BRANCH=5.x + - rvm: jruby-9.1.9.0 + env: LOGSTASH_BRANCH=feature/9000 + allow_failures: + - rvm: jruby-9.1.9.0 + fast_finish: true +install: true +script: ci/build.sh jdk: oraclejdk8 -before_install: [] diff --git a/Gemfile b/Gemfile index 2b03d18..93e5e5d 100644 --- a/Gemfile +++ b/Gemfile @@ -1,4 +1,10 @@ source 'https://rubygems.org/' -# Specify your gem's dependencies in logstash-mass_effect.gemspec gemspec + +logstash_path = "../../logstash" + +if Dir.exist?(logstash_path) && ENV["LOGSTASH_SOURCE"] == 1 + gem 'logstash-core', :path => "#{logstash_path}/logstash-core" + gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api" +end diff --git a/ci/build.sh b/ci/build.sh new file mode 100755 index 0000000..076e908 --- /dev/null +++ b/ci/build.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# version: 1 +######################################################## +# +# AUTOMATICALLY GENERATED! DO NOT EDIT +# +######################################################## +set -e + +echo "Starting build process in: `pwd`" +./ci/setup.sh + +if [[ -f "ci/run.sh" ]]; then + echo "Running custom build script in: `pwd`/ci/run.sh" + ./ci/run.sh +else + echo "Running default build scripts in: `pwd`/ci/build.sh" + bundle install + bundle exec rake vendor + bundle exec rspec spec +fi diff --git a/ci/run.sh b/ci/run.sh new file mode 100755 index 0000000..feaf36d --- /dev/null +++ b/ci/run.sh @@ -0,0 +1,3 @@ +bundle install +bundle exec rake vendor +bundle exec rspec spec && bundle exec rspec spec --tag integration diff --git a/ci/setup.sh b/ci/setup.sh new file mode 100755 index 0000000..835fa43 --- /dev/null +++ b/ci/setup.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# version: 1 +######################################################## +# +# AUTOMATICALLY GENERATED! DO NOT EDIT +# +######################################################## +set -e +if [ "$LOGSTASH_BRANCH" ]; then + echo "Building plugin using Logstash source" + BASE_DIR=`pwd` + echo "Checking out branch: $LOGSTASH_BRANCH" + git clone -b $LOGSTASH_BRANCH https://github.com/elastic/logstash.git ../../logstash --depth 1 + printf "Checked out Logstash revision: %s\n" "$(git -C ../../logstash rev-parse HEAD)" + cd ../../logstash + echo "Building plugins with Logstash version:" + cat versions.yml + echo "---" + # We need to build the jars for that specific version + echo "Running gradle assemble in: `pwd`" + ./gradlew assemble + cd $BASE_DIR + export LOGSTASH_SOURCE=1 +else + echo "Building plugin using released gems on rubygems" +fi From 2c555b84e3402f1f0e43a78ed6e375bea57e3c26 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Wed, 31 May 2017 16:45:01 -0400 Subject: [PATCH 09/40] Adjusting the build scripts to correctly load the logstash source and allow people to override it --- Gemfile | 5 +++-- ci/build.sh | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Gemfile b/Gemfile index 93e5e5d..32cc6fb 100644 --- a/Gemfile +++ b/Gemfile @@ -2,9 +2,10 @@ source 'https://rubygems.org/' gemspec -logstash_path = "../../logstash" +logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash" +use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1" -if Dir.exist?(logstash_path) && ENV["LOGSTASH_SOURCE"] == 1 +if Dir.exist?(logstash_path) && use_logstash_source gem 'logstash-core', :path => "#{logstash_path}/logstash-core" gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api" end diff --git a/ci/build.sh b/ci/build.sh index 076e908..06caffd 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -8,11 +8,11 @@ set -e echo "Starting build process in: `pwd`" -./ci/setup.sh +source ./ci/setup.sh if [[ -f "ci/run.sh" ]]; then echo "Running custom build script in: `pwd`/ci/run.sh" - ./ci/run.sh + source ./ci/run.sh else echo "Running default build scripts in: `pwd`/ci/build.sh" bundle install From 3d0858b7ba176e38a0f4fc9c56cdf1b8329000b3 Mon Sep 17 00:00:00 2001 From: Guy Boertje Date: Tue, 6 Jun 2017 19:11:57 +0100 Subject: [PATCH 10/40] Fixes as per comment in issue #49 (#50) * Fixes as per comment in issue #49 Fixes #49 * add comment about the large batch_timeout value --- spec/unit/outputs/redis_spec.rb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/spec/unit/outputs/redis_spec.rb b/spec/unit/outputs/redis_spec.rb index 7ed1220..bdff2b3 100644 --- a/spec/unit/outputs/redis_spec.rb +++ b/spec/unit/outputs/redis_spec.rb @@ -11,14 +11,15 @@ # TODO: refactor specs above and probably rely on a Redis mock to correctly test the code expected behaviour, the actual # tests agains Redis should be moved into integration tests. let(:key) { "thekey" } - let(:payload) { "somepayload"} - let(:event) { LogStash::Event.new({"message" => "test"}) } let(:config) { { "key" => key, "data_type" => "list", "batch" => true, "batch_events" => 50, + "batch_timeout" => 3600 * 24, + # ^ this a very large timeout value to prevent the Flush Timer thread in Stud::Buffer from calling flush + # it screws with the RSpec expect().to receive thread safety. } } let(:redis) { described_class.new(config) } @@ -27,12 +28,13 @@ redis.register expect(redis).to receive(:buffer_receive).exactly(10000).times.and_call_original expect(redis).to receive(:flush).exactly(200).times + expect(redis).not_to receive(:on_flush_error) # I was able to reproduce the LocalJumpError: unexpected next exception at around 50 # consicutive invocations. setting to 10000 should reproduce it for any environment # I have no clue at this point why this problem does not happen at every invocation - 1.upto(10000) do - expect{redis.receive(event)}.to_not raise_error + 10000.times do |i| + expect{redis.receive(LogStash::Event.new({"message" => "test-#{i}"}))}.to_not raise_error end end end From cd56ae004358b9b1543c5236108333fe81aa7532 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Tue, 13 Jun 2017 09:10:41 -0400 Subject: [PATCH 11/40] update .travis.yml for jruby9k jobs --- .travis.yml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 38f1e9d..09796f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,21 +2,17 @@ sudo: false language: ruby cache: bundler -env: +env: rvm: - - jruby-1.7.25 +- jruby-1.7.25 services: - - redis-server +- redis-server matrix: include: - - rvm: jruby-1.7.25 - env: LOGSTASH_BRANCH=master - - rvm: jruby-1.7.25 - env: LOGSTASH_BRANCH=5.x - - rvm: jruby-9.1.9.0 - env: LOGSTASH_BRANCH=feature/9000 - allow_failures: - - rvm: jruby-9.1.9.0 + - rvm: jruby-9.1.10.0 + env: LOGSTASH_BRANCH=master + - rvm: jruby-1.7.25 + env: LOGSTASH_BRANCH=5.x fast_finish: true install: true script: ci/build.sh From 8582267ba6bf8a88face75bbb666b4f3dc8b1a79 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 15 Jun 2017 20:01:00 -0400 Subject: [PATCH 12/40] update plugin header for better search results --- docs/index.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 9d6d5bc..2e2bb14 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -14,7 +14,7 @@ END - GENERATED VARIABLES, DO NOT EDIT! [id="plugins-{type}-{plugin}"] -=== Redis +=== Redis output plugin include::{include_path}/plugin_header.asciidoc[] From 31e02f467edc84dfe52b03e0e81eedfa779600d7 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 22 Jun 2017 22:30:41 -0400 Subject: [PATCH 13/40] [skip ci] Updating the plugin doc --- docs/index.asciidoc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 2e2bb14..056dc63 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -7,7 +7,7 @@ START - GENERATED VARIABLES, DO NOT EDIT! :version: %VERSION% :release_date: %RELEASE_DATE% :changelog_url: %CHANGELOG_URL% -:include_path: ../../../logstash/docs/include +:include_path: ../../../../logstash/docs/include /////////////////////////////////////////// END - GENERATED VARIABLES, DO NOT EDIT! /////////////////////////////////////////// @@ -33,7 +33,7 @@ For more information, see http://redis.io/[the Redis homepage] [id="plugins-{type}s-{plugin}-options"] ==== Redis Output Configuration Options -This plugin supports the following configuration options plus the <> described later. +This plugin supports the following configuration options plus the <> described later. [cols="<,<,<",options="header",] |======================================================================= @@ -54,7 +54,7 @@ This plugin supports the following configuration options plus the <> |<>|No |======================================================================= -Also see <> for a list of options supported by all +Also see <> for a list of options supported by all output plugins.   @@ -217,4 +217,5 @@ Redis initial connection timeout in seconds. -include::{include_path}/{type}.asciidoc[] +[id="plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] \ No newline at end of file From 888766e6fe9a3b1accfd2b4f309d5b3807acf4d2 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 22 Jun 2017 22:54:15 -0400 Subject: [PATCH 14/40] bump patch level for doc generation --- logstash-output-redis.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash-output-redis.gemspec b/logstash-output-redis.gemspec index bb3d1b9..c5e6731 100644 --- a/logstash-output-redis.gemspec +++ b/logstash-output-redis.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-output-redis' - s.version = '3.0.3' + s.version = '3.0.4' s.licenses = ['Apache License (2.0)'] s.summary = "This output will send events to a Redis queue using RPUSH" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From b767e3fe4d5075decdb5fcb020d6d2d24ad6503a Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Mon, 26 Jun 2017 21:24:16 -0400 Subject: [PATCH 15/40] [skip ci] Updating the plugin id in the doc to match the index in the docbook --- docs/index.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 056dc63..7bf03b0 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -12,7 +12,7 @@ START - GENERATED VARIABLES, DO NOT EDIT! END - GENERATED VARIABLES, DO NOT EDIT! /////////////////////////////////////////// -[id="plugins-{type}-{plugin}"] +[id="plugins-{type}s-{plugin}"] === Redis output plugin From e2a2664797895d947ddf8cbcf9dbf01a4bdc2d6e Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Wed, 26 Jul 2017 12:38:11 +0100 Subject: [PATCH 16/40] on travis test against 5.6 and 6.0 logstash-core --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 09796f1..594101c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,9 +11,11 @@ matrix: include: - rvm: jruby-9.1.10.0 env: LOGSTASH_BRANCH=master + - rvm: jruby-9.1.10.0 + env: LOGSTASH_BRANCH=6.x - rvm: jruby-1.7.25 - env: LOGSTASH_BRANCH=5.x + env: LOGSTASH_BRANCH=5.6 fast_finish: true install: true script: ci/build.sh -jdk: oraclejdk8 +jdk: oraclejdk8 \ No newline at end of file From a390527e8a875895018c36b9408d27e418987e8e Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Thu, 27 Jul 2017 17:13:27 +0100 Subject: [PATCH 17/40] Mark deprecated `name` and `queue` configuration parameters as obsolete Also, since queue is no longer available, we can mark `key` and `data_type` as required and remove validations at `register` time. --- CHANGELOG.md | 3 +++ docs/index.asciidoc | 23 ++--------------------- lib/logstash/outputs/redis.rb | 32 +++++--------------------------- logstash-output-redis.gemspec | 2 +- 4 files changed, 11 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86d4a0b..a4d29d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.0.0 + - Mark deprecated `name` and `queue` configuration parameters as obsolete + ## 3.0.3 - Fix logging method signature for #debug ## 3.0.2 diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 7bf03b0..9406057 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -119,8 +119,8 @@ Only supported for `list` Redis `data_type`. * Value can be any of: `list`, `channel` * There is no default value for this setting. -Either list or channel. If `redis_type` is list, then we will set -RPUSH to key. If `redis_type` is channel, then we will PUBLISH to `key`. +Either list or channel. If `data_type` is list, then we will set +RPUSH to key. If `data_type` is channel, then we will PUBLISH to `key`. [id="plugins-{type}s-{plugin}-db"] ===== `db` @@ -156,15 +156,6 @@ For example: The name of a Redis list or channel. Dynamic names are valid here, for example `logstash-%{type}`. -[id="plugins-{type}s-{plugin}-name"] -===== `name` (DEPRECATED) - - * DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions. - * Value type is <> - * Default value is `"default"` - -Name is used for logging in case there are multiple instances. - [id="plugins-{type}s-{plugin}-password"] ===== `password` @@ -181,16 +172,6 @@ Password to authenticate with. There is no authentication by default. The default port to connect on. Can be overridden on any hostname. -[id="plugins-{type}s-{plugin}-queue"] -===== `queue` (DEPRECATED) - - * DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions. - * Value type is <> - * There is no default value for this setting. - -The name of the Redis queue (we'll use RPUSH on this). Dynamic names are -valid here, for example `logstash-%{type}` - [id="plugins-{type}s-{plugin}-reconnect_interval"] ===== `reconnect_interval` diff --git a/lib/logstash/outputs/redis.rb b/lib/logstash/outputs/redis.rb index aa289a8..81e3c9b 100644 --- a/lib/logstash/outputs/redis.rb +++ b/lib/logstash/outputs/redis.rb @@ -21,8 +21,7 @@ class LogStash::Outputs::Redis < LogStash::Outputs::Base default :codec, "json" # Name is used for logging in case there are multiple instances. - config :name, :validate => :string, :default => 'default', - :deprecated => true + config :name, :validate => :string, :obsolete => "This option is obsolete" # The hostname(s) of your Redis server(s). Ports may be specified on any # hostname, which will override the global port config. @@ -51,17 +50,15 @@ class LogStash::Outputs::Redis < LogStash::Outputs::Base # Password to authenticate with. There is no authentication by default. config :password, :validate => :password - # The name of the Redis queue (we'll use RPUSH on this). Dynamic names are - # valid here, for example `logstash-%{type}` - config :queue, :validate => :string, :deprecated => true + config :queue, :validate => :string, :obsolete => "This option is obsolete. Use `key` and `data_type`." # The name of a Redis list or channel. Dynamic names are # valid here, for example `logstash-%{type}`. - config :key, :validate => :string, :required => false + config :key, :validate => :string, :required => true # Either list or channel. If `redis_type` is list, then we will set # RPUSH to key. If `redis_type` is channel, then we will PUBLISH to `key`. - config :data_type, :validate => [ "list", "channel" ], :required => false + config :data_type, :validate => [ "list", "channel" ], :required => true # Set to true if you want Redis to batch up values and send 1 RPUSH command # instead of one command per value to push on the list. Note that this only @@ -98,25 +95,6 @@ class LogStash::Outputs::Redis < LogStash::Outputs::Base def register require 'redis' - # TODO remove after setting key and data_type to true - if @queue - if @key or @data_type - raise RuntimeError.new( - "Cannot specify queue parameter and key or data_type" - ) - end - @key = @queue - @data_type = 'list' - end - - if not @key or not @data_type - raise RuntimeError.new( - "Must define queue, or key and data_type parameters" - ) - end - # end TODO - - if @batch if @data_type != "list" raise RuntimeError.new( @@ -224,7 +202,7 @@ def connect # A string used to identify a Redis instance in log messages def identity - @name || "redis://#{@password}@#{@current_host}:#{@current_port}/#{@db} #{@data_type}:#{@key}" + "redis://#{@password}@#{@current_host}:#{@current_port}/#{@db} #{@data_type}:#{@key}" end def send_to_redis(event, payload) diff --git a/logstash-output-redis.gemspec b/logstash-output-redis.gemspec index c5e6731..94040f5 100644 --- a/logstash-output-redis.gemspec +++ b/logstash-output-redis.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-output-redis' - s.version = '3.0.4' + s.version = '4.0.0' s.licenses = ['Apache License (2.0)'] s.summary = "This output will send events to a Redis queue using RPUSH" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 192a5fa3e9e12c5f312cae0e72b1e623b36dc541 Mon Sep 17 00:00:00 2001 From: Jordan Sissel Date: Tue, 15 Aug 2017 10:01:03 -0700 Subject: [PATCH 18/40] Version bump For https://github.com/elastic/logstash/issues/7993 [ci skip] --- CHANGELOG.md | 3 +++ logstash-output-redis.gemspec | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4d29d9..2163d9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.0.1 + - Fix some documentation issues + ## 4.0.0 - Mark deprecated `name` and `queue` configuration parameters as obsolete diff --git a/logstash-output-redis.gemspec b/logstash-output-redis.gemspec index 94040f5..400bf2d 100644 --- a/logstash-output-redis.gemspec +++ b/logstash-output-redis.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-output-redis' - s.version = '4.0.0' + s.version = '4.0.1' s.licenses = ['Apache License (2.0)'] s.summary = "This output will send events to a Redis queue using RPUSH" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 91421e66e8172699f4907d12d0ba9cfceb8805a1 Mon Sep 17 00:00:00 2001 From: Andrew Cholakian Date: Tue, 12 Sep 2017 11:37:01 -0500 Subject: [PATCH 19/40] Pin redis dep to 3.x.x since 4.x requires ruby 2+ syntax Fixes #54 --- CHANGELOG.md | 2 ++ logstash-output-redis.gemspec | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2163d9b..0a0b930 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## 4.0.2 + - Tighten gem deps for the 'redis' gem ## 4.0.1 - Fix some documentation issues diff --git a/logstash-output-redis.gemspec b/logstash-output-redis.gemspec index 400bf2d..02ad7f2 100644 --- a/logstash-output-redis.gemspec +++ b/logstash-output-redis.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-output-redis' - s.version = '4.0.1' + s.version = '4.0.2' s.licenses = ['Apache License (2.0)'] s.summary = "This output will send events to a Redis queue using RPUSH" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -22,7 +22,7 @@ Gem::Specification.new do |s| # Gem dependencies s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" - s.add_runtime_dependency 'redis' + s.add_runtime_dependency 'redis', '~> 3' s.add_runtime_dependency 'stud' s.add_development_dependency 'logstash-devutils' From b21ed5ec7b875ff2cb860e606150a85ab6f04cb4 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Fri, 27 Oct 2017 17:24:57 -0500 Subject: [PATCH 20/40] Travis - add 6.0 build, remove default JRuby 1.7 build, bump RVM versions --- .travis.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 594101c..3944068 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,20 +2,19 @@ sudo: false language: ruby cache: bundler -env: -rvm: -- jruby-1.7.25 services: - redis-server matrix: include: - - rvm: jruby-9.1.10.0 + - rvm: jruby-9.1.13.0 env: LOGSTASH_BRANCH=master - - rvm: jruby-9.1.10.0 + - rvm: jruby-9.1.13.0 env: LOGSTASH_BRANCH=6.x - - rvm: jruby-1.7.25 + - rvm: jruby-9.1.13.0 + env: LOGSTASH_BRANCH=6.0 + - rvm: jruby-1.7.27 env: LOGSTASH_BRANCH=5.6 fast_finish: true install: true script: ci/build.sh -jdk: oraclejdk8 \ No newline at end of file +jdk: oraclejdk8 From 4a2cc6efe262c1e88f3c08b1fd307ad7d7cbd396 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 7 Nov 2017 11:18:31 +0000 Subject: [PATCH 21/40] [skip ci] update gemspec summary --- CHANGELOG.md | 3 +++ logstash-output-redis.gemspec | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a0b930..50a151f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.0.3 + - Update gemspec summary + ## 4.0.2 - Tighten gem deps for the 'redis' gem ## 4.0.1 diff --git a/logstash-output-redis.gemspec b/logstash-output-redis.gemspec index 02ad7f2..32e9cb1 100644 --- a/logstash-output-redis.gemspec +++ b/logstash-output-redis.gemspec @@ -1,9 +1,9 @@ Gem::Specification.new do |s| s.name = 'logstash-output-redis' - s.version = '4.0.2' + s.version = '4.0.3' s.licenses = ['Apache License (2.0)'] - s.summary = "This output will send events to a Redis queue using RPUSH" + s.summary = "Sends events to a Redis queue using the `RPUSH` command" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" s.authors = ["Elastic"] s.email = 'info@elastic.co' From 021280170589eea53edfe2f4146e672641beb80b Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Mon, 8 Jan 2018 21:30:57 +0000 Subject: [PATCH 22/40] [skip ci] update license to 2018 --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 43976b7..2162c9b 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2012–2016 Elasticsearch +Copyright (c) 2012-2018 Elasticsearch Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From 23c61aaecbffcfdf1b692b6de4bf5a92176ea40e Mon Sep 17 00:00:00 2001 From: James McNeil Date: Fri, 6 Apr 2018 11:22:29 +0100 Subject: [PATCH 23/40] Adds ssl config boolean --- lib/logstash/outputs/redis.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/logstash/outputs/redis.rb b/lib/logstash/outputs/redis.rb index 81e3c9b..7d99641 100644 --- a/lib/logstash/outputs/redis.rb +++ b/lib/logstash/outputs/redis.rb @@ -41,6 +41,9 @@ class LogStash::Outputs::Redis < LogStash::Outputs::Base # The default port to connect on. Can be overridden on any hostname. config :port, :validate => :number, :default => 6379 + # SSL + config :ssl, :validate => :boolean, :default => false + # The Redis database number. config :db, :validate => :number, :default => 0 @@ -189,7 +192,8 @@ def connect :host => @current_host, :port => @current_port, :timeout => @timeout, - :db => @db + :db => @db, + :ssl => @ssl } @logger.debug("connection params", params) From 2ef3eb231d693862d1d108ea72c2bd9c41682b0c Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 6 Apr 2018 23:55:37 +0100 Subject: [PATCH 24/40] set default_codec doc attribute --- CHANGELOG.md | 3 +++ docs/index.asciidoc | 5 ++++- logstash-output-redis.gemspec | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50a151f..e071023 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.0.4 + - Docs: Set the default_codec doc attribute. + ## 4.0.3 - Update gemspec summary diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 9406057..abfc430 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -1,5 +1,6 @@ :plugin: redis :type: output +:default_codec: json /////////////////////////////////////////// START - GENERATED VARIABLES, DO NOT EDIT! @@ -199,4 +200,6 @@ Redis initial connection timeout in seconds. [id="plugins-{type}s-{plugin}-common-options"] -include::{include_path}/{type}.asciidoc[] \ No newline at end of file +include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/logstash-output-redis.gemspec b/logstash-output-redis.gemspec index 32e9cb1..44d6d31 100644 --- a/logstash-output-redis.gemspec +++ b/logstash-output-redis.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-output-redis' - s.version = '4.0.3' + s.version = '4.0.4' s.licenses = ['Apache License (2.0)'] s.summary = "Sends events to a Redis queue using the `RPUSH` command" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 6c55a5c7bda93e0be4e43a0c2761b8af45b185b9 Mon Sep 17 00:00:00 2001 From: James McNeil Date: Tue, 10 Apr 2018 10:09:11 +0100 Subject: [PATCH 25/40] adds SSL documentation --- docs/index.asciidoc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 9406057..ecc0b93 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -172,8 +172,16 @@ Password to authenticate with. There is no authentication by default. The default port to connect on. Can be overridden on any hostname. +[id="plugins-{type}s-{plugin}-ssl"] +===== `ssl` + + * Value type is <> + * Default value is `false` + +Enable SSL support. + [id="plugins-{type}s-{plugin}-reconnect_interval"] -===== `reconnect_interval` +===== `reconnect_interval` * Value type is <> * Default value is `1` From 8b557fd3574ac7500c9a1ef6f8e35176b53bf29d Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Thu, 3 Jan 2019 15:26:10 -0500 Subject: [PATCH 26/40] pin bundler version to < 2 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 3944068..8c2db63 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,3 +18,4 @@ matrix: install: true script: ci/build.sh jdk: oraclejdk8 +before_install: gem install bundler -v '< 2' From 4c7146d21ea9813bed917e8d5fccf079ddbf397f Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Fri, 4 Jan 2019 11:27:36 -0500 Subject: [PATCH 27/40] [skip ci] Travis: update LOGSTASH_BRANCH from 6.[0..4] to 6.5 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8c2db63..af88cc9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ matrix: - rvm: jruby-9.1.13.0 env: LOGSTASH_BRANCH=6.x - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=6.0 + env: LOGSTASH_BRANCH=6.5 - rvm: jruby-1.7.27 env: LOGSTASH_BRANCH=5.6 fast_finish: true From 8d7e5e6fe38cefee851c01940000a8ceac6ab2b7 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Thu, 10 Jan 2019 14:55:58 +0000 Subject: [PATCH 28/40] bump redis dependency and remove obsolete fields --- CHANGELOG.md | 4 ++++ lib/logstash/outputs/redis.rb | 5 ----- logstash-output-redis.gemspec | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e071023..e36aeac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 5.0.0 + - Removed obsolete fields `queue` and `name` + - Changed major version of redis library dependency to 4.x + ## 4.0.4 - Docs: Set the default_codec doc attribute. diff --git a/lib/logstash/outputs/redis.rb b/lib/logstash/outputs/redis.rb index 81e3c9b..3918764 100644 --- a/lib/logstash/outputs/redis.rb +++ b/lib/logstash/outputs/redis.rb @@ -20,9 +20,6 @@ class LogStash::Outputs::Redis < LogStash::Outputs::Base default :codec, "json" - # Name is used for logging in case there are multiple instances. - config :name, :validate => :string, :obsolete => "This option is obsolete" - # The hostname(s) of your Redis server(s). Ports may be specified on any # hostname, which will override the global port config. # If the hosts list is an array, Logstash will pick one random host to connect to, @@ -50,8 +47,6 @@ class LogStash::Outputs::Redis < LogStash::Outputs::Base # Password to authenticate with. There is no authentication by default. config :password, :validate => :password - config :queue, :validate => :string, :obsolete => "This option is obsolete. Use `key` and `data_type`." - # The name of a Redis list or channel. Dynamic names are # valid here, for example `logstash-%{type}`. config :key, :validate => :string, :required => true diff --git a/logstash-output-redis.gemspec b/logstash-output-redis.gemspec index 44d6d31..8c5dcc7 100644 --- a/logstash-output-redis.gemspec +++ b/logstash-output-redis.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-output-redis' - s.version = '4.0.4' + s.version = '5.0.0' s.licenses = ['Apache License (2.0)'] s.summary = "Sends events to a Redis queue using the `RPUSH` command" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -22,7 +22,7 @@ Gem::Specification.new do |s| # Gem dependencies s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" - s.add_runtime_dependency 'redis', '~> 3' + s.add_runtime_dependency 'redis', '~> 4' s.add_runtime_dependency 'stud' s.add_development_dependency 'logstash-devutils' From 92203ab42c97d83ff46414a99dbf271b8e8dd531 Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Wed, 13 Feb 2019 00:33:06 +0000 Subject: [PATCH 29/40] update matrix to include current targets [ci skip] --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index af88cc9..467ff5c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,9 +9,11 @@ matrix: - rvm: jruby-9.1.13.0 env: LOGSTASH_BRANCH=master - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=6.x + env: LOGSTASH_BRANCH=7.0 - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=6.5 + env: LOGSTASH_BRANCH=6.7 + - rvm: jruby-9.1.13.0 + env: LOGSTASH_BRANCH=6.6 - rvm: jruby-1.7.27 env: LOGSTASH_BRANCH=5.6 fast_finish: true From 40e23d4d3e572fe2e4e5d5f8ff465fdc7fe331eb Mon Sep 17 00:00:00 2001 From: boundless-thread <29227206+boundless-thread@users.noreply.github.com> Date: Tue, 25 Jun 2019 20:55:49 -0400 Subject: [PATCH 30/40] Update index.asciidoc for correct required args Documentation said `key` and `data_type` are not required, however there are no default values for those variables. Therefore, they are required. --- docs/index.asciidoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index abfc430..c598521 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -44,10 +44,10 @@ This plugin supports the following configuration options plus the <> |<>|No | <> |<>|No | <> |<>|No -| <> |<>, one of `["list", "channel"]`|No +| <> |<>, one of `["list", "channel"]`|Yes | <> |<>|No | <> |<>|No -| <> |<>|No +| <> |<>|Yes | <> |<>|No | <> |<>|No | <> |<>|No @@ -202,4 +202,4 @@ Redis initial connection timeout in seconds. [id="plugins-{type}s-{plugin}-common-options"] include::{include_path}/{type}.asciidoc[] -:default_codec!: \ No newline at end of file +:default_codec!: From 299b91e4b9fc33b3606cb40cc3cec1e82f266369 Mon Sep 17 00:00:00 2001 From: Colin Surprenant Date: Tue, 6 Aug 2019 14:22:23 -0400 Subject: [PATCH 31/40] [skip ci] Travis: switch to openjdk8 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 467ff5c..29c5b6c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,5 +19,5 @@ matrix: fast_finish: true install: true script: ci/build.sh -jdk: oraclejdk8 +jdk: openjdk8 before_install: gem install bundler -v '< 2' From 76a9a55adad291f7df2af13ff82d6c9761cdb602 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Fri, 11 Oct 2019 12:38:23 -0400 Subject: [PATCH 32/40] Bump version and add changelog entry --- CHANGELOG.md | 3 +++ logstash-output-redis.gemspec | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e36aeac..64ea752 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 5.1.0 + - Added basic support for SSL [#59](https://github.com/logstash-plugins/logstash-output-redis/pull/59) + ## 5.0.0 - Removed obsolete fields `queue` and `name` - Changed major version of redis library dependency to 4.x diff --git a/logstash-output-redis.gemspec b/logstash-output-redis.gemspec index 8c5dcc7..b34996a 100644 --- a/logstash-output-redis.gemspec +++ b/logstash-output-redis.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-output-redis' - s.version = '5.0.0' + s.version = '5.1.0' s.licenses = ['Apache License (2.0)'] s.summary = "Sends events to a Redis queue using the `RPUSH` command" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 4e2fcb06f06eb99f106ed3a99ce860ffbb7cbacc Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Fri, 11 Oct 2019 13:19:19 -0400 Subject: [PATCH 33/40] Remove 5.6 from travis tests Version 4 of the redis dropped support for ruby < 2.2.2, meaning that 5.6 tests will never pass --- .travis.yml | 2 -- CHANGELOG.md | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 29c5b6c..dc0f224 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,8 +14,6 @@ matrix: env: LOGSTASH_BRANCH=6.7 - rvm: jruby-9.1.13.0 env: LOGSTASH_BRANCH=6.6 - - rvm: jruby-1.7.27 - env: LOGSTASH_BRANCH=5.6 fast_finish: true install: true script: ci/build.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 64ea752..c8f07fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## 5.1.0 - - Added basic support for SSL [#59](https://github.com/logstash-plugins/logstash-output-redis/pull/59) + - Added basic support for SSL [#59](https://github.com/logstash-plugins/logstash-output-redis/pull/59) + - Fixed documentation of required settings [#61](https://github.com/logstash-plugins/logstash-output-redis/pull/61) ## 5.0.0 - Removed obsolete fields `queue` and `name` From e5abf6c3d61a231d55c6294eedcf9e506df72f21 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Mon, 14 Oct 2019 10:22:55 -0400 Subject: [PATCH 34/40] Pin logstash-core to 6.x and above --- logstash-output-redis.gemspec | 1 + 1 file changed, 1 insertion(+) diff --git a/logstash-output-redis.gemspec b/logstash-output-redis.gemspec index b34996a..e986d8e 100644 --- a/logstash-output-redis.gemspec +++ b/logstash-output-redis.gemspec @@ -21,6 +21,7 @@ Gem::Specification.new do |s| # Gem dependencies s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" + s.add_runtime_dependency 'logstash-core', '>= 6.0' s.add_runtime_dependency 'redis', '~> 4' s.add_runtime_dependency 'stud' From 499aecf465e4434bb90f624a87e8496db9f7ede3 Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Thu, 5 Mar 2020 16:51:27 +0100 Subject: [PATCH 35/40] Test: avoid insist gem usage (#65) for devutils 2.0 compatibility --- spec/integration/outputs/redis_spec.rb | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/spec/integration/outputs/redis_spec.rb b/spec/integration/outputs/redis_spec.rb index 4d531f7..26f2e0e 100644 --- a/spec/integration/outputs/redis_spec.rb +++ b/spec/integration/outputs/redis_spec.rb @@ -36,18 +36,18 @@ redis = Redis.new(:host => "127.0.0.1") # The list should contain the number of elements our agent pushed up. - insist { redis.llen(key) } == event_count + expect(redis.llen(key)).to eql event_count # Now check all events for order and correctness. event_count.times do |value| id, element = redis.blpop(key, 0) event = LogStash::Event.new(LogStash::Json.load(element)) - insist { event.get("sequence") } == value - insist { event.get("message") } == message + expect(event.get("sequence")).to eql value + expect(event.get("message")).to eql message end # The list should now be empty - insist { redis.llen(key) } == 0 + expect(redis.llen(key)).to eql 0 end end @@ -74,4 +74,3 @@ end end end - From fa2b320227e2b4af480a8fd8b874ff53825220a1 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 17 Mar 2020 15:41:55 +0000 Subject: [PATCH 36/40] [skip ci] updated apache license --- LICENSE | 209 +++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 199 insertions(+), 10 deletions(-) diff --git a/LICENSE b/LICENSE index 2162c9b..a80a3fd 100644 --- a/LICENSE +++ b/LICENSE @@ -1,13 +1,202 @@ -Copyright (c) 2012-2018 Elasticsearch -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ - http://www.apache.org/licenses/LICENSE-2.0 + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2020 Elastic and contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From 297897ff2d743e74049d4b3f5c9c1e94e93f2625 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Wed, 18 Mar 2020 16:14:14 +0000 Subject: [PATCH 37/40] update to centralized travis configuration --- .ci/run.sh | 9 +++++++++ .travis.yml | 23 ++--------------------- ci/build.sh | 21 --------------------- ci/run.sh | 3 --- ci/setup.sh | 26 -------------------------- 5 files changed, 11 insertions(+), 71 deletions(-) create mode 100755 .ci/run.sh delete mode 100755 ci/build.sh delete mode 100755 ci/run.sh delete mode 100755 ci/setup.sh diff --git a/.ci/run.sh b/.ci/run.sh new file mode 100755 index 0000000..60dc1c4 --- /dev/null +++ b/.ci/run.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# This is intended to be run inside the docker container as the command of the docker-compose. + +env + +set -ex + +jruby -rbundler/setup -S rspec -fd +jruby -rbundler/setup -S rspec -fd --tag integration diff --git a/.travis.yml b/.travis.yml index dc0f224..a50fc73 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,21 +1,2 @@ ---- -sudo: false -language: ruby -cache: bundler -services: -- redis-server -matrix: - include: - - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=master - - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=7.0 - - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=6.7 - - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=6.6 - fast_finish: true -install: true -script: ci/build.sh -jdk: openjdk8 -before_install: gem install bundler -v '< 2' +import: +- logstash-plugins/.ci:travis/travis.yml@1.x \ No newline at end of file diff --git a/ci/build.sh b/ci/build.sh deleted file mode 100755 index 06caffd..0000000 --- a/ci/build.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# version: 1 -######################################################## -# -# AUTOMATICALLY GENERATED! DO NOT EDIT -# -######################################################## -set -e - -echo "Starting build process in: `pwd`" -source ./ci/setup.sh - -if [[ -f "ci/run.sh" ]]; then - echo "Running custom build script in: `pwd`/ci/run.sh" - source ./ci/run.sh -else - echo "Running default build scripts in: `pwd`/ci/build.sh" - bundle install - bundle exec rake vendor - bundle exec rspec spec -fi diff --git a/ci/run.sh b/ci/run.sh deleted file mode 100755 index feaf36d..0000000 --- a/ci/run.sh +++ /dev/null @@ -1,3 +0,0 @@ -bundle install -bundle exec rake vendor -bundle exec rspec spec && bundle exec rspec spec --tag integration diff --git a/ci/setup.sh b/ci/setup.sh deleted file mode 100755 index 835fa43..0000000 --- a/ci/setup.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# version: 1 -######################################################## -# -# AUTOMATICALLY GENERATED! DO NOT EDIT -# -######################################################## -set -e -if [ "$LOGSTASH_BRANCH" ]; then - echo "Building plugin using Logstash source" - BASE_DIR=`pwd` - echo "Checking out branch: $LOGSTASH_BRANCH" - git clone -b $LOGSTASH_BRANCH https://github.com/elastic/logstash.git ../../logstash --depth 1 - printf "Checked out Logstash revision: %s\n" "$(git -C ../../logstash rev-parse HEAD)" - cd ../../logstash - echo "Building plugins with Logstash version:" - cat versions.yml - echo "---" - # We need to build the jars for that specific version - echo "Running gradle assemble in: `pwd`" - ./gradlew assemble - cd $BASE_DIR - export LOGSTASH_SOURCE=1 -else - echo "Building plugin using released gems on rubygems" -fi From 8138b7bfffe02d2823ad82ef49fe2b04a049b5d3 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Wed, 18 Mar 2020 17:41:14 +0000 Subject: [PATCH 38/40] re-add the redis server --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a50fc73..32b67f2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,2 +1,4 @@ import: -- logstash-plugins/.ci:travis/travis.yml@1.x \ No newline at end of file +- logstash-plugins/.ci:travis/travis.yml@1.x +services: +- redis-server From ad519e83f169a0d611b5f1d04445fb8e9b57a717 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Mon, 4 Jan 2021 12:49:21 +0000 Subject: [PATCH 39/40] [skip ci] update travis ci badge from .org to .com --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ce56241..5b57edd 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Logstash Plugin -[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-output-redis.svg)](https://travis-ci.org/logstash-plugins/logstash-output-redis) +[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-output-redis.svg)](https://travis-ci.com/logstash-plugins/logstash-output-redis) This is a plugin for [Logstash](https://github.com/elastic/logstash). From 5fdfd2157520e9c64a5f7d55ccc77f421b71cb7b Mon Sep 17 00:00:00 2001 From: Edmo Vamerlatti Costa <11836452+edmocosta@users.noreply.github.com> Date: Tue, 4 Jun 2024 11:04:01 +0200 Subject: [PATCH 40/40] Add support to SSL/TLS options (#69) This commit added support to SSL/TLS configurations (ssl_enabled, ssl_certificate_authorities, ssl_certificate, ssl_key, ssl_verification_mode, ssl_supported_protocols and ssl_cipher_suites) and fixed the integration tests. --- .ci/Dockerfile.redis_ssl | 7 ++ .ci/docker-compose.override.yml | 51 +++++++++ .ci/redis_ssl.entrypoint.sh | 5 + .ci/run.sh | 3 +- CHANGELOG.md | 10 ++ docs/index.asciidoc | 87 ++++++++++++++++ lib/logstash/outputs/redis.rb | 95 ++++++++++++++++- logstash-output-redis.gemspec | 2 +- spec/fixtures/certificates/ca.crt | 31 ++++++ spec/fixtures/certificates/ca.key | 52 ++++++++++ spec/fixtures/certificates/ca.txt | 1 + spec/fixtures/certificates/client.crt | 26 +++++ spec/fixtures/certificates/client.key | 28 +++++ spec/fixtures/certificates/gen-test-certs.sh | 60 +++++++++++ spec/fixtures/certificates/openssl.cnf | 6 ++ spec/fixtures/certificates/redis.crt | 25 +++++ spec/fixtures/certificates/redis.dh | 8 ++ spec/fixtures/certificates/redis.key | 28 +++++ spec/fixtures/certificates/server.crt | 26 +++++ spec/fixtures/certificates/server.key | 28 +++++ spec/integration/outputs/redis_spec.rb | 62 ++++++++++- spec/unit/outputs/redis_spec.rb | 104 ++++++++++++++++++- 22 files changed, 734 insertions(+), 11 deletions(-) create mode 100644 .ci/Dockerfile.redis_ssl create mode 100644 .ci/docker-compose.override.yml create mode 100755 .ci/redis_ssl.entrypoint.sh create mode 100644 spec/fixtures/certificates/ca.crt create mode 100644 spec/fixtures/certificates/ca.key create mode 100644 spec/fixtures/certificates/ca.txt create mode 100644 spec/fixtures/certificates/client.crt create mode 100644 spec/fixtures/certificates/client.key create mode 100755 spec/fixtures/certificates/gen-test-certs.sh create mode 100755 spec/fixtures/certificates/openssl.cnf create mode 100644 spec/fixtures/certificates/redis.crt create mode 100644 spec/fixtures/certificates/redis.dh create mode 100644 spec/fixtures/certificates/redis.key create mode 100644 spec/fixtures/certificates/server.crt create mode 100644 spec/fixtures/certificates/server.key diff --git a/.ci/Dockerfile.redis_ssl b/.ci/Dockerfile.redis_ssl new file mode 100644 index 0000000..334bd3b --- /dev/null +++ b/.ci/Dockerfile.redis_ssl @@ -0,0 +1,7 @@ +FROM redis + +COPY /spec/fixtures/certificates /certificates + +COPY ./.ci/redis_ssl.entrypoint.sh /redis_ssl.entrypoint.sh + +ENTRYPOINT ["/redis_ssl.entrypoint.sh"] \ No newline at end of file diff --git a/.ci/docker-compose.override.yml b/.ci/docker-compose.override.yml new file mode 100644 index 0000000..0aa5eea --- /dev/null +++ b/.ci/docker-compose.override.yml @@ -0,0 +1,51 @@ +version: '2.4' + +services: + logstash: + container_name: logstash + command: /usr/share/plugins/plugin/.ci/run.sh + environment: + - ELASTIC_STACK_VERSION=$ELASTIC_STACK_VERSION + depends_on: + - redis + - redis_ssl + networks: + app_net: + ipv4_address: 172.16.238.10 + ipv6_address: 2001:3984:3989::10 + redis: + image: redis + hostname: redis + container_name: redis + command: --port 16379 + ports: + - "16379:16379/tcp" + networks: + app_net: + ipv4_address: 172.16.238.161 + ipv6_address: 2001:3984:3989::161 + redis_ssl: + hostname: redis_ssl + container_name: redis_ssl + ports: + - "26379:26379/tcp" + networks: + app_net: + ipv4_address: 172.16.238.162 + ipv6_address: 2001:3984:3989::162 + build: + context: ../ + dockerfile: .ci/Dockerfile.redis_ssl + environment: + - PORT=26379 +networks: + app_net: + driver: bridge + enable_ipv6: true + ipam: + driver: default + config: + - subnet: 172.16.238.0/24 + gateway: 172.16.238.1 + - subnet: 2001:3984:3989::/64 + gateway: 2001:3984:3989::1 \ No newline at end of file diff --git a/.ci/redis_ssl.entrypoint.sh b/.ci/redis_ssl.entrypoint.sh new file mode 100755 index 0000000..0c104e6 --- /dev/null +++ b/.ci/redis_ssl.entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +set -e + +redis-server --tls-port $PORT --port 0 --tls-cert-file /certificates/redis.crt --tls-key-file /certificates/redis.key --tls-ca-cert-file /certificates/ca.crt \ No newline at end of file diff --git a/.ci/run.sh b/.ci/run.sh index 60dc1c4..c7071eb 100755 --- a/.ci/run.sh +++ b/.ci/run.sh @@ -5,5 +5,4 @@ env set -ex -jruby -rbundler/setup -S rspec -fd -jruby -rbundler/setup -S rspec -fd --tag integration +bundle exec rspec spec && bundle exec rspec --tag integration -fd 2>/dev/null diff --git a/CHANGELOG.md b/CHANGELOG.md index c8f07fc..03e2621 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## 5.2.0 + - Added support to SSL/TLS configurations [#69](https://github.com/logstash-plugins/logstash-output-redis/pull/69) + - `ssl_enabled` + - `ssl_certificate_authorities` + - `ssl_certificate` + - `ssl_key` + - `ssl_verification_mode` + - `ssl_supported_protocols` + - `ssl_cipher_suites` + ## 5.1.0 - Added basic support for SSL [#59](https://github.com/logstash-plugins/logstash-output-redis/pull/59) - Fixed documentation of required settings [#61](https://github.com/logstash-plugins/logstash-output-redis/pull/61) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 6c88d81..08fbaaf 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -52,6 +52,13 @@ This plugin supports the following configuration options plus the <> |<>|No | <> |<>|No | <> |<>|No +| <> |<>|No +| <> |list of <>|No +| <> |list of <>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>, one of `["full", "none"]`|No | <> |<>|No |======================================================================= @@ -197,6 +204,86 @@ Interval for reconnecting to failed Redis connections Shuffle the host list during Logstash startup. +[id="plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is <> + * There is no default value for this setting. + +Path to certificate in PEM format. This certificate will be presented +to the other part of the TLS connection. + +[id="plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is <> + * Default value is `[]` + +Validate the certificate chain against these authorities. +You can define multiple files. All the certificates will be read and added to the trust store. +The system CA path is automatically included. + +[id="plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is a list of <> + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is <> + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is <> + * There is no default value for this setting. + +SSL key path + +[id="plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is <> + * Default value is `nil` + +SSL key passphrase + +[id="plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is <> + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a secure connection. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another part in the TLS connection: + +`full` validates that the server certificate has an issue date that's within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + [id="plugins-{type}s-{plugin}-timeout"] ===== `timeout` diff --git a/lib/logstash/outputs/redis.rb b/lib/logstash/outputs/redis.rb index 8a80ca1..e4a934b 100644 --- a/lib/logstash/outputs/redis.rb +++ b/lib/logstash/outputs/redis.rb @@ -39,7 +39,32 @@ class LogStash::Outputs::Redis < LogStash::Outputs::Base config :port, :validate => :number, :default => 6379 # SSL - config :ssl, :validate => :boolean, :default => false + config :ssl_enabled, :validate => :boolean, :default => false + + # Validate the certificate chain against these authorities. You can define multiple files. + # All the certificates will be read and added to the trust store. + config :ssl_certificate_authorities, :validate => :path, :list => true + + # Options to verify the server's certificate. + # "full": validates that the provided certificate has an issue date that’s within the not_before and not_after dates; + # chains to a trusted Certificate Authority (CA); has a hostname or IP address that matches the names within the certificate. + # "none": performs no certificate validation. Disabling this severely compromises security (https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf) + config :ssl_verification_mode, :validate => %w[full none], :default => 'full' + + # SSL certificate path + config :ssl_certificate, :validate => :path + + # SSL key path + config :ssl_key, :validate => :path + + # SSL key passphrase + config :ssl_key_passphrase, :validate => :password, :default => nil + + # NOTE: the default setting [] uses SSL engine defaults + config :ssl_supported_protocols, :validate => %w[TLSv1.1 TLSv1.2 TLSv1.3], :default => [], :list => true + + # The list of ciphers suite to use + config :ssl_cipher_suites, :validate => :string, :list => true # The Redis database number. config :db, :validate => :number, :default => 0 @@ -93,6 +118,8 @@ class LogStash::Outputs::Redis < LogStash::Outputs::Base def register require 'redis' + validate_ssl_config! + if @batch if @data_type != "list" raise RuntimeError.new( @@ -188,8 +215,11 @@ def connect :port => @current_port, :timeout => @timeout, :db => @db, - :ssl => @ssl + :ssl => @ssl_enabled, } + + params[:ssl_params] = setup_ssl_params if @ssl_enabled + @logger.debug("connection params", params) if @password @@ -199,6 +229,67 @@ def connect Redis.new(params) end # def connect + def setup_ssl_params + require "openssl" + + params = {} + params[:cert_store] = ssl_certificate_store + + if @ssl_verification_mode == 'none' + params[:verify_mode] = OpenSSL::SSL::VERIFY_NONE + else + params[:verify_mode] = OpenSSL::SSL::VERIFY_PEER|OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT + end + + if @ssl_certificate + params[:cert] = OpenSSL::X509::Certificate.new(File.read(@ssl_certificate)) + if @ssl_key + # if we have an encrypted key and a password is not provided (nil) than OpenSSL::PKey::RSA + # prompts the user to enter a password interactively - we do not want to do that, + # for plain-text keys the default '' password argument gets simply ignored + params[:key] = OpenSSL::PKey::RSA.new(File.read(@ssl_key), @ssl_key_passphrase.value || '') + end + end + + params[:min_version] = :TLS1_1 + if @ssl_supported_protocols.any? + protocols = @ssl_supported_protocols.map { |v| v.delete('v').tr(".", "_").to_sym }.sort + params[:min_version] = protocols.first + params[:max_version] = protocols.last + end + + params[:ciphers] = @ssl_cipher_suites if @ssl_cipher_suites&.any? + params + end + + def ssl_certificate_store + cert_store = new_ssl_certificate_store + cert_store.set_default_paths + @ssl_certificate_authorities&.each do |cert| + cert_store.add_file(cert) + end + + cert_store + end + + def new_ssl_certificate_store + OpenSSL::X509::Store.new + end + + def validate_ssl_config! + unless @ssl_enabled + ignored_ssl_settings = original_params.select { |k| k != 'ssl_enabled' && k.start_with?('ssl_') } + @logger.warn("Configured SSL settings are not used when `ssl_enabled` is set to `false`: #{ignored_ssl_settings.keys}") if ignored_ssl_settings.any? + return + end + + if @ssl_certificate && !@ssl_key + raise LogStash::ConfigurationError, "Using an `ssl_certificate` requires an `ssl_key`" + elsif @ssl_key && !@ssl_certificate + raise LogStash::ConfigurationError, 'An `ssl_certificate` is required when using an `ssl_key`' + end + end + # A string used to identify a Redis instance in log messages def identity "redis://#{@password}@#{@current_host}:#{@current_port}/#{@db} #{@data_type}:#{@key}" diff --git a/logstash-output-redis.gemspec b/logstash-output-redis.gemspec index e986d8e..ca75225 100644 --- a/logstash-output-redis.gemspec +++ b/logstash-output-redis.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-output-redis' - s.version = '5.1.0' + s.version = '5.2.0' s.licenses = ['Apache License (2.0)'] s.summary = "Sends events to a Redis queue using the `RPUSH` command" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/fixtures/certificates/ca.crt b/spec/fixtures/certificates/ca.crt new file mode 100644 index 0000000..2c436b8 --- /dev/null +++ b/spec/fixtures/certificates/ca.crt @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFSzCCAzOgAwIBAgIUKvrJklkPPavWxOu4m3Lk6Odcii4wDQYJKoZIhvcNAQEL +BQAwNTETMBEGA1UECgwKUmVkaXMgVGVzdDEeMBwGA1UEAwwVQ2VydGlmaWNhdGUg +QXV0aG9yaXR5MB4XDTI0MDUxNjE4NDUzOVoXDTM0MDUxNDE4NDUzOVowNTETMBEG +A1UECgwKUmVkaXMgVGVzdDEeMBwGA1UEAwwVQ2VydGlmaWNhdGUgQXV0aG9yaXR5 +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAuQzq+B+eh8sPIUx0Do63 +maQLLRxjJB0jAw2dfoFTuEYl3Vb7jHMIOombPKayfABRV42fbV4yg4zup2PP/0yN +Sn9y13QpiYv/5ieBvADbRlmhtSqV1EKCotDaYE17xTabQ0oRSEOTi1UjrX9gFTXd +EzjJuE7WHTvwNaPTzUgQ5BwLlv3tE0Rt6WEZtwwgPVIe3K85cZ03cRONWd7Zmuw0 +jYgzd5LMmBWNqKyqI0BO4/lnRqOQdEExJYAClDJr9OsjgBGXsSbChvMX/OURXdvx +lterFEwqhSrC4AsNyqMY/tqy8+2k2heUk/N0VeCRE+ZPVfibYg9zNKNKlADAox3n +1rs463rnE4nbDk4A31sGPzr/1a7ZQPEIhSQ6lYpJfYaiEkB/SvfPGOnC7BRg7f72 +PpOrbTxlBrvQPwzg/aw0hrhiYkxvVSdz+jDfWY4cte59CqSCSXPEOiDtONs36Bxa +jTVtAKdFsR7cNyrKugntXPk6AfNSDMp/PbFfhMAJser3p9gJl8wPXSdOP9MSF3wD +Puy2Vc+CCAo/CloLfkNZ02GLDYfiiYgY3+elk7RHjgfi38lzqQJebExzXUm73lax +4bD2AHszRMTQ7UzGaFOh8RRjC3hSAl7DCDlG/bpQ+EVDXtFmhZieXmQ0blGYF0/T +mtzFAMJFeEZhq99HWq3mH20CAwEAAaNTMFEwHQYDVR0OBBYEFOTyqBIuWjseEIw0 +nTjZXA99Fwz3MB8GA1UdIwQYMBaAFOTyqBIuWjseEIw0nTjZXA99Fwz3MA8GA1Ud +EwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIBAAI6rbmPxRD1TUkSg7NbAb/3 +A1zTw49vK9AAtlvgbvkRh7WMWuQYmy8P2zrVbI5Xf2yDjk9ecQZq93h1I2+1pa10 +xTjhHSHNnEB0NbieTU0H8WUThdMqlLM+AwgdQ5u/ftkbc+dqA7LLeaPuchSL20hx +/BobLwP1aUMyKyxveUO8QO7QZReb1IkpjjhoqZY/pAgoPU1sYtBvPDgI5B+1Uo+M +EWc3DqqOVSIaVHKMQMS/e6QBJ5HTtLXtkLYitQ5OKxGO6fFrWG38c+0Pynem0aaD +eAWliV0b8ojr1l3B1GStrht8dSjjd8mrf0lsno6QmHo25wM22T2ETmml7FU/+oDJ +XW7937h/tNf/WKSzs+ZUV8cwkhhcBf/a7a4Eszw8JKfIrgMGrVMBrXoExY/c0GFB +esoN7lf2vy2WDCMs72aeflBTGZ2H/hi6u4/6sZ4kRoecvz7EDztXhC5i0Udtv8jC +wUnxZqwuzqTkuRsTas1oQvpXhf30hVlk+KFz0mPykPRWge8qOGUw+8omRhC6+41M +TJoHvo6/Gtz79oqwbAwNit3RybhGdwRc3/xLD1mOCtlfs5fCYnUTpFlS9lSS/gmz +ScI5Su8fCVuGIuHwqgzWDO13Q1ApqBQCUZBul1KsVXqNgLWNzUgq2v0Z4WbhvHWq +ktuOrK6AOmNndv9KJ4b4 +-----END CERTIFICATE----- diff --git a/spec/fixtures/certificates/ca.key b/spec/fixtures/certificates/ca.key new file mode 100644 index 0000000..43cadcf --- /dev/null +++ b/spec/fixtures/certificates/ca.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQC5DOr4H56Hyw8h +THQOjreZpAstHGMkHSMDDZ1+gVO4RiXdVvuMcwg6iZs8prJ8AFFXjZ9tXjKDjO6n +Y8//TI1Kf3LXdCmJi//mJ4G8ANtGWaG1KpXUQoKi0NpgTXvFNptDShFIQ5OLVSOt +f2AVNd0TOMm4TtYdO/A1o9PNSBDkHAuW/e0TRG3pYRm3DCA9Uh7crzlxnTdxE41Z +3tma7DSNiDN3ksyYFY2orKojQE7j+WdGo5B0QTElgAKUMmv06yOAEZexJsKG8xf8 +5RFd2/GW16sUTCqFKsLgCw3Koxj+2rLz7aTaF5ST83RV4JET5k9V+JtiD3M0o0qU +AMCjHefWuzjreucTidsOTgDfWwY/Ov/VrtlA8QiFJDqVikl9hqISQH9K988Y6cLs +FGDt/vY+k6ttPGUGu9A/DOD9rDSGuGJiTG9VJ3P6MN9Zjhy17n0KpIJJc8Q6IO04 +2zfoHFqNNW0Ap0WxHtw3Ksq6Ce1c+ToB81IMyn89sV+EwAmx6ven2AmXzA9dJ04/ +0xIXfAM+7LZVz4IICj8KWgt+Q1nTYYsNh+KJiBjf56WTtEeOB+LfyXOpAl5sTHNd +SbveVrHhsPYAezNExNDtTMZoU6HxFGMLeFICXsMIOUb9ulD4RUNe0WaFmJ5eZDRu +UZgXT9Oa3MUAwkV4RmGr30dareYfbQIDAQABAoICAAcsvguCxlXDhMGowjMyS2IU +ny5PlHR5ALuEkp+uCf3yElPDpJssdpw2a6NtXyw5ou2/GvhVl6XMGBC7aPwgDrZd +vKkn4thnajOVQo10hwvyO7fXf9mr/n6rlFv5hRv/YwO34nWn0Res5GY8sw//d6Pg +JsoTnG+jvEuIH5Us/yKpB7YCAGgn1g8faazUNofKIFI0JQkHrHeg53Edl9hO4ja+ +mig/s9kNtOFeWHZnbued6ugtPVOUl4A5t1/hmJf1afn0NKp0s0QiNsLiasr11Sch +E4wE6/0fzFB9fTpnGnf/KMMWarbeBS+7V75gp59a5aaSrgueR7w5vJGCefWZGj0D +FyK9duRHFKnBc/+4GC9tHt91o+YzwZq09/EbmescWdOmoHALUEryP/drAYwNHhji +GmEGxf84aLdMctWCPM3OrqLLVOHXol+C1E5i/r8chWKvy42n1kY1DN7ZA3tvJ9ND +l4CmFd0/15ugicPMwshCIh2aRMRc++2K9PHj7pucLT6vJMYVEJjlDe9lCwjrSbJa +l2zdcqQ/moFllGjx1hTZRpX5xHb/bWGJeOSmxa85BHk9wZyfpXr8K0D/Wu+x2iyd +likbSfRSTQN3NVkgZqGAOOLUVAMkQntoA6oNMWcGQx2YtAfN2rINHBSh4DJ/9PGV +C2lM24fmVvhMT8zm6vGBAoIBAQD6xqFL1D/2BzeFxB37FZMUrdHAF2O+oXOW0bg3 +ktbR+Mn8Ev+9keHv8WNRcTxKzkXDsbozfpR1FCEh3bJP07mvCjM3CgPmWLlokb/7 +DSkuhpdg1dBi78Az2Jec8ZEgIHaW5uciM6oALVoRy7/PJ1tnrCTgkiW/Mj9hRjF2 +aU4JSnKhAe1rns8Zi4wC19lfxT95M2+K6N8dqf19g7C6Ta7wgaSyyHDqqF2AqTcJ +MYOkg9wW4EKSh5hNPQeNqy6SRs60PYNTTKAlSb+hv5CzGU+d3BvQ3lAqFlIzqhSn +ZzqWOqdvt8/XLXKrG5/dDHx2WNktpNxf657lfdn0UCDMKt1NAoIBAQC858daVNec +Mg368Ctz5dcJk77Em8wavxNO4XOvTEpkxkwml7/RU6fCNut/PMKpw6CJhruvB+Uw +RkV+y+MuNEQSRDAYJgfB1yIIswf2r9QnLdcwGr8LzLbYhSzkLE0/zXClTCU/gLdB +qCrCagnK4oQLn2uXsws81t0tsd9wqpyfzVxEFL1vLl4iByplSneBB1PzFIjjOZSz +FH8AA14VYur6txLCHfK+pejG2rMkwWm5vhggTgwkEP8m+8igXcDqadFsrIfj/acd +uzAQTAWxr22xhrq9bk8ksZSf2cFkxnUUEeYLyD+w3j3aIP42zdvqw3EwFVPPLPCq +C8CpXniTQ7qhAoIBAHE/mj/ATlIw2CYUduWYzJ8eEAcLiQPhrW2CsAqIyXJxQ+YO +PmfBpaLSA+QXmv69QEwrysbOdwSYXo4IrCnYl5YwqQhGAZTYpIJQr74UJ3DXXy1f +4YeMdeP0ACPkA4HapzNmeyK66h7CXFaFCM3oHCMpXk38MgeyDWVEabnrvE39zN5h +HHLpeOU0W7o89/GEBjIZxFma6Idg6RTxj7HSuxnpshL66eNOvUsJH82LCsQUuCOs +iuPZUDIxSWP01kdAqsBCtze69udf8qZ8+D+pQocri3Q7sUAQbHbgbAtdVtvZCRwu +ijHGWG5lP0B4Dwzy+sDeHpdCtLM2rmLifK02/lECggEAY9vSUjoPUA5VvZVAIyAY +sINdkOka1/ix6DzdvokBuovorW4ChFFImS4XB0fDT6KONqT2iewOh5peGk0FZuNS +KKXLFrjj2OA0qYyaZRlFeQCOhGtfGom5DnQ36tZQb63WIktCVXNhbF/NWsBbNNjR +HKPFaIfJJPwgJ4ujphdKfF8+qu6bxS4prKtFCg53ZcnN4u2z/wntRU1MZWFiVsPX +m3kzfVH/hDCIALaThJaJoRqP/mPDnz0s5BqtT7i/xirx/hSmMSthMCJHohXN3MPz +pKioBoNNawREoMuno/IU6escRJYytPiGkcxnFel0m4F9UAG2MAp9niZgTigWbSKc +gQKCAQBykTYZ1LlUwxxx1BmdGBW5THKzjbo5Xp9amagtFfeFBF3c1/nnNCIXvbYk +PG9vyZ1on/Ic9CBzehHloLfjSRuBSNXkrpOnCeSTBRwwMXca0JbJJ85i6IGxhO6o +T6VP/mXU81LnIbRGsWVOGwZyvN0MAJH30zH7vAjs1e4qvUrT7B4lC8BqOdNAsqIq +R8j6N0BNSjCEumZG6LK4UMa0mYG0UFHiEZpjpK3qY1ddkouttPNVOE5FcdCko6YK ++HByMtgqlEMNFfTd673ScnxMyRgmlmlUL1RSfoBv5LTRp5a81BKUDD+82B5TUBY3 +oDGPtZqf+lH6or1EtANdnw3K5AyN +-----END PRIVATE KEY----- diff --git a/spec/fixtures/certificates/ca.txt b/spec/fixtures/certificates/ca.txt new file mode 100644 index 0000000..646c199 --- /dev/null +++ b/spec/fixtures/certificates/ca.txt @@ -0,0 +1 @@ +47A3D82860FAB19E7F26B8FF7043280173FB5832 diff --git a/spec/fixtures/certificates/client.crt b/spec/fixtures/certificates/client.crt new file mode 100644 index 0000000..54361a1 --- /dev/null +++ b/spec/fixtures/certificates/client.crt @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE----- +MIIEUjCCAjqgAwIBAgIUR6PYKGD6sZ5/Jrj/cEMoAXP7WDEwDQYJKoZIhvcNAQEL +BQAwNTETMBEGA1UECgwKUmVkaXMgVGVzdDEeMBwGA1UEAwwVQ2VydGlmaWNhdGUg +QXV0aG9yaXR5MCAXDTI0MDUxNjE4NDUzOVoYDzQ3NjIwNDEyMTg0NTM5WjArMRMw +EQYDVQQKDApSZWRpcyBUZXN0MRQwEgYDVQQDDAtDbGllbnQtb25seTCCASIwDQYJ +KoZIhvcNAQEBBQADggEPADCCAQoCggEBAMP1ceXuKItfVwCUvBdMgFh8mZP7ak/T +Abeo/Fh6uv5LnBt6eJ6uAlaXSmJReFiLOYH5c565bh/KeKMmCI1CzsiFhBZZuFDk +Rj8KkX9ux4aGj/O251YeuNy8HYw3spurrhc6EVNIhJvyaN2HprdPiIqYakbMf/HC +FSqeJzRbsF4NnXmTajGVowCKAnFsjSP8ARg4bDlbe3A02grgMqgyi3b5LXaNhjV8 +CRDbJL0qLfZCt2pQuot+UOQQUhjX4vAr7swd42OSAY7lPeIGmD3Le4/M9IBCSxjI +ZXuHCZTY7Gwu0JSmfM4hSl5WgaAAlHLj5Kw1WYOzgix3wnU9rMu5rC0CAwEAAaNi +MGAwCwYDVR0PBAQDAgWgMBEGCWCGSAGG+EIBAQQEAwIHgDAdBgNVHQ4EFgQUq4dH +nr2Dp8kXSlV4QUTp46T7tpgwHwYDVR0jBBgwFoAU5PKoEi5aOx4QjDSdONlcD30X +DPcwDQYJKoZIhvcNAQELBQADggIBAHqXEBwU+IzDrS5W0sANa2JMIFu3B4OigBQ5 +e+cFFQDfA8ZQL/PSLnUOWp4v2hsnw+Qh/OkUCq1Hv29IBhMml8s12FTRaAQo1x5X +28iJVsnzj/JsfsShbH7CE0qxlcbYmOsI0GJqAApoQ85/fXq1+BQOImBu9Q1iJhBK +rk8Vdh/noKJpqpZmA47Zb+JKrQMbqCYHd2qQxnbiKzTDznEQMMBNxELnXWWS5UQn +zevTeyYrP4eihWIOqGnUF/O0PoaMQuJk2x6vwDYyxQaAKuSoOnSRp9VmvcdirKJZ +Jj6w6XzeAennxwTFZK49nAX+KnglRqUOHIZAiKrVQTcgMnQOWf9D9hKF5pzJEhoj +QtFV490uqw1B08U5KoX2gHtcelhEw7V0gHv26NHuKJR5F7L0ZPDBchIr6yVIL4JU +pDALSKZEZMAyC0QYBwXL4tVHgY4MmlD5ep76yNKMSHFHImGmd/7zgIZz6NS10DE2 +KCemXnHWGO46kNP3Ebv5mpK4JpOH0zI1YAXhmib+U0rjfSUpA5hmOtWXk1bMgS+k +PljU730hsgaIR3/7xmDrLG1TPQTY51EO9R36egCOXwTzRerslgCygA+Rf1gFA0NO +xfS8fOFEmO1z/HHlESB7gsvusjfm2bhxPxf8v+djIuRwIwvXEVJqjMnC5OBTWOwP +3dgnw1I6 +-----END CERTIFICATE----- diff --git a/spec/fixtures/certificates/client.key b/spec/fixtures/certificates/client.key new file mode 100644 index 0000000..d704359 --- /dev/null +++ b/spec/fixtures/certificates/client.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDD9XHl7iiLX1cA +lLwXTIBYfJmT+2pP0wG3qPxYerr+S5wbeniergJWl0piUXhYizmB+XOeuW4fynij +JgiNQs7IhYQWWbhQ5EY/CpF/bseGho/ztudWHrjcvB2MN7Kbq64XOhFTSISb8mjd +h6a3T4iKmGpGzH/xwhUqnic0W7BeDZ15k2oxlaMAigJxbI0j/AEYOGw5W3twNNoK +4DKoMot2+S12jYY1fAkQ2yS9Ki32QrdqULqLflDkEFIY1+LwK+7MHeNjkgGO5T3i +Bpg9y3uPzPSAQksYyGV7hwmU2OxsLtCUpnzOIUpeVoGgAJRy4+SsNVmDs4Isd8J1 +PazLuawtAgMBAAECggEARelsDNnmmi7WFe16m59eDPPLQRoKKFeCPdmqyhBjZXVp +SP5tSUOQrv8D+UbStlhZmI78Wo0ShoIQfziDBY7nwm+sG+M/mJCQpU4qcbn+snaQ +piVDrJyCXLI6ont+m/5oun2rj7iIIzEdi6epaa4U60mMZRzxqrmRHqbVGt95cVwv +OjTlNANbrTc7lHXpySwasKxNe9hNOxpHC/GsnSnNUBjnNcmfBzlmd4tfbCl4SlpA +IrjrFzcD02LuIZe6mLG4f6F9gG4MIc5ck0jEPzfiucZT6VQgPR+5Qe/92nYZccLm +R1lnxsFSMrQjdkOYjAb/QTTcl4Fr6Bu1oeYoziFVuwKBgQDoztIRAAWHsvc2Lhkd +xsNT9xWD4kRZsj3RUia+d5ZuSiYZV1I5ITvhJqpfLv6zdmwOG1UTRfwqjtDU3tBX +JF7COfSWYnMcuIBeh4eee6zuXoQbaeGRgbK9Pe61Q9+4JX0LnZUXaVjelGrY6vFC +VpcwiC2GEmI+9sjYRaHnJzbrfwKBgQDXeuFTUKmSUOLt23taC7xnM6PUC4CDcIHl +evh663JumkWpAe3QEpODQYlagh8wYLSAS2WNcRgyjvZB+3mSEK1Ty/c9xoLViFnS +KJfNPvkMBDldQ/JJ9ATGgFu2pfKkM7ou4Z9mpqqY0i2A4ckBBEcEpS6BQFI/t3mQ +YeuuCKWuUwKBgQDRu2kR+aQBwR/nBpaH9cWIgkM2wgPzjpkUV18iHhg+mbxC/iDL +6P3J84xvHaZFxPzJpKP7LgRjzq+h5m1o5BIeBhor1NuBV1IGVzC9fQuo3ytCixu0 +e1SsGfxP/qqwec1yRm/HXJe8RZcQfnAE5H9mMHhanXs24BrGg8y8st5OPwKBgQDA +fNSsSHIycm9Fa7tVRQaYalj/Iwc6Y1amWKN/qrQeeVGhGEgIh4+ewPyiDXsvDDQy +Jyggodar02nIp4yCHsO3B41lcd+OQK98PSbeAlMXmO5lkjzuMz3Q1PkYwEVwyYSY +PJTYbioXOinL5+ZXMir+y1AvxfWzVYMSp2aRoMfgYQKBgQCC1oUswYrR39ObvfXD +ZPOdlv3Xp2jz7L0LOIiYggNXniiBq0IT7Xtu8FYNZ2vnMPi1PsQENqO89vlFeJ92 +5l2ZMfKceGjc3zxyWQzqPhXXbukBbQC1TYUP+ZxxghRmy5PCTn+siNW4UO1dtUh2 +s3QtMleyQEr8SusIpJEnG+/ErQ== +-----END PRIVATE KEY----- diff --git a/spec/fixtures/certificates/gen-test-certs.sh b/spec/fixtures/certificates/gen-test-certs.sh new file mode 100755 index 0000000..a84838a --- /dev/null +++ b/spec/fixtures/certificates/gen-test-certs.sh @@ -0,0 +1,60 @@ + +#!/bin/bash + +# COPIED/MODIFIED from the redis server gen-certs util +# https://github.com/redis/redis/blob/cc0091f0f9fe321948c544911b3ea71837cf86e3/utils/gen-test-certs.sh + +# Generate some test certificates which are used by the regression test suite: +# +# ca.{crt,key} Self signed CA certificate. +# redis.{crt,key} A certificate with no key usage/policy restrictions. +# client.{crt,key} A certificate restricted for SSL client usage. +# server.{crt,key} A certificate restricted for SSL server usage. +# redis.dh DH Params file. + +generate_cert() { + local name=$1 + local cn="$2" + local opts="$3" + + local keyfile=${name}.key + local certfile=${name}.crt + + [ -f $keyfile ] || openssl genrsa -out $keyfile 2048 + openssl req \ + -new -sha256 \ + -subj "/O=Redis Test/CN=$cn" \ + -key $keyfile | \ + openssl x509 \ + -req -sha256 \ + -CA ca.crt \ + -CAkey ca.key \ + -CAserial ca.txt \ + -CAcreateserial \ + -days 999999 \ + $opts \ + -out $certfile +} + +[ -f ca.key ] || openssl genrsa -out ca.key 4096 +openssl req \ + -x509 -new -nodes -sha256 \ + -key ca.key \ + -days 3650 \ + -subj '/O=Redis Test/CN=Certificate Authority' \ + -out ca.crt + +cat > openssl.cnf <<_END_ +[ server_cert ] +keyUsage = digitalSignature, keyEncipherment +nsCertType = server +[ client_cert ] +keyUsage = digitalSignature, keyEncipherment +nsCertType = client +_END_ + +generate_cert server "Server-only" "-extfile openssl.cnf -extensions server_cert" +generate_cert client "Client-only" "-extfile openssl.cnf -extensions client_cert" +generate_cert redis "Generic-cert" + +[ -f redis.dh ] || openssl dhparam -out redis.dh 2048 diff --git a/spec/fixtures/certificates/openssl.cnf b/spec/fixtures/certificates/openssl.cnf new file mode 100755 index 0000000..f56038a --- /dev/null +++ b/spec/fixtures/certificates/openssl.cnf @@ -0,0 +1,6 @@ +[ server_cert ] +keyUsage = digitalSignature, keyEncipherment +nsCertType = server +[ client_cert ] +keyUsage = digitalSignature, keyEncipherment +nsCertType = client diff --git a/spec/fixtures/certificates/redis.crt b/spec/fixtures/certificates/redis.crt new file mode 100644 index 0000000..b8725ec --- /dev/null +++ b/spec/fixtures/certificates/redis.crt @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIEMzCCAhugAwIBAgIUR6PYKGD6sZ5/Jrj/cEMoAXP7WDIwDQYJKoZIhvcNAQEL +BQAwNTETMBEGA1UECgwKUmVkaXMgVGVzdDEeMBwGA1UEAwwVQ2VydGlmaWNhdGUg +QXV0aG9yaXR5MCAXDTI0MDUxNjE4NDUzOVoYDzQ3NjIwNDEyMTg0NTM5WjAsMRMw +EQYDVQQKDApSZWRpcyBUZXN0MRUwEwYDVQQDDAxHZW5lcmljLWNlcnQwggEiMA0G +CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC3kvR6caLws6+HAFpxx3nrA0pK4AKa +tALoFRjWe+7eKqAuXquSJRXHoVPhVXATcMZ1oePPjfDw3LYcDOT6l2BC+ixqyvlC +7zngVNgFjpX9oOO6pkeLFre38R+u1U516L4H/MpiqB9S4/YtGV7HXZCCPctVQjGJ +6VuxSHBqxJBXPtBw3U11JfKMOHE8MNXXP9mc9tBAZDg4tWuP5fRZ3P4urnen2iTq +q/xpuoj7bXAfNNzhbKYEWCFJvbPqIGvDz8JXdywrUaOa4hRctNjqKz5feTwd6A8f +9DW87blj6i2/HwF3AZchOPs52O3vvHrw5zQS9qJlgAEq1nR/TZo6sQt1AgMBAAGj +QjBAMB0GA1UdDgQWBBT+stkMkr87nvQ7R2a9Ww1bkXnr/zAfBgNVHSMEGDAWgBTk +8qgSLlo7HhCMNJ042VwPfRcM9zANBgkqhkiG9w0BAQsFAAOCAgEAL5O9GgjXn4BU +8lUB137DNLXbpjt8qB4mSuKJ5sJAnkMW9g7IV7uXdolCH2wSSU4Akk+42w8ghf+T +ZHxInv5EWARd1hiT4mM/d1nf0V1HGYcUQKXCJTu9lxTq5rBCLF932jp3dd/P9g3N +JfASc3NNSk+VlTB4xlLCQ5b+X+13/TgHGOAqJn1VIYHAD37dyiJkr8S8w7RZuQYT +jhEt0S05zcnudM1Kl4RizQ+HlouTz1gNoEkaaC1sNmQHcBSP79xjvTrt2g/YFOsc +CY7TwCOmGNhKKC7Gm3Q/yiFj1/lbNq++XYwZrTNS5pknTaUK6LVNuJkYe3HMy4iq +LKLbG3sg426071SGbm2vXGrs/p4C02OLqCnL8mwXDxPQ8Px75deBlrXxvOvg1GTl +T+pqK4Hx888IavQVrXy7RsdtFAoRGhBWneU+qwF1Nlfvpy+zGOxs+X3+GRF/eclx +HWfdRD3huHD6PLyswgG+jBxdbuSaHGbHHBNFVAkK6W8sS9bp7D55cN0gizJglIxA +Z/S8EJwyVHkHVszZvzV4rOVGBiO5KnQrUcLVaXDL4lEKa/LuJR0Upec5MrLyZwfI +bURTlpO9zacz/e8hWN89lezJ0dCGLp30aQFmWPdjNh1VbMjJ9SeCEjPErTrbInpP +ciIyR/xHvB7P+Z7wQKDVd5F+dryq/aA= +-----END CERTIFICATE----- diff --git a/spec/fixtures/certificates/redis.dh b/spec/fixtures/certificates/redis.dh new file mode 100644 index 0000000..d68146e --- /dev/null +++ b/spec/fixtures/certificates/redis.dh @@ -0,0 +1,8 @@ +-----BEGIN DH PARAMETERS----- +MIIBDAKCAQEA221GevwdIvhYntSBvnUWI6GeMcQlUADx1WEMn1mjv4HcvIU1wYor +bPLj86jdS0lU5u3zdRZ/wuxmnM58hgetaXX8VNPUzfvYhyKInzwzBvfSQSDvFbY/ +aVdtIhUQ6ij1rSYqzcjVv3gxd2dZNSTT+dYzw/jKLJ9UI7J9PL3dkee/us6hLyDI +sfgM/arHcDxMRZvMHPHv4vwzIHt8/MqEgeN1BIm9941r8JTs4NRUk4zfSr7Bl5ff +QNiDWzN8RK5bSMcGn+XANWrQKoQyktNYaT9yAjIHNxvCPV4nM6UtyTSRMtcz1+Am +mPVONVR9RyPV+FxG6E0AjmukPfu2Z7JVnwIBAgICAOE= +-----END DH PARAMETERS----- diff --git a/spec/fixtures/certificates/redis.key b/spec/fixtures/certificates/redis.key new file mode 100644 index 0000000..48b463c --- /dev/null +++ b/spec/fixtures/certificates/redis.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC3kvR6caLws6+H +AFpxx3nrA0pK4AKatALoFRjWe+7eKqAuXquSJRXHoVPhVXATcMZ1oePPjfDw3LYc +DOT6l2BC+ixqyvlC7zngVNgFjpX9oOO6pkeLFre38R+u1U516L4H/MpiqB9S4/Yt +GV7HXZCCPctVQjGJ6VuxSHBqxJBXPtBw3U11JfKMOHE8MNXXP9mc9tBAZDg4tWuP +5fRZ3P4urnen2iTqq/xpuoj7bXAfNNzhbKYEWCFJvbPqIGvDz8JXdywrUaOa4hRc +tNjqKz5feTwd6A8f9DW87blj6i2/HwF3AZchOPs52O3vvHrw5zQS9qJlgAEq1nR/ +TZo6sQt1AgMBAAECggEAIjfKSmiULrnPpiJyuXKtTxk15+8JjTywrgIAtzoytEbN +xNbwtDUlciioMgBy+6yx0Ytzo2SZ3MvHRkNBC1TcJJ/tV5quC8P7zc0C10ZUvwoW +aNxFya9aX3caqMQp8+CGS4bbUrhz3kePw6hO+mp+XUiXdjT4dvSAmvMdwUkSfrBt +xtcnwiX4g355WwZKp4sqOSnhEQY0KT64hnx1RGAL3IGl6Kj2/V4uwkgbVVR/r2wH +ivSW6UA7Umnky2Liccg/QWDkY4OdHWJ2DP4sAZ+HjOBR1eKf+tVJBA40giM9uQ0a +pfGuOvOBaUz9voS70uYLy8aGZauoPzfhvzuRby6JAQKBgQDlk7qofud2+uGDohk8 +QPEz6Zb+h5UxItRI+5ZjylRzcafD03f3euNPCftUQFblQaiRlRN7NH5ozlpmbI5v +13YVUNH9toTawpiSRh5RVMtk6dTNUk80J1zhdXZi4nfw3VXVoJW7SQPB8hLSZ7OO +YTlKcH9PFg6eVeEZbyEddbS56QKBgQDMs8pSnnGPUdJiQRP11uXUYlBZSgINTM7T +AS2e9gXwPP6azTK9e2ky0Bhx+fW+7A4+aRjrJor/V5HCEEUUInHLq/GtG0Z8CmxO +oaXusSFKsqtVWauCVogo/2S3kTXjGY2++DIZnCc3q7njoXcwIamUxB1riBCyPLr+ +XzFV0TmBrQKBgQDjsimnhHY944ZX8CXnROH2Au/ncsHeAhVabW0zfDFWbptd9hc+ +NXvNoLDNVyLYLs7p1VmFEQNvH3s5quF6u9A6Q/aCFMyfy/FW4oSfVeKFLAYLvl8f +8wqD8uSAHu2mz1+aibL4FerMKTPRy1ijFnqkAf/CiLXnBHZFq5rEhUHCQQKBgFJo +zzsTroQUXK5RpMeEDGLEcbSApvLTyTAJeWRVh26AH8ASfCrsVB2wySaZiuWiAtn9 +VZrVmX/SsUt/YVAJzeZBArq+EFI/n/rH+RVvGggIhhGGWBXQ4GAg8Vs1iowy7W3E +XecRhnhTGjMZ/fHSm4LYsT9pN8+Jw35EtIfoA07BAoGACJWmulCP582Pdrn0Xrzx +mdCVtq1HZd5rUW/KA9fb2ht8A4i0ykZozADYu+eLZ5wu1hxEx/+IyNAhzp8OP1eC +DybKS0px957pesTR97hYOgV/xbQp3PhiBaQrgaVHd4AgeXxL39uMzj7tSZKQknfR +0rJRsJkplqO24hty/aQ5qO8= +-----END PRIVATE KEY----- diff --git a/spec/fixtures/certificates/server.crt b/spec/fixtures/certificates/server.crt new file mode 100644 index 0000000..72745b8 --- /dev/null +++ b/spec/fixtures/certificates/server.crt @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE----- +MIIEUjCCAjqgAwIBAgIUR6PYKGD6sZ5/Jrj/cEMoAXP7WDAwDQYJKoZIhvcNAQEL +BQAwNTETMBEGA1UECgwKUmVkaXMgVGVzdDEeMBwGA1UEAwwVQ2VydGlmaWNhdGUg +QXV0aG9yaXR5MCAXDTI0MDUxNjE4NDUzOVoYDzQ3NjIwNDEyMTg0NTM5WjArMRMw +EQYDVQQKDApSZWRpcyBUZXN0MRQwEgYDVQQDDAtTZXJ2ZXItb25seTCCASIwDQYJ +KoZIhvcNAQEBBQADggEPADCCAQoCggEBAMvxotag0WUqRJZ008eEBIQnqNBdCFm3 +18Px5rusxwGZ3Kchpi2VYxmKLTUwq5XQo29zHefvlnEFytG4SmvwK1tvW+zYfrgT +JWC+j40VcfqgTS2zesqu/zzHcEyWHqJBcMANkSBXNy6byp6zx0JbFqB4RUETqoXZ +5v4K3BWohMvAX+Ww78VIQtvc7hNbJZipCvoxShz9Q0B8lOJiDR2gXAVaZltOu+Hk +dRolHle+qkYslY27L70cheiQZvweRtC6NquTOSpg+ZDlezn4pHtPKlnxUa82ciZj +rqfpysaPLEzeuZ/MDuiyOZ4+h7b8/vyilKRupcV0bU+NfVFsWdvOnaECAwEAAaNi +MGAwCwYDVR0PBAQDAgWgMBEGCWCGSAGG+EIBAQQEAwIGQDAdBgNVHQ4EFgQUw851 +jwi3qF89kUF/SS30OWlCX3swHwYDVR0jBBgwFoAU5PKoEi5aOx4QjDSdONlcD30X +DPcwDQYJKoZIhvcNAQELBQADggIBAD0tyTntrFQbvF8bt1o3KhahY0SaHD1SdHWf +W81SujJkk3gqpOB/kP/JJUapQOyZ80seYUXtYNMXLWX2zXFoCMquZ7tjG352ybNV +Rko1/n9C0fyby+4rEFF281hkqaY1oiBNzwjXp1g6PAHO1zN5ZQAPcB9RLkOXxOnl +wPtHX5k6w433B2Xc6k0j5l1q6wK/H4mdD0nlNV05qf2o/Gl8GIAQEo/ZdqO5+aeb +8VqDcnQGe1MYhIJRTNYTYeBQ5Qe+caCm0vOm3dLqkdzfKUCwY+q6OGez7JnUJphK +VmDYfWAKLjZP7XTsck7oU5tCFWw/eKNr+wr8KwiLaAqmCUO68Pe2q0YTRu2k9fGj +cQ+YbZbl2MFoUAo37xdKsE1xMtsTBBezGsoq1/D9yJi1Yfp1KXyRFujbpEVtMTFG +X9YtxF4mfPxaguXrrthsMg04eLgG/D+kz7aYh3WydsWZnlCuJ+o6q5c+V0OGwmU+ +u7GrgyhNXGftQksf9BrNTwcngVFcJSe2xXLG1dDtQJTz8/KcNSIcdb4PPe6bP+st +035x2Z7e3JkqiDMNQew80JqeCeYIII9MeUknPHAX3FK9JWK27LhUosnstEFy5FHT +xR9BXUupIMJtaDdAICtSujMEm7NkKl38LMQ3UjGZl/jyLd+Y9UnmD3pDouMSW4Js +9YD5FxH9 +-----END CERTIFICATE----- diff --git a/spec/fixtures/certificates/server.key b/spec/fixtures/certificates/server.key new file mode 100644 index 0000000..562bb52 --- /dev/null +++ b/spec/fixtures/certificates/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDL8aLWoNFlKkSW +dNPHhASEJ6jQXQhZt9fD8ea7rMcBmdynIaYtlWMZii01MKuV0KNvcx3n75ZxBcrR +uEpr8Ctbb1vs2H64EyVgvo+NFXH6oE0ts3rKrv88x3BMlh6iQXDADZEgVzcum8qe +s8dCWxageEVBE6qF2eb+CtwVqITLwF/lsO/FSELb3O4TWyWYqQr6MUoc/UNAfJTi +Yg0doFwFWmZbTrvh5HUaJR5XvqpGLJWNuy+9HIXokGb8HkbQujarkzkqYPmQ5Xs5 ++KR7TypZ8VGvNnImY66n6crGjyxM3rmfzA7osjmePoe2/P78opSkbqXFdG1PjX1R +bFnbzp2hAgMBAAECggEACiMEV2+KfuE0I+gSRW/Ad3be7T3dihORJPkxYS1KHjBR +C0nY8Bm1cXXYr8TPt4Blejb2IuLJxwv9F/HVeaJYLlkVcCsV19WESWfpW89nwRe+ +BE4wJZI/h5xoXpck1kbjSTl3Z9yL5qw5zMv15MyBHjhiJVUaWAEz2fdNV4J++L9M +Ex4H/WvbmNLumam8bnsw3EUc5AEbsEIMBiKfIlNSo6aAco0iVh/wpdv/k78f5F3M +XQRtJLaVxn2i5W1BoXzUPTQGDFjd8+kuHHZBAEMjUuH9j+v4RlszEeSM0Eq213C+ +Aux89yl763tLFrL66a3866SUoKfdp0tkyOVVh3Sw8QKBgQD9TqdxwPqkFTEHyHsG +6Xrw+YzoWTv0ZaczwnRE+gXw8rZnkKm4zGbjk0EZMJW/l8mdvbsnhOlasaOBfc4a +jpqgvEZpU00V8p/lYqAec0X/qDhLm930T1nMVcatYVSZTPaNuvo6FePV7ches2zC +KVJgmKxwnYPHTubWo99KRDsIkQKBgQDOHKU7tfOkzadhuXBkiH6iPYL+S7H3yxAQ +QbS27V//6JHIyWP7tei0IwHAZcFR8sdeF2fApyZu7Za6J/7MgyjEimevNg4BeF8Y +fWLj9EoEEiHTtipl7tpsAI/RM2Gc4oSJ5LElmLzO9OkwnPWnP1oeClqPtjJ447Oq +h9ai49hMEQKBgQCAJ7wVlEH6o/ITcv2zaIrXwhnlL8hihM+MI1R51VkOJ23bhdgo +c9mWR9kGqosIzP2nPeejPd5mETuc9w5AZ2eZ7Mde4FM+yGrXPgeugDmo9+3Lawdn +ZIIVHEyRSY3opYST/kY8/vgvMsdOCk77CLbxwix0KW0TeDrX2z7MLysC0QKBgBnv +wt8zs1g+xNxoUHtmLcFOykgj0F5lQD07d3k5f8YdRxMkPJ+1MXZRnWvPojckvO/b +NCQrJIv6++erFsY3jw7Ed+21eKe4tQbI6IPU/udqPLV+i/FN0FCc9XwW4iay3ojX +rW8UvFWyvhFu/v6v2zBCj0jcXZeW18oS0/CliVdxAoGBANBIaB8pdLE+bp9YZGKU +glrlo11jdqJBYvidlPngZDVvIBSw3b4woK3m+r1+bDpQseDATmTDaaItdHSktmTw +fNGAZFyfZmXtmmuPJu16j62ML0YlRWUSAvbD3ntCIKs0zRjyywSOJs/gnaJKNIhB +Pj3xop1kUBPxx2Y9Kcu2bg2C +-----END PRIVATE KEY----- diff --git a/spec/integration/outputs/redis_spec.rb b/spec/integration/outputs/redis_spec.rb index 26f2e0e..c4d9bb6 100644 --- a/spec/integration/outputs/redis_spec.rb +++ b/spec/integration/outputs/redis_spec.rb @@ -6,8 +6,13 @@ describe LogStash::Outputs::Redis do + FIXTURES_PATH = File.expand_path('../../fixtures', File.dirname(__FILE__)) + PORT = 16379 + SSL_PORT = 26379 + context "integration tests", :integration => true do shared_examples_for "writing to redis list" do |extra_config| + let(:timeout) { 5 } let(:key) { 10.times.collect { rand(10).to_s }.join("") } let(:event_count) { Flores::Random.integer(0..10000) } let(:message) { Flores::Random.text(0..100) } @@ -15,7 +20,9 @@ { "key" => key, "data_type" => "list", - "host" => "localhost" + "host" => "redis", + "port" => PORT, + "timeout" => timeout } } let(:redis_config) { @@ -23,6 +30,19 @@ } let(:redis_output) { described_class.new(redis_config) } + let(:redis) do + ssl_enabled = redis_config['ssl_enabled'] == true + cli_config = { + :host => redis_config["host"], + :port => redis_config["port"] || PORT, + :timeout => timeout, + :ssl => ssl_enabled + } + + cli_config[:ssl_params] = redis_output.send(:setup_ssl_params) if ssl_enabled + Redis.new(cli_config) + end + before do redis_output.register event_count.times do |i| @@ -32,15 +52,17 @@ redis_output.close end - it "should successfully send all events to redis" do - redis = Redis.new(:host => "127.0.0.1") + after do + redis.del(key) + end + it "should successfully send all events to redis" do # The list should contain the number of elements our agent pushed up. expect(redis.llen(key)).to eql event_count # Now check all events for order and correctness. event_count.times do |value| - id, element = redis.blpop(key, 0) + id, element = redis.blpop(key, :timeout => timeout) event = LogStash::Event.new(LogStash::Json.load(element)) expect(event.get("sequence")).to eql value expect(event.get("message")).to eql message @@ -55,11 +77,41 @@ include_examples "writing to redis list" end + context "when SSL is enabled" do + context "with client certificate and key" do + ssl_config = { + "host" => "redis_ssl", + "port" => SSL_PORT, + "ssl_enabled" => true, + "ssl_certificate_authorities" => File.join(FIXTURES_PATH, 'certificates/ca.crt'), + "ssl_certificate" => File.join(FIXTURES_PATH, 'certificates/client.crt'), + "ssl_key" => File.join(FIXTURES_PATH, 'certificates/client.key') + } + + include_examples "writing to redis list", ssl_config + end + + context "with ssl_verification_mode => none" do + ssl_config = { + "host" => "redis_ssl", + "port" => SSL_PORT, + "ssl_enabled" => true, + "ssl_verification_mode" => "none", + "ssl_certificate" => File.join(FIXTURES_PATH, 'certificates/client.crt'), + "ssl_key" => File.join(FIXTURES_PATH, 'certificates/client.key') + } + + include_examples "writing to redis list", ssl_config + end + + end + context "when batch_mode is true" do batch_events = Flores::Random.integer(1..1000) batch_settings = { "batch" => true, - "batch_events" => batch_events + "batch_events" => batch_events, + "port" => PORT } include_examples "writing to redis list", batch_settings do diff --git a/spec/unit/outputs/redis_spec.rb b/spec/unit/outputs/redis_spec.rb index bdff2b3..1ada8f0 100644 --- a/spec/unit/outputs/redis_spec.rb +++ b/spec/unit/outputs/redis_spec.rb @@ -3,6 +3,7 @@ require "logstash/json" require "redis" require "flores/random" +require "flores/pki" describe LogStash::Outputs::Redis do @@ -38,5 +39,106 @@ end end end -end + context "with SSL enabled" do + let(:config) {{ "ssl_enabled" => true, "key" => "key", "data_type" => "list" }} + subject(:plugin) { described_class.new(config) } + + context "and not providing a certificate/key pair" do + it "registers without error" do + expect { plugin.register }.to_not raise_error + end + end + + context "and providing a certificate/key pair" do + let(:cert_key_pair) { Flores::PKI.generate } + let(:certificate) do + path = Tempfile.new('certificate').path + IO.write(path, cert_key_pair.first.to_s) + path + end + let(:key) do + path = Tempfile.new('key').path + IO.write(path, cert_key_pair[1].to_s) + path + end + let(:config) { super().merge("ssl_certificate" => certificate, "ssl_key" => key) } + + it "registers without error" do + expect { plugin.register }.to_not raise_error + end + end + + FIXTURES_PATH = File.expand_path('../../fixtures', File.dirname(__FILE__)) + + context "and plain-text certificate/key" do + let(:key_file) { File.join(FIXTURES_PATH, 'certificates/redis.key') } + let(:crt_file) { File.join(FIXTURES_PATH, 'certificates/redis.crt') } + let(:config) { super().merge("ssl_certificate" => crt_file, "ssl_key" => key_file) } + + it "registers without error" do + expect { plugin.register }.to_not raise_error + end + + context 'with password set' do + let(:config) { super().merge("ssl_key_passphrase" => 'ignored') } + + it "registers without error" do # password simply ignored + expect { plugin.register }.to_not raise_error + end + end + + context 'with supported protocol' do + let(:config) { super().merge("ssl_supported_protocols" => %w[TLSv1.2 TLSv1.3]) } + + it 'configures minimum TLS version' do + plugin.register + ssl_params = plugin.send(:setup_ssl_params) + expect(ssl_params).to match(a_hash_including(:min_version => :TLS1_2, :max_version => :TLS1_3)) + end + end + end + + context "with only ssl_certificate set" do + let(:config) { super().merge("ssl_certificate" => File.join(FIXTURES_PATH, 'certificates/redis.crt')) } + + it "should raise a configuration error to request also `ssl_key`" do + expect { plugin.register }.to raise_error(LogStash::ConfigurationError, /Using an `ssl_certificate` requires an `ssl_key`/) + end + end + + context "with only ssl_key set" do + let(:config) { super().merge("ssl_key" => File.join(FIXTURES_PATH, 'certificates/redis.key')) } + + it "should raise a configuration error to request also `ssl_key`" do + expect { plugin.register }.to raise_error(LogStash::ConfigurationError, /An `ssl_certificate` is required when using an `ssl_key`/) + end + end + + context "with ssl_certificate_authorities" do + let(:certificate_path) { File.join(FIXTURES_PATH, 'certificates/redis.crt') } + let(:config) do + super().merge('ssl_certificate_authorities' => [certificate_path]) + end + + it "sets cert_store values" do + ssl_store = double(OpenSSL::X509::Store.new) + allow(ssl_store).to receive(:set_default_paths) + allow(ssl_store).to receive(:add_file) + allow(subject).to receive(:new_ssl_certificate_store).and_return(ssl_store) + subject.send :setup_ssl_params + expect(ssl_store).to have_received(:add_file).with(certificate_path) + end + end + + context "CAs certificates" do + it "includes openssl default paths" do + ssl_store = double(OpenSSL::X509::Store.new) + allow(ssl_store).to receive(:set_default_paths) + allow(plugin).to receive(:new_ssl_certificate_store).and_return(ssl_store) + subject.send :setup_ssl_params + expect(ssl_store).to have_received(:set_default_paths) + end + end + end +end