diff --git a/.ci/Dockerfile.redis_ssl b/.ci/Dockerfile.redis_ssl new file mode 100644 index 0000000..334bd3b --- /dev/null +++ b/.ci/Dockerfile.redis_ssl @@ -0,0 +1,7 @@ +FROM redis + +COPY /spec/fixtures/certificates /certificates + +COPY ./.ci/redis_ssl.entrypoint.sh /redis_ssl.entrypoint.sh + +ENTRYPOINT ["/redis_ssl.entrypoint.sh"] \ No newline at end of file diff --git a/.ci/docker-compose.override.yml b/.ci/docker-compose.override.yml new file mode 100644 index 0000000..0aa5eea --- /dev/null +++ b/.ci/docker-compose.override.yml @@ -0,0 +1,51 @@ +version: '2.4' + +services: + logstash: + container_name: logstash + command: /usr/share/plugins/plugin/.ci/run.sh + environment: + - ELASTIC_STACK_VERSION=$ELASTIC_STACK_VERSION + depends_on: + - redis + - redis_ssl + networks: + app_net: + ipv4_address: 172.16.238.10 + ipv6_address: 2001:3984:3989::10 + redis: + image: redis + hostname: redis + container_name: redis + command: --port 16379 + ports: + - "16379:16379/tcp" + networks: + app_net: + ipv4_address: 172.16.238.161 + ipv6_address: 2001:3984:3989::161 + redis_ssl: + hostname: redis_ssl + container_name: redis_ssl + ports: + - "26379:26379/tcp" + networks: + app_net: + ipv4_address: 172.16.238.162 + ipv6_address: 2001:3984:3989::162 + build: + context: ../ + dockerfile: .ci/Dockerfile.redis_ssl + environment: + - PORT=26379 +networks: + app_net: + driver: bridge + enable_ipv6: true + ipam: + driver: default + config: + - subnet: 172.16.238.0/24 + gateway: 172.16.238.1 + - subnet: 2001:3984:3989::/64 + gateway: 2001:3984:3989::1 \ No newline at end of file diff --git a/.ci/redis_ssl.entrypoint.sh b/.ci/redis_ssl.entrypoint.sh new file mode 100755 index 0000000..0c104e6 --- /dev/null +++ b/.ci/redis_ssl.entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +set -e + +redis-server --tls-port $PORT --port 0 --tls-cert-file /certificates/redis.crt --tls-key-file /certificates/redis.key --tls-ca-cert-file /certificates/ca.crt \ No newline at end of file diff --git a/.ci/run.sh b/.ci/run.sh new file mode 100755 index 0000000..c7071eb --- /dev/null +++ b/.ci/run.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# This is intended to be run inside the docker container as the command of the docker-compose. + +env + +set -ex + +bundle exec rspec spec && bundle exec rspec --tag integration -fd 2>/dev/null diff --git a/.travis.yml b/.travis.yml index 2220b24..32b67f2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,4 @@ -sudo: false -language: ruby -cache: bundler +import: +- logstash-plugins/.ci:travis/travis.yml@1.x services: - - redis-server -rvm: - - jruby-1.7.23 -script: - - bundle exec rspec spec && bundle exec rspec spec --tag integration +- redis-server diff --git a/CHANGELOG.md b/CHANGELOG.md index b53b225..03e2621 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,43 @@ +## 5.2.0 + - Added support to SSL/TLS configurations [#69](https://github.com/logstash-plugins/logstash-output-redis/pull/69) + - `ssl_enabled` + - `ssl_certificate_authorities` + - `ssl_certificate` + - `ssl_key` + - `ssl_verification_mode` + - `ssl_supported_protocols` + - `ssl_cipher_suites` + +## 5.1.0 + - Added basic support for SSL [#59](https://github.com/logstash-plugins/logstash-output-redis/pull/59) + - Fixed documentation of required settings [#61](https://github.com/logstash-plugins/logstash-output-redis/pull/61) + +## 5.0.0 + - Removed obsolete fields `queue` and `name` + - Changed major version of redis library dependency to 4.x + +## 4.0.4 + - Docs: Set the default_codec doc attribute. + +## 4.0.3 + - Update gemspec summary + +## 4.0.2 + - Tighten gem deps for the 'redis' gem +## 4.0.1 + - Fix some documentation issues + +## 4.0.0 + - Mark deprecated `name` and `queue` configuration parameters as obsolete + +## 3.0.3 + - Fix logging method signature for #debug +## 3.0.2 + - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99 +## 3.0.1 + - Republish all the gems under jruby. +## 3.0.0 + - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141 # 2.0.5 - Fix LocalJumpError exception see https://github.com/logstash-plugins/logstash-output-redis/pull/27 # 2.0.4 diff --git a/Gemfile b/Gemfile index d926697..32cc6fb 100644 --- a/Gemfile +++ b/Gemfile @@ -1,2 +1,11 @@ source 'https://rubygems.org/' -gemspec \ No newline at end of file + +gemspec + +logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash" +use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1" + +if Dir.exist?(logstash_path) && use_logstash_source + gem 'logstash-core', :path => "#{logstash_path}/logstash-core" + gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api" +end diff --git a/LICENSE b/LICENSE index 43976b7..a80a3fd 100644 --- a/LICENSE +++ b/LICENSE @@ -1,13 +1,202 @@ -Copyright (c) 2012–2016 Elasticsearch -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ - http://www.apache.org/licenses/LICENSE-2.0 + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2020 Elastic and contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index ce56241..5b57edd 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Logstash Plugin -[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-output-redis.svg)](https://travis-ci.org/logstash-plugins/logstash-output-redis) +[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-output-redis.svg)](https://travis-ci.com/logstash-plugins/logstash-output-redis) This is a plugin for [Logstash](https://github.com/elastic/logstash). diff --git a/docs/index.asciidoc b/docs/index.asciidoc new file mode 100644 index 0000000..08fbaaf --- /dev/null +++ b/docs/index.asciidoc @@ -0,0 +1,300 @@ +:plugin: redis +:type: output +:default_codec: json + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: %VERSION% +:release_date: %RELEASE_DATE% +:changelog_url: %CHANGELOG_URL% +:include_path: ../../../../logstash/docs/include +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="plugins-{type}s-{plugin}"] + +=== Redis output plugin + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This output will send events to a Redis queue using RPUSH. +The RPUSH command is supported in Redis v0.0.7+. Using +PUBLISH to a channel requires at least v1.3.8+. +While you may be able to make these Redis versions work, +the best performance and stability will be found in more +recent stable versions. Versions 2.6.0+ are recommended. + +For more information, see http://redis.io/[the Redis homepage] + + +[id="plugins-{type}s-{plugin}-options"] +==== Redis Output Configuration Options + +This plugin supports the following configuration options plus the <> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>, one of `["list", "channel"]`|Yes +| <> |<>|No +| <> |<>|No +| <> |<>|Yes +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |list of <>|No +| <> |list of <>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>, one of `["full", "none"]`|No +| <> |<>|No +|======================================================================= + +Also see <> for a list of options supported by all +output plugins. + +  + +[id="plugins-{type}s-{plugin}-batch"] +===== `batch` + + * Value type is <> + * Default value is `false` + +Set to true if you want Redis to batch up values and send 1 RPUSH command +instead of one command per value to push on the list. Note that this only +works with `data_type="list"` mode right now. + +If true, we send an RPUSH every "batch_events" events or +"batch_timeout" seconds (whichever comes first). +Only supported for `data_type` is "list". + +[id="plugins-{type}s-{plugin}-batch_events"] +===== `batch_events` + + * Value type is <> + * Default value is `50` + +If batch is set to true, the number of events we queue up for an RPUSH. + +[id="plugins-{type}s-{plugin}-batch_timeout"] +===== `batch_timeout` + + * Value type is <> + * Default value is `5` + +If batch is set to true, the maximum amount of time between RPUSH commands +when there are pending events to flush. + +[id="plugins-{type}s-{plugin}-congestion_interval"] +===== `congestion_interval` + + * Value type is <> + * Default value is `1` + +How often to check for congestion. Default is one second. +Zero means to check on every event. + +[id="plugins-{type}s-{plugin}-congestion_threshold"] +===== `congestion_threshold` + + * Value type is <> + * Default value is `0` + +In case Redis `data_type` is `list` and has more than `@congestion_threshold` items, +block until someone consumes them and reduces congestion, otherwise if there are +no consumers Redis will run out of memory, unless it was configured with OOM protection. +But even with OOM protection, a single Redis list can block all other users of Redis, +until Redis CPU consumption reaches the max allowed RAM size. +A default value of 0 means that this limit is disabled. +Only supported for `list` Redis `data_type`. + +[id="plugins-{type}s-{plugin}-data_type"] +===== `data_type` + + * Value can be any of: `list`, `channel` + * There is no default value for this setting. + +Either list or channel. If `data_type` is list, then we will set +RPUSH to key. If `data_type` is channel, then we will PUBLISH to `key`. + +[id="plugins-{type}s-{plugin}-db"] +===== `db` + + * Value type is <> + * Default value is `0` + +The Redis database number. + +[id="plugins-{type}s-{plugin}-host"] +===== `host` + + * Value type is <> + * Default value is `["127.0.0.1"]` + +The hostname(s) of your Redis server(s). Ports may be specified on any +hostname, which will override the global port config. +If the hosts list is an array, Logstash will pick one random host to connect to, +if that host is disconnected it will then pick another. + +For example: +[source,ruby] + "127.0.0.1" + ["127.0.0.1", "127.0.0.2"] + ["127.0.0.1:6380", "127.0.0.1"] + +[id="plugins-{type}s-{plugin}-key"] +===== `key` + + * Value type is <> + * There is no default value for this setting. + +The name of a Redis list or channel. Dynamic names are +valid here, for example `logstash-%{type}`. + +[id="plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is <> + * There is no default value for this setting. + +Password to authenticate with. There is no authentication by default. + +[id="plugins-{type}s-{plugin}-port"] +===== `port` + + * Value type is <> + * Default value is `6379` + +The default port to connect on. Can be overridden on any hostname. + +[id="plugins-{type}s-{plugin}-ssl"] +===== `ssl` + + * Value type is <> + * Default value is `false` + +Enable SSL support. + +[id="plugins-{type}s-{plugin}-reconnect_interval"] +===== `reconnect_interval` + + * Value type is <> + * Default value is `1` + +Interval for reconnecting to failed Redis connections + +[id="plugins-{type}s-{plugin}-shuffle_hosts"] +===== `shuffle_hosts` + + * Value type is <> + * Default value is `true` + +Shuffle the host list during Logstash startup. + +[id="plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + + * Value type is <> + * There is no default value for this setting. + +Path to certificate in PEM format. This certificate will be presented +to the other part of the TLS connection. + +[id="plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is <> + * Default value is `[]` + +Validate the certificate chain against these authorities. +You can define multiple files. All the certificates will be read and added to the trust store. +The system CA path is automatically included. + +[id="plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + + * Value type is a list of <> + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + +[id="plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is <> + * Default value is `false` + +Enable SSL (must be set for other `ssl_` options to take effect). + +[id="plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + + * Value type is <> + * There is no default value for this setting. + +SSL key path + +[id="plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + + * Value type is <> + * Default value is `nil` + +SSL key passphrase + +[id="plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is <> + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a secure connection. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another part in the TLS connection: + +`full` validates that the server certificate has an issue date that's within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +[id="plugins-{type}s-{plugin}-timeout"] +===== `timeout` + + * Value type is <> + * Default value is `5` + +Redis initial connection timeout in seconds. + + + +[id="plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/lib/logstash/outputs/redis.rb b/lib/logstash/outputs/redis.rb index 78cf6d6..e4a934b 100644 --- a/lib/logstash/outputs/redis.rb +++ b/lib/logstash/outputs/redis.rb @@ -20,12 +20,10 @@ class LogStash::Outputs::Redis < LogStash::Outputs::Base default :codec, "json" - # Name is used for logging in case there are multiple instances. - config :name, :validate => :string, :default => 'default', - :deprecated => true - # The hostname(s) of your Redis server(s). Ports may be specified on any # hostname, which will override the global port config. + # If the hosts list is an array, Logstash will pick one random host to connect to, + # if that host is disconnected it will then pick another. # # For example: # [source,ruby] @@ -40,6 +38,34 @@ class LogStash::Outputs::Redis < LogStash::Outputs::Base # The default port to connect on. Can be overridden on any hostname. config :port, :validate => :number, :default => 6379 + # SSL + config :ssl_enabled, :validate => :boolean, :default => false + + # Validate the certificate chain against these authorities. You can define multiple files. + # All the certificates will be read and added to the trust store. + config :ssl_certificate_authorities, :validate => :path, :list => true + + # Options to verify the server's certificate. + # "full": validates that the provided certificate has an issue date that’s within the not_before and not_after dates; + # chains to a trusted Certificate Authority (CA); has a hostname or IP address that matches the names within the certificate. + # "none": performs no certificate validation. Disabling this severely compromises security (https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf) + config :ssl_verification_mode, :validate => %w[full none], :default => 'full' + + # SSL certificate path + config :ssl_certificate, :validate => :path + + # SSL key path + config :ssl_key, :validate => :path + + # SSL key passphrase + config :ssl_key_passphrase, :validate => :password, :default => nil + + # NOTE: the default setting [] uses SSL engine defaults + config :ssl_supported_protocols, :validate => %w[TLSv1.1 TLSv1.2 TLSv1.3], :default => [], :list => true + + # The list of ciphers suite to use + config :ssl_cipher_suites, :validate => :string, :list => true + # The Redis database number. config :db, :validate => :number, :default => 0 @@ -49,17 +75,13 @@ class LogStash::Outputs::Redis < LogStash::Outputs::Base # Password to authenticate with. There is no authentication by default. config :password, :validate => :password - # The name of the Redis queue (we'll use RPUSH on this). Dynamic names are - # valid here, for example `logstash-%{type}` - config :queue, :validate => :string, :deprecated => true - # The name of a Redis list or channel. Dynamic names are # valid here, for example `logstash-%{type}`. - config :key, :validate => :string, :required => false + config :key, :validate => :string, :required => true # Either list or channel. If `redis_type` is list, then we will set # RPUSH to key. If `redis_type` is channel, then we will PUBLISH to `key`. - config :data_type, :validate => [ "list", "channel" ], :required => false + config :data_type, :validate => [ "list", "channel" ], :required => true # Set to true if you want Redis to batch up values and send 1 RPUSH command # instead of one command per value to push on the list. Note that this only @@ -96,24 +118,7 @@ class LogStash::Outputs::Redis < LogStash::Outputs::Base def register require 'redis' - # TODO remove after setting key and data_type to true - if @queue - if @key or @data_type - raise RuntimeError.new( - "Cannot specify queue parameter and key or data_type" - ) - end - @key = @queue - @data_type = 'list' - end - - if not @key or not @data_type - raise RuntimeError.new( - "Must define queue, or key and data_type parameters" - ) - end - # end TODO - + validate_ssl_config! if @batch if @data_type != "list" @@ -209,9 +214,13 @@ def connect :host => @current_host, :port => @current_port, :timeout => @timeout, - :db => @db + :db => @db, + :ssl => @ssl_enabled, } - @logger.debug(params) + + params[:ssl_params] = setup_ssl_params if @ssl_enabled + + @logger.debug("connection params", params) if @password params[:password] = @password.value @@ -220,9 +229,70 @@ def connect Redis.new(params) end # def connect + def setup_ssl_params + require "openssl" + + params = {} + params[:cert_store] = ssl_certificate_store + + if @ssl_verification_mode == 'none' + params[:verify_mode] = OpenSSL::SSL::VERIFY_NONE + else + params[:verify_mode] = OpenSSL::SSL::VERIFY_PEER|OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT + end + + if @ssl_certificate + params[:cert] = OpenSSL::X509::Certificate.new(File.read(@ssl_certificate)) + if @ssl_key + # if we have an encrypted key and a password is not provided (nil) than OpenSSL::PKey::RSA + # prompts the user to enter a password interactively - we do not want to do that, + # for plain-text keys the default '' password argument gets simply ignored + params[:key] = OpenSSL::PKey::RSA.new(File.read(@ssl_key), @ssl_key_passphrase.value || '') + end + end + + params[:min_version] = :TLS1_1 + if @ssl_supported_protocols.any? + protocols = @ssl_supported_protocols.map { |v| v.delete('v').tr(".", "_").to_sym }.sort + params[:min_version] = protocols.first + params[:max_version] = protocols.last + end + + params[:ciphers] = @ssl_cipher_suites if @ssl_cipher_suites&.any? + params + end + + def ssl_certificate_store + cert_store = new_ssl_certificate_store + cert_store.set_default_paths + @ssl_certificate_authorities&.each do |cert| + cert_store.add_file(cert) + end + + cert_store + end + + def new_ssl_certificate_store + OpenSSL::X509::Store.new + end + + def validate_ssl_config! + unless @ssl_enabled + ignored_ssl_settings = original_params.select { |k| k != 'ssl_enabled' && k.start_with?('ssl_') } + @logger.warn("Configured SSL settings are not used when `ssl_enabled` is set to `false`: #{ignored_ssl_settings.keys}") if ignored_ssl_settings.any? + return + end + + if @ssl_certificate && !@ssl_key + raise LogStash::ConfigurationError, "Using an `ssl_certificate` requires an `ssl_key`" + elsif @ssl_key && !@ssl_certificate + raise LogStash::ConfigurationError, 'An `ssl_certificate` is required when using an `ssl_key`' + end + end + # A string used to identify a Redis instance in log messages def identity - @name || "redis://#{@password}@#{@current_host}:#{@current_port}/#{@db} #{@data_type}:#{@key}" + "redis://#{@password}@#{@current_host}:#{@current_port}/#{@db} #{@data_type}:#{@key}" end def send_to_redis(event, payload) diff --git a/logstash-output-redis.gemspec b/logstash-output-redis.gemspec index fd1ba01..ca75225 100644 --- a/logstash-output-redis.gemspec +++ b/logstash-output-redis.gemspec @@ -1,9 +1,9 @@ Gem::Specification.new do |s| s.name = 'logstash-output-redis' - s.version = '2.0.5' + s.version = '5.2.0' s.licenses = ['Apache License (2.0)'] - s.summary = "This output will send events to a Redis queue using RPUSH" + s.summary = "Sends events to a Redis queue using the `RPUSH` command" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" s.authors = ["Elastic"] s.email = 'info@elastic.co' @@ -11,7 +11,7 @@ Gem::Specification.new do |s| s.require_paths = ["lib"] # Files - s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT'] + s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"] # Tests s.test_files = s.files.grep(%r{^(test|spec|features)/}) @@ -20,9 +20,10 @@ Gem::Specification.new do |s| s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" } # Gem dependencies - s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0" + s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" + s.add_runtime_dependency 'logstash-core', '>= 6.0' - s.add_runtime_dependency 'redis' + s.add_runtime_dependency 'redis', '~> 4' s.add_runtime_dependency 'stud' s.add_development_dependency 'logstash-devutils' diff --git a/spec/fixtures/certificates/ca.crt b/spec/fixtures/certificates/ca.crt new file mode 100644 index 0000000..2c436b8 --- /dev/null +++ b/spec/fixtures/certificates/ca.crt @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFSzCCAzOgAwIBAgIUKvrJklkPPavWxOu4m3Lk6Odcii4wDQYJKoZIhvcNAQEL +BQAwNTETMBEGA1UECgwKUmVkaXMgVGVzdDEeMBwGA1UEAwwVQ2VydGlmaWNhdGUg +QXV0aG9yaXR5MB4XDTI0MDUxNjE4NDUzOVoXDTM0MDUxNDE4NDUzOVowNTETMBEG +A1UECgwKUmVkaXMgVGVzdDEeMBwGA1UEAwwVQ2VydGlmaWNhdGUgQXV0aG9yaXR5 +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAuQzq+B+eh8sPIUx0Do63 +maQLLRxjJB0jAw2dfoFTuEYl3Vb7jHMIOombPKayfABRV42fbV4yg4zup2PP/0yN +Sn9y13QpiYv/5ieBvADbRlmhtSqV1EKCotDaYE17xTabQ0oRSEOTi1UjrX9gFTXd +EzjJuE7WHTvwNaPTzUgQ5BwLlv3tE0Rt6WEZtwwgPVIe3K85cZ03cRONWd7Zmuw0 +jYgzd5LMmBWNqKyqI0BO4/lnRqOQdEExJYAClDJr9OsjgBGXsSbChvMX/OURXdvx +lterFEwqhSrC4AsNyqMY/tqy8+2k2heUk/N0VeCRE+ZPVfibYg9zNKNKlADAox3n +1rs463rnE4nbDk4A31sGPzr/1a7ZQPEIhSQ6lYpJfYaiEkB/SvfPGOnC7BRg7f72 +PpOrbTxlBrvQPwzg/aw0hrhiYkxvVSdz+jDfWY4cte59CqSCSXPEOiDtONs36Bxa +jTVtAKdFsR7cNyrKugntXPk6AfNSDMp/PbFfhMAJser3p9gJl8wPXSdOP9MSF3wD +Puy2Vc+CCAo/CloLfkNZ02GLDYfiiYgY3+elk7RHjgfi38lzqQJebExzXUm73lax +4bD2AHszRMTQ7UzGaFOh8RRjC3hSAl7DCDlG/bpQ+EVDXtFmhZieXmQ0blGYF0/T +mtzFAMJFeEZhq99HWq3mH20CAwEAAaNTMFEwHQYDVR0OBBYEFOTyqBIuWjseEIw0 +nTjZXA99Fwz3MB8GA1UdIwQYMBaAFOTyqBIuWjseEIw0nTjZXA99Fwz3MA8GA1Ud +EwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIBAAI6rbmPxRD1TUkSg7NbAb/3 +A1zTw49vK9AAtlvgbvkRh7WMWuQYmy8P2zrVbI5Xf2yDjk9ecQZq93h1I2+1pa10 +xTjhHSHNnEB0NbieTU0H8WUThdMqlLM+AwgdQ5u/ftkbc+dqA7LLeaPuchSL20hx +/BobLwP1aUMyKyxveUO8QO7QZReb1IkpjjhoqZY/pAgoPU1sYtBvPDgI5B+1Uo+M +EWc3DqqOVSIaVHKMQMS/e6QBJ5HTtLXtkLYitQ5OKxGO6fFrWG38c+0Pynem0aaD +eAWliV0b8ojr1l3B1GStrht8dSjjd8mrf0lsno6QmHo25wM22T2ETmml7FU/+oDJ +XW7937h/tNf/WKSzs+ZUV8cwkhhcBf/a7a4Eszw8JKfIrgMGrVMBrXoExY/c0GFB +esoN7lf2vy2WDCMs72aeflBTGZ2H/hi6u4/6sZ4kRoecvz7EDztXhC5i0Udtv8jC +wUnxZqwuzqTkuRsTas1oQvpXhf30hVlk+KFz0mPykPRWge8qOGUw+8omRhC6+41M +TJoHvo6/Gtz79oqwbAwNit3RybhGdwRc3/xLD1mOCtlfs5fCYnUTpFlS9lSS/gmz +ScI5Su8fCVuGIuHwqgzWDO13Q1ApqBQCUZBul1KsVXqNgLWNzUgq2v0Z4WbhvHWq +ktuOrK6AOmNndv9KJ4b4 +-----END CERTIFICATE----- diff --git a/spec/fixtures/certificates/ca.key b/spec/fixtures/certificates/ca.key new file mode 100644 index 0000000..43cadcf --- /dev/null +++ b/spec/fixtures/certificates/ca.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQC5DOr4H56Hyw8h +THQOjreZpAstHGMkHSMDDZ1+gVO4RiXdVvuMcwg6iZs8prJ8AFFXjZ9tXjKDjO6n +Y8//TI1Kf3LXdCmJi//mJ4G8ANtGWaG1KpXUQoKi0NpgTXvFNptDShFIQ5OLVSOt +f2AVNd0TOMm4TtYdO/A1o9PNSBDkHAuW/e0TRG3pYRm3DCA9Uh7crzlxnTdxE41Z +3tma7DSNiDN3ksyYFY2orKojQE7j+WdGo5B0QTElgAKUMmv06yOAEZexJsKG8xf8 +5RFd2/GW16sUTCqFKsLgCw3Koxj+2rLz7aTaF5ST83RV4JET5k9V+JtiD3M0o0qU +AMCjHefWuzjreucTidsOTgDfWwY/Ov/VrtlA8QiFJDqVikl9hqISQH9K988Y6cLs +FGDt/vY+k6ttPGUGu9A/DOD9rDSGuGJiTG9VJ3P6MN9Zjhy17n0KpIJJc8Q6IO04 +2zfoHFqNNW0Ap0WxHtw3Ksq6Ce1c+ToB81IMyn89sV+EwAmx6ven2AmXzA9dJ04/ +0xIXfAM+7LZVz4IICj8KWgt+Q1nTYYsNh+KJiBjf56WTtEeOB+LfyXOpAl5sTHNd +SbveVrHhsPYAezNExNDtTMZoU6HxFGMLeFICXsMIOUb9ulD4RUNe0WaFmJ5eZDRu +UZgXT9Oa3MUAwkV4RmGr30dareYfbQIDAQABAoICAAcsvguCxlXDhMGowjMyS2IU +ny5PlHR5ALuEkp+uCf3yElPDpJssdpw2a6NtXyw5ou2/GvhVl6XMGBC7aPwgDrZd +vKkn4thnajOVQo10hwvyO7fXf9mr/n6rlFv5hRv/YwO34nWn0Res5GY8sw//d6Pg +JsoTnG+jvEuIH5Us/yKpB7YCAGgn1g8faazUNofKIFI0JQkHrHeg53Edl9hO4ja+ +mig/s9kNtOFeWHZnbued6ugtPVOUl4A5t1/hmJf1afn0NKp0s0QiNsLiasr11Sch +E4wE6/0fzFB9fTpnGnf/KMMWarbeBS+7V75gp59a5aaSrgueR7w5vJGCefWZGj0D +FyK9duRHFKnBc/+4GC9tHt91o+YzwZq09/EbmescWdOmoHALUEryP/drAYwNHhji +GmEGxf84aLdMctWCPM3OrqLLVOHXol+C1E5i/r8chWKvy42n1kY1DN7ZA3tvJ9ND +l4CmFd0/15ugicPMwshCIh2aRMRc++2K9PHj7pucLT6vJMYVEJjlDe9lCwjrSbJa +l2zdcqQ/moFllGjx1hTZRpX5xHb/bWGJeOSmxa85BHk9wZyfpXr8K0D/Wu+x2iyd +likbSfRSTQN3NVkgZqGAOOLUVAMkQntoA6oNMWcGQx2YtAfN2rINHBSh4DJ/9PGV +C2lM24fmVvhMT8zm6vGBAoIBAQD6xqFL1D/2BzeFxB37FZMUrdHAF2O+oXOW0bg3 +ktbR+Mn8Ev+9keHv8WNRcTxKzkXDsbozfpR1FCEh3bJP07mvCjM3CgPmWLlokb/7 +DSkuhpdg1dBi78Az2Jec8ZEgIHaW5uciM6oALVoRy7/PJ1tnrCTgkiW/Mj9hRjF2 +aU4JSnKhAe1rns8Zi4wC19lfxT95M2+K6N8dqf19g7C6Ta7wgaSyyHDqqF2AqTcJ +MYOkg9wW4EKSh5hNPQeNqy6SRs60PYNTTKAlSb+hv5CzGU+d3BvQ3lAqFlIzqhSn +ZzqWOqdvt8/XLXKrG5/dDHx2WNktpNxf657lfdn0UCDMKt1NAoIBAQC858daVNec +Mg368Ctz5dcJk77Em8wavxNO4XOvTEpkxkwml7/RU6fCNut/PMKpw6CJhruvB+Uw +RkV+y+MuNEQSRDAYJgfB1yIIswf2r9QnLdcwGr8LzLbYhSzkLE0/zXClTCU/gLdB +qCrCagnK4oQLn2uXsws81t0tsd9wqpyfzVxEFL1vLl4iByplSneBB1PzFIjjOZSz +FH8AA14VYur6txLCHfK+pejG2rMkwWm5vhggTgwkEP8m+8igXcDqadFsrIfj/acd +uzAQTAWxr22xhrq9bk8ksZSf2cFkxnUUEeYLyD+w3j3aIP42zdvqw3EwFVPPLPCq +C8CpXniTQ7qhAoIBAHE/mj/ATlIw2CYUduWYzJ8eEAcLiQPhrW2CsAqIyXJxQ+YO +PmfBpaLSA+QXmv69QEwrysbOdwSYXo4IrCnYl5YwqQhGAZTYpIJQr74UJ3DXXy1f +4YeMdeP0ACPkA4HapzNmeyK66h7CXFaFCM3oHCMpXk38MgeyDWVEabnrvE39zN5h +HHLpeOU0W7o89/GEBjIZxFma6Idg6RTxj7HSuxnpshL66eNOvUsJH82LCsQUuCOs +iuPZUDIxSWP01kdAqsBCtze69udf8qZ8+D+pQocri3Q7sUAQbHbgbAtdVtvZCRwu +ijHGWG5lP0B4Dwzy+sDeHpdCtLM2rmLifK02/lECggEAY9vSUjoPUA5VvZVAIyAY +sINdkOka1/ix6DzdvokBuovorW4ChFFImS4XB0fDT6KONqT2iewOh5peGk0FZuNS +KKXLFrjj2OA0qYyaZRlFeQCOhGtfGom5DnQ36tZQb63WIktCVXNhbF/NWsBbNNjR +HKPFaIfJJPwgJ4ujphdKfF8+qu6bxS4prKtFCg53ZcnN4u2z/wntRU1MZWFiVsPX +m3kzfVH/hDCIALaThJaJoRqP/mPDnz0s5BqtT7i/xirx/hSmMSthMCJHohXN3MPz +pKioBoNNawREoMuno/IU6escRJYytPiGkcxnFel0m4F9UAG2MAp9niZgTigWbSKc +gQKCAQBykTYZ1LlUwxxx1BmdGBW5THKzjbo5Xp9amagtFfeFBF3c1/nnNCIXvbYk +PG9vyZ1on/Ic9CBzehHloLfjSRuBSNXkrpOnCeSTBRwwMXca0JbJJ85i6IGxhO6o +T6VP/mXU81LnIbRGsWVOGwZyvN0MAJH30zH7vAjs1e4qvUrT7B4lC8BqOdNAsqIq +R8j6N0BNSjCEumZG6LK4UMa0mYG0UFHiEZpjpK3qY1ddkouttPNVOE5FcdCko6YK ++HByMtgqlEMNFfTd673ScnxMyRgmlmlUL1RSfoBv5LTRp5a81BKUDD+82B5TUBY3 +oDGPtZqf+lH6or1EtANdnw3K5AyN +-----END PRIVATE KEY----- diff --git a/spec/fixtures/certificates/ca.txt b/spec/fixtures/certificates/ca.txt new file mode 100644 index 0000000..646c199 --- /dev/null +++ b/spec/fixtures/certificates/ca.txt @@ -0,0 +1 @@ +47A3D82860FAB19E7F26B8FF7043280173FB5832 diff --git a/spec/fixtures/certificates/client.crt b/spec/fixtures/certificates/client.crt new file mode 100644 index 0000000..54361a1 --- /dev/null +++ b/spec/fixtures/certificates/client.crt @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE----- +MIIEUjCCAjqgAwIBAgIUR6PYKGD6sZ5/Jrj/cEMoAXP7WDEwDQYJKoZIhvcNAQEL +BQAwNTETMBEGA1UECgwKUmVkaXMgVGVzdDEeMBwGA1UEAwwVQ2VydGlmaWNhdGUg +QXV0aG9yaXR5MCAXDTI0MDUxNjE4NDUzOVoYDzQ3NjIwNDEyMTg0NTM5WjArMRMw +EQYDVQQKDApSZWRpcyBUZXN0MRQwEgYDVQQDDAtDbGllbnQtb25seTCCASIwDQYJ +KoZIhvcNAQEBBQADggEPADCCAQoCggEBAMP1ceXuKItfVwCUvBdMgFh8mZP7ak/T +Abeo/Fh6uv5LnBt6eJ6uAlaXSmJReFiLOYH5c565bh/KeKMmCI1CzsiFhBZZuFDk +Rj8KkX9ux4aGj/O251YeuNy8HYw3spurrhc6EVNIhJvyaN2HprdPiIqYakbMf/HC +FSqeJzRbsF4NnXmTajGVowCKAnFsjSP8ARg4bDlbe3A02grgMqgyi3b5LXaNhjV8 +CRDbJL0qLfZCt2pQuot+UOQQUhjX4vAr7swd42OSAY7lPeIGmD3Le4/M9IBCSxjI +ZXuHCZTY7Gwu0JSmfM4hSl5WgaAAlHLj5Kw1WYOzgix3wnU9rMu5rC0CAwEAAaNi +MGAwCwYDVR0PBAQDAgWgMBEGCWCGSAGG+EIBAQQEAwIHgDAdBgNVHQ4EFgQUq4dH +nr2Dp8kXSlV4QUTp46T7tpgwHwYDVR0jBBgwFoAU5PKoEi5aOx4QjDSdONlcD30X +DPcwDQYJKoZIhvcNAQELBQADggIBAHqXEBwU+IzDrS5W0sANa2JMIFu3B4OigBQ5 +e+cFFQDfA8ZQL/PSLnUOWp4v2hsnw+Qh/OkUCq1Hv29IBhMml8s12FTRaAQo1x5X +28iJVsnzj/JsfsShbH7CE0qxlcbYmOsI0GJqAApoQ85/fXq1+BQOImBu9Q1iJhBK +rk8Vdh/noKJpqpZmA47Zb+JKrQMbqCYHd2qQxnbiKzTDznEQMMBNxELnXWWS5UQn +zevTeyYrP4eihWIOqGnUF/O0PoaMQuJk2x6vwDYyxQaAKuSoOnSRp9VmvcdirKJZ +Jj6w6XzeAennxwTFZK49nAX+KnglRqUOHIZAiKrVQTcgMnQOWf9D9hKF5pzJEhoj +QtFV490uqw1B08U5KoX2gHtcelhEw7V0gHv26NHuKJR5F7L0ZPDBchIr6yVIL4JU +pDALSKZEZMAyC0QYBwXL4tVHgY4MmlD5ep76yNKMSHFHImGmd/7zgIZz6NS10DE2 +KCemXnHWGO46kNP3Ebv5mpK4JpOH0zI1YAXhmib+U0rjfSUpA5hmOtWXk1bMgS+k +PljU730hsgaIR3/7xmDrLG1TPQTY51EO9R36egCOXwTzRerslgCygA+Rf1gFA0NO +xfS8fOFEmO1z/HHlESB7gsvusjfm2bhxPxf8v+djIuRwIwvXEVJqjMnC5OBTWOwP +3dgnw1I6 +-----END CERTIFICATE----- diff --git a/spec/fixtures/certificates/client.key b/spec/fixtures/certificates/client.key new file mode 100644 index 0000000..d704359 --- /dev/null +++ b/spec/fixtures/certificates/client.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDD9XHl7iiLX1cA +lLwXTIBYfJmT+2pP0wG3qPxYerr+S5wbeniergJWl0piUXhYizmB+XOeuW4fynij +JgiNQs7IhYQWWbhQ5EY/CpF/bseGho/ztudWHrjcvB2MN7Kbq64XOhFTSISb8mjd +h6a3T4iKmGpGzH/xwhUqnic0W7BeDZ15k2oxlaMAigJxbI0j/AEYOGw5W3twNNoK +4DKoMot2+S12jYY1fAkQ2yS9Ki32QrdqULqLflDkEFIY1+LwK+7MHeNjkgGO5T3i +Bpg9y3uPzPSAQksYyGV7hwmU2OxsLtCUpnzOIUpeVoGgAJRy4+SsNVmDs4Isd8J1 +PazLuawtAgMBAAECggEARelsDNnmmi7WFe16m59eDPPLQRoKKFeCPdmqyhBjZXVp +SP5tSUOQrv8D+UbStlhZmI78Wo0ShoIQfziDBY7nwm+sG+M/mJCQpU4qcbn+snaQ +piVDrJyCXLI6ont+m/5oun2rj7iIIzEdi6epaa4U60mMZRzxqrmRHqbVGt95cVwv +OjTlNANbrTc7lHXpySwasKxNe9hNOxpHC/GsnSnNUBjnNcmfBzlmd4tfbCl4SlpA +IrjrFzcD02LuIZe6mLG4f6F9gG4MIc5ck0jEPzfiucZT6VQgPR+5Qe/92nYZccLm +R1lnxsFSMrQjdkOYjAb/QTTcl4Fr6Bu1oeYoziFVuwKBgQDoztIRAAWHsvc2Lhkd +xsNT9xWD4kRZsj3RUia+d5ZuSiYZV1I5ITvhJqpfLv6zdmwOG1UTRfwqjtDU3tBX +JF7COfSWYnMcuIBeh4eee6zuXoQbaeGRgbK9Pe61Q9+4JX0LnZUXaVjelGrY6vFC +VpcwiC2GEmI+9sjYRaHnJzbrfwKBgQDXeuFTUKmSUOLt23taC7xnM6PUC4CDcIHl +evh663JumkWpAe3QEpODQYlagh8wYLSAS2WNcRgyjvZB+3mSEK1Ty/c9xoLViFnS +KJfNPvkMBDldQ/JJ9ATGgFu2pfKkM7ou4Z9mpqqY0i2A4ckBBEcEpS6BQFI/t3mQ +YeuuCKWuUwKBgQDRu2kR+aQBwR/nBpaH9cWIgkM2wgPzjpkUV18iHhg+mbxC/iDL +6P3J84xvHaZFxPzJpKP7LgRjzq+h5m1o5BIeBhor1NuBV1IGVzC9fQuo3ytCixu0 +e1SsGfxP/qqwec1yRm/HXJe8RZcQfnAE5H9mMHhanXs24BrGg8y8st5OPwKBgQDA +fNSsSHIycm9Fa7tVRQaYalj/Iwc6Y1amWKN/qrQeeVGhGEgIh4+ewPyiDXsvDDQy +Jyggodar02nIp4yCHsO3B41lcd+OQK98PSbeAlMXmO5lkjzuMz3Q1PkYwEVwyYSY +PJTYbioXOinL5+ZXMir+y1AvxfWzVYMSp2aRoMfgYQKBgQCC1oUswYrR39ObvfXD +ZPOdlv3Xp2jz7L0LOIiYggNXniiBq0IT7Xtu8FYNZ2vnMPi1PsQENqO89vlFeJ92 +5l2ZMfKceGjc3zxyWQzqPhXXbukBbQC1TYUP+ZxxghRmy5PCTn+siNW4UO1dtUh2 +s3QtMleyQEr8SusIpJEnG+/ErQ== +-----END PRIVATE KEY----- diff --git a/spec/fixtures/certificates/gen-test-certs.sh b/spec/fixtures/certificates/gen-test-certs.sh new file mode 100755 index 0000000..a84838a --- /dev/null +++ b/spec/fixtures/certificates/gen-test-certs.sh @@ -0,0 +1,60 @@ + +#!/bin/bash + +# COPIED/MODIFIED from the redis server gen-certs util +# https://github.com/redis/redis/blob/cc0091f0f9fe321948c544911b3ea71837cf86e3/utils/gen-test-certs.sh + +# Generate some test certificates which are used by the regression test suite: +# +# ca.{crt,key} Self signed CA certificate. +# redis.{crt,key} A certificate with no key usage/policy restrictions. +# client.{crt,key} A certificate restricted for SSL client usage. +# server.{crt,key} A certificate restricted for SSL server usage. +# redis.dh DH Params file. + +generate_cert() { + local name=$1 + local cn="$2" + local opts="$3" + + local keyfile=${name}.key + local certfile=${name}.crt + + [ -f $keyfile ] || openssl genrsa -out $keyfile 2048 + openssl req \ + -new -sha256 \ + -subj "/O=Redis Test/CN=$cn" \ + -key $keyfile | \ + openssl x509 \ + -req -sha256 \ + -CA ca.crt \ + -CAkey ca.key \ + -CAserial ca.txt \ + -CAcreateserial \ + -days 999999 \ + $opts \ + -out $certfile +} + +[ -f ca.key ] || openssl genrsa -out ca.key 4096 +openssl req \ + -x509 -new -nodes -sha256 \ + -key ca.key \ + -days 3650 \ + -subj '/O=Redis Test/CN=Certificate Authority' \ + -out ca.crt + +cat > openssl.cnf <<_END_ +[ server_cert ] +keyUsage = digitalSignature, keyEncipherment +nsCertType = server +[ client_cert ] +keyUsage = digitalSignature, keyEncipherment +nsCertType = client +_END_ + +generate_cert server "Server-only" "-extfile openssl.cnf -extensions server_cert" +generate_cert client "Client-only" "-extfile openssl.cnf -extensions client_cert" +generate_cert redis "Generic-cert" + +[ -f redis.dh ] || openssl dhparam -out redis.dh 2048 diff --git a/spec/fixtures/certificates/openssl.cnf b/spec/fixtures/certificates/openssl.cnf new file mode 100755 index 0000000..f56038a --- /dev/null +++ b/spec/fixtures/certificates/openssl.cnf @@ -0,0 +1,6 @@ +[ server_cert ] +keyUsage = digitalSignature, keyEncipherment +nsCertType = server +[ client_cert ] +keyUsage = digitalSignature, keyEncipherment +nsCertType = client diff --git a/spec/fixtures/certificates/redis.crt b/spec/fixtures/certificates/redis.crt new file mode 100644 index 0000000..b8725ec --- /dev/null +++ b/spec/fixtures/certificates/redis.crt @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIEMzCCAhugAwIBAgIUR6PYKGD6sZ5/Jrj/cEMoAXP7WDIwDQYJKoZIhvcNAQEL +BQAwNTETMBEGA1UECgwKUmVkaXMgVGVzdDEeMBwGA1UEAwwVQ2VydGlmaWNhdGUg +QXV0aG9yaXR5MCAXDTI0MDUxNjE4NDUzOVoYDzQ3NjIwNDEyMTg0NTM5WjAsMRMw +EQYDVQQKDApSZWRpcyBUZXN0MRUwEwYDVQQDDAxHZW5lcmljLWNlcnQwggEiMA0G +CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC3kvR6caLws6+HAFpxx3nrA0pK4AKa +tALoFRjWe+7eKqAuXquSJRXHoVPhVXATcMZ1oePPjfDw3LYcDOT6l2BC+ixqyvlC +7zngVNgFjpX9oOO6pkeLFre38R+u1U516L4H/MpiqB9S4/YtGV7HXZCCPctVQjGJ +6VuxSHBqxJBXPtBw3U11JfKMOHE8MNXXP9mc9tBAZDg4tWuP5fRZ3P4urnen2iTq +q/xpuoj7bXAfNNzhbKYEWCFJvbPqIGvDz8JXdywrUaOa4hRctNjqKz5feTwd6A8f +9DW87blj6i2/HwF3AZchOPs52O3vvHrw5zQS9qJlgAEq1nR/TZo6sQt1AgMBAAGj +QjBAMB0GA1UdDgQWBBT+stkMkr87nvQ7R2a9Ww1bkXnr/zAfBgNVHSMEGDAWgBTk +8qgSLlo7HhCMNJ042VwPfRcM9zANBgkqhkiG9w0BAQsFAAOCAgEAL5O9GgjXn4BU +8lUB137DNLXbpjt8qB4mSuKJ5sJAnkMW9g7IV7uXdolCH2wSSU4Akk+42w8ghf+T +ZHxInv5EWARd1hiT4mM/d1nf0V1HGYcUQKXCJTu9lxTq5rBCLF932jp3dd/P9g3N +JfASc3NNSk+VlTB4xlLCQ5b+X+13/TgHGOAqJn1VIYHAD37dyiJkr8S8w7RZuQYT +jhEt0S05zcnudM1Kl4RizQ+HlouTz1gNoEkaaC1sNmQHcBSP79xjvTrt2g/YFOsc +CY7TwCOmGNhKKC7Gm3Q/yiFj1/lbNq++XYwZrTNS5pknTaUK6LVNuJkYe3HMy4iq +LKLbG3sg426071SGbm2vXGrs/p4C02OLqCnL8mwXDxPQ8Px75deBlrXxvOvg1GTl +T+pqK4Hx888IavQVrXy7RsdtFAoRGhBWneU+qwF1Nlfvpy+zGOxs+X3+GRF/eclx +HWfdRD3huHD6PLyswgG+jBxdbuSaHGbHHBNFVAkK6W8sS9bp7D55cN0gizJglIxA +Z/S8EJwyVHkHVszZvzV4rOVGBiO5KnQrUcLVaXDL4lEKa/LuJR0Upec5MrLyZwfI +bURTlpO9zacz/e8hWN89lezJ0dCGLp30aQFmWPdjNh1VbMjJ9SeCEjPErTrbInpP +ciIyR/xHvB7P+Z7wQKDVd5F+dryq/aA= +-----END CERTIFICATE----- diff --git a/spec/fixtures/certificates/redis.dh b/spec/fixtures/certificates/redis.dh new file mode 100644 index 0000000..d68146e --- /dev/null +++ b/spec/fixtures/certificates/redis.dh @@ -0,0 +1,8 @@ +-----BEGIN DH PARAMETERS----- +MIIBDAKCAQEA221GevwdIvhYntSBvnUWI6GeMcQlUADx1WEMn1mjv4HcvIU1wYor +bPLj86jdS0lU5u3zdRZ/wuxmnM58hgetaXX8VNPUzfvYhyKInzwzBvfSQSDvFbY/ +aVdtIhUQ6ij1rSYqzcjVv3gxd2dZNSTT+dYzw/jKLJ9UI7J9PL3dkee/us6hLyDI +sfgM/arHcDxMRZvMHPHv4vwzIHt8/MqEgeN1BIm9941r8JTs4NRUk4zfSr7Bl5ff +QNiDWzN8RK5bSMcGn+XANWrQKoQyktNYaT9yAjIHNxvCPV4nM6UtyTSRMtcz1+Am +mPVONVR9RyPV+FxG6E0AjmukPfu2Z7JVnwIBAgICAOE= +-----END DH PARAMETERS----- diff --git a/spec/fixtures/certificates/redis.key b/spec/fixtures/certificates/redis.key new file mode 100644 index 0000000..48b463c --- /dev/null +++ b/spec/fixtures/certificates/redis.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC3kvR6caLws6+H +AFpxx3nrA0pK4AKatALoFRjWe+7eKqAuXquSJRXHoVPhVXATcMZ1oePPjfDw3LYc +DOT6l2BC+ixqyvlC7zngVNgFjpX9oOO6pkeLFre38R+u1U516L4H/MpiqB9S4/Yt +GV7HXZCCPctVQjGJ6VuxSHBqxJBXPtBw3U11JfKMOHE8MNXXP9mc9tBAZDg4tWuP +5fRZ3P4urnen2iTqq/xpuoj7bXAfNNzhbKYEWCFJvbPqIGvDz8JXdywrUaOa4hRc +tNjqKz5feTwd6A8f9DW87blj6i2/HwF3AZchOPs52O3vvHrw5zQS9qJlgAEq1nR/ +TZo6sQt1AgMBAAECggEAIjfKSmiULrnPpiJyuXKtTxk15+8JjTywrgIAtzoytEbN +xNbwtDUlciioMgBy+6yx0Ytzo2SZ3MvHRkNBC1TcJJ/tV5quC8P7zc0C10ZUvwoW +aNxFya9aX3caqMQp8+CGS4bbUrhz3kePw6hO+mp+XUiXdjT4dvSAmvMdwUkSfrBt +xtcnwiX4g355WwZKp4sqOSnhEQY0KT64hnx1RGAL3IGl6Kj2/V4uwkgbVVR/r2wH +ivSW6UA7Umnky2Liccg/QWDkY4OdHWJ2DP4sAZ+HjOBR1eKf+tVJBA40giM9uQ0a +pfGuOvOBaUz9voS70uYLy8aGZauoPzfhvzuRby6JAQKBgQDlk7qofud2+uGDohk8 +QPEz6Zb+h5UxItRI+5ZjylRzcafD03f3euNPCftUQFblQaiRlRN7NH5ozlpmbI5v +13YVUNH9toTawpiSRh5RVMtk6dTNUk80J1zhdXZi4nfw3VXVoJW7SQPB8hLSZ7OO +YTlKcH9PFg6eVeEZbyEddbS56QKBgQDMs8pSnnGPUdJiQRP11uXUYlBZSgINTM7T +AS2e9gXwPP6azTK9e2ky0Bhx+fW+7A4+aRjrJor/V5HCEEUUInHLq/GtG0Z8CmxO +oaXusSFKsqtVWauCVogo/2S3kTXjGY2++DIZnCc3q7njoXcwIamUxB1riBCyPLr+ +XzFV0TmBrQKBgQDjsimnhHY944ZX8CXnROH2Au/ncsHeAhVabW0zfDFWbptd9hc+ +NXvNoLDNVyLYLs7p1VmFEQNvH3s5quF6u9A6Q/aCFMyfy/FW4oSfVeKFLAYLvl8f +8wqD8uSAHu2mz1+aibL4FerMKTPRy1ijFnqkAf/CiLXnBHZFq5rEhUHCQQKBgFJo +zzsTroQUXK5RpMeEDGLEcbSApvLTyTAJeWRVh26AH8ASfCrsVB2wySaZiuWiAtn9 +VZrVmX/SsUt/YVAJzeZBArq+EFI/n/rH+RVvGggIhhGGWBXQ4GAg8Vs1iowy7W3E +XecRhnhTGjMZ/fHSm4LYsT9pN8+Jw35EtIfoA07BAoGACJWmulCP582Pdrn0Xrzx +mdCVtq1HZd5rUW/KA9fb2ht8A4i0ykZozADYu+eLZ5wu1hxEx/+IyNAhzp8OP1eC +DybKS0px957pesTR97hYOgV/xbQp3PhiBaQrgaVHd4AgeXxL39uMzj7tSZKQknfR +0rJRsJkplqO24hty/aQ5qO8= +-----END PRIVATE KEY----- diff --git a/spec/fixtures/certificates/server.crt b/spec/fixtures/certificates/server.crt new file mode 100644 index 0000000..72745b8 --- /dev/null +++ b/spec/fixtures/certificates/server.crt @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE----- +MIIEUjCCAjqgAwIBAgIUR6PYKGD6sZ5/Jrj/cEMoAXP7WDAwDQYJKoZIhvcNAQEL +BQAwNTETMBEGA1UECgwKUmVkaXMgVGVzdDEeMBwGA1UEAwwVQ2VydGlmaWNhdGUg +QXV0aG9yaXR5MCAXDTI0MDUxNjE4NDUzOVoYDzQ3NjIwNDEyMTg0NTM5WjArMRMw +EQYDVQQKDApSZWRpcyBUZXN0MRQwEgYDVQQDDAtTZXJ2ZXItb25seTCCASIwDQYJ +KoZIhvcNAQEBBQADggEPADCCAQoCggEBAMvxotag0WUqRJZ008eEBIQnqNBdCFm3 +18Px5rusxwGZ3Kchpi2VYxmKLTUwq5XQo29zHefvlnEFytG4SmvwK1tvW+zYfrgT +JWC+j40VcfqgTS2zesqu/zzHcEyWHqJBcMANkSBXNy6byp6zx0JbFqB4RUETqoXZ +5v4K3BWohMvAX+Ww78VIQtvc7hNbJZipCvoxShz9Q0B8lOJiDR2gXAVaZltOu+Hk +dRolHle+qkYslY27L70cheiQZvweRtC6NquTOSpg+ZDlezn4pHtPKlnxUa82ciZj +rqfpysaPLEzeuZ/MDuiyOZ4+h7b8/vyilKRupcV0bU+NfVFsWdvOnaECAwEAAaNi +MGAwCwYDVR0PBAQDAgWgMBEGCWCGSAGG+EIBAQQEAwIGQDAdBgNVHQ4EFgQUw851 +jwi3qF89kUF/SS30OWlCX3swHwYDVR0jBBgwFoAU5PKoEi5aOx4QjDSdONlcD30X +DPcwDQYJKoZIhvcNAQELBQADggIBAD0tyTntrFQbvF8bt1o3KhahY0SaHD1SdHWf +W81SujJkk3gqpOB/kP/JJUapQOyZ80seYUXtYNMXLWX2zXFoCMquZ7tjG352ybNV +Rko1/n9C0fyby+4rEFF281hkqaY1oiBNzwjXp1g6PAHO1zN5ZQAPcB9RLkOXxOnl +wPtHX5k6w433B2Xc6k0j5l1q6wK/H4mdD0nlNV05qf2o/Gl8GIAQEo/ZdqO5+aeb +8VqDcnQGe1MYhIJRTNYTYeBQ5Qe+caCm0vOm3dLqkdzfKUCwY+q6OGez7JnUJphK +VmDYfWAKLjZP7XTsck7oU5tCFWw/eKNr+wr8KwiLaAqmCUO68Pe2q0YTRu2k9fGj +cQ+YbZbl2MFoUAo37xdKsE1xMtsTBBezGsoq1/D9yJi1Yfp1KXyRFujbpEVtMTFG +X9YtxF4mfPxaguXrrthsMg04eLgG/D+kz7aYh3WydsWZnlCuJ+o6q5c+V0OGwmU+ +u7GrgyhNXGftQksf9BrNTwcngVFcJSe2xXLG1dDtQJTz8/KcNSIcdb4PPe6bP+st +035x2Z7e3JkqiDMNQew80JqeCeYIII9MeUknPHAX3FK9JWK27LhUosnstEFy5FHT +xR9BXUupIMJtaDdAICtSujMEm7NkKl38LMQ3UjGZl/jyLd+Y9UnmD3pDouMSW4Js +9YD5FxH9 +-----END CERTIFICATE----- diff --git a/spec/fixtures/certificates/server.key b/spec/fixtures/certificates/server.key new file mode 100644 index 0000000..562bb52 --- /dev/null +++ b/spec/fixtures/certificates/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDL8aLWoNFlKkSW +dNPHhASEJ6jQXQhZt9fD8ea7rMcBmdynIaYtlWMZii01MKuV0KNvcx3n75ZxBcrR +uEpr8Ctbb1vs2H64EyVgvo+NFXH6oE0ts3rKrv88x3BMlh6iQXDADZEgVzcum8qe +s8dCWxageEVBE6qF2eb+CtwVqITLwF/lsO/FSELb3O4TWyWYqQr6MUoc/UNAfJTi +Yg0doFwFWmZbTrvh5HUaJR5XvqpGLJWNuy+9HIXokGb8HkbQujarkzkqYPmQ5Xs5 ++KR7TypZ8VGvNnImY66n6crGjyxM3rmfzA7osjmePoe2/P78opSkbqXFdG1PjX1R +bFnbzp2hAgMBAAECggEACiMEV2+KfuE0I+gSRW/Ad3be7T3dihORJPkxYS1KHjBR +C0nY8Bm1cXXYr8TPt4Blejb2IuLJxwv9F/HVeaJYLlkVcCsV19WESWfpW89nwRe+ +BE4wJZI/h5xoXpck1kbjSTl3Z9yL5qw5zMv15MyBHjhiJVUaWAEz2fdNV4J++L9M +Ex4H/WvbmNLumam8bnsw3EUc5AEbsEIMBiKfIlNSo6aAco0iVh/wpdv/k78f5F3M +XQRtJLaVxn2i5W1BoXzUPTQGDFjd8+kuHHZBAEMjUuH9j+v4RlszEeSM0Eq213C+ +Aux89yl763tLFrL66a3866SUoKfdp0tkyOVVh3Sw8QKBgQD9TqdxwPqkFTEHyHsG +6Xrw+YzoWTv0ZaczwnRE+gXw8rZnkKm4zGbjk0EZMJW/l8mdvbsnhOlasaOBfc4a +jpqgvEZpU00V8p/lYqAec0X/qDhLm930T1nMVcatYVSZTPaNuvo6FePV7ches2zC +KVJgmKxwnYPHTubWo99KRDsIkQKBgQDOHKU7tfOkzadhuXBkiH6iPYL+S7H3yxAQ +QbS27V//6JHIyWP7tei0IwHAZcFR8sdeF2fApyZu7Za6J/7MgyjEimevNg4BeF8Y +fWLj9EoEEiHTtipl7tpsAI/RM2Gc4oSJ5LElmLzO9OkwnPWnP1oeClqPtjJ447Oq +h9ai49hMEQKBgQCAJ7wVlEH6o/ITcv2zaIrXwhnlL8hihM+MI1R51VkOJ23bhdgo +c9mWR9kGqosIzP2nPeejPd5mETuc9w5AZ2eZ7Mde4FM+yGrXPgeugDmo9+3Lawdn +ZIIVHEyRSY3opYST/kY8/vgvMsdOCk77CLbxwix0KW0TeDrX2z7MLysC0QKBgBnv +wt8zs1g+xNxoUHtmLcFOykgj0F5lQD07d3k5f8YdRxMkPJ+1MXZRnWvPojckvO/b +NCQrJIv6++erFsY3jw7Ed+21eKe4tQbI6IPU/udqPLV+i/FN0FCc9XwW4iay3ojX +rW8UvFWyvhFu/v6v2zBCj0jcXZeW18oS0/CliVdxAoGBANBIaB8pdLE+bp9YZGKU +glrlo11jdqJBYvidlPngZDVvIBSw3b4woK3m+r1+bDpQseDATmTDaaItdHSktmTw +fNGAZFyfZmXtmmuPJu16j62ML0YlRWUSAvbD3ntCIKs0zRjyywSOJs/gnaJKNIhB +Pj3xop1kUBPxx2Y9Kcu2bg2C +-----END PRIVATE KEY----- diff --git a/spec/integration/outputs/redis_spec.rb b/spec/integration/outputs/redis_spec.rb index e934e64..c4d9bb6 100644 --- a/spec/integration/outputs/redis_spec.rb +++ b/spec/integration/outputs/redis_spec.rb @@ -6,8 +6,13 @@ describe LogStash::Outputs::Redis do + FIXTURES_PATH = File.expand_path('../../fixtures', File.dirname(__FILE__)) + PORT = 16379 + SSL_PORT = 26379 + context "integration tests", :integration => true do shared_examples_for "writing to redis list" do |extra_config| + let(:timeout) { 5 } let(:key) { 10.times.collect { rand(10).to_s }.join("") } let(:event_count) { Flores::Random.integer(0..10000) } let(:message) { Flores::Random.text(0..100) } @@ -15,7 +20,9 @@ { "key" => key, "data_type" => "list", - "host" => "localhost" + "host" => "redis", + "port" => PORT, + "timeout" => timeout } } let(:redis_config) { @@ -23,6 +30,19 @@ } let(:redis_output) { described_class.new(redis_config) } + let(:redis) do + ssl_enabled = redis_config['ssl_enabled'] == true + cli_config = { + :host => redis_config["host"], + :port => redis_config["port"] || PORT, + :timeout => timeout, + :ssl => ssl_enabled + } + + cli_config[:ssl_params] = redis_output.send(:setup_ssl_params) if ssl_enabled + Redis.new(cli_config) + end + before do redis_output.register event_count.times do |i| @@ -32,22 +52,24 @@ redis_output.close end - it "should successfully send all events to redis" do - redis = Redis.new(:host => "127.0.0.1") + after do + redis.del(key) + end + it "should successfully send all events to redis" do # The list should contain the number of elements our agent pushed up. - insist { redis.llen(key) } == event_count + expect(redis.llen(key)).to eql event_count # Now check all events for order and correctness. event_count.times do |value| - id, element = redis.blpop(key, 0) + id, element = redis.blpop(key, :timeout => timeout) event = LogStash::Event.new(LogStash::Json.load(element)) - insist { event["sequence"] } == value - insist { event["message"] } == message + expect(event.get("sequence")).to eql value + expect(event.get("message")).to eql message end # The list should now be empty - insist { redis.llen(key) } == 0 + expect(redis.llen(key)).to eql 0 end end @@ -55,11 +77,41 @@ include_examples "writing to redis list" end + context "when SSL is enabled" do + context "with client certificate and key" do + ssl_config = { + "host" => "redis_ssl", + "port" => SSL_PORT, + "ssl_enabled" => true, + "ssl_certificate_authorities" => File.join(FIXTURES_PATH, 'certificates/ca.crt'), + "ssl_certificate" => File.join(FIXTURES_PATH, 'certificates/client.crt'), + "ssl_key" => File.join(FIXTURES_PATH, 'certificates/client.key') + } + + include_examples "writing to redis list", ssl_config + end + + context "with ssl_verification_mode => none" do + ssl_config = { + "host" => "redis_ssl", + "port" => SSL_PORT, + "ssl_enabled" => true, + "ssl_verification_mode" => "none", + "ssl_certificate" => File.join(FIXTURES_PATH, 'certificates/client.crt'), + "ssl_key" => File.join(FIXTURES_PATH, 'certificates/client.key') + } + + include_examples "writing to redis list", ssl_config + end + + end + context "when batch_mode is true" do batch_events = Flores::Random.integer(1..1000) batch_settings = { "batch" => true, - "batch_events" => batch_events + "batch_events" => batch_events, + "port" => PORT } include_examples "writing to redis list", batch_settings do @@ -74,4 +126,3 @@ end end end - diff --git a/spec/unit/outputs/redis_spec.rb b/spec/unit/outputs/redis_spec.rb index 7ed1220..1ada8f0 100644 --- a/spec/unit/outputs/redis_spec.rb +++ b/spec/unit/outputs/redis_spec.rb @@ -3,6 +3,7 @@ require "logstash/json" require "redis" require "flores/random" +require "flores/pki" describe LogStash::Outputs::Redis do @@ -11,14 +12,15 @@ # TODO: refactor specs above and probably rely on a Redis mock to correctly test the code expected behaviour, the actual # tests agains Redis should be moved into integration tests. let(:key) { "thekey" } - let(:payload) { "somepayload"} - let(:event) { LogStash::Event.new({"message" => "test"}) } let(:config) { { "key" => key, "data_type" => "list", "batch" => true, "batch_events" => 50, + "batch_timeout" => 3600 * 24, + # ^ this a very large timeout value to prevent the Flush Timer thread in Stud::Buffer from calling flush + # it screws with the RSpec expect().to receive thread safety. } } let(:redis) { described_class.new(config) } @@ -27,14 +29,116 @@ redis.register expect(redis).to receive(:buffer_receive).exactly(10000).times.and_call_original expect(redis).to receive(:flush).exactly(200).times + expect(redis).not_to receive(:on_flush_error) # I was able to reproduce the LocalJumpError: unexpected next exception at around 50 # consicutive invocations. setting to 10000 should reproduce it for any environment # I have no clue at this point why this problem does not happen at every invocation - 1.upto(10000) do - expect{redis.receive(event)}.to_not raise_error + 10000.times do |i| + expect{redis.receive(LogStash::Event.new({"message" => "test-#{i}"}))}.to_not raise_error end end end -end + context "with SSL enabled" do + let(:config) {{ "ssl_enabled" => true, "key" => "key", "data_type" => "list" }} + subject(:plugin) { described_class.new(config) } + + context "and not providing a certificate/key pair" do + it "registers without error" do + expect { plugin.register }.to_not raise_error + end + end + + context "and providing a certificate/key pair" do + let(:cert_key_pair) { Flores::PKI.generate } + let(:certificate) do + path = Tempfile.new('certificate').path + IO.write(path, cert_key_pair.first.to_s) + path + end + let(:key) do + path = Tempfile.new('key').path + IO.write(path, cert_key_pair[1].to_s) + path + end + let(:config) { super().merge("ssl_certificate" => certificate, "ssl_key" => key) } + + it "registers without error" do + expect { plugin.register }.to_not raise_error + end + end + + FIXTURES_PATH = File.expand_path('../../fixtures', File.dirname(__FILE__)) + + context "and plain-text certificate/key" do + let(:key_file) { File.join(FIXTURES_PATH, 'certificates/redis.key') } + let(:crt_file) { File.join(FIXTURES_PATH, 'certificates/redis.crt') } + let(:config) { super().merge("ssl_certificate" => crt_file, "ssl_key" => key_file) } + + it "registers without error" do + expect { plugin.register }.to_not raise_error + end + + context 'with password set' do + let(:config) { super().merge("ssl_key_passphrase" => 'ignored') } + + it "registers without error" do # password simply ignored + expect { plugin.register }.to_not raise_error + end + end + + context 'with supported protocol' do + let(:config) { super().merge("ssl_supported_protocols" => %w[TLSv1.2 TLSv1.3]) } + + it 'configures minimum TLS version' do + plugin.register + ssl_params = plugin.send(:setup_ssl_params) + expect(ssl_params).to match(a_hash_including(:min_version => :TLS1_2, :max_version => :TLS1_3)) + end + end + end + + context "with only ssl_certificate set" do + let(:config) { super().merge("ssl_certificate" => File.join(FIXTURES_PATH, 'certificates/redis.crt')) } + + it "should raise a configuration error to request also `ssl_key`" do + expect { plugin.register }.to raise_error(LogStash::ConfigurationError, /Using an `ssl_certificate` requires an `ssl_key`/) + end + end + + context "with only ssl_key set" do + let(:config) { super().merge("ssl_key" => File.join(FIXTURES_PATH, 'certificates/redis.key')) } + + it "should raise a configuration error to request also `ssl_key`" do + expect { plugin.register }.to raise_error(LogStash::ConfigurationError, /An `ssl_certificate` is required when using an `ssl_key`/) + end + end + + context "with ssl_certificate_authorities" do + let(:certificate_path) { File.join(FIXTURES_PATH, 'certificates/redis.crt') } + let(:config) do + super().merge('ssl_certificate_authorities' => [certificate_path]) + end + + it "sets cert_store values" do + ssl_store = double(OpenSSL::X509::Store.new) + allow(ssl_store).to receive(:set_default_paths) + allow(ssl_store).to receive(:add_file) + allow(subject).to receive(:new_ssl_certificate_store).and_return(ssl_store) + subject.send :setup_ssl_params + expect(ssl_store).to have_received(:add_file).with(certificate_path) + end + end + + context "CAs certificates" do + it "includes openssl default paths" do + ssl_store = double(OpenSSL::X509::Store.new) + allow(ssl_store).to receive(:set_default_paths) + allow(plugin).to receive(:new_ssl_certificate_store).and_return(ssl_store) + subject.send :setup_ssl_params + expect(ssl_store).to have_received(:set_default_paths) + end + end + end +end