From 3dace7164108a8ee2e8fcf95d03021ce38621e28 Mon Sep 17 00:00:00 2001 From: Keith Duncan Date: Sat, 24 Jan 2015 23:55:47 +1100 Subject: [PATCH 01/23] Add histogram method --- lib/statsd.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/statsd.rb b/lib/statsd.rb index cbd3e539..8322e11 100644 --- a/lib/statsd.rb +++ b/lib/statsd.rb @@ -109,6 +109,12 @@ def time(stat, sample_rate=1) result end + # Sends a histogram measurement for the given stat to the statsd server. The + # sample_rate determines what percentage of the time this report is sent. The + # statsd server then uses the sample_rate to correctly track the average + # for the stat. + def histogram(stat, value, sample_rate=1); send stat, value, 'h', sample_rate end + private def sampled(sample_rate) From 1d28d610b6529d097914138ebc5c55f54ce91b28 Mon Sep 17 00:00:00 2001 From: Keith Duncan Date: Sun, 25 Jan 2015 00:08:44 +1100 Subject: [PATCH 02/23] Bump version --- statsd-ruby.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statsd-ruby.gemspec b/statsd-ruby.gemspec index 1b53551..faf3b7e 100644 --- a/statsd-ruby.gemspec +++ b/statsd-ruby.gemspec @@ -5,7 +5,7 @@ Gem::Specification.new do |s| s.name = %q{statsd-ruby} - s.version = "0.3.0.github.5" + s.version = "0.4.0.github" s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= s.authors = ["Rein Henrichs"] From 8145eed68ca9b0c14ee1068d8315bf4027927cff Mon Sep 17 00:00:00 2001 From: Ryan Tomayko Date: Tue, 31 Mar 2015 04:47:02 -0400 Subject: [PATCH 03/23] Avoid loading openssl and securerandom libs unless needed --- lib/statsd.rb | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/statsd.rb b/lib/statsd.rb index 8322e11..22c5674 100644 --- a/lib/statsd.rb +++ b/lib/statsd.rb @@ -1,5 +1,3 @@ -require 'openssl' -require 'securerandom' require 'socket' require 'time' require 'zlib' @@ -32,9 +30,6 @@ def initialize(host, port, key = nil) #characters that will be replaced with _ in stat names RESERVED_CHARS_REGEX = /[\:\|\@]/ - # Digest object as a constant - SHA256 = OpenSSL::Digest::SHA256.new - class << self # Set to any standard logger instance (including stdlib's Logger) to enable # stat logging using logger.debug @@ -150,11 +145,22 @@ def select_host(stat) end def signed_payload(key, message) + sha256 = Statsd.setup_openssl payload = timestamp + nonce + message - signature = OpenSSL::HMAC.digest(SHA256, key, payload) + signature = OpenSSL::HMAC.digest(sha256, key, payload) signature + payload end + # defer loading openssl and securerandom unless needed. this shaves ~10ms off + # of baseline require load time for environments that don't require message signing. + def self.setup_openssl + @sha256 ||= begin + require 'securerandom' + require 'openssl' + OpenSSL::Digest::SHA256.new + end + end + def timestamp [Time.now.to_i].pack("Q<") end From d3a7ccbf296db67e34db0568bdfbddea42e1b6c4 Mon Sep 17 00:00:00 2001 From: Ryan Tomayko Date: Tue, 31 Mar 2015 04:47:24 -0400 Subject: [PATCH 04/23] Also shouldn't need extended 'time' module stuff here Time.now is built in. Loading the time library adds some stuff like Time#iso8601 but doesn't seem to be used here. --- lib/statsd.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/statsd.rb b/lib/statsd.rb index 22c5674..1c4a9dd 100644 --- a/lib/statsd.rb +++ b/lib/statsd.rb @@ -1,5 +1,4 @@ require 'socket' -require 'time' require 'zlib' # = Statsd: A Statsd client (https://github.com/etsy/statsd) From dfbf6469dbf236bd5dae7101e00ce2172e27afe9 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Tue, 5 May 2015 10:38:06 +0200 Subject: [PATCH 05/23] Connect dat UDP socket --- lib/statsd.rb | 62 ++++++++++++++++++++++----------------------------- 1 file changed, 27 insertions(+), 35 deletions(-) diff --git a/lib/statsd.rb b/lib/statsd.rb index 1c4a9dd..dd06228 100644 --- a/lib/statsd.rb +++ b/lib/statsd.rb @@ -14,13 +14,20 @@ # statsd = Statsd.new('localhost').tap{|sd| sd.namespace = 'account'} # statsd.increment 'activate' class Statsd - class Host - attr_reader :ip, :port, :key - def initialize(host, port, key = nil) - @ip = Addrinfo.ip(host).ip_address - @port = port + class RubyUdpClient + attr_reader :key, :sock + + def initialize(address, port, key = nil) + @sock = UDPSocket.new + @sock.connect(address, port) @key = key end + + def send(msg) + sock.write(msg) + rescue => boom + nil + end end # A namespace to prepend to all statsd calls. @@ -29,22 +36,19 @@ def initialize(host, port, key = nil) #characters that will be replaced with _ in stat names RESERVED_CHARS_REGEX = /[\:\|\@]/ - class << self - # Set to any standard logger instance (including stdlib's Logger) to enable - # stat logging using logger.debug - attr_accessor :logger + def initialize(client_class = nil) + @shards = [] + @client_class = client_class || RubyUdpClient end - # @param [String] host your statsd host - # @param [Integer] port your statsd port - def initialize(host, port=8125, key=nil) - @hosts = [] - add_host(host, port, key) + def self.simple(addr, port = nil) + self.new.add_shard(addr, port) end - def add_host(host, port = nil, key = nil) - host, port = host.split(':') if host.include?(':') - @hosts << Host.new(host, port.to_i, key) + def add_shard(addr, port = nil, key = nil) + addr, port = addr.split(':') if addr.include?(':') + @shards << @client_class.new(addr, port.to_i, key) + self end # Sends an increment (count = 1) for the given stat to the statsd server. @@ -120,26 +124,16 @@ def send(stat, delta, type, sample_rate=1) prefix = "#{@namespace}." unless @namespace.nil? stat = stat.to_s.gsub('::', '.').gsub(RESERVED_CHARS_REGEX, '_') msg = "#{prefix}#{stat}:#{delta}|#{type}#{'|@' << sample_rate.to_s if sample_rate < 1}" - send_to_socket(select_host(stat), msg) - end - end - - def send_to_socket(host, message) - self.class.logger.debug {"Statsd: #{message}"} if self.class.logger - if host.key.nil? - socket.send(message, 0, host.ip, host.port) - else - socket.send(signed_payload(host.key, message), 0, host.ip, host.port) + shard = select_shard(stat) + shard.send(shard.key ? signed_payload(shard.key, msg) : msg) end - rescue => boom - self.class.logger.error {"Statsd: #{boom.class} #{boom}"} if self.class.logger end - def select_host(stat) - if @hosts.size == 1 - @hosts.first + def select_shard(stat) + if @shards.size == 1 + @shards.first else - @hosts[Zlib.crc32(stat) % @hosts.size] + @shards[Zlib.crc32(stat) % @shards.size] end end @@ -167,6 +161,4 @@ def timestamp def nonce SecureRandom.random_bytes(4) end - - def socket; @socket ||= UDPSocket.new end end From 5068097ded3e965ea589968f6ebc252c27e1c6ef Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Tue, 5 May 2015 12:02:08 +0200 Subject: [PATCH 06/23] Add reader --- lib/statsd.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/statsd.rb b/lib/statsd.rb index dd06228..ab081dc 100644 --- a/lib/statsd.rb +++ b/lib/statsd.rb @@ -33,6 +33,9 @@ def send(msg) # A namespace to prepend to all statsd calls. attr_accessor :namespace + # All the endpoints where StatsD will report metrics + attr_reader :shards + #characters that will be replaced with _ in stat names RESERVED_CHARS_REGEX = /[\:\|\@]/ From 8ddc37490681394e8a59f3f793ccdea3f87bd694 Mon Sep 17 00:00:00 2001 From: Charlie Somerville Date: Thu, 7 May 2015 12:07:57 +1000 Subject: [PATCH 07/23] use addrinfo to find correct protocol family to create socket with --- lib/statsd.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/statsd.rb b/lib/statsd.rb index ab081dc..b9e0a8e 100644 --- a/lib/statsd.rb +++ b/lib/statsd.rb @@ -18,8 +18,9 @@ class RubyUdpClient attr_reader :key, :sock def initialize(address, port, key = nil) - @sock = UDPSocket.new - @sock.connect(address, port) + addrinfo = Addrinfo.ip(address) + @sock = UDPSocket.new(addrinfo.pfamily) + @sock.connect(addrinfo.ip_address, port) @key = key end From 403ec412681c532e1179e54f60023061a48e29af Mon Sep 17 00:00:00 2001 From: Adam Roben Date: Fri, 25 Sep 2015 14:32:07 -0400 Subject: [PATCH 08/23] Freeze metric type strings This saves one String allocation per recorded metric. --- lib/statsd.rb | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/statsd.rb b/lib/statsd.rb index b9e0a8e..33cd137 100644 --- a/lib/statsd.rb +++ b/lib/statsd.rb @@ -40,6 +40,11 @@ def send(msg) #characters that will be replaced with _ in stat names RESERVED_CHARS_REGEX = /[\:\|\@]/ + COUNTER_TYPE = "c".freeze + TIMING_TYPE = "ms".freeze + GAUGE_TYPE = "g".freeze + HISTOGRAM_TYPE = "h".freeze + def initialize(client_class = nil) @shards = [] @client_class = client_class || RubyUdpClient @@ -74,7 +79,7 @@ def decrement(stat, sample_rate=1); count stat, -1, sample_rate end # @param [String] stat stat name # @param [Integer] count count # @param [Integer] sample_rate sample rate, 1 for always - def count(stat, count, sample_rate=1); send stat, count, 'c', sample_rate end + def count(stat, count, sample_rate=1); send stat, count, COUNTER_TYPE, sample_rate end # Sends an arbitary gauge value for the given stat to the statsd server. # @@ -83,7 +88,7 @@ def count(stat, count, sample_rate=1); send stat, count, 'c', sample_rate end # @example Report the current user count: # $statsd.gauge('user.count', User.count) def gauge(stat, value) - send stat, value, 'g' + send stat, value, GAUGE_TYPE end # Sends a timing (in ms) for the given stat to the statsd server. The @@ -94,7 +99,7 @@ def gauge(stat, value) # @param stat stat name # @param [Integer] ms timing in milliseconds # @param [Integer] sample_rate sample rate, 1 for always - def timing(stat, ms, sample_rate=1); send stat, ms, 'ms', sample_rate end + def timing(stat, ms, sample_rate=1); send stat, ms, TIMING_TYPE, sample_rate end # Reports execution time of the provided block using {#timing}. # @@ -115,7 +120,7 @@ def time(stat, sample_rate=1) # sample_rate determines what percentage of the time this report is sent. The # statsd server then uses the sample_rate to correctly track the average # for the stat. - def histogram(stat, value, sample_rate=1); send stat, value, 'h', sample_rate end + def histogram(stat, value, sample_rate=1); send stat, value, HISTOGRAM_TYPE, sample_rate end private From 1a3e931dfa79dcab87de298d78fac2f2eb3468fc Mon Sep 17 00:00:00 2001 From: Adam Roben Date: Fri, 25 Sep 2015 14:35:11 -0400 Subject: [PATCH 09/23] Precompute the namespace prefix This saves one String allocation per recorded metric. --- lib/statsd.rb | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/statsd.rb b/lib/statsd.rb index 33cd137..674c01f 100644 --- a/lib/statsd.rb +++ b/lib/statsd.rb @@ -32,7 +32,12 @@ def send(msg) end # A namespace to prepend to all statsd calls. - attr_accessor :namespace + attr_reader :namespace + + def namespace=(namespace) + @namespace = namespace + @prefix = namespace ? "#{@namespace}." : "".freeze + end # All the endpoints where StatsD will report metrics attr_reader :shards @@ -48,6 +53,7 @@ def send(msg) def initialize(client_class = nil) @shards = [] @client_class = client_class || RubyUdpClient + self.namespace = nil end def self.simple(addr, port = nil) @@ -130,9 +136,8 @@ def sampled(sample_rate) def send(stat, delta, type, sample_rate=1) sampled(sample_rate) do - prefix = "#{@namespace}." unless @namespace.nil? stat = stat.to_s.gsub('::', '.').gsub(RESERVED_CHARS_REGEX, '_') - msg = "#{prefix}#{stat}:#{delta}|#{type}#{'|@' << sample_rate.to_s if sample_rate < 1}" + msg = "#{@prefix}#{stat}:#{delta}|#{type}#{'|@' << sample_rate.to_s if sample_rate < 1}" shard = select_shard(stat) shard.send(shard.key ? signed_payload(shard.key, msg) : msg) end From 184c039e8de8d9dd4cb35fa5c2d5c8c5c4d4e656 Mon Sep 17 00:00:00 2001 From: Adam Roben Date: Fri, 25 Sep 2015 14:36:16 -0400 Subject: [PATCH 10/23] Use #gsub! instead of #gsub to clean up metric names This saves one String allocation per recorded metric. --- lib/statsd.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/statsd.rb b/lib/statsd.rb index 674c01f..dbd4e58 100644 --- a/lib/statsd.rb +++ b/lib/statsd.rb @@ -136,7 +136,9 @@ def sampled(sample_rate) def send(stat, delta, type, sample_rate=1) sampled(sample_rate) do - stat = stat.to_s.gsub('::', '.').gsub(RESERVED_CHARS_REGEX, '_') + stat = stat.to_s.dup + stat.gsub!('::', '.') + stat.gsub!(RESERVED_CHARS_REGEX, '_') msg = "#{@prefix}#{stat}:#{delta}|#{type}#{'|@' << sample_rate.to_s if sample_rate < 1}" shard = select_shard(stat) shard.send(shard.key ? signed_payload(shard.key, msg) : msg) From f733126a4a7c1e7c1b93e704759b66caf5365c7e Mon Sep 17 00:00:00 2001 From: Adam Roben Date: Fri, 25 Sep 2015 14:37:32 -0400 Subject: [PATCH 11/23] Use a regex for replacing :: in metric names This saves two String allocations per recorded metric. --- lib/statsd.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/statsd.rb b/lib/statsd.rb index dbd4e58..e1633fe 100644 --- a/lib/statsd.rb +++ b/lib/statsd.rb @@ -45,6 +45,8 @@ def namespace=(namespace) #characters that will be replaced with _ in stat names RESERVED_CHARS_REGEX = /[\:\|\@]/ + COLON_COLON_REGEX = /::/ + COUNTER_TYPE = "c".freeze TIMING_TYPE = "ms".freeze GAUGE_TYPE = "g".freeze @@ -137,7 +139,7 @@ def sampled(sample_rate) def send(stat, delta, type, sample_rate=1) sampled(sample_rate) do stat = stat.to_s.dup - stat.gsub!('::', '.') + stat.gsub!(COLON_COLON_REGEX, '.') stat.gsub!(RESERVED_CHARS_REGEX, '_') msg = "#{@prefix}#{stat}:#{delta}|#{type}#{'|@' << sample_rate.to_s if sample_rate < 1}" shard = select_shard(stat) From 38e59ef4cca4e3507aa1cd8bc703293a109f12a8 Mon Sep 17 00:00:00 2001 From: Adam Roben Date: Fri, 25 Sep 2015 14:38:46 -0400 Subject: [PATCH 12/23] Use frozen strings for cleaning metric names This saves two String allocations per recorded metric. --- lib/statsd.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/statsd.rb b/lib/statsd.rb index e1633fe..c7c214a 100644 --- a/lib/statsd.rb +++ b/lib/statsd.rb @@ -139,8 +139,8 @@ def sampled(sample_rate) def send(stat, delta, type, sample_rate=1) sampled(sample_rate) do stat = stat.to_s.dup - stat.gsub!(COLON_COLON_REGEX, '.') - stat.gsub!(RESERVED_CHARS_REGEX, '_') + stat.gsub!(COLON_COLON_REGEX, ".".freeze) + stat.gsub!(RESERVED_CHARS_REGEX, "_".freeze) msg = "#{@prefix}#{stat}:#{delta}|#{type}#{'|@' << sample_rate.to_s if sample_rate < 1}" shard = select_shard(stat) shard.send(shard.key ? signed_payload(shard.key, msg) : msg) From bdc19c37f8517190fa6b591a681c4a83c5030641 Mon Sep 17 00:00:00 2001 From: Adam Roben Date: Fri, 25 Sep 2015 14:40:44 -0400 Subject: [PATCH 13/23] Build up message string incrementally This saves one String allocation per recorded metric. --- lib/statsd.rb | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/statsd.rb b/lib/statsd.rb index c7c214a..770e361 100644 --- a/lib/statsd.rb +++ b/lib/statsd.rb @@ -141,7 +141,19 @@ def send(stat, delta, type, sample_rate=1) stat = stat.to_s.dup stat.gsub!(COLON_COLON_REGEX, ".".freeze) stat.gsub!(RESERVED_CHARS_REGEX, "_".freeze) - msg = "#{@prefix}#{stat}:#{delta}|#{type}#{'|@' << sample_rate.to_s if sample_rate < 1}" + + msg = "" + msg << @prefix + msg << stat + msg << ":".freeze + msg << delta.to_s + msg << "|".freeze + msg << type + if sample_rate < 1 + msg << "|@".freeze + msg << sample_rate.to_s + end + shard = select_shard(stat) shard.send(shard.key ? signed_payload(shard.key, msg) : msg) end From cda3b858f3a1ab7121c7fdd06dc7a58e53f6bc7e Mon Sep 17 00:00:00 2001 From: Adam Roben Date: Tue, 29 Sep 2015 10:17:49 -0400 Subject: [PATCH 14/23] Get rid of COLON_COLON_REGEX constant According to @charliesome this is no more efficient than using the literal inline. --- lib/statsd.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/statsd.rb b/lib/statsd.rb index 770e361..55ac840 100644 --- a/lib/statsd.rb +++ b/lib/statsd.rb @@ -45,8 +45,6 @@ def namespace=(namespace) #characters that will be replaced with _ in stat names RESERVED_CHARS_REGEX = /[\:\|\@]/ - COLON_COLON_REGEX = /::/ - COUNTER_TYPE = "c".freeze TIMING_TYPE = "ms".freeze GAUGE_TYPE = "g".freeze @@ -139,7 +137,7 @@ def sampled(sample_rate) def send(stat, delta, type, sample_rate=1) sampled(sample_rate) do stat = stat.to_s.dup - stat.gsub!(COLON_COLON_REGEX, ".".freeze) + stat.gsub!(/::/, ".".freeze) stat.gsub!(RESERVED_CHARS_REGEX, "_".freeze) msg = "" From 80ac4de0deee3dc60ece3960acabc37b96e9c7de Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Tue, 12 Jan 2016 14:31:27 +0100 Subject: [PATCH 15/23] statsd: Abstract the Secure Client All the secure client implementation has been moved into the SecureUDPClient class; the `StatsD` core no longer has any secure-related features. Hence, the secure client must be initialized directly with the shared key, or with `StatsD#add_shard`, which has been updated to transparently forward its arguments to the client constructor. The hashing/HMAC is now performed right before writing to the socket, which allows for greater composability. --- lib/statsd.rb | 83 +++++++++++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 36 deletions(-) diff --git a/lib/statsd.rb b/lib/statsd.rb index 55ac840..4454b64 100644 --- a/lib/statsd.rb +++ b/lib/statsd.rb @@ -14,14 +14,52 @@ # statsd = Statsd.new('localhost').tap{|sd| sd.namespace = 'account'} # statsd.increment 'activate' class Statsd - class RubyUdpClient - attr_reader :key, :sock + class SecureUDPClient < UDPClient + def initialize(address, port, key) + super(address, port) + @key = key + end + + def send(msg) + super(signed_payload(msg)) + end + + private + # defer loading openssl and securerandom unless needed. this shaves ~10ms off + # of baseline require load time for environments that don't require message signing. + def self.setup_openssl + @sha256 ||= begin + require 'securerandom' + require 'openssl' + OpenSSL::Digest::SHA256.new + end + end + + def signed_payload(message) + sha256 = SecureUDPClient.setup_openssl + payload = timestamp + nonce + message + signature = OpenSSL::HMAC.digest(sha256, @key, payload) + signature + payload + end + + def timestamp + [Time.now.to_i].pack("Q<") + end + + def nonce + SecureRandom.random_bytes(4) + end + end + + class UDPClient + attr_reader :sock - def initialize(address, port, key = nil) + def initialize(address, port = nil) + address, port = address.split(':') if address.include?(':') addrinfo = Addrinfo.ip(address) + @sock = UDPSocket.new(addrinfo.pfamily) @sock.connect(addrinfo.ip_address, port) - @key = key end def send(msg) @@ -52,7 +90,7 @@ def namespace=(namespace) def initialize(client_class = nil) @shards = [] - @client_class = client_class || RubyUdpClient + @client_class = client_class || UDPClient self.namespace = nil end @@ -60,9 +98,8 @@ def self.simple(addr, port = nil) self.new.add_shard(addr, port) end - def add_shard(addr, port = nil, key = nil) - addr, port = addr.split(':') if addr.include?(':') - @shards << @client_class.new(addr, port.to_i, key) + def add_shard(*args) + @shards << @client_class.new(*args) self end @@ -129,7 +166,6 @@ def time(stat, sample_rate=1) def histogram(stat, value, sample_rate=1); send stat, value, HISTOGRAM_TYPE, sample_rate end private - def sampled(sample_rate) yield unless sample_rate < 1 and rand > sample_rate end @@ -140,7 +176,7 @@ def send(stat, delta, type, sample_rate=1) stat.gsub!(/::/, ".".freeze) stat.gsub!(RESERVED_CHARS_REGEX, "_".freeze) - msg = "" + msg = String.new msg << @prefix msg << stat msg << ":".freeze @@ -153,7 +189,7 @@ def send(stat, delta, type, sample_rate=1) end shard = select_shard(stat) - shard.send(shard.key ? signed_payload(shard.key, msg) : msg) + shard.send(msg) end end @@ -164,29 +200,4 @@ def select_shard(stat) @shards[Zlib.crc32(stat) % @shards.size] end end - - def signed_payload(key, message) - sha256 = Statsd.setup_openssl - payload = timestamp + nonce + message - signature = OpenSSL::HMAC.digest(sha256, key, payload) - signature + payload - end - - # defer loading openssl and securerandom unless needed. this shaves ~10ms off - # of baseline require load time for environments that don't require message signing. - def self.setup_openssl - @sha256 ||= begin - require 'securerandom' - require 'openssl' - OpenSSL::Digest::SHA256.new - end - end - - def timestamp - [Time.now.to_i].pack("Q<") - end - - def nonce - SecureRandom.random_bytes(4) - end end From dba7e7ac026121a74d830b62c94e40bcb261d802 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Tue, 12 Jan 2016 14:31:50 +0100 Subject: [PATCH 16/23] statsd: Allow buffering for the clients The new `Buffer` wrapper allow wrapping any UDP clients in a "buffering mode" that batches incoming metrics into a buffer and eventually flushes them to the statsd-compatible server. The receiving server *must* support the multi-metric StatsD protocol (namely, parsing several metrics in the same packet, separated by newlines). Both the original StatsD and now Brubeck support this. By sending several metrics in a single packet, we reduce the amount of network operations (both in the client and the server) and the amount of context switches to the kernel UDP stack, significantly increasing the throughput of the server implementation. The default maximum packet size has been set at 512 bytes, which is a "safe size" to reliably transmit an atomic packet through the internet: The MTU on the internet is 576, and the size of the IPv4 header is 20 bytes, and the UDP header 8 bytes. This leaves 548 bytes available for user data. We cap it at 512 to add some headspace (most other UDP-based protocols, such as DNS, do the same thing). --- lib/statsd.rb | 80 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 62 insertions(+), 18 deletions(-) diff --git a/lib/statsd.rb b/lib/statsd.rb index 4454b64..0bd8f6f 100644 --- a/lib/statsd.rb +++ b/lib/statsd.rb @@ -14,6 +14,24 @@ # statsd = Statsd.new('localhost').tap{|sd| sd.namespace = 'account'} # statsd.increment 'activate' class Statsd + class UDPClient + attr_reader :sock + + def initialize(address, port = nil) + address, port = address.split(':') if address.include?(':') + addrinfo = Addrinfo.ip(address) + + @sock = UDPSocket.new(addrinfo.pfamily) + @sock.connect(addrinfo.ip_address, port) + end + + def send(msg) + sock.write(msg) + rescue SystemCallError + nil + end + end + class SecureUDPClient < UDPClient def initialize(address, port, key) super(address, port) @@ -51,24 +69,6 @@ def nonce end end - class UDPClient - attr_reader :sock - - def initialize(address, port = nil) - address, port = address.split(':') if address.include?(':') - addrinfo = Addrinfo.ip(address) - - @sock = UDPSocket.new(addrinfo.pfamily) - @sock.connect(addrinfo.ip_address, port) - end - - def send(msg) - sock.write(msg) - rescue => boom - nil - end - end - # A namespace to prepend to all statsd calls. attr_reader :namespace @@ -103,6 +103,25 @@ def add_shard(*args) self end + def enable_buffering(buffer_size = nil) + return if @buffering + @shards.map! { |client| Buffer.new(client, buffer_size) } + @buffering = true + end + + def disable_buffering + return unless @buffering + flush_all + @shards.map! { |client| client.base_client } + @buffering = false + end + + def flush_all + return unless @buffering + @shards.each { |client| client.flush } + end + + # Sends an increment (count = 1) for the given stat to the statsd server. # # @param stat (see #count) @@ -200,4 +219,29 @@ def select_shard(stat) @shards[Zlib.crc32(stat) % @shards.size] end end + + class Buffer + DEFAULT_BUFFER_CAP = 512 + + attr_reader :base_client + + def initialize(client, buffer_cap = nil) + @base_client = client + @buffer = String.new + @buffer_cap = buffer_cap || DEFAULT_BUFFER_CAP + end + + def flush + return unless @buffer.bytesize > 0 + @base_client.send(@buffer) + @buffer.clear + end + + def send(msg) + flush if @buffer.bytesize + msg.bytesize >= @buffer_cap + @buffer << msg + @buffer << "\n".freeze + nil + end + end end From 6cc02da694c8ca3dc5853e8c35bbe8d1351fcc30 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Wed, 13 Jan 2016 17:01:49 +0100 Subject: [PATCH 17/23] Add flush count statistics --- lib/statsd.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/statsd.rb b/lib/statsd.rb index 0bd8f6f..0881fb2 100644 --- a/lib/statsd.rb +++ b/lib/statsd.rb @@ -224,17 +224,20 @@ class Buffer DEFAULT_BUFFER_CAP = 512 attr_reader :base_client + attr_accessor :flush_count def initialize(client, buffer_cap = nil) @base_client = client @buffer = String.new @buffer_cap = buffer_cap || DEFAULT_BUFFER_CAP + @flush_count = 0 end def flush return unless @buffer.bytesize > 0 @base_client.send(@buffer) @buffer.clear + @flush_count += 1 end def send(msg) From 220ed6290f1a5817ad28828e98bea0446518050a Mon Sep 17 00:00:00 2001 From: John Nunemaker Date: Mon, 1 Aug 2016 10:35:52 -0400 Subject: [PATCH 18/23] Move Statsd into GitHub module To avoid collision with other top level Statsd classes. Also gets the specs passing again and adds a gemfile to make it easier to work on. --- .gitignore | 3 + Gemfile | 2 + README.rdoc | 4 +- Rakefile | 28 ----- lib/github/statsd.rb | 255 +++++++++++++++++++++++++++++++++++++++++++ lib/statsd.rb | 250 ------------------------------------------ spec/helper.rb | 2 +- spec/statsd_spec.rb | 78 +++++-------- statsd-ruby.gemspec | 24 +--- 9 files changed, 293 insertions(+), 353 deletions(-) create mode 100644 Gemfile create mode 100644 lib/github/statsd.rb delete mode 100644 lib/statsd.rb diff --git a/.gitignore b/.gitignore index 2288186..bf9cfef 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,6 @@ pkg # # For vim: #*.swp + +# deps should be good enough in Gemfile and gemspec that this isn't needed +Gemfile.lock diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..3be9c3c --- /dev/null +++ b/Gemfile @@ -0,0 +1,2 @@ +source "https://rubygems.org/" +gemspec diff --git a/README.rdoc b/README.rdoc index 29588e9..4843c55 100644 --- a/README.rdoc +++ b/README.rdoc @@ -5,7 +5,7 @@ A Ruby statsd client (https://github.com/etsy/statsd) = Installing Bundler: - gem "statsd-ruby", :require => "statsd" + gem "statsd-ruby", :require => "github/statsd" = Testing @@ -14,7 +14,7 @@ Run the specs with rake spec Run the specs and include live integration specs with LIVE=true rake spec. Note: This will test over a real UDP socket. == Contributing to statsd - + * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet * Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it * Fork the project diff --git a/Rakefile b/Rakefile index 342e4b4..eb1f8d0 100644 --- a/Rakefile +++ b/Rakefile @@ -1,23 +1,6 @@ require 'rubygems' require 'rake' -require 'jeweler' -Jeweler::Tasks.new do |gem| - # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options - gem.name = "statsd-ruby" - gem.homepage = "http://github.com/reinh/statsd" - gem.license = "MIT" - gem.summary = %Q{A Statsd client in Ruby} - gem.description = %Q{A Statsd client in Ruby} - gem.email = "rein@phpfog.com" - gem.authors = ["Rein Henrichs"] - gem.add_development_dependency "minitest", ">= 0" - gem.add_development_dependency "yard", "~> 0.6.0" - gem.add_development_dependency "jeweler", "~> 1.5.2" - gem.add_development_dependency "rcov", ">= 0" -end -Jeweler::RubygemsDotOrgTasks.new - require 'rake/testtask' Rake::TestTask.new(:spec) do |spec| spec.libs << 'lib' << 'spec' @@ -25,15 +8,4 @@ Rake::TestTask.new(:spec) do |spec| spec.verbose = true end -require 'rcov/rcovtask' -Rcov::RcovTask.new do |spec| - spec.libs << 'lib' << 'spec' - spec.pattern = 'spec/**/*_spec.rb' - spec.verbose = true - spec.rcov_opts << "--exclude spec,gems" -end - task :default => :spec - -require 'yard' -YARD::Rake::YardocTask.new diff --git a/lib/github/statsd.rb b/lib/github/statsd.rb new file mode 100644 index 0000000..56730e8 --- /dev/null +++ b/lib/github/statsd.rb @@ -0,0 +1,255 @@ +require 'socket' +require 'zlib' + +module GitHub + # = Statsd: A Statsd client (https://github.com/etsy/statsd) + # + # @example Set up a global Statsd client for a server on localhost:8125 + # $statsd = Statsd.new 'localhost', 8125 + # @example Send some stats + # $statsd.increment 'garets' + # $statsd.timing 'glork', 320 + # @example Use {#time} to time the execution of a block + # $statsd.time('account.activate') { @account.activate! } + # @example Create a namespaced statsd client and increment 'account.activate' + # statsd = Statsd.new('localhost').tap{|sd| sd.namespace = 'account'} + # statsd.increment 'activate' + class Statsd + class UDPClient + attr_reader :sock + + def initialize(address, port = nil) + address, port = address.split(':') if address.include?(':') + addrinfo = Addrinfo.ip(address) + + @sock = UDPSocket.new(addrinfo.pfamily) + @sock.connect(addrinfo.ip_address, port) + end + + def send(msg) + sock.write(msg) + rescue SystemCallError + nil + end + end + + class SecureUDPClient < UDPClient + def initialize(address, port, key) + super(address, port) + @key = key + end + + def send(msg) + super(signed_payload(msg)) + end + + private + # defer loading openssl and securerandom unless needed. this shaves ~10ms off + # of baseline require load time for environments that don't require message signing. + def self.setup_openssl + @sha256 ||= begin + require 'securerandom' + require 'openssl' + OpenSSL::Digest::SHA256.new + end + end + + def signed_payload(message) + sha256 = SecureUDPClient.setup_openssl + payload = timestamp + nonce + message + signature = OpenSSL::HMAC.digest(sha256, @key, payload) + signature + payload + end + + def timestamp + [Time.now.to_i].pack("Q<") + end + + def nonce + SecureRandom.random_bytes(4) + end + end + + # A namespace to prepend to all statsd calls. + attr_reader :namespace + + def namespace=(namespace) + @namespace = namespace + @prefix = namespace ? "#{@namespace}." : "".freeze + end + + # All the endpoints where StatsD will report metrics + attr_reader :shards + + # The client class used to initialize shard instances and send metrics. + attr_reader :client_class + + #characters that will be replaced with _ in stat names + RESERVED_CHARS_REGEX = /[\:\|\@]/ + + COUNTER_TYPE = "c".freeze + TIMING_TYPE = "ms".freeze + GAUGE_TYPE = "g".freeze + HISTOGRAM_TYPE = "h".freeze + + def initialize(client_class = nil) + @shards = [] + @client_class = client_class || UDPClient + self.namespace = nil + end + + def self.simple(addr, port = nil) + self.new.add_shard(addr, port) + end + + def add_shard(*args) + @shards << @client_class.new(*args) + self + end + + def enable_buffering(buffer_size = nil) + return if @buffering + @shards.map! { |client| Buffer.new(client, buffer_size) } + @buffering = true + end + + def disable_buffering + return unless @buffering + flush_all + @shards.map! { |client| client.base_client } + @buffering = false + end + + def flush_all + return unless @buffering + @shards.each { |client| client.flush } + end + + + # Sends an increment (count = 1) for the given stat to the statsd server. + # + # @param stat (see #count) + # @param sample_rate (see #count) + # @see #count + def increment(stat, sample_rate=1); count stat, 1, sample_rate end + + # Sends a decrement (count = -1) for the given stat to the statsd server. + # + # @param stat (see #count) + # @param sample_rate (see #count) + # @see #count + def decrement(stat, sample_rate=1); count stat, -1, sample_rate end + + # Sends an arbitrary count for the given stat to the statsd server. + # + # @param [String] stat stat name + # @param [Integer] count count + # @param [Integer] sample_rate sample rate, 1 for always + def count(stat, count, sample_rate=1); send stat, count, COUNTER_TYPE, sample_rate end + + # Sends an arbitary gauge value for the given stat to the statsd server. + # + # @param [String] stat stat name. + # @param [Numeric] gauge value. + # @example Report the current user count: + # $statsd.gauge('user.count', User.count) + def gauge(stat, value) + send stat, value, GAUGE_TYPE + end + + # Sends a timing (in ms) for the given stat to the statsd server. The + # sample_rate determines what percentage of the time this report is sent. The + # statsd server then uses the sample_rate to correctly track the average + # timing for the stat. + # + # @param stat stat name + # @param [Integer] ms timing in milliseconds + # @param [Integer] sample_rate sample rate, 1 for always + def timing(stat, ms, sample_rate=1); send stat, ms, TIMING_TYPE, sample_rate end + + # Reports execution time of the provided block using {#timing}. + # + # @param stat (see #timing) + # @param sample_rate (see #timing) + # @yield The operation to be timed + # @see #timing + # @example Report the time (in ms) taken to activate an account + # $statsd.time('account.activate') { @account.activate! } + def time(stat, sample_rate=1) + start = Time.now + result = yield + timing(stat, ((Time.now - start) * 1000).round(5), sample_rate) + result + end + + # Sends a histogram measurement for the given stat to the statsd server. The + # sample_rate determines what percentage of the time this report is sent. The + # statsd server then uses the sample_rate to correctly track the average + # for the stat. + def histogram(stat, value, sample_rate=1); send stat, value, HISTOGRAM_TYPE, sample_rate end + + private + def sampled(sample_rate) + yield unless sample_rate < 1 and rand > sample_rate + end + + def send(stat, delta, type, sample_rate=1) + sampled(sample_rate) do + stat = stat.to_s.dup + stat.gsub!(/::/, ".".freeze) + stat.gsub!(RESERVED_CHARS_REGEX, "_".freeze) + + msg = String.new + msg << @prefix + msg << stat + msg << ":".freeze + msg << delta.to_s + msg << "|".freeze + msg << type + if sample_rate < 1 + msg << "|@".freeze + msg << sample_rate.to_s + end + + shard = select_shard(stat) + shard.send(msg) + end + end + + def select_shard(stat) + if @shards.size == 1 + @shards.first + else + @shards[Zlib.crc32(stat) % @shards.size] + end + end + + class Buffer + DEFAULT_BUFFER_CAP = 512 + + attr_reader :base_client + attr_accessor :flush_count + + def initialize(client, buffer_cap = nil) + @base_client = client + @buffer = String.new + @buffer_cap = buffer_cap || DEFAULT_BUFFER_CAP + @flush_count = 0 + end + + def flush + return unless @buffer.bytesize > 0 + @base_client.send(@buffer) + @buffer.clear + @flush_count += 1 + end + + def send(msg) + flush if @buffer.bytesize + msg.bytesize >= @buffer_cap + @buffer << msg + @buffer << "\n".freeze + nil + end + end + end +end diff --git a/lib/statsd.rb b/lib/statsd.rb deleted file mode 100644 index 0881fb2..0000000 --- a/lib/statsd.rb +++ /dev/null @@ -1,250 +0,0 @@ -require 'socket' -require 'zlib' - -# = Statsd: A Statsd client (https://github.com/etsy/statsd) -# -# @example Set up a global Statsd client for a server on localhost:8125 -# $statsd = Statsd.new 'localhost', 8125 -# @example Send some stats -# $statsd.increment 'garets' -# $statsd.timing 'glork', 320 -# @example Use {#time} to time the execution of a block -# $statsd.time('account.activate') { @account.activate! } -# @example Create a namespaced statsd client and increment 'account.activate' -# statsd = Statsd.new('localhost').tap{|sd| sd.namespace = 'account'} -# statsd.increment 'activate' -class Statsd - class UDPClient - attr_reader :sock - - def initialize(address, port = nil) - address, port = address.split(':') if address.include?(':') - addrinfo = Addrinfo.ip(address) - - @sock = UDPSocket.new(addrinfo.pfamily) - @sock.connect(addrinfo.ip_address, port) - end - - def send(msg) - sock.write(msg) - rescue SystemCallError - nil - end - end - - class SecureUDPClient < UDPClient - def initialize(address, port, key) - super(address, port) - @key = key - end - - def send(msg) - super(signed_payload(msg)) - end - - private - # defer loading openssl and securerandom unless needed. this shaves ~10ms off - # of baseline require load time for environments that don't require message signing. - def self.setup_openssl - @sha256 ||= begin - require 'securerandom' - require 'openssl' - OpenSSL::Digest::SHA256.new - end - end - - def signed_payload(message) - sha256 = SecureUDPClient.setup_openssl - payload = timestamp + nonce + message - signature = OpenSSL::HMAC.digest(sha256, @key, payload) - signature + payload - end - - def timestamp - [Time.now.to_i].pack("Q<") - end - - def nonce - SecureRandom.random_bytes(4) - end - end - - # A namespace to prepend to all statsd calls. - attr_reader :namespace - - def namespace=(namespace) - @namespace = namespace - @prefix = namespace ? "#{@namespace}." : "".freeze - end - - # All the endpoints where StatsD will report metrics - attr_reader :shards - - #characters that will be replaced with _ in stat names - RESERVED_CHARS_REGEX = /[\:\|\@]/ - - COUNTER_TYPE = "c".freeze - TIMING_TYPE = "ms".freeze - GAUGE_TYPE = "g".freeze - HISTOGRAM_TYPE = "h".freeze - - def initialize(client_class = nil) - @shards = [] - @client_class = client_class || UDPClient - self.namespace = nil - end - - def self.simple(addr, port = nil) - self.new.add_shard(addr, port) - end - - def add_shard(*args) - @shards << @client_class.new(*args) - self - end - - def enable_buffering(buffer_size = nil) - return if @buffering - @shards.map! { |client| Buffer.new(client, buffer_size) } - @buffering = true - end - - def disable_buffering - return unless @buffering - flush_all - @shards.map! { |client| client.base_client } - @buffering = false - end - - def flush_all - return unless @buffering - @shards.each { |client| client.flush } - end - - - # Sends an increment (count = 1) for the given stat to the statsd server. - # - # @param stat (see #count) - # @param sample_rate (see #count) - # @see #count - def increment(stat, sample_rate=1); count stat, 1, sample_rate end - - # Sends a decrement (count = -1) for the given stat to the statsd server. - # - # @param stat (see #count) - # @param sample_rate (see #count) - # @see #count - def decrement(stat, sample_rate=1); count stat, -1, sample_rate end - - # Sends an arbitrary count for the given stat to the statsd server. - # - # @param [String] stat stat name - # @param [Integer] count count - # @param [Integer] sample_rate sample rate, 1 for always - def count(stat, count, sample_rate=1); send stat, count, COUNTER_TYPE, sample_rate end - - # Sends an arbitary gauge value for the given stat to the statsd server. - # - # @param [String] stat stat name. - # @param [Numeric] gauge value. - # @example Report the current user count: - # $statsd.gauge('user.count', User.count) - def gauge(stat, value) - send stat, value, GAUGE_TYPE - end - - # Sends a timing (in ms) for the given stat to the statsd server. The - # sample_rate determines what percentage of the time this report is sent. The - # statsd server then uses the sample_rate to correctly track the average - # timing for the stat. - # - # @param stat stat name - # @param [Integer] ms timing in milliseconds - # @param [Integer] sample_rate sample rate, 1 for always - def timing(stat, ms, sample_rate=1); send stat, ms, TIMING_TYPE, sample_rate end - - # Reports execution time of the provided block using {#timing}. - # - # @param stat (see #timing) - # @param sample_rate (see #timing) - # @yield The operation to be timed - # @see #timing - # @example Report the time (in ms) taken to activate an account - # $statsd.time('account.activate') { @account.activate! } - def time(stat, sample_rate=1) - start = Time.now - result = yield - timing(stat, ((Time.now - start) * 1000).round(5), sample_rate) - result - end - - # Sends a histogram measurement for the given stat to the statsd server. The - # sample_rate determines what percentage of the time this report is sent. The - # statsd server then uses the sample_rate to correctly track the average - # for the stat. - def histogram(stat, value, sample_rate=1); send stat, value, HISTOGRAM_TYPE, sample_rate end - - private - def sampled(sample_rate) - yield unless sample_rate < 1 and rand > sample_rate - end - - def send(stat, delta, type, sample_rate=1) - sampled(sample_rate) do - stat = stat.to_s.dup - stat.gsub!(/::/, ".".freeze) - stat.gsub!(RESERVED_CHARS_REGEX, "_".freeze) - - msg = String.new - msg << @prefix - msg << stat - msg << ":".freeze - msg << delta.to_s - msg << "|".freeze - msg << type - if sample_rate < 1 - msg << "|@".freeze - msg << sample_rate.to_s - end - - shard = select_shard(stat) - shard.send(msg) - end - end - - def select_shard(stat) - if @shards.size == 1 - @shards.first - else - @shards[Zlib.crc32(stat) % @shards.size] - end - end - - class Buffer - DEFAULT_BUFFER_CAP = 512 - - attr_reader :base_client - attr_accessor :flush_count - - def initialize(client, buffer_cap = nil) - @base_client = client - @buffer = String.new - @buffer_cap = buffer_cap || DEFAULT_BUFFER_CAP - @flush_count = 0 - end - - def flush - return unless @buffer.bytesize > 0 - @base_client.send(@buffer) - @buffer.clear - @flush_count += 1 - end - - def send(msg) - flush if @buffer.bytesize + msg.bytesize >= @buffer_cap - @buffer << msg - @buffer << "\n".freeze - nil - end - end -end diff --git a/spec/helper.rb b/spec/helper.rb index caffc48..5e5c524 100644 --- a/spec/helper.rb +++ b/spec/helper.rb @@ -3,7 +3,7 @@ $LOAD_PATH.unshift(File.dirname(__FILE__)) $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib')) -require 'statsd' +require 'github/statsd' require 'logger' class FakeUDPSocket diff --git a/spec/statsd_spec.rb b/spec/statsd_spec.rb index ab6c582..2a63ac3 100644 --- a/spec/statsd_spec.rb +++ b/spec/statsd_spec.rb @@ -1,39 +1,39 @@ -require 'helper' +require_relative './helper' -describe Statsd do +describe GitHub::Statsd do before do - @statsd = Statsd.new('localhost', 1234) + @statsd = GitHub::Statsd.new(FakeUDPSocket) + @statsd.add_shard class << @statsd public :sampled # we need to test this - attr_reader :host, :port # we also need to test this - def socket; @socket ||= FakeUDPSocket.new end end end - after { @statsd.socket.clear } + after { @statsd.shards.first.clear } describe "#initialize" do - it "should set the host and port" do - @statsd.host.must_equal 'localhost' - @statsd.port.must_equal 1234 + it "should default client class to UDPClient" do + statsd = GitHub::Statsd.new + statsd.client_class.must_equal GitHub::Statsd::UDPClient end - it "should default the port to 8125" do - Statsd.new('localhost').instance_variable_get('@port').must_equal 8125 + it "should allow changing client class" do + statsd = GitHub::Statsd.new(FakeUDPSocket) + statsd.client_class.must_equal FakeUDPSocket end end describe "#increment" do it "should format the message according to the statsd spec" do @statsd.increment('foobar') - @statsd.socket.recv.must_equal ['foobar:1|c'] + @statsd.shards.first.recv.must_equal ['foobar:1|c'] end describe "with a sample rate" do before { class << @statsd; def rand; 0; end; end } # ensure delivery it "should format the message according to the statsd spec" do @statsd.increment('foobar', 0.5) - @statsd.socket.recv.must_equal ['foobar:1|c|@0.5'] + @statsd.shards.first.recv.must_equal ['foobar:1|c|@0.5'] end end end @@ -41,14 +41,14 @@ def socket; @socket ||= FakeUDPSocket.new end describe "#decrement" do it "should format the message according to the statsd spec" do @statsd.decrement('foobar') - @statsd.socket.recv.must_equal ['foobar:-1|c'] + @statsd.shards.first.recv.must_equal ['foobar:-1|c'] end describe "with a sample rate" do before { class << @statsd; def rand; 0; end; end } # ensure delivery it "should format the message according to the statsd spec" do @statsd.decrement('foobar', 0.5) - @statsd.socket.recv.must_equal ['foobar:-1|c|@0.5'] + @statsd.shards.first.recv.must_equal ['foobar:-1|c|@0.5'] end end end @@ -56,14 +56,14 @@ def socket; @socket ||= FakeUDPSocket.new end describe "#timing" do it "should format the message according to the statsd spec" do @statsd.timing('foobar', 500) - @statsd.socket.recv.must_equal ['foobar:500|ms'] + @statsd.shards.first.recv.must_equal ['foobar:500|ms'] end describe "with a sample rate" do before { class << @statsd; def rand; 0; end; end } # ensure delivery it "should format the message according to the statsd spec" do @statsd.timing('foobar', 500, 0.5) - @statsd.socket.recv.must_equal ['foobar:500|ms|@0.5'] + @statsd.shards.first.recv.must_equal ['foobar:500|ms|@0.5'] end end end @@ -71,7 +71,7 @@ def socket; @socket ||= FakeUDPSocket.new end describe "#time" do it "should format the message according to the statsd spec" do @statsd.time('foobar') { sleep(0.001); 'test' } - data = @statsd.socket.recv + data = @statsd.shards.first.recv key, value, unit = data.first.split(/[:|]/) key.must_equal "foobar" value.must_match /^\d\.\d{3}$/ @@ -88,7 +88,7 @@ def socket; @socket ||= FakeUDPSocket.new end it "should format the message according to the statsd spec" do result = @statsd.time('foobar', 0.5) { sleep(0.001); 'test' } - data = @statsd.socket.recv + data = @statsd.shards.first.recv key, value, unit, frequency = data.first.split(/[:|]/) key.must_equal "foobar" value.must_match /^\d\.\d{3}$/ @@ -132,42 +132,20 @@ def socket; @socket ||= FakeUDPSocket.new end it "should add namespace to increment" do @statsd.increment('foobar') - @statsd.socket.recv.must_equal ['service.foobar:1|c'] + @statsd.shards.first.recv.must_equal ['service.foobar:1|c'] end it "should add namespace to decrement" do @statsd.decrement('foobar') - @statsd.socket.recv.must_equal ['service.foobar:-1|c'] + @statsd.shards.first.recv.must_equal ['service.foobar:-1|c'] end it "should add namespace to timing" do @statsd.timing('foobar', 500) - @statsd.socket.recv.must_equal ['service.foobar:500|ms'] + @statsd.shards.first.recv.must_equal ['service.foobar:500|ms'] end end - describe "with logging" do - require 'stringio' - before { Statsd.logger = Logger.new(@log = StringIO.new)} - - it "should write to the log in debug" do - Statsd.logger.level = Logger::DEBUG - - @statsd.increment('foobar') - - @log.string.must_match "Statsd: foobar:1|c" - end - - it "should not write to the log unless debug" do - Statsd.logger.level = Logger::INFO - - @statsd.increment('foobar') - - @log.string.must_be_empty - end - - end - describe "stat names" do it "should accept anything as stat" do @@ -175,29 +153,29 @@ def socket; @socket ||= FakeUDPSocket.new end end it "should replace ruby constant delimeter with graphite package name" do - class Statsd::SomeClass; end - @statsd.increment(Statsd::SomeClass, 1) + class GitHub::Statsd::SomeClass; end + @statsd.increment(GitHub::Statsd::SomeClass, 1) - @statsd.socket.recv.must_equal ['Statsd.SomeClass:1|c'] + @statsd.shards.first.recv.must_equal ['GitHub.Statsd.SomeClass:1|c'] end it "should replace statsd reserved chars in the stat name" do @statsd.increment('ray@hostname.blah|blah.blah:blah', 1) - @statsd.socket.recv.must_equal ['ray_hostname.blah_blah.blah_blah:1|c'] + @statsd.shards.first.recv.must_equal ['ray_hostname.blah_blah.blah_blah:1|c'] end end end -describe Statsd do +describe GitHub::Statsd do describe "with a real UDP socket" do it "should actually send stuff over the socket" do socket = UDPSocket.new host, port = 'localhost', 12345 socket.bind(host, port) - statsd = Statsd.new(host, port) + statsd = GitHub::Statsd.new(host, port) statsd.increment('foobar') message = socket.recvfrom(16).first message.must_equal 'foobar:1|c' diff --git a/statsd-ruby.gemspec b/statsd-ruby.gemspec index faf3b7e..03fa904 100644 --- a/statsd-ruby.gemspec +++ b/statsd-ruby.gemspec @@ -32,26 +32,6 @@ Gem::Specification.new do |s| s.require_paths = ["lib"] s.rubygems_version = %q{1.3.9.1} s.summary = %q{A Statsd client in Ruby} - - if s.respond_to? :specification_version then - s.specification_version = 3 - - if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then - s.add_development_dependency(%q, [">= 0"]) - s.add_development_dependency(%q, ["~> 0.6.0"]) - s.add_development_dependency(%q, ["~> 1.5.2"]) - s.add_development_dependency(%q, [">= 0"]) - else - s.add_dependency(%q, [">= 0"]) - s.add_dependency(%q, ["~> 0.6.0"]) - s.add_dependency(%q, ["~> 1.5.2"]) - s.add_dependency(%q, [">= 0"]) - end - else - s.add_dependency(%q, [">= 0"]) - s.add_dependency(%q, ["~> 0.6.0"]) - s.add_dependency(%q, ["~> 1.5.2"]) - s.add_dependency(%q, [">= 0"]) - end + s.add_development_dependency "rake", "~> 11.2" + s.add_development_dependency "minitest", "~> 5.9" end - From c56f990b467cbbb34f55a78de6dabfdb1bbfee63 Mon Sep 17 00:00:00 2001 From: John Nunemaker Date: Mon, 1 Aug 2016 10:48:25 -0400 Subject: [PATCH 19/23] Just check for decimals instead of exactly 3 When checking 3 explicitly we sometimes get failures because 1.1.to_s is "1.1" instead of "1.10". According to vmg, one or more should be fine and we want more than 2 points of precision, so it is better to change the test than the implementation. --- spec/statsd_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/statsd_spec.rb b/spec/statsd_spec.rb index 2a63ac3..14d0b21 100644 --- a/spec/statsd_spec.rb +++ b/spec/statsd_spec.rb @@ -74,7 +74,7 @@ class << @statsd data = @statsd.shards.first.recv key, value, unit = data.first.split(/[:|]/) key.must_equal "foobar" - value.must_match /^\d\.\d{3}$/ + value.must_match /^\d\.\d+$/ unit.must_equal "ms" end @@ -91,7 +91,7 @@ class << @statsd data = @statsd.shards.first.recv key, value, unit, frequency = data.first.split(/[:|]/) key.must_equal "foobar" - value.must_match /^\d\.\d{3}$/ + value.must_match /^\d\.\d+$/ unit.must_equal "ms" frequency.must_equal "@0.5" end From 341c6cda9ae56ec3bdce98dd41b5e5879f8c6bd2 Mon Sep 17 00:00:00 2001 From: John Nunemaker Date: Mon, 1 Aug 2016 13:47:54 -0400 Subject: [PATCH 20/23] Correct file path for statsd.rb in gemspec --- statsd-ruby.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statsd-ruby.gemspec b/statsd-ruby.gemspec index 03fa904..6b9a7de 100644 --- a/statsd-ruby.gemspec +++ b/statsd-ruby.gemspec @@ -22,7 +22,7 @@ Gem::Specification.new do |s| "README.rdoc", "Rakefile", "VERSION", - "lib/statsd.rb", + "lib/github/statsd.rb", "spec/helper.rb", "spec/statsd_spec.rb", "statsd-ruby.gemspec" From 6d391388948d84cd393716e919895bbb66025e2d Mon Sep 17 00:00:00 2001 From: John Nunemaker Date: Tue, 2 Aug 2016 10:12:52 -0400 Subject: [PATCH 21/23] Stop ignoring Gemfile.lock https://github.com/github/statsd-ruby/pull/21#discussion_r73161227 --- .gitignore | 2 -- Gemfile.lock | 21 +++++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 Gemfile.lock diff --git a/.gitignore b/.gitignore index bf9cfef..b862e19 100644 --- a/.gitignore +++ b/.gitignore @@ -41,5 +41,3 @@ pkg # For vim: #*.swp -# deps should be good enough in Gemfile and gemspec that this isn't needed -Gemfile.lock diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..32e1ff6 --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,21 @@ +PATH + remote: . + specs: + statsd-ruby (0.4.0.github) + +GEM + remote: https://rubygems.org/ + specs: + minitest (5.9.0) + rake (11.2.2) + +PLATFORMS + ruby + +DEPENDENCIES + minitest (~> 5.9) + rake (~> 11.2) + statsd-ruby! + +BUNDLED WITH + 1.11.2 From 82b310597e8cca2db047c2213472ed02f2ee2c64 Mon Sep 17 00:00:00 2001 From: John Nunemaker Date: Fri, 5 Aug 2016 09:47:23 -0400 Subject: [PATCH 22/23] Add stastd file for backwards compatibility temporarily --- lib/statsd.rb | 1 + 1 file changed, 1 insertion(+) create mode 100644 lib/statsd.rb diff --git a/lib/statsd.rb b/lib/statsd.rb new file mode 100644 index 0000000..d9d52d4 --- /dev/null +++ b/lib/statsd.rb @@ -0,0 +1 @@ +require "github/statsd" From d478cc7ba92261c19e8968daa44b02f1da4b4a0e Mon Sep 17 00:00:00 2001 From: John Nunemaker Date: Mon, 8 Aug 2016 09:59:36 -0400 Subject: [PATCH 23/23] Remove temporary file No longer needed. --- lib/statsd.rb | 1 - 1 file changed, 1 deletion(-) delete mode 100644 lib/statsd.rb diff --git a/lib/statsd.rb b/lib/statsd.rb deleted file mode 100644 index d9d52d4..0000000 --- a/lib/statsd.rb +++ /dev/null @@ -1 +0,0 @@ -require "github/statsd"