diff --git a/backends/graphite.js b/backends/graphite.js index 998e3824..bb6737fb 100644 --- a/backends/graphite.js +++ b/backends/graphite.js @@ -24,6 +24,7 @@ var debug; var flushInterval; var graphiteHost; var graphitePort; +var flush_counts; // prefix configuration var globalPrefix; @@ -104,10 +105,14 @@ var flush_stats = function graphite_flush(ts, metrics) { if (legacyNamespace === true) { statString += namespace.join(".") + globalSuffix + valuePerSecond + ts_suffix; - statString += 'stats_counts.' + key + globalSuffix + value + ts_suffix; + if (flush_counts) { + statString += 'stats_counts.' + key + globalSuffix + value + ts_suffix; + } } else { statString += namespace.concat('rate').join(".") + globalSuffix + valuePerSecond + ts_suffix; - statString += namespace.concat('count').join(".") + globalSuffix + value + ts_suffix; + if (flush_counts) { + statString += namespace.concat('count').join(".") + globalSuffix + value + ts_suffix; + } } numStats += 1; @@ -235,6 +240,8 @@ exports.init = function graphite_init(startup_time, config, events) { flushInterval = config.flushInterval; + flush_counts = typeof(config.flush_counts) === "undefined" ? true : config.flush_counts; + events.on('flush', flush_stats); events.on('status', backend_status); diff --git a/debian/changelog b/debian/changelog index 1c49d2ae..a1340bd1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,7 +1,20 @@ +statsd (0.6.0-1) unstable; urgency=low + + * Non-maintainer upload. + * Fix lintian error messages regarding init script removal but not + installing them, using adduser but not defining the dependencies, + using /var/run/ in debian/dirs even though it can be a tmpfs, init script + not depending on $remote_fs, spelling mistakes + * removing depedency on ${misc:Depends} as this got only added as lintian + used to complain about not having it, it's not anymore + * Align version number to git tag versions + + -- Frederic Jaeckel Tue, 20 Aug 2013 14:03:10 +0200 + statsd (0.0.6-1) unstable; urgency=low * Update packaging for 0.0.6 - * Bump nodejs dependancy to 0.6 per pcakage.json + * Bump nodejs dependency to 0.6 per package.json -- Kiall Mac Innes Thu, 27 Jun 2013 19:17:00 +0100 diff --git a/debian/control b/debian/control index a1731e39..53dff2d2 100644 --- a/debian/control +++ b/debian/control @@ -7,7 +7,7 @@ Build-Depends: debhelper (>= 8.0.0) Package: statsd Architecture: all -Depends: ${misc:Depends}, nodejs (>= 0.6) +Depends: nodejs (>= 0.6), adduser Description: Stats aggregation daemon A network daemon for aggregating statistics (counters and timers), rolling them up, then sending them to graphite. diff --git a/debian/dirs b/debian/dirs index dc8b5569..4d2c5fca 100644 --- a/debian/dirs +++ b/debian/dirs @@ -1,2 +1 @@ var/log/statsd -var/run/statsd diff --git a/debian/postinst b/debian/postinst index 797e7755..6016c745 100755 --- a/debian/postinst +++ b/debian/postinst @@ -3,6 +3,11 @@ set -e if [ "$1" = configure ]; then + # Automatically added by dh_installinit + if [ -x "/etc/init.d/statsd" ]; then + update-rc.d statsd defaults >/dev/null + invoke-rc.d statsd start || exit $? + fi if ! getent passwd _statsd > /dev/null; then adduser --system --quiet --home /nonexistent --no-create-home \ diff --git a/debian/statsd.init b/debian/statsd.init index d9348541..09007f58 100644 --- a/debian/statsd.init +++ b/debian/statsd.init @@ -1,8 +1,8 @@ #! /bin/sh ### BEGIN INIT INFO # Provides: statsd -# Required-Start: $network $local_fs -# Required-Stop: +# Required-Start: $remote_fs $network $local_fs +# Required-Stop: $remote_fs # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 ### END INIT INFO @@ -41,6 +41,13 @@ CHDIR="/usr/share/statsd" # Depend on lsb-base (>= 3.0-6) to ensure that this file is present. . /lib/lsb/init-functions +# Create PIDDIR on runtime +if [ ! -d /var/run/$NAME ]; +then + mkdir /var/run/$NAME + chown $USER /var/run/$NAME +fi + # # Function that starts the daemon/service # diff --git a/docs/backend.md b/docs/backend.md index 32839726..8a9a5ed4 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -42,6 +42,7 @@ queues and third-party services. - [monitis backend](https://github.com/jeremiahshirk/statsd-monitis-backend) - [opentsdb backend](https://github.com/emurphy/statsd-opentsdb-backend) - [socket.io-backend](https://github.com/Chatham/statsd-socket.io) +- [stackdriver backend](https://github.com/Stackdriver/stackdriver-statsd-backend) - [statsd-backend](https://github.com/dynmeth/statsd-backend) - [statsd http backend](https://github.com/bmhatfield/statsd-http-backend) - [statsd aggregation backend](https://github.com/wanelo/gossip_girl) diff --git a/docs/graphite.md b/docs/graphite.md index 568438f6..fa6aeb4b 100644 --- a/docs/graphite.md +++ b/docs/graphite.md @@ -89,7 +89,7 @@ This means: * For all other databases, average the values (mean) when rolling up data, and store a None if less than 30% of the datapoints were received -Pay close attention to xFilesFactor: if your flush interval is long enough so there are not enough samples to satisfy this minimum factor, your data would simply be lost in the first downsampling cycle. However, setting a very low factor would also produce a misleading result, since you would probably agree that if you only have a single 10-second mean value sample reported in a 10-minute timeframe, this single sample alone should not normally be downsampled into a 10-minute mean value. For counts, however, every count should count ;-), hence the zero factor. +Pay close attention to xFilesFactor: if your flush interval is not long enough so there are not enough samples to satisfy this minimum factor, your data would simply be lost in the first downsampling cycle. However, setting a very low factor would also produce a misleading result, since you would probably agree that if you only have a single 10-second mean value sample reported in a 10-minute timeframe, this single sample alone should not normally be downsampled into a 10-minute mean value. For counts, however, every count should count ;-), hence the zero factor. **Notes:** diff --git a/exampleConfig.js b/exampleConfig.js index 6c257e53..a5406c38 100644 --- a/exampleConfig.js +++ b/exampleConfig.js @@ -25,6 +25,11 @@ Optional Variables: mgmt_address: address to run the management TCP interface on [default: 0.0.0.0] mgmt_port: port to run the management TCP interface on [default: 8126] + title : Allows for overriding the process title. [default: statsd] + if set to false, will not override the process title and let the OS set it. + The length of the title has to be less than or equal to the binary name + cli arguments + NOTE: This does not work on Mac's with node versions prior to v0.10 + healthStatus: default health status to be returned and statsd process starts ['up' or 'down', default: 'up'] dumpMessages: log all incoming messages flushInterval: interval (in ms) to flush to Graphite @@ -32,6 +37,8 @@ Optional Variables: (can be a single value or list of floating-point values) negative values mean to use "top" Nth percentile(s) values [%, default: 90] + flush_counts: send stats_counts metrics [default: true] + keyFlush: log the most frequently sent keys [object, default: undefined] interval: how often to log frequent keys [ms, default: 0] percent: percentage of frequent keys to log [%, default: 100] diff --git a/examples/statsd.go b/examples/statsd.go index 243e8074..f419e0d2 100644 --- a/examples/statsd.go +++ b/examples/statsd.go @@ -154,10 +154,11 @@ func (client *StatsdClient) UpdateStats(stats []string, delta int, sampleRate fl func (client *StatsdClient) Send(data map[string]string, sampleRate float32) { sampledData := make(map[string]string) if sampleRate < 1 { - r := rand.New(rand.sampleRateNewSource(time.Now().Unix())) - if rNum := r.Float32(); rNum <= sampleRateate { + r := rand.New(rand.NewSource(time.Now().Unix())) + rNum := r.Float32(); + if rNum <= sampleRate { for stat,value := range data { - sampledUpdateString := fmt.forSprintf("%s|@%f", value, sampleRate) + sampledUpdateString := fmt.Sprintf("%s|@%f", value, sampleRate) sampledData[stat] = sampledUpdateString } } diff --git a/lib/helpers.js b/lib/helpers.js new file mode 100644 index 00000000..9e7ab956 --- /dev/null +++ b/lib/helpers.js @@ -0,0 +1,38 @@ +/** + * Public: test function to filter out malformed packets + * + * Parameters: + * + * fields - Array of packet data (e.g. [ '100', 'ms', '@0.1' ]) + * + * Returns true for a valid packet and false otherwise + */ +function is_valid_packet(fields) { + + // test for existing metrics type + if (fields[1] === undefined) { + return false; + } + // filter out invalid metrics values + else if (fields[1] == 'g') { + if (!fields[0].match(/^([\-\+\d\.]+$)/)) { + return false; + } else { + return true; + } + } + else if (!fields[0].match(/^([\d\.]+$)/)) { + return false; + } + // filter out malformed sample rates + else if (fields[2] && !fields[2].match(/^@([\d\.]+$)/)) { + return false; + } + // looks like we're good + else { + return true; + } + +}; + +exports.is_valid_packet = is_valid_packet; diff --git a/lib/process_mgmt.js b/lib/process_mgmt.js new file mode 100644 index 00000000..26b634e0 --- /dev/null +++ b/lib/process_mgmt.js @@ -0,0 +1,27 @@ +var util = require('util'); + +var conf; + +exports.init = function(config) { + conf = config; + exports.set_title(config); + + process.on('SIGTERM', function() { + if (conf.debug) { + util.log('Starting Final Flush'); + } + healthStatus = 'down'; + process.exit(); + }); + +} + +exports.set_title = function(config) { + if (config.title !== undefined) { + if (config.title) { + process.title = config.title; + } + } else { + process.title = 'statsd'; + } +} diff --git a/stats.js b/stats.js index b2e2cca0..628a3ce7 100644 --- a/stats.js +++ b/stats.js @@ -4,11 +4,13 @@ var dgram = require('dgram') , util = require('util') , net = require('net') , config = require('./lib/config') + , helpers = require('./lib/helpers') , fs = require('fs') , events = require('events') , logger = require('./lib/logger') , set = require('./lib/set') , pm = require('./lib/process_metrics') + , process_mgmt = require('./lib/process_mgmt') , mgmt = require('./lib/mgmt_console'); @@ -42,6 +44,7 @@ function loadBackend(config, name) { } } + // global for conf var conf; @@ -137,6 +140,9 @@ var l; config.configFile(process.argv[2], function (config, oldConfig) { conf = config; + + process_mgmt.init(config); + l = new logger.Logger(config.log || {}); // setup config for stats prefix @@ -193,22 +199,16 @@ config.configFile(process.argv[2], function (config, oldConfig) { for (var i = 0; i < bits.length; i++) { var sampleRate = 1; var fields = bits[i].split("|"); - if (fields[2]) { - if (fields[2].match(/^@([\d\.]+)/)) { - sampleRate = Number(fields[2].match(/^@([\d\.]+)/)[1]); - } else { - l.log('Bad line: ' + fields + ' in msg "' + metrics[midx] +'"; has invalid sample rate'); - counters[bad_lines_seen]++; - stats.messages.bad_lines_seen++; - continue; - } - } - if (fields[1] === undefined) { + if (!helpers.is_valid_packet(fields)) { l.log('Bad line: ' + fields + ' in msg "' + metrics[midx] +'"'); counters[bad_lines_seen]++; stats.messages.bad_lines_seen++; continue; } + if (fields[2]) { + sampleRate = Number(fields[2].match(/^@([\d\.]+)/)[1]); + } + var metric_type = fields[1].trim(); if (metric_type === "ms") { if (! timers[key]) { @@ -302,7 +302,7 @@ config.configFile(process.argv[2], function (config, oldConfig) { backendEvents.emit('status', function(err, name, stat, val) { if (err) { l.log("Failed to read stats for backend " + - name + ": " + err); + name + ": " + err); } else { stat_writer(name, stat, val); } @@ -410,16 +410,6 @@ config.configFile(process.argv[2], function (config, oldConfig) { } }); -process.title = 'statsd'; - -process.on('SIGTERM', function() { - if (conf.debug) { - util.log('Starting Final Flush'); - } - healthStatus = 'down'; - process.exit(); -}); - process.on('exit', function () { flushMetrics(); }); diff --git a/test/helpers_tests.js b/test/helpers_tests.js new file mode 100644 index 00000000..b2f2c524 --- /dev/null +++ b/test/helpers_tests.js @@ -0,0 +1,59 @@ +var helpers = require('../lib/helpers'); + +module.exports = { + + no_metrics_field: function (test) { + var res = helpers.is_valid_packet(['foo', undefined]); + test.equals(res, false); + test.done(); + }, + + wrong_formated_metric_value: function (test) { + var res = helpers.is_valid_packet(['0,345345', 'ms']); + test.equals(res, false); + test.done(); + }, + + wrong_formated_sampling_value: function (test) { + var res = helpers.is_valid_packet(['345345', 'ms', '0,456456']); + test.equals(res, false); + test.done(); + }, + + counter_deltas_positive_are_not_valid: function (test) { + var res = helpers.is_valid_packet(['+10', 'c']); + test.equals(res, false); + test.done(); + }, + + counter_deltas_negative_are_not_valid: function (test) { + var res = helpers.is_valid_packet(['-10', 'c']); + test.equals(res, false); + test.done(); + }, + + gauges_delta_positive_are_valid: function (test) { + var res = helpers.is_valid_packet(['+10', 'g']); + test.equals(res, true); + test.done(); + }, + + gauges_delta_negative_are_valid: function (test) { + var res = helpers.is_valid_packet(['-10', 'g']); + test.equals(res, true); + test.done(); + }, + + correct_packet: function (test) { + var res = helpers.is_valid_packet(['345345', 'ms', '@1.0']); + test.equals(res, true); + test.done(); + }, + + correct_packet_with_small_sampling: function (test) { + var res = helpers.is_valid_packet(['100', 'ms', '@0.1']); + test.equals(res, true); + test.done(); + } + +}; diff --git a/test/process_mgmt_tests.js b/test/process_mgmt_tests.js new file mode 100644 index 00000000..6e7ab391 --- /dev/null +++ b/test/process_mgmt_tests.js @@ -0,0 +1,51 @@ +var process_mgmt = require('../lib/process_mgmt') + , os = require('os'); + +var config = {} + , can_set_title = true; + +module.exports = { + + setUp: function(callback) { + config = {}; + version_number = process.version.split(".")[1]; + platform = os.platform(); + can_set_title = (version_number >= 10 || platform != 'darwin'); + callback(); + }, + + test_setting_title: function(test){ + if (can_set_title) { + test.expect(1); + process_title = process.title; + config.title = "test-statsd"; + process_mgmt.set_title(config); + test.ok(process.title == config.title, "Can set a title that is less than or equal to the process title length"); + } else { + console.log("Not running this test, due to this being a node version before v0.10 and a Darwin os"); + } + test.done(); + }, + + test_no_title: function(test){ + test.expect(1); + process_title = process.title; + config.title = false; + process_mgmt.set_title(config); + test.ok(process_title == process.title, "A config.title of false should not override the default node process.title"); + test.done(); + }, + + test_default_title: function(test){ + if (can_set_title) { + test.expect(1); + default_title = 'statsd'; + process_mgmt.set_title(config); + test.ok(process.title == default_title, "If no config.title option set, set the process.title to statsd"); + } else { + console.log("Not running this test, due to this being a node version before v0.10 and a Darwin os"); + } + test.done(); + } + +};