* merge conflict

backends/graphite.js

@@ -24,6 +24,7 @@ var debug;
 var flushInterval;
 var graphiteHost;
 var graphitePort;
+var flush_counts;
 
 // prefix configuration
 var globalPrefix;
@@ -104,10 +105,14 @@ var flush_stats = function graphite_flush(ts, metrics) {
 
     if (legacyNamespace === true) {
       statString += namespace.join(".")   + globalSuffix + valuePerSecond + ts_suffix;
-      statString += 'stats_counts.' + key + globalSuffix + value          + ts_suffix;
+      if (flush_counts) {
+        statString += 'stats_counts.' + key + globalSuffix + value          + ts_suffix;
+      }
     } else {
       statString += namespace.concat('rate').join(".")  + globalSuffix + valuePerSecond + ts_suffix;
-      statString += namespace.concat('count').join(".") + globalSuffix + value          + ts_suffix;
+      if (flush_counts) {
+        statString += namespace.concat('count').join(".") + globalSuffix + value          + ts_suffix;
+      }
     }
 
     numStats += 1;
@@ -235,6 +240,8 @@ exports.init = function graphite_init(startup_time, config, events) {
 
   flushInterval = config.flushInterval;
 
+  flush_counts = typeof(config.flush_counts) === "undefined" ? true : config.flush_counts;
+
   events.on('flush', flush_stats);
   events.on('status', backend_status);
 

debian/changelog

@@ -1,7 +1,20 @@
+statsd (0.6.0-1) unstable; urgency=low
+
+  * Non-maintainer upload.
+  * Fix lintian error messages regarding init script removal but not
+    installing them, using adduser but not defining the dependencies,
+    using /var/run/ in debian/dirs even though it can be a tmpfs, init script
+    not depending on $remote_fs, spelling mistakes
+  * removing depedency on ${misc:Depends} as this got only added as lintian
+    used to complain about not having it, it's not anymore
+  * Align version number to git tag versions
+
+ -- Frederic Jaeckel <[email protected]>  Tue, 20 Aug 2013 14:03:10 +0200
+
 statsd (0.0.6-1) unstable; urgency=low
 
   * Update packaging for 0.0.6
-  * Bump nodejs dependancy to 0.6 per pcakage.json
+  * Bump nodejs dependency to 0.6 per package.json
 
  -- Kiall Mac Innes <[email protected]>  Thu, 27 Jun 2013 19:17:00 +0100
 

debian/control

@@ -7,7 +7,7 @@ Build-Depends: debhelper (>= 8.0.0)
 
 Package: statsd
 Architecture: all
-Depends: ${misc:Depends}, nodejs (>= 0.6)
+Depends: nodejs (>= 0.6), adduser
 Description: Stats aggregation daemon
   A network daemon for aggregating statistics (counters and timers),
   rolling them up, then sending them to graphite.

debian/dirs

@@ -1,2 +1 @@
 var/log/statsd
-var/run/statsd

debian/postinst

@@ -3,6 +3,11 @@
 set -e
 
 if [ "$1" = configure ]; then
+  # Automatically added by dh_installinit
+  if [ -x "/etc/init.d/statsd" ]; then
+    update-rc.d statsd defaults >/dev/null
+    invoke-rc.d statsd start || exit $?
+  fi
 
   if ! getent passwd _statsd > /dev/null; then
     adduser --system --quiet --home /nonexistent --no-create-home \

debian/statsd.init

@@ -1,8 +1,8 @@
 #! /bin/sh
 ### BEGIN INIT INFO
 # Provides:          statsd
-# Required-Start:    $network $local_fs
-# Required-Stop:
+# Required-Start:    $remote_fs $network $local_fs
+# Required-Stop:     $remote_fs
 # Default-Start:     2 3 4 5
 # Default-Stop:      0 1 6
 ### END INIT INFO
@@ -41,6 +41,13 @@ CHDIR="/usr/share/statsd"
 # Depend on lsb-base (>= 3.0-6) to ensure that this file is present.
 . /lib/lsb/init-functions
 
+# Create PIDDIR on runtime
+if [ ! -d /var/run/$NAME ];
+then
+  mkdir /var/run/$NAME
+  chown $USER /var/run/$NAME
+fi
+
 #
 # Function that starts the daemon/service
 #

docs/backend.md

@@ -42,6 +42,7 @@ queues and third-party services.
 - [monitis backend](https://github.com/jeremiahshirk/statsd-monitis-backend)
 - [opentsdb backend](https://github.com/emurphy/statsd-opentsdb-backend)
 - [socket.io-backend](https://github.com/Chatham/statsd-socket.io)
+- [stackdriver backend](https://github.com/Stackdriver/stackdriver-statsd-backend)
 - [statsd-backend](https://github.com/dynmeth/statsd-backend)
 - [statsd http backend](https://github.com/bmhatfield/statsd-http-backend)
 - [statsd aggregation backend](https://github.com/wanelo/gossip_girl)

docs/graphite.md

@@ -89,7 +89,7 @@ This means:
 * For all other databases, average the values (mean) when rolling up data, and
   store a None if less than 30% of the datapoints were received
 
-Pay close attention to xFilesFactor: if your flush interval is long enough so there are not enough samples to satisfy this minimum factor, your data would simply be lost in the first downsampling cycle. However, setting a very low factor would also produce a misleading result, since you would probably agree that if you only have a single 10-second mean value sample reported in a 10-minute timeframe, this single sample alone should not normally be downsampled into a 10-minute mean value. For counts, however, every count should count ;-), hence the zero factor.
+Pay close attention to xFilesFactor: if your flush interval is not long enough so there are not enough samples to satisfy this minimum factor, your data would simply be lost in the first downsampling cycle. However, setting a very low factor would also produce a misleading result, since you would probably agree that if you only have a single 10-second mean value sample reported in a 10-minute timeframe, this single sample alone should not normally be downsampled into a 10-minute mean value. For counts, however, every count should count ;-), hence the zero factor.
 
 **Notes:**
 

exampleConfig.js

@@ -25,13 +25,20 @@ Optional Variables:
   mgmt_address:     address to run the management TCP interface on
                     [default: 0.0.0.0]
   mgmt_port:        port to run the management TCP interface on [default: 8126]
+  title :           Allows for overriding the process title. [default: statsd]
+                    if set to false, will not override the process title and let the OS set it.
+                    The length of the title has to be less than or equal to the binary name + cli arguments
+                    NOTE: This does not work on Mac's with node versions prior to v0.10
+
   healthStatus:     default health status to be returned and statsd process starts ['up' or 'down', default: 'up']
   dumpMessages:     log all incoming messages
   flushInterval:    interval (in ms) to flush to Graphite
   percentThreshold: for time information, calculate the Nth percentile(s)
                     (can be a single value or list of floating-point values)
                     negative values mean to use "top" Nth percentile(s) values
                     [%, default: 90]
+  flush_counts:     send stats_counts metrics [default: true]
+
   keyFlush:         log the most frequently sent keys [object, default: undefined]
     interval:       how often to log frequent keys [ms, default: 0]
     percent:        percentage of frequent keys to log [%, default: 100]

examples/statsd.go

@@ -154,10 +154,11 @@ func (client *StatsdClient) UpdateStats(stats []string, delta int, sampleRate fl
 func (client *StatsdClient) Send(data map[string]string, sampleRate float32) {
 	sampledData := make(map[string]string)
 	if sampleRate < 1 {
-		r := rand.New(rand.sampleRateNewSource(time.Now().Unix()))
-		if rNum := r.Float32(); rNum <= sampleRateate {
+		r := rand.New(rand.NewSource(time.Now().Unix()))
+		rNum := r.Float32(); 
+		if rNum <= sampleRate {
 			for stat,value := range data {
-				sampledUpdateString := fmt.forSprintf("%s|@%f", value, sampleRate)
+				sampledUpdateString := fmt.Sprintf("%s|@%f", value, sampleRate)
 				sampledData[stat] = sampledUpdateString
 			}
 		}

lib/helpers.js

@@ -0,0 +1,38 @@
+/**
+ * Public: test function to filter out malformed packets
+ *
+ * Parameters:
+ *
+ *   fields - Array of packet data (e.g. [ '100', 'ms', '@0.1' ])
+ *
+ * Returns true for a valid packet and false otherwise
+ */
+function is_valid_packet(fields) {
+
+    // test for existing metrics type
+    if (fields[1] === undefined) {
+        return false;
+    }
+    // filter out invalid metrics values
+    else if (fields[1] == 'g') {
+        if (!fields[0].match(/^([\-\+\d\.]+$)/)) {
+            return false;
+        } else {
+            return true;
+        }
+    }
+    else if (!fields[0].match(/^([\d\.]+$)/)) {
+        return false;
+    }
+    // filter out malformed sample rates
+    else if (fields[2] && !fields[2].match(/^@([\d\.]+$)/)) {
+        return false;
+    }
+    // looks like we're good
+    else {
+        return true;
+    }
+
+};
+
+exports.is_valid_packet = is_valid_packet;

lib/process_mgmt.js

@@ -0,0 +1,27 @@
+var util = require('util');
+
+var conf;
+
+exports.init = function(config) {
+  conf = config;
+  exports.set_title(config);
+
+  process.on('SIGTERM', function() {
+   if (conf.debug) {
+     util.log('Starting Final Flush');
+   }
+   healthStatus = 'down';
+   process.exit();
+  });
+
+}
+
+exports.set_title = function(config) {
+ if (config.title !== undefined) {
+   if (config.title) {
+       process.title = config.title;
+   }
+ } else {
+   process.title = 'statsd';
+ }
+}

stats.js

@@ -4,11 +4,13 @@ var dgram  = require('dgram')
   , util    = require('util')
   , net    = require('net')
   , config = require('./lib/config')
+  , helpers = require('./lib/helpers')
   , fs     = require('fs')
   , events = require('events')
   , logger = require('./lib/logger')
   , set = require('./lib/set')
   , pm = require('./lib/process_metrics')
+  , process_mgmt = require('./lib/process_mgmt')
   , mgmt = require('./lib/mgmt_console');
 
 
@@ -42,6 +44,7 @@ function loadBackend(config, name) {
   }
 }
 
+
 // global for conf
 var conf;
 
@@ -137,6 +140,9 @@ var l;
 
 config.configFile(process.argv[2], function (config, oldConfig) {
   conf = config;
+
+  process_mgmt.init(config);
+
   l = new logger.Logger(config.log || {});
 
   // setup config for stats prefix
@@ -193,22 +199,16 @@ config.configFile(process.argv[2], function (config, oldConfig) {
         for (var i = 0; i < bits.length; i++) {
           var sampleRate = 1;
           var fields = bits[i].split("|");
-          if (fields[2]) {
-            if (fields[2].match(/^@([\d\.]+)/)) {
-              sampleRate = Number(fields[2].match(/^@([\d\.]+)/)[1]);
-            } else {
-              l.log('Bad line: ' + fields + ' in msg "' + metrics[midx] +'"; has invalid sample rate');
-              counters[bad_lines_seen]++;
-              stats.messages.bad_lines_seen++;
-              continue;
-            }
-          }
-          if (fields[1] === undefined) {
+          if (!helpers.is_valid_packet(fields)) {
               l.log('Bad line: ' + fields + ' in msg "' + metrics[midx] +'"');
               counters[bad_lines_seen]++;
               stats.messages.bad_lines_seen++;
               continue;
           }
+          if (fields[2]) {
+            sampleRate = Number(fields[2].match(/^@([\d\.]+)/)[1]);
+          }
+
           var metric_type = fields[1].trim();
           if (metric_type === "ms") {
             if (! timers[key]) {
@@ -302,7 +302,7 @@ config.configFile(process.argv[2], function (config, oldConfig) {
             backendEvents.emit('status', function(err, name, stat, val) {
               if (err) {
                 l.log("Failed to read stats for backend " +
-                         name + ": " + err);
+                        name + ": " + err);
               } else {
                 stat_writer(name, stat, val);
               }
@@ -410,16 +410,6 @@ config.configFile(process.argv[2], function (config, oldConfig) {
   }
 });
 
-process.title = 'statsd';
-
-process.on('SIGTERM', function() {
-  if (conf.debug) {
-    util.log('Starting Final Flush');
-  }
-  healthStatus = 'down';
-  process.exit();
-});
-
 process.on('exit', function () {
   flushMetrics();
 });

test/helpers_tests.js

@@ -0,0 +1,59 @@
+var helpers = require('../lib/helpers');
+
+module.exports = {
+
+  no_metrics_field: function (test) {
+    var res = helpers.is_valid_packet(['foo', undefined]);
+    test.equals(res, false);
+    test.done();
+  },
+
+  wrong_formated_metric_value: function (test) {
+    var res = helpers.is_valid_packet(['0,345345', 'ms']);
+    test.equals(res, false);
+    test.done();
+  },
+
+  wrong_formated_sampling_value: function (test) {
+    var res = helpers.is_valid_packet(['345345', 'ms', '0,456456']);
+    test.equals(res, false);
+    test.done();
+  },
+
+  counter_deltas_positive_are_not_valid: function (test) {
+    var res = helpers.is_valid_packet(['+10', 'c']);
+    test.equals(res, false);
+    test.done();
+  },
+
+  counter_deltas_negative_are_not_valid: function (test) {
+    var res = helpers.is_valid_packet(['-10', 'c']);
+    test.equals(res, false);
+    test.done();
+  },
+
+  gauges_delta_positive_are_valid: function (test) {
+    var res = helpers.is_valid_packet(['+10', 'g']);
+    test.equals(res, true);
+    test.done();
+  },
+
+  gauges_delta_negative_are_valid: function (test) {
+    var res = helpers.is_valid_packet(['-10', 'g']);
+    test.equals(res, true);
+    test.done();
+  },
+
+  correct_packet: function (test) {
+    var res = helpers.is_valid_packet(['345345', 'ms', '@1.0']);
+    test.equals(res, true);
+    test.done();
+  },
+
+  correct_packet_with_small_sampling: function (test) {
+    var res = helpers.is_valid_packet(['100', 'ms', '@0.1']);
+    test.equals(res, true);
+    test.done();
+  }
+
+};