From 2d8aaf0ece9d0bdcc7dd49597d3244882a394ce8 Mon Sep 17 00:00:00 2001 From: Mike Heffner Date: Mon, 24 Sep 2012 14:12:13 -0400 Subject: [PATCH 01/21] Stat key name sanitization is now configurable at the top-level. Setting keyNameSanitize to false pushes the requirement of sanitizing key names to the backends. This permits backends that have less strict character set requirements to take advantage of an expanded stat name character set. The default behavior remains the same as collisions in key name space are not handled if two different stat names map to the same sanitized key name. --- backends/graphite.js | 28 +++++++++++++++++++++++----- exampleConfig.js | 3 +++ stats.js | 20 ++++++++++++++++---- test/graphite_tests.js | 19 +++++++++++++++++++ 4 files changed, 61 insertions(+), 9 deletions(-) diff --git a/backends/graphite.js b/backends/graphite.js index 41b6e32..5d7d8bb 100644 --- a/backends/graphite.js +++ b/backends/graphite.js @@ -34,6 +34,7 @@ var prefixTimer; var prefixGauge; var prefixSet; var globalSuffix; +var globalKeySanitize = true; // set up namespaces var legacyNamespace = true; @@ -98,15 +99,27 @@ var flush_stats = function graphite_flush(ts, metrics) { var timer_data = metrics.timer_data; var statsd_metrics = metrics.statsd_metrics; + // Sanitize key for graphite if not done globally + function sk(key) { + if (globalKeySanitize) { + return key; + } else { + return key.replace(/\s+/g, '_') + .replace(/\//g, '-') + .replace(/[^a-zA-Z_\-0-9\.]/g, ''); + } + }; + for (key in counters) { - var namespace = counterNamespace.concat(key); var value = counters[key]; var valuePerSecond = counter_rates[key]; // pre-calculated "per second" rate + var keyName = sk(key); + var namespace = counterNamespace.concat(keyName); if (legacyNamespace === true) { statString += namespace.join(".") + globalSuffix + valuePerSecond + ts_suffix; if (flush_counts) { - statString += 'stats_counts.' + key + globalSuffix + value + ts_suffix; + statString += 'stats_counts.' + keyName + globalSuffix + value + ts_suffix; } } else { statString += namespace.concat('rate').join(".") + globalSuffix + valuePerSecond + ts_suffix; @@ -119,8 +132,9 @@ var flush_stats = function graphite_flush(ts, metrics) { } for (key in timer_data) { - var namespace = timerNamespace.concat(key); + var namespace = timerNamespace.concat(sk(key)); var the_key = namespace.join("."); + for (timer_data_key in timer_data[key]) { if (typeof(timer_data[key][timer_data_key]) === 'number') { statString += the_key + '.' + timer_data_key + globalSuffix + timer_data[key][timer_data_key] + ts_suffix; @@ -138,13 +152,13 @@ var flush_stats = function graphite_flush(ts, metrics) { } for (key in gauges) { - var namespace = gaugesNamespace.concat(key); + var namespace = gaugesNamespace.concat(sk(key)); statString += namespace.join(".") + globalSuffix + gauges[key] + ts_suffix; numStats += 1; } for (key in sets) { - var namespace = setsNamespace.concat(key); + var namespace = setsNamespace.concat(sk(key)); statString += namespace.join(".") + '.count' + globalSuffix + sets[key].values().length + ts_suffix; numStats += 1; } @@ -238,6 +252,10 @@ exports.init = function graphite_init(startup_time, config, events) { graphiteStats.flush_time = 0; graphiteStats.flush_length = 0; + if (config.keyNameSanitize !== undefined) { + globalKeySanitize = config.keyNameSanitize; + } + flushInterval = config.flushInterval; flush_counts = typeof(config.flush_counts) === "undefined" ? true : config.flush_counts; diff --git a/exampleConfig.js b/exampleConfig.js index a5406c3..ff77fce 100644 --- a/exampleConfig.js +++ b/exampleConfig.js @@ -52,6 +52,9 @@ Optional Variables: deleteCounters: don't send values to graphite for inactive counters, as opposed to sending 0 [default: false] prefixStats: prefix to use for the statsd statistics data for this running instance of statsd [default: statsd] applies to both legacy and new namespacing + keyNameSanitize: sanitize all stat names on ingress [default: true] + If disabled, it is up to the backends to sanitize keynames + as appropriate per their storage requirements. console: prettyprint: whether to prettyprint the console backend diff --git a/stats.js b/stats.js index 628a3ce..e8eef81 100644 --- a/stats.js +++ b/stats.js @@ -28,6 +28,7 @@ var flushInterval, keyFlushInt, server, mgmtServer; var startup_time = Math.round(new Date().getTime() / 1000); var backendEvents = new events.EventEmitter(); var healthStatus = config.healthStatus || 'up'; +var keyNameSanitize = true; // Load and init the backend from the backends/ directory. function loadBackend(config, name) { @@ -135,6 +136,16 @@ var stats = { } }; +function sanitizeKeyName(key) { + if (keyNameSanitize) { + return key.replace(/\s+/g, '_') + .replace(/\//g, '-') + .replace(/[^a-zA-Z_\-0-9\.]/g, ''); + } else { + return key; + } +} + // Global for the logger var l; @@ -156,6 +167,10 @@ config.configFile(process.argv[2], function (config, oldConfig) { counters[bad_lines_seen] = 0; counters[packets_received] = 0; + if (config.keyNameSanitize !== undefined) { + keyNameSanitize = config.keyNameSanitize; + } + if (server === undefined) { // key counting @@ -180,10 +195,7 @@ config.configFile(process.argv[2], function (config, oldConfig) { l.log(metrics[midx].toString()); } var bits = metrics[midx].toString().split(':'); - var key = bits.shift() - .replace(/\s+/g, '_') - .replace(/\//g, '-') - .replace(/[^a-zA-Z_\-0-9\.]/g, ''); + var key = sanitizeKeyName(bits.shift()); if (keyFlushInterval > 0) { if (! keyCounter[key]) { diff --git a/test/graphite_tests.js b/test/graphite_tests.js index b93a6ae..503b138 100644 --- a/test/graphite_tests.js +++ b/test/graphite_tests.js @@ -358,5 +358,24 @@ module.exports = { }); }); }); + }, + + metric_names_are_sanitized: function(test) { + var me = this; + this.acceptor.once('connection', function(c) { + statsd_send('fo/o:250|c',me.sock,'127.0.0.1',8125,function(){ + statsd_send('b ar:250|c',me.sock,'127.0.0.1',8125,function(){ + statsd_send('foo+bar:250|c',me.sock,'127.0.0.1',8125,function(){ + collect_for(me.acceptor, me.myflush, function(strings){ + var str = strings.join(); + test.ok(str.indexOf('fo-o') !== -1, "Did not map 'fo/o' => 'fo-o'"); + test.ok(str.indexOf('b_ar') !== -1, "Did not map 'b ar' => 'b_ar'"); + test.ok(str.indexOf('foobar') !== -1, "Did not map 'foo+bar' => 'foobar'"); + test.done(); + }); + }); + }); + }); + }); } } From a7426b32d10f73ea638090480452a9068632868e Mon Sep 17 00:00:00 2001 From: shaylang Date: Mon, 22 Sep 2014 14:47:06 +0300 Subject: [PATCH 02/21] Fix merge conflicts with master --- stats.js | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/stats.js b/stats.js index dbd1c52..abd3fd6 100644 --- a/stats.js +++ b/stats.js @@ -27,12 +27,9 @@ var flushInterval, keyFlushInt, serverLoaded, mgmtServer; var startup_time = Math.round(new Date().getTime() / 1000); var backendEvents = new events.EventEmitter(); var healthStatus = config.healthStatus || 'up'; -<<<<<<< HEAD var old_timestamp = 0; var timestamp_lag_namespace; -======= var keyNameSanitize = true; ->>>>>>> pr/155 // Load and init the backend from the backends/ directory. function loadBackend(config, name) { @@ -194,15 +191,11 @@ config.configFile(process.argv[2], function (config) { counters[bad_lines_seen] = 0; counters[packets_received] = 0; -<<<<<<< HEAD - if (!serverLoaded) { -======= if (config.keyNameSanitize !== undefined) { keyNameSanitize = config.keyNameSanitize; } - if (server === undefined) { ->>>>>>> pr/155 + if (!serverLoaded) { // key counting var keyFlushInterval = Number((config.keyFlush && config.keyFlush.interval) || 0); From 494ed3029bad61ddfc76f0838b563cb17ba44168 Mon Sep 17 00:00:00 2001 From: shaylang Date: Mon, 22 Sep 2014 14:51:10 +0300 Subject: [PATCH 03/21] Fix merge conflicts with master --- backends/graphite.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/backends/graphite.js b/backends/graphite.js index a0295e7..09e7b48 100644 --- a/backends/graphite.js +++ b/backends/graphite.js @@ -32,11 +32,8 @@ var prefixTimer; var prefixGauge; var prefixSet; var globalSuffix; -<<<<<<< HEAD var prefixStats; -======= var globalKeySanitize = true; ->>>>>>> pr/155 // set up namespaces var legacyNamespace = true; From dc5bd4247c6aabc39ef474e71326bc4fae65197f Mon Sep 17 00:00:00 2001 From: shaylang Date: Mon, 22 Sep 2014 16:31:42 +0300 Subject: [PATCH 04/21] increase timeout due to sporadic failures --- test/graphite_tests.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/graphite_tests.js b/test/graphite_tests.js index b677432..971af4a 100644 --- a/test/graphite_tests.js +++ b/test/graphite_tests.js @@ -366,7 +366,7 @@ module.exports = { statsd_send('fo/o:250|c',me.sock,'127.0.0.1',8125,function(){ statsd_send('b ar:250|c',me.sock,'127.0.0.1',8125,function(){ statsd_send('foo+bar:250|c',me.sock,'127.0.0.1',8125,function(){ - collect_for(me.acceptor, me.myflush, function(strings){ + collect_for(me.acceptor, me.myflush * 2, function(strings){ var str = strings.join(); test.ok(str.indexOf('fo-o') !== -1, "Did not map 'fo/o' => 'fo-o'"); test.ok(str.indexOf('b_ar') !== -1, "Did not map 'b ar' => 'b_ar'"); From a1b9b2169d26cfdd449c9279128edfa835a299f3 Mon Sep 17 00:00:00 2001 From: Patrick Koch Date: Tue, 2 Sep 2014 20:02:13 +0000 Subject: [PATCH 05/21] bumping for v0.7.2 patch release Let's see if travis will upload a release to npm for us. --- Changelog.md | 3 +++ package.json | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 7210191..5c62ab0 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,8 @@ # Changelog +## v0.7.2 (09/02/2014) +- Fixes to detecting valid packets + ## v0.7.1 (02/06/2014) - move contributing information into CONTRIBUTING.md - Updates winser to v0.1.6 diff --git a/package.json b/package.json index 48eb0e5..6248209 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "type": "git", "url": "https://github.com/etsy/statsd.git" }, - "version": "0.7.1", + "version": "0.7.2", "dependencies": { }, "devDependencies": { From 523ae8330d0390081346d9dd2b355126e2a8019e Mon Sep 17 00:00:00 2001 From: Anton Rudeshko Date: Sun, 27 Jul 2014 11:33:52 +0400 Subject: [PATCH 06/21] Added IDEA project files to .gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 3c3629e..7d618a4 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ node_modules + +# WebStorm / IntelliJ IDEA project files +.idea +*.iml From 2006226adf1ba29c4e1dea39384f2ed379993742 Mon Sep 17 00:00:00 2001 From: Anton Rudeshko Date: Mon, 28 Jul 2014 18:53:17 +0400 Subject: [PATCH 07/21] Removed unused variable --- backends/graphite.js | 1 - 1 file changed, 1 deletion(-) diff --git a/backends/graphite.js b/backends/graphite.js index f988702..8a9adbf 100644 --- a/backends/graphite.js +++ b/backends/graphite.js @@ -27,7 +27,6 @@ var flush_counts; // prefix configuration var globalPrefix; -var prefixPersecond; var prefixCounter; var prefixTimer; var prefixGauge; From d47e6d71d116ba8c1e40c0e6490ef28861896c70 Mon Sep 17 00:00:00 2001 From: Anton Rudeshko Date: Mon, 28 Jul 2014 18:58:30 +0400 Subject: [PATCH 08/21] Fix `prefixStats` global variable leaking `prefixStats` is leaked through stats.js and was in global scope of event emitter without failing graphite backend by pure accident. Now it is read from config. --- backends/graphite.js | 3 +++ stats.js | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/backends/graphite.js b/backends/graphite.js index 8a9adbf..34cd299 100644 --- a/backends/graphite.js +++ b/backends/graphite.js @@ -32,6 +32,7 @@ var prefixTimer; var prefixGauge; var prefixSet; var globalSuffix; +var prefixStats; // set up namespaces var legacyNamespace = true; @@ -188,6 +189,7 @@ exports.init = function graphite_init(startup_time, config, events, logger) { prefixSet = config.graphite.prefixSet; globalSuffix = config.graphite.globalSuffix; legacyNamespace = config.graphite.legacyNamespace; + prefixStats = config.prefixStats; // set defaults for prefixes & suffix globalPrefix = globalPrefix !== undefined ? globalPrefix : "stats"; @@ -195,6 +197,7 @@ exports.init = function graphite_init(startup_time, config, events, logger) { prefixTimer = prefixTimer !== undefined ? prefixTimer : "timers"; prefixGauge = prefixGauge !== undefined ? prefixGauge : "gauges"; prefixSet = prefixSet !== undefined ? prefixSet : "sets"; + prefixStats = prefixStats !== undefined ? prefixStats : "statsd"; legacyNamespace = legacyNamespace !== undefined ? legacyNamespace : true; // In order to unconditionally add this string, it either needs to be diff --git a/stats.js b/stats.js index ccfe676..ed7b599 100644 --- a/stats.js +++ b/stats.js @@ -151,7 +151,7 @@ config.configFile(process.argv[2], function (config, oldConfig) { l = new logger.Logger(config.log || {}); // setup config for stats prefix - prefixStats = config.prefixStats; + var prefixStats = config.prefixStats; prefixStats = prefixStats !== undefined ? prefixStats : "statsd"; //setup the names for the stats stored in counters{} bad_lines_seen = prefixStats + ".bad_lines_seen"; From 7e8efc2c856a4ee5bb4ee1b412d2fed01c3a3be5 Mon Sep 17 00:00:00 2001 From: Anton Rudeshko Date: Mon, 28 Jul 2014 19:47:27 +0400 Subject: [PATCH 09/21] Removed unused parameter --- stats.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stats.js b/stats.js index ed7b599..bea7a07 100644 --- a/stats.js +++ b/stats.js @@ -143,7 +143,7 @@ var stats = { // Global for the logger var l; -config.configFile(process.argv[2], function (config, oldConfig) { +config.configFile(process.argv[2], function (config) { conf = config; process_mgmt.init(config); From be5c23404d7270e1d3d4342daa8c72dd3824316f Mon Sep 17 00:00:00 2001 From: Patrick Koch Date: Thu, 11 Sep 2014 18:41:35 -0400 Subject: [PATCH 10/21] fix typo in Gauges section --- docs/metric_types.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/metric_types.md b/docs/metric_types.md index 75b68d3..e839cd0 100644 --- a/docs/metric_types.md +++ b/docs/metric_types.md @@ -82,7 +82,7 @@ StatsD now also supports gauges, arbitrary values, which can be recorded. gaugor:333|g If the gauge is not updated at the next flush, it will send the previous value. You can opt to send -no metric at all for this gauge, by setting `config.deleteGauge` +no metric at all for this gauge, by setting `config.deleteGauges` Adding a sign to the gauge value will change the value, rather than setting it. From 88c8fce6a8b056ca9fd0173a8700f6db3be01e93 Mon Sep 17 00:00:00 2001 From: Ben Burry Date: Mon, 15 Sep 2014 21:33:32 +0000 Subject: [PATCH 11/21] Convert listen server to loadable module, defaulting to udp --- servers/udp.js | 8 ++++++++ stats.js | 28 +++++++++++++++++++++------- 2 files changed, 29 insertions(+), 7 deletions(-) create mode 100644 servers/udp.js diff --git a/servers/udp.js b/servers/udp.js new file mode 100644 index 0000000..b03bc32 --- /dev/null +++ b/servers/udp.js @@ -0,0 +1,8 @@ +var dgram = require('dgram'); + +exports.init = function(config, callback){ + var udp_version = config.address_ipv6 ? 'udp6' : 'udp4'; + var server = dgram.createSocket(udp_version, callback); + server.bind(config.port || 8125, config.address || undefined); + return true; +}; diff --git a/stats.js b/stats.js index bea7a07..3a615f7 100644 --- a/stats.js +++ b/stats.js @@ -1,7 +1,6 @@ /*jshint node:true, laxcomma:true */ -var dgram = require('dgram') - , util = require('util') +var util = require('util') , net = require('net') , config = require('./lib/config') , helpers = require('./lib/helpers') @@ -24,7 +23,7 @@ var sets = {}; var counter_rates = {}; var timer_data = {}; var pctThreshold = null; -var flushInterval, keyFlushInt, server, mgmtServer; +var flushInterval, keyFlushInt, serverLoaded, mgmtServer; var startup_time = Math.round(new Date().getTime() / 1000); var backendEvents = new events.EventEmitter(); var healthStatus = config.healthStatus || 'up'; @@ -46,6 +45,21 @@ function loadBackend(config, name) { } } +// Load and init the server from the servers/ directory. +function startServer(config, name, callback) { + var servermod = require(name); + + if (config.debug) { + l.log("Loading server: " + name, 'DEBUG'); + } + + var ret = servermod.init(config, callback); + if (!ret) { + l.log("Failed to load server: " + name); + process.exit(1); + } +} + // global for conf var conf; @@ -162,13 +176,14 @@ config.configFile(process.argv[2], function (config) { counters[bad_lines_seen] = 0; counters[packets_received] = 0; - if (server === undefined) { + if (!serverLoaded) { // key counting var keyFlushInterval = Number((config.keyFlush && config.keyFlush.interval) || 0); - var udp_version = config.address_ipv6 ? 'udp6' : 'udp4'; - server = dgram.createSocket(udp_version, function (msg, rinfo) { + // The default server is UDP + var server = config.server || './servers/udp' + serverLoaded = startServer(config, server, function (msg, rinfo) { backendEvents.emit('packet', msg, rinfo); counters[packets_received]++; var packet_data = msg.toString(); @@ -355,7 +370,6 @@ config.configFile(process.argv[2], function (config) { }); }); - server.bind(config.port || 8125, config.address || undefined); mgmtServer.listen(config.mgmt_port || 8126, config.mgmt_address || undefined); util.log("server is up"); From 9eab749f782907fcfbcecfc224851ab3ddefa3a7 Mon Sep 17 00:00:00 2001 From: Ben Burry Date: Mon, 15 Sep 2014 21:33:54 +0000 Subject: [PATCH 12/21] Add tcp loadable server module --- servers/tcp.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 servers/tcp.js diff --git a/servers/tcp.js b/servers/tcp.js new file mode 100644 index 0000000..e094a32 --- /dev/null +++ b/servers/tcp.js @@ -0,0 +1,16 @@ +var net = require('net'); + +exports.init = function(config, callback){ + var server = net.createServer(function(stream) { + stream.setEncoding('ascii'); + + stream.on('data', function(data) { + var rinfo = stream.address(); + rinfo.size = data.length; + callback(data, rinfo); + }); + }); + + server.listen(config.port || 8125, config.address || undefined); + return true; +}; From 15bf64ad43b8a43da68b139cadb5e3b7b6cf74be Mon Sep 17 00:00:00 2001 From: Ben Burry Date: Mon, 15 Sep 2014 23:28:40 +0000 Subject: [PATCH 13/21] Represent remote address in tcp server rinfo, not local --- servers/tcp.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/servers/tcp.js b/servers/tcp.js index e094a32..c1ee3af 100644 --- a/servers/tcp.js +++ b/servers/tcp.js @@ -1,13 +1,18 @@ var net = require('net'); +function rinfo(tcpstream, data) { + this.address = tcpstream.remoteAddress; + this.port = tcpstream.remotePort; + this.family = tcpstream.address().family; + this.size = data.length; +} + exports.init = function(config, callback){ var server = net.createServer(function(stream) { stream.setEncoding('ascii'); stream.on('data', function(data) { - var rinfo = stream.address(); - rinfo.size = data.length; - callback(data, rinfo); + callback(data, new rinfo(stream, data)); }); }); From f2102aeb7130c3dc378d0431da499568b62aac12 Mon Sep 17 00:00:00 2001 From: Ben Burry Date: Tue, 16 Sep 2014 00:26:47 +0000 Subject: [PATCH 14/21] Rename server init to 'start' to reflect reality --- servers/tcp.js | 2 +- servers/udp.js | 2 +- stats.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/servers/tcp.js b/servers/tcp.js index c1ee3af..3c2e651 100644 --- a/servers/tcp.js +++ b/servers/tcp.js @@ -7,7 +7,7 @@ function rinfo(tcpstream, data) { this.size = data.length; } -exports.init = function(config, callback){ +exports.start = function(config, callback){ var server = net.createServer(function(stream) { stream.setEncoding('ascii'); diff --git a/servers/udp.js b/servers/udp.js index b03bc32..a0a3072 100644 --- a/servers/udp.js +++ b/servers/udp.js @@ -1,6 +1,6 @@ var dgram = require('dgram'); -exports.init = function(config, callback){ +exports.start = function(config, callback){ var udp_version = config.address_ipv6 ? 'udp6' : 'udp4'; var server = dgram.createSocket(udp_version, callback); server.bind(config.port || 8125, config.address || undefined); diff --git a/stats.js b/stats.js index 3a615f7..280f3eb 100644 --- a/stats.js +++ b/stats.js @@ -53,7 +53,7 @@ function startServer(config, name, callback) { l.log("Loading server: " + name, 'DEBUG'); } - var ret = servermod.init(config, callback); + var ret = servermod.start(config, callback); if (!ret) { l.log("Failed to load server: " + name); process.exit(1); From 6f7f566751a0a52594a918812b447c02ea1c6b79 Mon Sep 17 00:00:00 2001 From: Ben Burry Date: Tue, 16 Sep 2014 00:27:54 +0000 Subject: [PATCH 15/21] Add basic test for each of udp and tcp server module --- test/server_tests.js | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 test/server_tests.js diff --git a/test/server_tests.js b/test/server_tests.js new file mode 100644 index 0000000..b426ba9 --- /dev/null +++ b/test/server_tests.js @@ -0,0 +1,42 @@ +var dgram = require('dgram'), + net = require('net'); + +var config = { + address: '127.0.0.1', + port: 8125 +}; +var msg = "This is a test\r\n"; + +module.exports = { + udp_data_received: function(test) { + test.expect(3); + var server = require('../servers/udp'); + var started = server.start(config, function(data, rinfo) { + test.equal(msg, data.toString()); + test.equal(msg.length, rinfo.size); + test.done(); + }); + test.ok(started); + + var buf = new Buffer(msg); + var sock = dgram.createSocket('udp4'); + sock.send(buf, 0, buf.length, config.port, config.address, function(err, bytes) { + sock.close(); + }); + }, + tcp_data_received: function(test) { + test.expect(3); + var server = require('../servers/tcp'); + var started = server.start(config, function(data, rinfo) { + test.equal(msg, data.toString()); + test.equal(msg.length, rinfo.size); + test.done(); + }); + test.ok(started); + + var client = net.connect(config.port, config.address, function() { + client.write(msg); + client.end(); + }); + } +} From 6984d5d37dede44aaa7b06c2b46f74bc0d1ebfa7 Mon Sep 17 00:00:00 2001 From: Ben Burry Date: Tue, 16 Sep 2014 00:35:26 +0000 Subject: [PATCH 16/21] Update docs and example config to reflect new tcp server option --- README.md | 9 +++++---- exampleConfig.js | 8 ++++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 9f14b96..74b129c 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ StatsD [![Build Status][travis-ci_status_img]][travis-ci_statsd] ====== A network daemon that runs on the [Node.js][node] platform and -listens for statistics, like counters and timers, sent over [UDP][udp] -and sends aggregates to one or more pluggable backend services (e.g., +listens for statistics, like counters and timers, sent over [UDP][udp] or +[TCP][tcp] and sends aggregates to one or more pluggable backend services (e.g., [Graphite][graphite]). We ([Etsy][etsy]) [blogged][blog post] about how it works and why we created it. @@ -46,12 +46,12 @@ Installation and Configuration Usage ------- -The basic line protocol expects metrics to be sent via UDP in the format: +The basic line protocol expects metrics to be sent in the format: :| So the simplest way to send in metrics from your command line if you have -StatsD running on localhost would be: +StatsD running with the default UDP server on localhost would be: echo "foo:1|c" | nc -u -w0 127.0.0.1 8125 @@ -104,6 +104,7 @@ Meta [counting-timing]: http://code.flickr.com/blog/2008/10/27/counting-timing/ [Flicker-StatsD]: https://github.com/iamcal/Flickr-StatsD [udp]: http://en.wikipedia.org/wiki/User_Datagram_Protocol +[tcp]: http://en.wikipedia.org/wiki/Transmission_Control_Protocol [docs_metric_types]: https://github.com/etsy/statsd/blob/master/docs/metric_types.md [docs_graphite]: https://github.com/etsy/statsd/blob/master/docs/graphite.md [docs_backend]: https://github.com/etsy/statsd/blob/master/docs/backend.md diff --git a/exampleConfig.js b/exampleConfig.js index 174dd09..b0053a5 100644 --- a/exampleConfig.js +++ b/exampleConfig.js @@ -15,10 +15,14 @@ Optional Variables: the default graphite backend will be loaded. * example for console and graphite: [ "./backends/console", "./backends/graphite" ] + server: the server to load. The server must exist by name in the directory + servers/. If not specified, the default udp server will be loaded. + * example for tcp server: + "./servers/tcp" debug: debug flag [default: false] - address: address to listen on over UDP [default: 0.0.0.0] + address: address to listen on [default: 0.0.0.0] address_ipv6: defines if the address is an IPv4 or IPv6 address [true or false, default: false] - port: port to listen for messages on over UDP [default: 8125] + port: port to listen for messages on [default: 8125] mgmt_address: address to run the management TCP interface on [default: 0.0.0.0] mgmt_port: port to run the management TCP interface on [default: 8126] From f630e15f04451ade70b5c028796344e1b23debd6 Mon Sep 17 00:00:00 2001 From: Ben Burry Date: Tue, 16 Sep 2014 21:16:11 +0000 Subject: [PATCH 17/21] Add brief docs around server callback function sig --- stats.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/stats.js b/stats.js index 280f3eb..eb57671 100644 --- a/stats.js +++ b/stats.js @@ -46,6 +46,10 @@ function loadBackend(config, name) { } // Load and init the server from the servers/ directory. +// The callback mimics the dgram 'message' event parameters (msg, rinfo) +// msg: the message received by the server. may contain more than one metric +// rinfo: contains remote address information and message length +// (attributes are .address, .port, .family, .size - you're welcome) function startServer(config, name, callback) { var servermod = require(name); From 4e7a60049e70b7c227f833d251f913d6e1edf8f7 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Wed, 17 Sep 2014 10:04:39 -0700 Subject: [PATCH 18/21] Add servers directory to debian package --- debian/statsd.install | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/statsd.install b/debian/statsd.install index 174d59e..21b225b 100644 --- a/debian/statsd.install +++ b/debian/statsd.install @@ -2,4 +2,5 @@ stats.js /usr/share/statsd lib/*.js /usr/share/statsd/lib backends/*.js /usr/share/statsd/backends lib/*.js /usr/share/statsd/lib +servers/*.js /usr/share/statsd/servers debian/localConfig.js /etc/statsd From c7e6421d1e7b284a94372acf455a2e18b4d997a8 Mon Sep 17 00:00:00 2001 From: Mike Heffner Date: Mon, 24 Sep 2012 14:12:13 -0400 Subject: [PATCH 19/21] Stat key name sanitization is now configurable at the top-level. Setting keyNameSanitize to false pushes the requirement of sanitizing key names to the backends. This permits backends that have less strict character set requirements to take advantage of an expanded stat name character set. The default behavior remains the same as collisions in key name space are not handled if two different stat names map to the same sanitized key name. --- backends/graphite.js | 28 ++++++++-- config.js | 113 +++++++++++++++++++++++++++++++++++++++++ exampleConfig.js | 3 ++ stats.js | 21 ++++++-- test/graphite_tests.js | 19 +++++++ test/stam | 1 + 6 files changed, 175 insertions(+), 10 deletions(-) create mode 100644 config.js create mode 100644 test/stam diff --git a/backends/graphite.js b/backends/graphite.js index 34cd299..09e7b48 100644 --- a/backends/graphite.js +++ b/backends/graphite.js @@ -33,6 +33,7 @@ var prefixGauge; var prefixSet; var globalSuffix; var prefixStats; +var globalKeySanitize = true; // set up namespaces var legacyNamespace = true; @@ -97,15 +98,27 @@ var flush_stats = function graphite_flush(ts, metrics) { var timer_data = metrics.timer_data; var statsd_metrics = metrics.statsd_metrics; + // Sanitize key for graphite if not done globally + function sk(key) { + if (globalKeySanitize) { + return key; + } else { + return key.replace(/\s+/g, '_') + .replace(/\//g, '-') + .replace(/[^a-zA-Z_\-0-9\.]/g, ''); + } + }; + for (key in counters) { - var namespace = counterNamespace.concat(key); var value = counters[key]; var valuePerSecond = counter_rates[key]; // pre-calculated "per second" rate + var keyName = sk(key); + var namespace = counterNamespace.concat(keyName); if (legacyNamespace === true) { statString += namespace.join(".") + globalSuffix + valuePerSecond + ts_suffix; if (flush_counts) { - statString += 'stats_counts.' + key + globalSuffix + value + ts_suffix; + statString += 'stats_counts.' + keyName + globalSuffix + value + ts_suffix; } } else { statString += namespace.concat('rate').join(".") + globalSuffix + valuePerSecond + ts_suffix; @@ -118,8 +131,9 @@ var flush_stats = function graphite_flush(ts, metrics) { } for (key in timer_data) { - var namespace = timerNamespace.concat(key); + var namespace = timerNamespace.concat(sk(key)); var the_key = namespace.join("."); + for (timer_data_key in timer_data[key]) { if (typeof(timer_data[key][timer_data_key]) === 'number') { statString += the_key + '.' + timer_data_key + globalSuffix + timer_data[key][timer_data_key] + ts_suffix; @@ -137,13 +151,13 @@ var flush_stats = function graphite_flush(ts, metrics) { } for (key in gauges) { - var namespace = gaugesNamespace.concat(key); + var namespace = gaugesNamespace.concat(sk(key)); statString += namespace.join(".") + globalSuffix + gauges[key] + ts_suffix; numStats += 1; } for (key in sets) { - var namespace = setsNamespace.concat(key); + var namespace = setsNamespace.concat(sk(key)); statString += namespace.join(".") + '.count' + globalSuffix + sets[key].values().length + ts_suffix; numStats += 1; } @@ -239,6 +253,10 @@ exports.init = function graphite_init(startup_time, config, events, logger) { graphiteStats.flush_time = 0; graphiteStats.flush_length = 0; + if (config.keyNameSanitize !== undefined) { + globalKeySanitize = config.keyNameSanitize; + } + flushInterval = config.flushInterval; flush_counts = typeof(config.flush_counts) === "undefined" ? true : config.flush_counts; diff --git a/config.js b/config.js new file mode 100644 index 0000000..7e03fe4 --- /dev/null +++ b/config.js @@ -0,0 +1,113 @@ +/* +Graphite Required Variables: + +(Leave these unset to avoid sending stats to Graphite. + Set debug flag and leave these unset to run in 'dry' debug mode - + useful for testing statsd clients without a Graphite server.) + + graphiteHost: hostname or IP of Graphite server + graphitePort: port of Graphite server + +Optional Variables: + + backends: an array of backends to load. Each backend must exist + by name in the directory backends/. If not specified, + the default graphite backend will be loaded. + * example for console and graphite: + [ "./backends/console", "./backends/graphite" ] + server: the server to load. The server must exist by name in the directory + servers/. If not specified, the default udp server will be loaded. + * example for tcp server: + "./servers/tcp" + debug: debug flag [default: false] + address: address to listen on [default: 0.0.0.0] + address_ipv6: defines if the address is an IPv4 or IPv6 address [true or false, default: false] + port: port to listen for messages on [default: 8125] + mgmt_address: address to run the management TCP interface on + [default: 0.0.0.0] + mgmt_port: port to run the management TCP interface on [default: 8126] + title: Allows for overriding the process title. [default: statsd] + if set to false, will not override the process title and let the OS set it. + The length of the title has to be less than or equal to the binary name + cli arguments + NOTE: This does not work on Mac's with node versions prior to v0.10 + + healthStatus: default health status to be returned and statsd process starts ['up' or 'down', default: 'up'] + dumpMessages: log all incoming messages + flushInterval: interval (in ms) to flush metrics to each backend + percentThreshold: for time information, calculate the Nth percentile(s) + (can be a single value or list of floating-point values) + negative values mean to use "top" Nth percentile(s) values + [%, default: 90] + flush_counts: send stats_counts metrics [default: true] + + keyFlush: log the most frequently sent keys [object, default: undefined] + interval: how often to log frequent keys [ms, default: 0] + percent: percentage of frequent keys to log [%, default: 100] + log: location of log file for frequent keys [default: STDOUT] + deleteIdleStats: don't send values to graphite for inactive counters, sets, gauges, or timers + as opposed to sending 0. For gauges, this unsets the gauge (instead of sending + the previous value). Can be individually overriden. [default: false] + deleteGauges: don't send values to graphite for inactive gauges, as opposed to sending the previous value [default: false] + deleteTimers: don't send values to graphite for inactive timers, as opposed to sending 0 [default: false] + deleteSets: don't send values to graphite for inactive sets, as opposed to sending 0 [default: false] + deleteCounters: don't send values to graphite for inactive counters, as opposed to sending 0 [default: false] + prefixStats: prefix to use for the statsd statistics data for this running instance of statsd [default: statsd] + applies to both legacy and new namespacing + keyNameSanitize: sanitize all stat names on ingress [default: true] + If disabled, it is up to the backends to sanitize keynames + as appropriate per their storage requirements. + + console: + prettyprint: whether to prettyprint the console backend + output [true or false, default: true] + + log: log settings [object, default: undefined] + backend: where to log: stdout or syslog [string, default: stdout] + application: name of the application for syslog [string, default: statsd] + level: log level for [node-]syslog [string, default: LOG_INFO] + + graphite: + legacyNamespace: use the legacy namespace [default: true] + globalPrefix: global prefix to use for sending stats to graphite [default: "stats"] + prefixCounter: graphite prefix for counter metrics [default: "counters"] + prefixTimer: graphite prefix for timer metrics [default: "timers"] + prefixGauge: graphite prefix for gauge metrics [default: "gauges"] + prefixSet: graphite prefix for set metrics [default: "sets"] + globalSuffix: global suffix to use for sending stats to graphite [default: ""] + This is particularly useful for sending per host stats by + settings this value to: require('os').hostname().split('.')[0] + + repeater: an array of hashes of the for host: and port: + that details other statsd servers to which the received + packets should be "repeated" (duplicated to). + e.g. [ { host: '10.10.10.10', port: 8125 }, + { host: 'observer', port: 88125 } ] + + repeaterProtocol: whether to use udp4 or udp6 for repeaters. + ["udp4" or "udp6", default: "udp4"] + + histogram: for timers, an array of mappings of strings (to match metrics) and + corresponding ordered non-inclusive upper limits of bins. + For all matching metrics, histograms are maintained over + time by writing the frequencies for all bins. + 'inf' means infinity. A lower limit of 0 is assumed. + default: [], meaning no histograms for any timer. + First match wins. examples: + * histogram to only track render durations, with unequal + class intervals and catchall for outliers: + [ { metric: 'render', bins: [ 0.01, 0.1, 1, 10, 'inf'] } ] + * histogram for all timers except 'foo' related, + equal class interval and catchall for outliers: + [ { metric: 'foo', bins: [] }, + { metric: '', bins: [ 50, 100, 150, 200, 'inf'] } ] + + automaticConfigReload: whether to watch the config file and reload it when it + changes. The default is true. Set this to false to disable. +*/ +{ + graphitePort: 2003 +, graphiteHost: "127.0.0.1" +, port: 8125 +, keyNameSanitize: false +, backends: [ "./backends/graphite", "./backends/console" ] +} diff --git a/exampleConfig.js b/exampleConfig.js index b0053a5..598f2ec 100644 --- a/exampleConfig.js +++ b/exampleConfig.js @@ -53,6 +53,9 @@ Optional Variables: deleteCounters: don't send values to graphite for inactive counters, as opposed to sending 0 [default: false] prefixStats: prefix to use for the statsd statistics data for this running instance of statsd [default: statsd] applies to both legacy and new namespacing + keyNameSanitize: sanitize all stat names on ingress [default: true] + If disabled, it is up to the backends to sanitize keynames + as appropriate per their storage requirements. console: prettyprint: whether to prettyprint the console backend diff --git a/stats.js b/stats.js index eb57671..b8762b6 100644 --- a/stats.js +++ b/stats.js @@ -29,6 +29,7 @@ var backendEvents = new events.EventEmitter(); var healthStatus = config.healthStatus || 'up'; var old_timestamp = 0; var timestamp_lag_namespace; +var keyNameSanitize = true; // Load and init the backend from the backends/ directory. function loadBackend(config, name) { @@ -158,6 +159,16 @@ var stats = { } }; +function sanitizeKeyName(key) { + if (keyNameSanitize) { + return key.replace(/\s+/g, '_') + .replace(/\//g, '-') + .replace(/[^a-zA-Z_\-0-9\.]/g, ''); + } else { + return key; + } +} + // Global for the logger var l; @@ -180,8 +191,11 @@ config.configFile(process.argv[2], function (config) { counters[bad_lines_seen] = 0; counters[packets_received] = 0; - if (!serverLoaded) { + if (config.keyNameSanitize !== undefined) { + keyNameSanitize = config.keyNameSanitize; + } + if (!serverLoaded) { // key counting var keyFlushInterval = Number((config.keyFlush && config.keyFlush.interval) || 0); @@ -205,10 +219,7 @@ config.configFile(process.argv[2], function (config) { l.log(metrics[midx].toString()); } var bits = metrics[midx].toString().split(':'); - var key = bits.shift() - .replace(/\s+/g, '_') - .replace(/\//g, '-') - .replace(/[^a-zA-Z_\-0-9\.]/g, ''); + var key = sanitizeKeyName(bits.shift()); if (keyFlushInterval > 0) { if (! keyCounter[key]) { diff --git a/test/graphite_tests.js b/test/graphite_tests.js index a034f31..971af4a 100644 --- a/test/graphite_tests.js +++ b/test/graphite_tests.js @@ -358,5 +358,24 @@ module.exports = { }); }); }); + }, + + metric_names_are_sanitized: function(test) { + var me = this; + this.acceptor.once('connection', function(c) { + statsd_send('fo/o:250|c',me.sock,'127.0.0.1',8125,function(){ + statsd_send('b ar:250|c',me.sock,'127.0.0.1',8125,function(){ + statsd_send('foo+bar:250|c',me.sock,'127.0.0.1',8125,function(){ + collect_for(me.acceptor, me.myflush * 2, function(strings){ + var str = strings.join(); + test.ok(str.indexOf('fo-o') !== -1, "Did not map 'fo/o' => 'fo-o'"); + test.ok(str.indexOf('b_ar') !== -1, "Did not map 'b ar' => 'b_ar'"); + test.ok(str.indexOf('foobar') !== -1, "Did not map 'foo+bar' => 'foobar'"); + test.done(); + }); + }); + }); + }); + }); } } diff --git a/test/stam b/test/stam new file mode 100644 index 0000000..6b1f07d --- /dev/null +++ b/test/stam @@ -0,0 +1 @@ +stats.counters.statsd.bad_lines_seen.rate 0 1411387496,stats.counters.statsd.bad_lines_seen.count 0 1411387496,stats.counters.statsd.packets_received.rate 15 1411387496,stats.counters.statsd.packets_received.count 3 1411387496,stats.counters.fo-o.rate 1250 1411387496,stats.counters.fo-o.count 250 1411387496,stats.counters.b_ar.rate 1250 1411387496,stats.counters.b_ar.count 250 1411387496,stats.counters.foobar.rate 1250 1411387496,stats.counters.foobar.count 250 1411387496,stats.gauges.statsd.timestamp_lag -0.2 1411387496,stats.statsd.numStats 6 1411387496,stats.statsd.graphiteStats.calculationtime 0 1411387496,stats.statsd.processing_time 0 1411387496,stats.statsd.graphiteStats.last_exception 1411387495 1411387496,stats.statsd.graphiteStats.last_flush 1411387496 1411387496,stats.statsd.graphiteStats.flush_time 1 1411387496,stats.statsd.graphiteStats.flush_length 587 1411387496, \ No newline at end of file From 55be359068e80d8d7635c14219fa856801edfeca Mon Sep 17 00:00:00 2001 From: shaylang Date: Mon, 22 Sep 2014 14:47:06 +0300 Subject: [PATCH 20/21] merge from PR155 --- config.js | 113 ------------------------------------------------------ test/stam | 1 - 2 files changed, 114 deletions(-) delete mode 100644 config.js delete mode 100644 test/stam diff --git a/config.js b/config.js deleted file mode 100644 index 7e03fe4..0000000 --- a/config.js +++ /dev/null @@ -1,113 +0,0 @@ -/* -Graphite Required Variables: - -(Leave these unset to avoid sending stats to Graphite. - Set debug flag and leave these unset to run in 'dry' debug mode - - useful for testing statsd clients without a Graphite server.) - - graphiteHost: hostname or IP of Graphite server - graphitePort: port of Graphite server - -Optional Variables: - - backends: an array of backends to load. Each backend must exist - by name in the directory backends/. If not specified, - the default graphite backend will be loaded. - * example for console and graphite: - [ "./backends/console", "./backends/graphite" ] - server: the server to load. The server must exist by name in the directory - servers/. If not specified, the default udp server will be loaded. - * example for tcp server: - "./servers/tcp" - debug: debug flag [default: false] - address: address to listen on [default: 0.0.0.0] - address_ipv6: defines if the address is an IPv4 or IPv6 address [true or false, default: false] - port: port to listen for messages on [default: 8125] - mgmt_address: address to run the management TCP interface on - [default: 0.0.0.0] - mgmt_port: port to run the management TCP interface on [default: 8126] - title: Allows for overriding the process title. [default: statsd] - if set to false, will not override the process title and let the OS set it. - The length of the title has to be less than or equal to the binary name + cli arguments - NOTE: This does not work on Mac's with node versions prior to v0.10 - - healthStatus: default health status to be returned and statsd process starts ['up' or 'down', default: 'up'] - dumpMessages: log all incoming messages - flushInterval: interval (in ms) to flush metrics to each backend - percentThreshold: for time information, calculate the Nth percentile(s) - (can be a single value or list of floating-point values) - negative values mean to use "top" Nth percentile(s) values - [%, default: 90] - flush_counts: send stats_counts metrics [default: true] - - keyFlush: log the most frequently sent keys [object, default: undefined] - interval: how often to log frequent keys [ms, default: 0] - percent: percentage of frequent keys to log [%, default: 100] - log: location of log file for frequent keys [default: STDOUT] - deleteIdleStats: don't send values to graphite for inactive counters, sets, gauges, or timers - as opposed to sending 0. For gauges, this unsets the gauge (instead of sending - the previous value). Can be individually overriden. [default: false] - deleteGauges: don't send values to graphite for inactive gauges, as opposed to sending the previous value [default: false] - deleteTimers: don't send values to graphite for inactive timers, as opposed to sending 0 [default: false] - deleteSets: don't send values to graphite for inactive sets, as opposed to sending 0 [default: false] - deleteCounters: don't send values to graphite for inactive counters, as opposed to sending 0 [default: false] - prefixStats: prefix to use for the statsd statistics data for this running instance of statsd [default: statsd] - applies to both legacy and new namespacing - keyNameSanitize: sanitize all stat names on ingress [default: true] - If disabled, it is up to the backends to sanitize keynames - as appropriate per their storage requirements. - - console: - prettyprint: whether to prettyprint the console backend - output [true or false, default: true] - - log: log settings [object, default: undefined] - backend: where to log: stdout or syslog [string, default: stdout] - application: name of the application for syslog [string, default: statsd] - level: log level for [node-]syslog [string, default: LOG_INFO] - - graphite: - legacyNamespace: use the legacy namespace [default: true] - globalPrefix: global prefix to use for sending stats to graphite [default: "stats"] - prefixCounter: graphite prefix for counter metrics [default: "counters"] - prefixTimer: graphite prefix for timer metrics [default: "timers"] - prefixGauge: graphite prefix for gauge metrics [default: "gauges"] - prefixSet: graphite prefix for set metrics [default: "sets"] - globalSuffix: global suffix to use for sending stats to graphite [default: ""] - This is particularly useful for sending per host stats by - settings this value to: require('os').hostname().split('.')[0] - - repeater: an array of hashes of the for host: and port: - that details other statsd servers to which the received - packets should be "repeated" (duplicated to). - e.g. [ { host: '10.10.10.10', port: 8125 }, - { host: 'observer', port: 88125 } ] - - repeaterProtocol: whether to use udp4 or udp6 for repeaters. - ["udp4" or "udp6", default: "udp4"] - - histogram: for timers, an array of mappings of strings (to match metrics) and - corresponding ordered non-inclusive upper limits of bins. - For all matching metrics, histograms are maintained over - time by writing the frequencies for all bins. - 'inf' means infinity. A lower limit of 0 is assumed. - default: [], meaning no histograms for any timer. - First match wins. examples: - * histogram to only track render durations, with unequal - class intervals and catchall for outliers: - [ { metric: 'render', bins: [ 0.01, 0.1, 1, 10, 'inf'] } ] - * histogram for all timers except 'foo' related, - equal class interval and catchall for outliers: - [ { metric: 'foo', bins: [] }, - { metric: '', bins: [ 50, 100, 150, 200, 'inf'] } ] - - automaticConfigReload: whether to watch the config file and reload it when it - changes. The default is true. Set this to false to disable. -*/ -{ - graphitePort: 2003 -, graphiteHost: "127.0.0.1" -, port: 8125 -, keyNameSanitize: false -, backends: [ "./backends/graphite", "./backends/console" ] -} diff --git a/test/stam b/test/stam deleted file mode 100644 index 6b1f07d..0000000 --- a/test/stam +++ /dev/null @@ -1 +0,0 @@ -stats.counters.statsd.bad_lines_seen.rate 0 1411387496,stats.counters.statsd.bad_lines_seen.count 0 1411387496,stats.counters.statsd.packets_received.rate 15 1411387496,stats.counters.statsd.packets_received.count 3 1411387496,stats.counters.fo-o.rate 1250 1411387496,stats.counters.fo-o.count 250 1411387496,stats.counters.b_ar.rate 1250 1411387496,stats.counters.b_ar.count 250 1411387496,stats.counters.foobar.rate 1250 1411387496,stats.counters.foobar.count 250 1411387496,stats.gauges.statsd.timestamp_lag -0.2 1411387496,stats.statsd.numStats 6 1411387496,stats.statsd.graphiteStats.calculationtime 0 1411387496,stats.statsd.processing_time 0 1411387496,stats.statsd.graphiteStats.last_exception 1411387495 1411387496,stats.statsd.graphiteStats.last_flush 1411387496 1411387496,stats.statsd.graphiteStats.flush_time 1 1411387496,stats.statsd.graphiteStats.flush_length 587 1411387496, \ No newline at end of file From 0eb2517b04d645fe58a5d65c32b6f63a0eca9658 Mon Sep 17 00:00:00 2001 From: shaylang Date: Tue, 23 Sep 2014 18:43:08 +0300 Subject: [PATCH 21/21] Metric name sanitization moved to graphite backend. it is now the responsibility of each backend developer to take care for sanitization --- backends/graphite.js | 9 --------- exampleConfig.js | 3 --- stats.js | 16 +--------------- 3 files changed, 1 insertion(+), 27 deletions(-) diff --git a/backends/graphite.js b/backends/graphite.js index 09e7b48..20b73f0 100644 --- a/backends/graphite.js +++ b/backends/graphite.js @@ -33,7 +33,6 @@ var prefixGauge; var prefixSet; var globalSuffix; var prefixStats; -var globalKeySanitize = true; // set up namespaces var legacyNamespace = true; @@ -100,13 +99,9 @@ var flush_stats = function graphite_flush(ts, metrics) { // Sanitize key for graphite if not done globally function sk(key) { - if (globalKeySanitize) { - return key; - } else { return key.replace(/\s+/g, '_') .replace(/\//g, '-') .replace(/[^a-zA-Z_\-0-9\.]/g, ''); - } }; for (key in counters) { @@ -253,10 +248,6 @@ exports.init = function graphite_init(startup_time, config, events, logger) { graphiteStats.flush_time = 0; graphiteStats.flush_length = 0; - if (config.keyNameSanitize !== undefined) { - globalKeySanitize = config.keyNameSanitize; - } - flushInterval = config.flushInterval; flush_counts = typeof(config.flush_counts) === "undefined" ? true : config.flush_counts; diff --git a/exampleConfig.js b/exampleConfig.js index 598f2ec..b0053a5 100644 --- a/exampleConfig.js +++ b/exampleConfig.js @@ -53,9 +53,6 @@ Optional Variables: deleteCounters: don't send values to graphite for inactive counters, as opposed to sending 0 [default: false] prefixStats: prefix to use for the statsd statistics data for this running instance of statsd [default: statsd] applies to both legacy and new namespacing - keyNameSanitize: sanitize all stat names on ingress [default: true] - If disabled, it is up to the backends to sanitize keynames - as appropriate per their storage requirements. console: prettyprint: whether to prettyprint the console backend diff --git a/stats.js b/stats.js index e515afe..4f9f21c 100644 --- a/stats.js +++ b/stats.js @@ -29,7 +29,6 @@ var backendEvents = new events.EventEmitter(); var healthStatus = config.healthStatus || 'up'; var old_timestamp = 0; var timestamp_lag_namespace; -var keyNameSanitize = true; // Load and init the backend from the backends/ directory. function loadBackend(config, name) { @@ -159,16 +158,6 @@ var stats = { } }; -function sanitizeKeyName(key) { - if (keyNameSanitize) { - return key.replace(/\s+/g, '_') - .replace(/\//g, '-') - .replace(/[^a-zA-Z_\-0-9\.]/g, ''); - } else { - return key; - } -} - // Global for the logger var l; @@ -191,9 +180,6 @@ config.configFile(process.argv[2], function (config) { counters[bad_lines_seen] = 0; counters[packets_received] = 0; - if (config.keyNameSanitize !== undefined) { - keyNameSanitize = config.keyNameSanitize; - } if (!serverLoaded) { // key counting var keyFlushInterval = Number((config.keyFlush && config.keyFlush.interval) || 0); @@ -218,7 +204,7 @@ config.configFile(process.argv[2], function (config) { l.log(metrics[midx].toString()); } var bits = metrics[midx].toString().split(':'); - var key = sanitizeKeyName(bits.shift()); + var key = bits.shift(); if (keyFlushInterval > 0) { if (! keyCounter[key]) {