adding source files for Elasticsearch meetup at GrubHub 12-13-2016

bytes_in_per_backend.json

@@ -0,0 +1,16 @@
+[
+  {
+    "_id": "Bytes-in-per-Backend",
+    "_type": "visualization",
+    "_source": {
+      "title": "Bytes in per Backend",
+      "visState": "{\"title\":\"Bytes in per Backend\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(*,split='haproxy_stats.pxname:10',metric='avg:haproxy_stats.bin').derivative().label(label='$1 ',regex='.*pxname:(.*?)\\\\s.*$').precision(0).max(0)\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  }
+]

bytes_out_per_backend.json

@@ -0,0 +1,16 @@
+[
+  {
+    "_id": "Bytes-out-per-Backend",
+    "_type": "visualization",
+    "_source": {
+      "title": "Bytes out per Backend",
+      "visState": "{\"type\":\"timelion\",\"title\":\"Bytes out per Backend\",\"params\":{\"expression\":\".es(*,split='haproxy_stats.pxname:10',metric='avg:haproxy_stats.bout').derivative().label(label='$1 ',regex='.*pxname:(.*?)\\\\s.*$').precision(0).max(0)\",\"interval\":\"auto\"}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{}"
+      }
+    }
+  }
+]

connections_by_lb.json

@@ -0,0 +1,17 @@
+[
+  {
+    "_id": "Connections-by-LB",
+    "_type": "visualization",
+    "_source": {
+      "title": "Connections by LB",
+      "visState": "{\"title\":\"Connections by LB\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false,\"legendPosition\":\"right\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"haproxy_stats.scur\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"region\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"row\":false}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"http_poller_metadata.name\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "savedSearchId": "haproxy_all",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"filter\":[]}"
+      }
+    }
+  }
+]

connections_per_backend.json

@@ -0,0 +1,17 @@
+[
+  {
+    "_id": "Connections-per-Backend",
+    "_type": "visualization",
+    "_source": {
+      "title": "Connections per Backend",
+      "visState": "{\"title\":\"Connections per Backend\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":false,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{},\"legendPosition\":\"right\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"haproxy_stats.scur\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"region\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\",\"row\":false}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"haproxy_stats.pxname\",\"exclude\":{\"pattern\":\"bt_vip\"},\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "savedSearchId": "haproxy_all",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"filter\":[]}"
+      }
+    }
+  }
+]

connections_per_server.json

@@ -0,0 +1,17 @@
+[
+  {
+    "_id": "Connections-per-Tagserve",
+    "_type": "visualization",
+    "_source": {
+      "title": "Connections per Tagserve",
+      "visState": "{\"title\":\"Connections per Tagserve\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"defaultYExtents\":false,\"drawLinesBetweenPoints\":false,\"interpolate\":\"linear\",\"legendPosition\":\"right\",\"radiusRatio\":9,\"scale\":\"linear\",\"setYExtents\":false,\"shareYAxis\":true,\"showCircles\":true,\"smoothLines\":false,\"times\":[],\"yAxis\":{}},\"aggs\":[{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"1\",\"enabled\":true,\"type\":\"median\",\"schema\":\"metric\",\"params\":{\"field\":\"haproxy_stats.rate\",\"percents\":[50]}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"haproxy_stats.svname\",\"exclude\":{\"pattern\":\"(FRONT|BACK)END\"},\"size\":70,\"order\":\"desc\",\"orderBy\":\"1.50\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"radius\",\"params\":{\"field\":\"haproxy_stats.scur\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "savedSearchId": "haproxy_all",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"filter\":[]}"
+      }
+    }
+  }
+]

down_backend_server.json

@@ -0,0 +1,17 @@
+[
+  {
+    "_id": "Down-Backend-Servers",
+    "_type": "visualization",
+    "_source": {
+      "title": "Down Backend Servers",
+      "visState": "{\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"region\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"row\":false}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"haproxy_stats.pxname\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "savedSearchId": "Backend-Not-Ok",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"filter\":[]}"
+      }
+    }
+  }
+]

kibana-dashboard.json

@@ -0,0 +1,19 @@
+[
+  {
+    "_id": "US-Load-Balancer",
+    "_type": "dashboard",
+    "_source": {
+      "title": "US Load Balancer",
+      "hits": 0,
+      "description": "",
+      "panelsJSON": "[{\"col\":1,\"id\":\"Connections-by-LB\",\"panelIndex\":1,\"row\":1,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Connections-per-Backend\",\"panelIndex\":2,\"row\":1,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Connections-per-Tagserve\",\"panelIndex\":3,\"row\":4,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Down-Backend-Servers\",\"panelIndex\":4,\"row\":4,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Bytes-in-per-Backend\",\"panelIndex\":5,\"row\":8,\"size_x\":6,\"size_y\":2,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Bytes-out-per-Backend\",\"panelIndex\":6,\"row\":8,\"size_x\":6,\"size_y\":2,\"type\":\"visualization\"}]",
+      "optionsJSON": "{\"darkTheme\":true}",
+      "uiStateJSON": "{\"P-1\":{\"vis\":{\"legendOpen\":true}},\"P-3\":{\"vis\":{\"legendOpen\":true}},\"P-4\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}",
+      "version": 1,
+      "timeRestore": false,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"region:we1 OR region:ea1\"}}}]}"
+      }
+    }
+  }
+]

logstash.conf

@@ -0,0 +1,46 @@
+input {
+  http_poller {
+    urls => {
+      lb06 => "http://lb06/status;csv"
+    }
+    tags => haproxy_stats
+    codec => plain
+    metadata_target => http_poller_metadata
+    interval => 60
+  }
+}
+
+filter {
+  if "haproxy_stats" in [tags] {
+    split {}
+    # We can't read the haproxy csv header, so we define it statically
+    # This is because we're working line by line, and so have no header context
+    csv {
+       target => "haproxy_stats"
+       columns => [ pxname,svname,qcur,qmax,scur,smax,slim,stot,bin,bout,dreq,dresp,ereq,econ,eresp,wretr,wredis,status,weight,act,bck,chkfail,chkdown,lastchg,downtime,qlimit,pid,iid,sid,throttle,lbtot,tracked,type,rate,rate_lim,rate_max,check_status,check_code,check_duration,hrsp_1xx,hrsp_2xx,hrsp_3xx,hrsp_4xx,hrsp_5xx,hrsp_other,hanafail,req_rate,req_rate_max,req_tot,cli_abrt,srv_abrt,comp_in,comp_out,comp_byp,comp_rsp,lastsess,last_chk,last_agt,qtime,ctime,rtime,ttime ]
+    }
+    # Drop the haproxy CSV header, which always has this special value
+    if [haproxy_stats][pxname] == "# pxname" {
+      drop{}
+    }
+    # We no longer need the message field as the CSV filter has created separate
+    # fields for data.
+    mutate {
+      remove_field => message
+    }
+    # Same as the cast we did for apache
+    ruby {
+      code => "h=event['haproxy_stats']; h.each {|k,v| h[k] = v.to_f if v =~ /\A-?[0-9\.]+\Z/}"
+    }
+  }
+}
+
+output {
+  if "haproxy_stats" in [tags] {
+    elasticsearch {
+      hosts => ["es01:9200", "es02:9200", "es03:9200", "es04:9200"]
+      index => "haproxy-%{+YYYY.MM.dd}"
+    }
+  }
+}
+