diff --git a/DataHandler.py b/DataHandler.py index 3ca9df7..143f70f 100644 --- a/DataHandler.py +++ b/DataHandler.py @@ -138,7 +138,7 @@ def __operateNode__(self, nodeID, nodeData): nnameid = nname.replace(':', '') if nnameid == nodeGateway: if 'tq' in nvalue: - siteDict['averages']['gateway_tq'].append(nvalue['tq']) + siteDict['averages']['gateway_uplink_tq'].append(nvalue['tq']) elif nnameid == nodeGatewayNexthop: if 'tq' in nvalue: siteDict['averages']['gateway_nexthop_tq'].append(nvalue['tq']) diff --git a/GraphiteManager.py b/GraphiteManager.py deleted file mode 100644 index 42c3b33..0000000 --- a/GraphiteManager.py +++ /dev/null @@ -1,65 +0,0 @@ -# -# (c) 2015 dray -# -# This script is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License or any later version. -# -# This script is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY. See the -# GNU General Public License for more details. -# -# For a copy of the GNU General Public License -# see . -# - -import socket -import time - -class GraphiteManager: - def __init__(self,server,port,domain): - self.server = server - self.port = port - self.domain = domain - self.message = "" - - def prepareMessage(self, data): - for i in data['nodes']: - self.__addHieraDictMessage__(data['nodes'][i], "node.%s" % i) - try: - self.__addDictMessage__("nodes."+self.domain+".firmware.%s.count",data['firmwarecount']) - self.__addDictMessage__("nodes."+self.domain+".branch.%s.count", data['branchcount']) - self.__addDictMessage__("nodes."+self.domain+".hardware.%s.count", data['hardwarecount']) - - self.__addSingleMessage__("nodes."+self.domain+".autoupdate.count",data['autoupdate']) - self.__addSingleMessage__("nodes."+self.domain+".location.count",data['locationcount']) - self.__addSingleMessage__("nodes."+self.domain+".total.count",data['nodecount']) - self.__addSingleMessage__("nodes."+self.domain+".totalclient.count",data['totalclients']) - self.__addSingleMessage__("nodes."+self.domain+".clients_per_node",data['totalclients']/float(data['nodecount'])) - except: - pass - - def send(self): - sock = socket.socket() - sock.connect((self.server, int(self.port))) - sock.sendall(self.message.encode()) - sock.close() - pass - - def printout(self): - print(self.message) - - def __addSingleMessage__(self,key,value): - self.message += "%s %s %d\n" %(key,value, int(time.time())) - - def __addDictMessage__(self,key,dict): - for i in dict: - self.__addSingleMessage__(key % i, dict[i]) - - def __addHieraDictMessage__(self,data, path =''): - for k, v in data.iteritems(): - if isinstance(v, dict): - self.__addHieraDictMessage__(v, path + '.' + k if len(path) > 0 else k) - else: - self.__addSingleMessage__(path + '.' + k, v) - diff --git a/JsonManager.py b/JsonManager.py deleted file mode 100644 index 4999e7e..0000000 --- a/JsonManager.py +++ /dev/null @@ -1,316 +0,0 @@ -# -# (c) 2015 dray -# -# This script is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License or any later version. -# -# This script is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY. See the -# GNU General Public License for more details. -# -# For a copy of the GNU General Public License -# see . -# - -from subprocess import check_output -import json, sys, os.path, socket, zlib, ctypes -class JsonManager: - def __init__(self): - self.advStats = {} - self.json158 = [] - self.json159 = [] - self.json160 = [] - self.result = {} - self.__readAdvancedNodesFile__(os.path.dirname(os.path.realpath(__file__)) + '/advnodes') - - def getRespondd(self, interface, command): - _clib = ctypes.cdll.LoadLibrary("libc.so.6") - try: - if_id = _clib.if_nametoindex(interface) - except OSError: - return {} - - sock = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) - - message = 'GET ' + command - sock.sendto(message, ('ff02::2', 1001, 0, if_id)) - - sock.settimeout(0.75) # wait 750 ms w/o any response before finish - - # receive - responses = {} - while True: - try: - buffer, address = sock.recvfrom(2048) - except socket.timeout: - break - - try: - data = zlib.decompress(buffer, -15) - nodeinfo = json.loads(data.decode('utf-8'))[command] - responses[nodeinfo['node_id']] = nodeinfo - except (zlib.error, UnicodeDecodeError, ValueError): - continue - - - return responses - - def loadJsonFromRespondd(self, batif): - self.json158 = self.getRespondd(batif, 'nodeinfo') - self.json159 = self.getRespondd(batif, 'statistics') - self.json160 = self.getRespondd(batif, 'neighbours') - - - def processJson158(self): - self.result["autoupdate"] = 0 - for id in self.json158: - node = self.json158[id] - nodeID = id.replace(':','') - - # Check for advanced-stats feature - if nodeID not in self.advStats: - if 'advanced-stats' in node and 'store-stats' in node['advanced-stats'] and node['advanced-stats']['store-stats'] == True: - self.__addNode__(nodeID, True) - else: - self.__addNode__(nodeID, False) - - # Nodes/Firmware - if 'software' in node: - if 'firmware' in node['software']: - firmware = node['software']['firmware']['release'] - self.__incCounter__('firmwarecount',firmware) - if 'autoupdater' in node['software']: - if 'branch' in node['software']['autoupdater']: - branch = node['software']['autoupdater']['branch'] - self.__incCounter__('branchcount',branch) - - if node['software']['autoupdater']['enabled']: - self.__incCounter__('autoupdate') - - if 'hardware' in node: - hardware = node['hardware']['model'] - self.__incCounter__('hardwarecount',hardware) - - if 'location' in node: - self.__incCounter__('locationcount') - - # do advanced stats stuff in 158 - if self.advStats[nodeID]['enabled'] == True: - - # generate mapping for interface names - if 'network' in node and 'mesh' in node['network'] and 'bat0' in node['network']['mesh'] and 'interfaces' in node['network']['mesh']['bat0']: - self.advStats[nodeID]['if_mapping'] = {} - for k, v in node['network']['mesh']['bat0']['interfaces'].iteritems(): - for i, mac in enumerate(v): - self.advStats[nodeID]['if_mapping'][mac] = k + '_' + str(i) - - - - - self.result['nodecount'] = len(self.json158) - - - def processJson159(self): - self.result['nodes'] = {} - self.result['totalclients']=0 - for id in self.json159: - - node = self.json159[id] - nodeID = node['node_id'] - - - # Client/Node - - self.result['nodes'][nodeID] = {} - try: - if 'clients' in node: - self.result['nodes'][nodeID]["count"] = node['clients']['total'] - self.result['totalclients'] += node['clients']['total'] - except: - sys.stderr.write("Error %s" % sys.exc_info()[0]) - - try: - if nodeID in self.advStats and self.advStats[nodeID]['enabled'] == True: - self.result['nodes'][nodeID].update(self.processAdvancedStats159(node)) - except: - sys.stderr.write("Error %s" % sys.exc_info()[0]) - - - def processJson160(self): - for id, node in self.json160.iteritems(): - nodeID = id.replace(':','') - if nodeID in self.advStats and self.advStats[nodeID]['enabled'] == True: - try: - if 'wifi' in node: - self.result['nodes'][nodeID]['wifi'] = self.__wifiAndBatmanStats__(nodeID, node['wifi'], ['noise', 'inactive', 'signal']) - if 'batadv' in node: - self.result['nodes'][nodeID]['batadv'] = self.__wifiAndBatmanStats__(nodeID, node['batadv'], ['tq', 'lastseen']) - except: - pass -# sys.stderr.write("Error %s" % sys.exc_info()[0]) - - - def processAdvancedStats159(self, node): - advancedStats = {} - - #add data, where no procession or conversion is needed - entries = [ - 'uptime', - 'idletime', - 'loadavg', - [ 'memory', - [ - 'cached', - 'buffers', - 'total', - 'free' - ] - ], - [ 'clients', - [ - 'total', - 'wifi' - ] - ], - [ 'processes', - [ - 'running', - 'total' - ] - ], - 'rootfs_usage' - ] - - advancedStats.update(self.__cherryPickEntries__(node,entries)) - - # add traffic stats - if 'traffic' in node: - advancedStats['traffic'] = {} - if 'rx' in node['traffic'] and 'tx' in node['traffic']: - advancedStats['traffic']['node'] = self.__ifStats__(node['traffic']['rx'], node['traffic']['tx']) - if 'mgmt_rx' in node['traffic'] and 'mgmt_tx' in node['traffic']: - advancedStats['traffic']['managed'] = self.__ifStats__(node['traffic']['mgmt_rx'], node['traffic']['mgmt_tx']) - if 'forward' in node['traffic']: - advancedStats['traffic']['forward'] = self.__ifStats__(node['traffic']['forward']) - # add vpn stats - if 'mesh_vpn' in node: - advancedStats['mesh_vpn'] = self.__vpnStats__(node['mesh_vpn']) - if 'gateway' in node: - advancedStats['bat_gw_id'] = node['gateway'].replace(':','') - if 'gateway_nexthop' in node: - advancedStats['bat_gw_next_hop_id'] = node['gateway_nexthop'].replace(':','') - return advancedStats - - - def __vpnStats__(self,data): - dataStats = {} - if 'groups' in data: - for gname, group in data['groups'].iteritems(): - dataStats[gname] = {} - if 'peers' in group: - for pname, peer in group['peers'].iteritems(): - if peer and 'established' in peer: - dataStats[gname][pname] = peer['established'] - else: - dataStats[gname][pname] = 0 - return dataStats - - - def __ifStats__(self,rx,tx = None): - mapping = { - 'bytes' : 'if_octets', - 'dropped' : 'if_dropped', - 'packets' : 'if_packets' - } - ifaceStats = {} - for k, v in mapping.iteritems(): - if rx and k in rx or tx and k in tx: - ifaceStats[v] = {} - if rx and tx: - if k in rx: - ifaceStats[v]['rx'] = rx[k] - if k in tx: - ifaceStats[v]['tx'] = tx[k] - elif k in rx: - ifaceStats[v] = rx[k] - return ifaceStats - - - def __cherryPickEntries__(self, data, entries): - dataStats = {} - for entry in entries: - if isinstance(entry, list): - if entry[0] in data: - dataStats[entry[0]] = (self.__cherryPickEntries__(data[entry[0]], entry[1])) - else: - if entry in data: - dataStats[entry] = data[entry] - return dataStats - - - def __getIfName__(self, id, ifmac): - if ifmac in self.advStats[id]['if_mapping']: - return self.advStats[id]['if_mapping'][ifmac] - else: - return ifmac.replace(':', '_') - - - def __wifiAndBatmanStats__(self, id, data, keys): - dataStats = { - 'count' : 0 - } - #print("mark", self.result['nodes'][id]) - gwid = self.result['nodes'][id]['bat_gw_id'] if 'bat_gw_id' in self.result['nodes'][id] else None - for if_id, if_val in data.iteritems(): - if_id_print = self.__getIfName__(id, if_id) - if if_val and 'neighbours' in if_val: - for neigh_id, neigh_val in data[if_id]['neighbours'].iteritems(): - dataStats['count'] += 1 - if gwid == neigh_id.replace(':',''): - if 'gateways' not in dataStats: - dataStats['gateways'] = {'count' : 0} - dataStats['gateways']['count'] += 1 - dataStats['gateways'][neigh_id.replace(':','_')] = self.__cherryPickEntries__(neigh_val, keys) - else: - if if_id_print not in dataStats: - dataStats[if_id_print] = {'count' : 0} - dataStats[if_id_print]['count'] += 1 - dataStats[if_id_print][neigh_id.replace(':','_')] = self.__cherryPickEntries__(neigh_val, keys) - return dataStats - - - def __readAdvancedNodesFile__(self,filename): - if os.path.isfile(filename): - with open(filename) as f: - for line in f: - self.__addNode__(line.strip(),True) - - def __addNode__(self,nodeID,advStats = False): - if nodeID not in self.advStats: - self.advStats[nodeID] = { - 'enabled' : advStats - } - - def __incCounter__(self, key, value=None): - - if value is None: - if key not in self.result: - self.result[key] = 0 - self.result[key]+=1 - else: - value = self.___cleanstr___(value) - if key not in self.result: - self.result[key] = {} - if value in self.result[key]: - self.result[key][value]+=1 - else: - self.result[key][value]=1 - - - def ___cleanstr___(self, cleanstr): - specialChars = [" ","+",".","\\","/","-"] - for char in specialChars: - cleanstr = cleanstr.replace(char,"_") - cleanstr = cleanstr.replace(":","") - return cleanstr diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c82b64d --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Simon Wüllhorst + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/main.py b/main.py old mode 100644 new mode 100755 index e5a664c..63f689c --- a/main.py +++ b/main.py @@ -1,47 +1,51 @@ #!/usr/bin/env python3 # -*- coding: utf8 -*- -# -# (c) 2015 dray -# -# This script is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License or any later version. -# -# This script is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY. See the -# GNU General Public License for more details. -# -# For a copy of the GNU General Public License -# see . -# -# + +# MIT License + +# Copyright (c) 2017 Simon Wüllhorst + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. import argparse -from JsonManager import JsonManager -from GraphiteManager import GraphiteManager - -parser = argparse.ArgumentParser(description='This Script gets information about Freifunk Muenster') -parser.add_argument('--server', required=True, help='Server') -parser.add_argument('--port', required=True, help='Port', default=2003) -parser.add_argument('--batif', required=True, help='Batman interface', default='bat0') -parser.add_argument('--domain', help='Freifunk Domäne', default='legacy') -parser.add_argument('--local', help='Load local json files (alfred_158.json,alfred_159.json)', action='store_true') -parser.add_argument('--print-only', help='Print only', action='store_true') -args = parser.parse_args() - -jsonManager = JsonManager() -if args.local: - jsonManager.loadJson() -else: - jsonManager.loadJsonFromRespondd(args.batif) -jsonManager.processJson158() -jsonManager.processJson159() -jsonManager.processJson160() - -graphiteManager = GraphiteManager(args.server, args.port, args.domain) -graphiteManager.prepareMessage(jsonManager.result) - -if args.print_only: - graphiteManager.printout() -else: - graphiteManager.send() +from JsonHandler import JsonHandler +from DataHandler import DataHandler +from GraphiteHandler import GraphiteHandler + +def main(): + args = __parseArguments__() + config = JsonHandler('./config.json') + rawJson = JsonHandler(args.hopglass_raw) + handler = DataHandler(rawJson.data, config.data) + handler.convert() + graphiteHandler = GraphiteHandler(args.server, args.port) + graphiteHandler.prepareMessage(handler.domains, handler.nodes) + + +def __parseArguments__(): + parser = argparse.ArgumentParser(description='This Script is a link between Hopglass-Server and Graphite.') + parser.add_argument('--server', required=False, help='Graphite Server', default='127.0.0.1') + parser.add_argument('--port', required=False, help='Graphite Port', default=2003) + parser.add_argument('--hopglass-raw', help='Hopglass raw.json source.', default='./raw.json') + parser.add_argument('--print-only', help='Print only', action='store_true') + + return parser.parse_args() + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/test.py b/test.py deleted file mode 100755 index 63f689c..0000000 --- a/test.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf8 -*- - -# MIT License - -# Copyright (c) 2017 Simon Wüllhorst - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import argparse -from JsonHandler import JsonHandler -from DataHandler import DataHandler -from GraphiteHandler import GraphiteHandler - -def main(): - args = __parseArguments__() - config = JsonHandler('./config.json') - rawJson = JsonHandler(args.hopglass_raw) - handler = DataHandler(rawJson.data, config.data) - handler.convert() - graphiteHandler = GraphiteHandler(args.server, args.port) - graphiteHandler.prepareMessage(handler.domains, handler.nodes) - - -def __parseArguments__(): - parser = argparse.ArgumentParser(description='This Script is a link between Hopglass-Server and Graphite.') - parser.add_argument('--server', required=False, help='Graphite Server', default='127.0.0.1') - parser.add_argument('--port', required=False, help='Graphite Port', default=2003) - parser.add_argument('--hopglass-raw', help='Hopglass raw.json source.', default='./raw.json') - parser.add_argument('--print-only', help='Print only', action='store_true') - - return parser.parse_args() - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/update.sh b/update.sh deleted file mode 100755 index 977c294..0000000 --- a/update.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -parallel=0 -path=$(realpath $(dirname $0)) -for i in /proc/sys/net/ipv4/conf/bat*; do - ((parallel++)) - num=${i#*bat} - python $path/main.py --server=148.251.101.196 --port=2003 --domain=domaene-${num} --batif=bat${num} & - - if ((parallel >= 16)) - then - parallel=0 - wait - fi -done -wait