-
Notifications
You must be signed in to change notification settings - Fork 3
/
Copy pathpcap_stats.py
executable file
·1015 lines (873 loc) · 49.9 KB
/
pcap_stats.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env python3
"""Print statistics of a pcap file or packets arriving on an interface."""
from __future__ import annotations
#Dedicated to the memory of Alan Paller, whose vision of a secure
#Internet gave thousands of us a chance to learn and grow.
__version__ = '0.0.63'
__author__ = 'William Stearns'
__copyright__ = 'Copyright 2021, William Stearns'
__credits__ = ['William Stearns']
__email__ = '[email protected]'
__license__ = 'GPL 3.0'
__maintainer__ = 'William Stearns'
__status__ = 'Production' #Prototype, Development or Production
import os
import sys
import time
import tempfile
import errno
import gzip #Lets us read from gzip-compressed pcap files
import bz2 #Lets us read from bzip2-compressed pcap files
import re #Regex matching on UUID hostnames
import binascii
from typing import List, Optional, Union, cast
from db_lib import add_to_db_dict, buffer_merges, select_key
try:
#from scapy.all import *
from scapy.all import ARP, DNS, Dot3, Ether, GRE, ICMP, IP, IPv6, LLC, Loopback, NBNSQueryResponse, NBTDatagram, Scapy_Exception, STP, TCP, UDP, sniff # pylint: disable=no-name-in-module
from scapy.config import conf
except ImportError:
sys.stderr.write('Unable to load the scapy library. Perhaps run sudo apt install python3-pip || sudo yum install python3-pip ; sudo pip3 install scapy ?\n')
sys.stderr.flush()
sys.exit(1)
try:
from scapy.contrib import erspan # pylint: disable=unused-import
erspan_loaded = True
except ImportError:
sys.stderr.write('Unable to load the erspan module from the scapy library contrib section. Perhaps run sudo apt install python3-pip || sudo yum install python3-pip ; sudo pip3 install scapy ?\n')
sys.stderr.flush()
erspan_loaded = False
#sys.exit(1)
try:
from passer_lib import DNS_extract, explode_ip, generate_meta_from_packet
except ImportError:
sys.stderr.write('Unable to load the passer_lib library.\n')
sys.stderr.flush()
sys.exit(1)
def debug_out(output_string: str) -> None:
"""Send debuging output to stderr."""
if cl_args['devel']:
sys.stderr.write(output_string + '\n')
sys.stderr.flush()
def mkdir_p(path: str) -> None:
"""Create an entire directory branch. Will not complain if the directory already exists."""
if not os.path.isdir(path):
try:
os.makedirs(path)
except OSError as exc:
if exc.errno == errno.EEXIST and os.path.isdir(path):
pass
elif exc.errno == errno.EEXIST:
sys.stderr.write(path + ' exists, so we cannot create it as a directory. Exiting.\n')
sys.stderr.flush()
sys.exit(1)
else:
raise
def force_string(raw_string: Union[bytes|str|List]) -> str:
"""Make sure the returned object is a string."""
if sys.version_info > (3, 0): #Python 3
if isinstance(raw_string, bytes):
retval = cast(str, raw_string.decode("utf-8", 'replace'))
elif isinstance(raw_string, str):
retval = cast(str, raw_string)
elif isinstance(raw_string, list):
retval = cast(str, ' '.join([force_string(listitem) for listitem in raw_string]))
#print(str(type(raw_string)))
#print("huh:" + str(raw_string))
#sys.exit()
else:
print(str(type(raw_string)))
print("huh:" + str(raw_string))
sys.exit()
retval = str(raw_string)
else:
retval = str(raw_string)
return retval
def open_bzip2_file_to_tmp_file(bzip2_filename: str) -> str:
"""Open up a bzip2 file to a temporary file and return that filename."""
tmp_fd, tmp_path = tempfile.mkstemp()
try:
with os.fdopen(tmp_fd, 'wb') as tmp_h, bz2.BZ2File(bzip2_filename, 'rb') as compressed_file:
for data in iter(lambda: compressed_file.read(100 * 1024), b''):
tmp_h.write(data)
return tmp_path
except:
sys.stderr.write("While expanding bzip2 file, unable to write to " + str(tmp_path) + ', exiting.\n')
raise
def open_gzip_file_to_tmp_file(gzip_filename: str) -> str:
"""Open up a gzip file to a temporary file and return that filename."""
tmp_fd, tmp_path = tempfile.mkstemp()
try:
with os.fdopen(tmp_fd, 'wb') as tmp_h, gzip.GzipFile(gzip_filename, 'rb') as compressed_file:
for data in iter(lambda: compressed_file.read(100 * 1024), b''):
tmp_h.write(data)
return tmp_path
except:
sys.stderr.write("While expanding gzip file, unable to write to " + str(tmp_path) + ', exiting.\n')
raise
def packet_layers(pkt) -> List:
"""Returns a list of packet layers."""
layers = []
counter = 0
while True:
layer = pkt.getlayer(counter)
if layer is not None:
#print(layer.name)
layers.append(layer.name)
else:
break
counter += 1
return layers
#Sample return ['Ethernet', 'IP', 'TCP']
def packet_len(packet, whichlayer) -> int: # pylint: disable=unused-argument
"""Finds the appropriate length of the packet based on user preference."""
#FIXME - only the IP/IPv6 option works at the moment.
#print(str(len(p)) + " _6_ " + str(p[IPv6].plen))
if cl_args['length'] == 'ip':
if packet.haslayer(IP):
#p[IP].len returns the IP header onwards, but not the ethernet header above it.
pack_len = packet[IP].len
elif packet.haslayer(IPv6):
#p[IPv6].plen returns the IP header onwards, but not the ethernet header above it.
pack_len = packet[IPv6].plen
elif packet.haslayer(ARP):
#p[ARP].len returns the ARP header onwards, but not the ethernet header above it.
pack_len = packet[ARP].plen
#FIXME - add else: here to handle other types
#elif cl_args['length'] == 'layer':
# pack_len = 0
# if packet.haslayer(IP):
# #p[IP].len returns the IP header onwards, but not the ethernet header above it.
# pack_len = packet[whichlayer].len
# elif packet.haslayer(IPv6):
# #p[IPv6].plen returns the IP header onwards, but not the ethernet header above it.
# pack_len = packet[whichlayer].plen
#elif cl_args['length'] == 'payload':
# pack_len = 0
#elif cl_args['length'] == 'ethernet':
# pack_len = 0
# #len(p) returns the full size of the packet including the ethernet header (and possibly 40 more bytes?)
else:
pack_len = 0
return pack_len
def inc_stats(one_label: str, length: int) -> None:
"""Increment the packet count and total size for this layer."""
if "p_stats" not in inc_stats.__dict__:
inc_stats.p_stats = {'count': [0, 0]} # type: ignore
if one_label not in inc_stats.p_stats: # type: ignore
inc_stats.p_stats[one_label] = [0, 0] # type: ignore
inc_stats.p_stats[one_label][0] += 1 # type: ignore
inc_stats.p_stats[one_label][1] += length # type: ignore
def processpacket(p):
"""Extract statistics from a single packet."""
if "field_filter" not in processpacket.__dict__:
processpacket.field_filter = {'count': '', 'ARP': 'arp',
'DHCP6 IA Address Option (IA TA or IA NA suboption)': 'udp port 546 or udp port 547', 'DHCP6 Preference Option': 'udp port 546 or udp port 547', 'DHCP6 Server Identifier Option': 'udp port 546 or udp port 547', 'DHCP6 Status Code Option': 'udp port 546 or udp port 547', 'DHCPv6 Confirm Message': 'udp port 546 or udp port 547', 'DHCPv6 Reply Message': 'udp port 546 or udp port 547',
'DNS': 'port 53 or udp port 5353 or udp port 5355', 'ESP': 'ip proto esp',
'ICMPv6 Neighbor Discovery - Neighbor Advertisement': 'icmp6[0] == 136', 'icmp6_136.0': 'icmp6[0] == 136', 'ICMPv6 Neighbor Discovery - Neighbor Solicitation': 'icmp6[0] == 135', 'icmp6_135.0': 'icmp6[0] == 135', 'ICMPv6 Neighbor Discovery - Router Advertisement': 'icmp6[0] == 134', 'icmp6_134.0': 'icmp6[0] == 134', 'ICMPv6 Neighbor Discovery - Router Solicitation': 'icmp6[0] == 133', 'IP': 'ip', 'ICMP': 'icmp', 'IPv6': 'ip6', 'ISAKMP': 'port 500 or udp port 4500',
'Loopback': '-i lo',
'NTPHeader': 'udp port 123', 'RIP_entry': 'udp port 520', 'RIP_header': 'udp port 520', 'SNMP': 'port 161 or port 162', 'TCP': 'tcp', 'TFTP_Read_Request': 'port 69', 'TFTP_opcode': 'port 69', 'UDP': 'udp', '802.1Q': 'vlan',
'TCP_FLAGS_': 'tcp[12:2] & 0x01ff = 0x0000', 'TCP_FLAGS_F': 'tcp[12:2] & 0x01ff = 0x0001', 'TCP_FLAGS_S': 'tcp[12:2] & 0x01ff = 0x0002', 'TCP_FLAGS_R': 'tcp[12:2] & 0x01ff = 0x0004', 'TCP_FLAGS_SR': 'tcp[12:2] & 0x01ff = 0x0006', 'TCP_FLAGS_RP': 'tcp[12:2] & 0x01ff = 0x000C',
'TCP_FLAGS_A': 'tcp[12:2] & 0x01ff = 0x0010', 'TCP_FLAGS_FA': 'tcp[12:2] & 0x01ff = 0x0011', 'TCP_FLAGS_SA': 'tcp[12:2] & 0x01ff = 0x0012', 'TCP_FLAGS_RA': 'tcp[12:2] & 0x01ff = 0x0014', 'TCP_FLAGS_FRA': 'tcp[12:2] & 0x01ff = 0x0015', 'TCP_FLAGS_PA': 'tcp[12:2] & 0x01ff = 0x0018', 'TCP_FLAGS_FPA': 'tcp[12:2] & 0x01ff = 0x0019', 'TCP_FLAGS_RPA': 'tcp[12:2] & 0x01ff = 0x001C',
'TCP_FLAGS_U': 'tcp[12:2] & 0x01ff = 0x0020', 'TCP_FLAGS_SU': 'tcp[12:2] & 0x01ff = 0x0022', 'TCP_FLAGS_FPU': 'tcp[12:2] & 0x01ff = 0x0029', 'TCP_FLAGS_FSPU': 'tcp[12:2] & 0x01ff = 0x002b',
'TCP_FLAGS_SAU': 'tcp[12:2] & 0x01ff = 0x0032', 'TCP_FLAGS_FSRPAU': 'tcp[12:2] & 0x01ff = 0x003f',
'TCP_FLAGS_AE': 'tcp[12:2] & 0x01ff = 0x0050', 'TCP_FLAGS_FAE': 'tcp[12:2] & 0x01ff = 0x0051', 'TCP_FLAGS_SAE': 'tcp[12:2] & 0x01ff = 0x0052',
'TCP_FLAGS_AC': 'tcp[12:2] & 0x01ff = 0x0090', 'TCP_FLAGS_RAC': 'tcp[12:2] & 0x01ff = 0x0094', 'TCP_FLAGS_SPAC': 'tcp[12:2] & 0x01ff = 0x0095', 'TCP_FLAGS_PAC': 'tcp[12:2] & 0x01ff = 0x0098',
'TCP_FLAGS_SEC': 'tcp[12:2] & 0x01ff = 0x00c2', 'TCP_FLAGS_FSPEC': 'tcp[12:2] & 0x01ff = 0x00cb',
'TCP_FLAGS_SAEC': 'tcp[12:2] & 0x01ff = 0x00d2',
'TCP_FLAGS_AN': 'tcp[12:2] & 0x01ff = 0x0110', 'TCP_FLAGS_FAN': 'tcp[12:2] & 0x01ff = 0x0111', 'TCP_FLAGS_RAN': 'tcp[12:2] & 0x01ff = 0x0114',
'TCP_FLAGS_FSRPAUEN': 'tcp[12:2] & 0x01ff = 0x017f'}
#Scapy flags
#N NS 0x0100 (ECN Nonce - concealment protection. See https://en.wikipedia.org/wiki/Transmission_Control_Protocol)
#C CWR 0x80
#E ECN 0x40
#U URG 0x20
#A ACK 0x10
#P PSH 0x08
#R RST 0x04
#S SYN 0x02
#F FIN 0x01
if "tcp_server_ports" not in processpacket.__dict__:
processpacket.tcp_server_ports = [7, 13, 21, 22, 23, 25, 53, 79, 80, 88, 102, 110, 111, 113, 135, 139, 143, 389, 443, 445, 465, 514, 631, 902, 990, 993, 995, 1001, 1119, 1433, 1521, 1723, 2222, 2598, 3128, 3306, 3389, 3724, 5000, 5001, 5060, 5223, 5228, 5432, 5601, 5900, 7070, 7680, 8008, 8009, 8080, 8088, 8333, 8443, 9200, 9443, 13000, 49152]
if "udp_server_ports" not in processpacket.__dict__:
processpacket.udp_server_ports = [7, 53, 67, 123, 137, 138, 192, 443, 1900, 2190, 5002, 5353, 5355, 8200, 16384, 16385, 16386, 17500, 19305, 56833, 60682, 62988]
if "local_ips" not in processpacket.__dict__:
processpacket.local_ips = set([])
if "minstamp" not in processpacket.__dict__:
processpacket.minstamp = None #None, or float holding the minimum timestamp (usually from the first packet) of any encountered packet
if "maxstamp" not in processpacket.__dict__:
processpacket.maxstamp = None #None, or float holding the maximum timestamp (usually from the last packet) of any encountered packet
if "ipv4s_for_mac" not in processpacket.__dict__:
processpacket.ipv4s_for_mac = {} #Key is a mac address, value is a set of IPv4 address strings
if "ipv6s_for_mac" not in processpacket.__dict__:
processpacket.ipv6s_for_mac = {} #Key is a mac address, value is a set of IPv4 address strings
if "debug_shown" not in processpacket.__dict__:
processpacket.debug_shown = [] #List of strings we've already shown
#We reinstate this to hold all the writes for hostnames, to be later submitted with add_to_db_dict
if 'dns_for_ip' not in processpacket.__dict__:
processpacket.dns_for_ip = {}
#This key_value_dict is in this form:
#{
# key1: [value1, value2, value3...],
# key2: [value4],
# key3: [value5, value6]
#}
if 'ports_for_ip' not in processpacket.__dict__: #Dictionary; keys are IP addresses, values are sets of ports used by this IP (specifically, the ports at the IP's end of the connection)
processpacket.ports_for_ip = {}
if 'cast_type' not in processpacket.__dict__: #Dictionary; keys are IP addresses, values are one of 'broadcast', 'multicast', (or unicast, though we don't remember this as it's the default.)
processpacket.cast_type = {}
p_layers = packet_layers(p)
prefs = {'nxdomain': False, 'devel': cl_args['devel'], 'quit': False}
dests = {'unhandled': None}
meta = generate_meta_from_packet(p, prefs, dests)
if p.haslayer(IP):
i_layer = p.getlayer(IP)
p_len = packet_len(p, IP)
proto = str(i_layer.proto)
ttl = int(i_layer.ttl)
sIP = i_layer.src
dIP = i_layer.dst
label = 'ip4_' + sIP
inc_stats(label, p_len)
processpacket.field_filter[label] = 'host ' + sIP
label = 'ip4_' + dIP
inc_stats(label, p_len)
processpacket.field_filter[label] = 'host ' + dIP
elif p.haslayer(IPv6):
i_layer = p.getlayer(IPv6)
p_len = packet_len(p, IPv6)
proto = str(i_layer.nh)
ttl = int(i_layer.hlim)
sIP = i_layer.src
dIP = i_layer.dst
label = 'ip6_' + sIP
inc_stats(label, p_len)
processpacket.field_filter[label] = 'ip6 host ' + sIP
label = 'ip6_' + dIP
inc_stats(label, p_len)
processpacket.field_filter[label] = 'ip6 host ' + dIP
elif p.haslayer(ARP):
i_layer = None
p_len = packet_len(p, ARP)
proto = None
ttl = -1
sIP = p.getlayer(ARP).psrc
dIP = p.getlayer(ARP).pdst
elif (p.haslayer(Ether) and p[Ether].type == 0x888E): #EAPOL packet; PAE=0x888E
i_layer = None
p_len = 0 #FIXME
proto = None
ttl = -1
sIP = None
dIP = None
elif p.haslayer(LLC) or p.haslayer(STP):
i_layer = None
p_len = 0 #FIXME
proto = None
ttl = -1
sIP = None
dIP = None
elif p_layers == ['Ethernet', 'Raw']:
i_layer = None
p_len = 0 #FIXME
proto = None
ttl = -1
sIP = None
dIP = None
else:
if p.haslayer(Ether):
print("Ethernet type")
print(p[Ether].type)
p.show()
sys.exit(99)
i_layer = None
p_len = 0 #FIXME
proto = None
ttl = -1
sIP = None
dIP = None
inc_stats('count', p_len)
if sIP and sIP not in ('0.0.0.0', '::') and not sIP.startswith(('169.254.', 'fe80:')) and p.haslayer(Ether): #We remember all the IPv4 and IPv6 addresses associated with a particular mac to decide later whether a mac address is a router.
sMAC = p.getlayer(Ether).src
if ':' in sIP:
if sMAC not in processpacket.ipv6s_for_mac:
processpacket.ipv6s_for_mac[sMAC] = set([])
processpacket.ipv6s_for_mac[sMAC].add(sIP)
else:
if sMAC not in processpacket.ipv4s_for_mac:
processpacket.ipv4s_for_mac[sMAC] = set([])
processpacket.ipv4s_for_mac[sMAC].add(sIP)
for a_layer in p_layers:
if a_layer not in ignore_layers:
inc_stats(a_layer, p_len)
if p.haslayer(Ether):
if p[Ether].dst == 'ff:ff:ff:ff:ff:ff':
label = 'ethernet_broadcast'
inc_stats(label, p_len)
processpacket.field_filter[label] = 'ether broadcast'
if dIP and (p.haslayer(IPv6) or p.haslayer(IP)):
processpacket.cast_type[dIP] = 'broadcast'
elif p[Ether].dst.startswith(('01:00:5e:', '33:33:', '01:80:c2:')):
label = 'ethernet_multicast'
inc_stats(label, p_len)
processpacket.field_filter[label] = 'ether multicast'
if dIP and (p.haslayer(IPv6) or p.haslayer(IP)):
processpacket.cast_type[dIP] = 'multicast'
else:
label = 'ethernet_unicast'
inc_stats(label, p_len)
processpacket.field_filter[label] = 'not (ether broadcast) and not (ether multicast)'
elif p.haslayer(Loopback) or p.haslayer(Dot3):
pass
else:
#p.show()
#sys.exit(34)
label = 'non_ethernet'
inc_stats(label, p_len)
#processpacket.field_filter[label] = '...' #unsure
if p.haslayer(DNS) and (isinstance(p[DNS], DNS)):
dns_tuples = DNS_extract(p, meta, prefs, dests)
#print(str(dns_tuples))
for one_tuple in dns_tuples:
if one_tuple[0] == 'DN':
if one_tuple[2] in ('A', 'AAAA', 'PTR', 'CNAME'):
if one_tuple[1] and one_tuple[3]: #Check that the IP and hostname fields are not empty
ip_string = force_string(one_tuple[1])
dns_hostname = force_string(one_tuple[3])
if ip_string not in processpacket.dns_for_ip:
processpacket.dns_for_ip[ip_string] = []
if dns_hostname not in processpacket.dns_for_ip[ip_string]:
processpacket.dns_for_ip[ip_string].append(dns_hostname)
#FIXME - keep above, as below is too many calls and slows down the program
#buffer_merges(ip_hostnames_db, ip_string, [dns_hostname], 50)
#else:
# print(str(one_tuple))
elif one_tuple[0] in ('US', 'UC', 'TS', 'IP'):
pass
else:
debug_out(str(one_tuple))
#Now that we've added any dns entries, let's see if the source or dest IPs could be converted to hostnames so we have hostname entries too.
if sIP and sIP in processpacket.dns_for_ip:
for one_name in processpacket.dns_for_ip[sIP]:
label = 'name_' + one_name
inc_stats(label, p_len)
processpacket.field_filter[label] = 'host ' + one_name
#FIXME - keep above, because select_key below is pulling entries that haven't been stored yet.
#if sIP and ip_hostnames_db:
# all_hostnames = select_key(ip_hostnames_db[0], sIP) #We only generate these "name_..." entries for recently added hostnames, not everything in the archives
# if all_hostnames:
# for one_name in all_hostnames:
# label = 'name_' + one_name
# inc_stats(label, p_len)
# processpacket.field_filter[label] = 'host ' + one_name
if dIP and dIP in processpacket.dns_for_ip:
for one_name in processpacket.dns_for_ip[dIP]:
label = 'name_' + one_name
inc_stats(label, p_len)
processpacket.field_filter[label] = 'host ' + one_name
#FIXME - keep above, because select_key below is pulling entries that haven't been stored yet.
#if dIP and ip_hostnames_db:
# all_hostnames = select_key(ip_hostnames_db[0], dIP) #We only generate these "name_..." entries for recently added hostnames, not everything in the archives
# if all_hostnames:
# for one_name in all_hostnames:
# label = 'name_' + one_name
# inc_stats(label, p_len)
# processpacket.field_filter[label] = 'host ' + one_name
if sIP != '::':
if ttl == 255: #The system is sending a broadcast - it must be local
processpacket.local_ips.add(sIP)
elif ttl in (64, 128): #The ttl appears to be what it would be set to if it was on the local network. Flag as a local IP.
processpacket.local_ips.add(sIP)
elif ttl == 1 and p.haslayer(UDP) and p.getlayer(UDP).dport in (5353, 5355): #The ttl appears to be what it would be set to if it was on the local network. Flag as a local IP.
processpacket.local_ips.add(sIP)
if p.haslayer(ARP):
if p.getlayer(ARP).op == 1: #1 is a request ("who-has")
processpacket.local_ips.add(sIP)
#Note, the destination IP is likely "broadcast" in a request
elif p.getlayer(ARP).op == 2: #2 is a reply ("is-at")
processpacket.local_ips.add(sIP)
processpacket.local_ips.add(dIP)
else:
p.show()
sys.exit(98)
#Good for debugging
#if sIP and ttl != -1:
# label = 'ipttl_' + sIP + '_' + str(ttl)
# inc_stats(label, p_len)
# if ':' in sIP:
# processpacket.field_filter[label] = 'src host ' + sIP + ' and ip6[7] = ' + str(ttl)
# else:
# processpacket.field_filter[label] = 'src host ' + sIP + ' and ip[8] = ' + str(ttl)
p_epoch: float = p.time #Seconds and microseconds since the epoch of this packet
if processpacket.minstamp is None or p_epoch < processpacket.minstamp:
processpacket.minstamp = p_epoch
if processpacket.maxstamp is None or p_epoch > processpacket.maxstamp:
processpacket.maxstamp = p_epoch
#Details: https://stackoverflow.com/questions/46276152/scapy-timestamp-measurement-for-outgoing-packets
#p_gmt = time.gmtime(p_epoch) #Same timestamp, but in struct_time format for GMT
#print(p_gmt) #time.struct_time(tm_year=2021, tm_mon=7, tm_mday=20, tm_hour=21, tm_min=41, tm_sec=3, tm_wday=1, tm_yday=201, tm_isdst=0)
#print(time.asctime(p_gmt)) #Tue Jul 20 21:41:03 2021
if p.haslayer(TCP):
t_layer = p.getlayer(TCP)
#p.show()
#sys.exit(97)
if t_layer.flags == 'S' and t_layer.dport not in processpacket.tcp_server_ports: #Following blocks try to identify which end is the "server" port.
debug_out("Adding " + str(t_layer.dport) + " to tcp_server_ports")
processpacket.tcp_server_ports.append(t_layer.dport)
elif t_layer.flags == 'SA' and t_layer.sport not in processpacket.tcp_server_ports:
debug_out("Adding " + str(t_layer.sport) + " to tcp_server_ports")
processpacket.tcp_server_ports.append(t_layer.sport)
if t_layer.sport in processpacket.tcp_server_ports:
label = 'tcp_' + str(t_layer.sport)
inc_stats(label, p_len)
processpacket.field_filter[label] = 'tcp port ' + str(t_layer.sport)
elif t_layer.dport in processpacket.tcp_server_ports:
label = 'tcp_' + str(t_layer.dport)
inc_stats(label, p_len)
processpacket.field_filter[label] = 'tcp port ' + str(t_layer.dport)
#elif t_layer.sport in tcp_ignore_ports and t_layer.dport in tcp_ignore_ports: #Was used for early troubleshooting, no longer needed.
# pass
else:
warning = "No tcp server port: " + str(t_layer.sport) + " " + str(t_layer.dport)
if warning not in processpacket.debug_shown:
debug_out(warning)
processpacket.debug_shown.append(warning)
#p.show()
#sys.exit(96)
if sIP and sIP != '0.0.0.0':
if sIP not in processpacket.ports_for_ip:
processpacket.ports_for_ip[sIP] = set()
processpacket.ports_for_ip[sIP].add('tcp_' + str(t_layer.sport))
if dIP and dIP != '0.0.0.0':
if dIP not in processpacket.ports_for_ip:
processpacket.ports_for_ip[dIP] = set()
processpacket.ports_for_ip[dIP].add('tcp_' + str(t_layer.dport))
label = 'TCP_FLAGS_' + str(t_layer.flags)
inc_stats(label, p_len)
elif p.haslayer(UDP):
u_layer = p.getlayer(UDP)
if u_layer.sport in processpacket.udp_server_ports: #Following blocks try to identify which end is the "server" port.
label = 'udp_' + str(u_layer.sport)
inc_stats(label, p_len)
processpacket.field_filter[label] = 'udp port ' + str(u_layer.sport)
elif u_layer.dport in processpacket.udp_server_ports:
label = 'udp_' + str(u_layer.dport)
inc_stats(label, p_len)
processpacket.field_filter[label] = 'udp port ' + str(u_layer.dport)
elif u_layer.sport >= 33434 and u_layer.sport < 33524: #Special case traceroute if we didn't find it in the fixed ports above
label = 'udp_' + str(u_layer.sport)
inc_stats(label, p_len)
processpacket.field_filter[label] = 'udp port ' + str(u_layer.sport)
elif u_layer.dport >= 33434 and u_layer.dport < 33524:
label = 'udp_' + str(u_layer.dport)
inc_stats(label, p_len)
processpacket.field_filter[label] = 'udp port ' + str(u_layer.dport)
#elif u_layer.sport in udp_ignore_ports and u_layer.dport in udp_ignore_ports: #Was used for early troubleshooting, no longer needed.
# pass
else:
label = 'udp_' + str(u_layer.sport)
inc_stats(label, p_len)
processpacket.field_filter[label] = 'udp port ' + str(u_layer.sport)
label = 'udp_' + str(u_layer.dport)
inc_stats(label, p_len)
processpacket.field_filter[label] = 'udp port ' + str(u_layer.dport)
#debug_out("No udp server port")
#p.show()
#sys.exit(95)
if sIP and sIP != '0.0.0.0':
if sIP not in processpacket.ports_for_ip:
processpacket.ports_for_ip[sIP] = set()
processpacket.ports_for_ip[sIP].add('udp_' + str(u_layer.sport))
if dIP and dIP != '0.0.0.0':
if dIP not in processpacket.ports_for_ip:
processpacket.ports_for_ip[dIP] = set()
processpacket.ports_for_ip[dIP].add('udp_' + str(u_layer.dport))
#To test, run "nmblookup some_netbios_hostname" or "smbutil lookup some_netbios_hostname"
if u_layer.sport == 137 and p.haslayer(NBNSQueryResponse):
netbios_hostname = p[NBNSQueryResponse].RR_NAME.rstrip().rstrip(nullbyte).decode('UTF-8')
netbios_address = p[NBNSQueryResponse].NB_ADDRESS.rstrip() #Apparently .decode('UTF-8') is not needed for a string
buffer_merges(ip_netbios_db, netbios_address, [netbios_hostname], 50)
if u_layer.sport == 138 and p.haslayer(NBTDatagram):
netbios_hostname = p[NBTDatagram].SourceName.rstrip().decode('UTF-8')
buffer_merges(ip_netbios_db, sIP, [netbios_hostname], 50)
elif p.haslayer(ICMP):
i_layer = p.getlayer(ICMP)
label = 'icmp_' + str(i_layer.type) + '.' + str(i_layer.code)
inc_stats(label, p_len)
processpacket.field_filter[label] = 'icmptype = ' + str(i_layer.type) + ' and icmpcode = ' + str(i_layer.code)
#p.show()
#sys.exit(94)
#IPv6 doesn't have a dedicated ICMPv6 layer, so we need to key off the IPv6 next_header value of 58 for ICMPv6
elif p.haslayer(IPv6) and p.getlayer(IPv6).nh == 58:
i_layer = p.getlayer('IPv6').payload
label = 'icmp6_' + str(i_layer.type) + '.' + str(i_layer.code)
inc_stats(label, p_len)
#processpacket.field_filter[label] = 'icmptype = ' + str(i_layer.type) + ' and icmpcode = ' + str(i_layer.code) #I don't believe this works on ipv6
#If we want to get down into the individual icmp types:
#debug_out(label)
#if i_layer.type == 1: #'ICMPv6 Destination Unreachable'
# debug_out('dest_unreach')
#elif i_layer.type == 3: #'ICMPv6 Time Exceeded'
# debug_out('time exceeded')
#elif i_layer.type == 134: #'ICMPv6 Neighbor Discovery - Router Advertisement'
# debug_out('router_adv')
#elif i_layer.type == 135: #'ICMPv6 Neighbor Discovery - Neighbor Solicitation'
# debug_out('neighbor_sol')
#elif i_layer.type == 136: #'ICMPv6 Neighbor Discovery - Neighbor Advertisement'
# debug_out('neighbor_adv')
#else:
# p.show()
# debug_out(p_layers)
# sys.exit(93)
elif p.haslayer(ARP) or p.haslayer(LLC):
pass
elif p.haslayer(IP) and p[IP].proto == 47 and p.haslayer(GRE):
#FIXME - we're not parsing the _inner_ IP layer - see passer.py IPv4/UDPv4/ipencap for approach.
label = 'proto_' + str(proto) + "_" + str(p[GRE].proto)
inc_stats(label, p_len)
processpacket.field_filter[label] = 'ip proto ' + str(proto)
if p[GRE].proto == 2048: #0x800==2048==IPv4
if p[GRE].payload:
encap_packet = p[GRE].payload
processpacket(encap_packet)
#p.show()
#sys.exit(31)
elif p[GRE].proto == 35006: #0x88be==35006==ERSPAN_type_II
#https://datatracker.ietf.org/doc/html/draft-foschiano-erspan-00
pass
elif p[GRE].proto == 8939: #0x22eb==8939==ERSPAN_type_III
#https://datatracker.ietf.org/doc/html/draft-foschiano-erspan-00
pass
elif proto:
label = 'proto_' + str(proto)
inc_stats(label, p_len)
processpacket.field_filter[label] = 'ip proto ' + str(proto)
elif p.haslayer(IPv6): #use "ip6 proto"
pass #FIXME
elif p_layers == ['Ethernet', 'Raw']:
pass
elif (p.haslayer(Ether) and p[Ether].type == 0x888E): #EAPOL packet; PAE=0x888E
pass
else:
debug_out("Non-udp-tcp")
p.show()
sys.exit(92)
def hints_for(proto_desc: str, list_of_hints: List) -> str:
"""For a given protocol, return the appropriate hint information to go in field 5 of the output."""
hint_return = ''
if proto_desc in hints:
hint_return = hints[proto_desc]
elif proto_desc.startswith(('ip4_169.254.',)):
hint_return = 'link_local/unable_to_get_address'
elif proto_desc.startswith(('ip6_fe80:')):
hint_return = 'link_local_address'
elif proto_desc.startswith(('ip4_10.', 'ip4_172.16.', 'ip4_172.17.', 'ip4_172.18.', 'ip4_172.19.', 'ip4_172.20.', 'ip4_172.21.', 'ip4_172.22.', 'ip4_172.23.', 'ip4_172.24.', 'ip4_172.25.', 'ip4_172.26.', 'ip4_172.27.', 'ip4_172.28.', 'ip4_172.29.', 'ip4_172.30.', 'ip4_172.31.', 'ip4_192.168.')):
hint_return = 'rfc1918/reserved'
elif proto_desc.startswith(('ip4_17.')):
hint_return = 'apple'
elif proto_desc.startswith(('ip4_73.')):
hint_return = 'comcast'
for one_hint in list_of_hints:
if hint_return and one_hint:
hint_return += ' ' + one_hint
elif one_hint:
hint_return = one_hint
return hint_return
def print_stats(mincount_to_show: int, minsize_to_show: int, out_format: str, source_string: str) -> None:
"""Show statistics"""
if "p_stats" in inc_stats.__dict__:
if out_format == 'html':
print('<html>')
print('<head>')
print('<title>pcap_stats for ' + source_string + '</title>')
print('</head>')
print('<body>')
print('<table border=1>')
print('<tr><th colspan=6 bgcolor="#ffffff">Pcap Statistics for ' + source_string + '</th></tr>')
print("<tr><th colspan=5>Begin_time: " + time.asctime(time.gmtime(int(processpacket.minstamp))) + ", End_time: " + time.asctime(time.gmtime(int(processpacket.maxstamp))) + ", Elapsed_time: " + str(processpacket.maxstamp - processpacket.minstamp) + " seconds</th></tr>") # type: ignore
print('<tr><th>Count</th><th>Bytes</th><th>Description</th><th>BPF expression</th><th>Hint</th></tr>')
#print(inc_stats.p_stats)
#print("Local_IPs: " + str(sorted(processpacket.local_ips)))
elif out_format == 'ascii':
print("Begin_time: " + time.asctime(time.gmtime(int(processpacket.minstamp)))) # type: ignore
print("End_time: " + time.asctime(time.gmtime(int(processpacket.maxstamp)))) # type: ignore
print("Elapsed_time: " + str(processpacket.maxstamp - processpacket.minstamp)) # type: ignore
for one_key in sorted(inc_stats.p_stats.keys()): # type: ignore
hints_list: List = []
desc: str = one_key.replace(' ', '_')
orig_ip: str = desc.replace('ip4_', '').replace('ip6_', '')
full_ip = explode_ip(orig_ip, {}, {})
if desc.startswith(('ip4_', 'ip6_')) and orig_ip in processpacket.ports_for_ip: # type: ignore
if len(processpacket.ports_for_ip[orig_ip]) == 1: # type: ignore
for sole_port in processpacket.ports_for_ip[orig_ip]: # type: ignore
break #Just retrieve the sole entry in the set
hints_list.append('sole_port:' + str(sole_port)) # pylint: disable=undefined-loop-variable
elif len(processpacket.ports_for_ip[orig_ip]) == 2 and 'udp_53' in processpacket.ports_for_ip[orig_ip] and 'tcp_53' in processpacket.ports_for_ip[orig_ip]: # type: ignore
hints_list.append('sole_port:udp_53_and_tcp_53')
else:
hints_list.append(str(len(processpacket.ports_for_ip[orig_ip])) + ' ports') # type: ignore
if desc.startswith(('ip4_', 'ip6_')) and orig_ip in processpacket.cast_type: # type: ignore
hints_list.append('cast:' + processpacket.cast_type[orig_ip]) # type: ignore
##FIXME - break into NB: (queried) and nb: (historic) (still needed?)
if ip_netbios_db:
netbios_hostname_list = select_key(ip_netbios_db, full_ip)
if netbios_hostname_list:
hints_list.append('NB:' + str(netbios_hostname_list))
#FIXME - break into DNS: (queried) and dns: (historic) by dropping next block entirely... (still needed?)
if ip_hostnames_db:
all_hostnames = select_key(ip_hostnames_db[0], full_ip)
else:
all_hostnames = []
if all_hostnames:
#Display recent hostnames.
if display_uuid_hosts:
orig_ip_list = all_hostnames
else:
orig_ip_list = [x for x in all_hostnames if not re.match(uuid_match, x)]
if orig_ip_list:
hints_list.append('DNS:' + str(orig_ip_list))
recent_lookup_set = set(orig_ip_list)
else:
recent_lookup_set = set()
if len(ip_hostnames_db) > 1: #Note, this only handles case of [current, archive], not with multiple archive dbs
old_hostnames = select_key(ip_hostnames_db[1], full_ip)
if old_hostnames:
#Display any archive hostnames.
if display_uuid_hosts:
orig_ip_list = old_hostnames
else:
orig_ip_list = [x for x in old_hostnames if not re.match(uuid_match, x)]
if orig_ip_list:
#Remove recent dns entries from this so we show just not-recently-looked-up hostnames after "dns:"
just_old_hostnames = set(orig_ip_list) - recent_lookup_set
if just_old_hostnames:
hints_list.append('dns:' + str(list(just_old_hostnames)))
if inc_stats.p_stats[one_key][0] >= mincount_to_show and inc_stats.p_stats[one_key][1] >= minsize_to_show: # type: ignore
if orig_ip in processpacket.local_ips: # type: ignore
hints_list.insert(0, 'local')
for _, ipv4_set in processpacket.ipv4s_for_mac.items(): # type: ignore #_ was originally one_mac, but we don't actually use it.
if orig_ip in ipv4_set and len(ipv4_set) > 10:
hints_list.insert(1, 'ipv4router:' + str(list(ipv4_set)[0:10]) + '..., ' + str(len(ipv4_set)) + ' entries.')
elif orig_ip in ipv4_set and len(ipv4_set) > 1:
hints_list.insert(1, 'ipv4router:' + str(ipv4_set))
for _, ipv6_set in processpacket.ipv6s_for_mac.items(): # type: ignore
if orig_ip in ipv6_set and len(ipv6_set) > 10:
hints_list.insert(1, 'ipv6router:' + str(list(ipv6_set)[0:10]) + '..., ' + str(len(ipv6_set)) + ' entries.')
elif orig_ip in ipv6_set and len(ipv6_set) > 1:
hints_list.insert(1, 'ipv6router:' + str(ipv6_set))
try:
if out_format == 'html':
print("<tr><td align=right>{0:}</td><td align=right>{1}</td><td>{2}</td><td>{3}</td><td>{4}</td></tr>".format(inc_stats.p_stats[one_key][0], inc_stats.p_stats[one_key][1], desc, processpacket.field_filter.get(one_key, ''), hints_for(desc, hints_list))) # type: ignore # pylint: disable=consider-using-f-string
elif out_format == 'ascii':
print("{0:>10d} {1:>13d} {2:60s} {3:48s} {4:30s}".format(inc_stats.p_stats[one_key][0], inc_stats.p_stats[one_key][1], desc, processpacket.field_filter.get(one_key, ''), hints_for(desc, hints_list))) # type: ignore # pylint: disable=consider-using-f-string
except BrokenPipeError:
sys.exit(0)
if out_format == 'html':
print('</table>')
print('</body></html>')
else:
sys.stderr.write('It does not appear any packets were read. Exiting.\n')
sys.stderr.flush()
def process_packet_source(if_name: Optional[str], pcap_source: Optional[str], user_args: dict):
"""Process the packets in a single source file, interface, or stdin."""
source_file = None
close_temp = False
delete_temp = False
#We have an interface to sniff on
if if_name:
debug_out('Reading packets from interface ' + if_name)
try:
if user_args['count']:
sniff(store=0, iface=if_name, filter=user_args['bpf'], count=user_args['count'], prn=lambda x: processpacket(x)) # pylint: disable=unnecessary-lambda
else:
sniff(store=0, iface=if_name, filter=user_args['bpf'], prn=lambda x: processpacket(x)) # pylint: disable=unnecessary-lambda
except ((Scapy_Exception, PermissionError)):
sys.stderr.write("Unable to open interface " + str(if_name) + ' . Permission error? Perhaps runs as root or under sudo? Exiting.\n')
raise
#Read from stdin
elif pcap_source in ('-', None, ''):
debug_out('Reading packets from stdin.')
tmp_packets = tempfile.NamedTemporaryFile(delete=True) # pylint: disable=consider-using-with
tmp_packets.write(sys.stdin.buffer.read())
tmp_packets.flush()
source_file = tmp_packets.name
close_temp = True
#Set up source packet file; next 2 sections check for and handle compressed file extensions first, then final "else" treats the source as a pcap file
elif cast(str, pcap_source).endswith('.bz2'):
debug_out('Reading bzip2 compressed packets from file ' + cast(str, pcap_source))
source_file = open_bzip2_file_to_tmp_file(cast(str, pcap_source))
delete_temp = True
elif cast(str, pcap_source).endswith('.gz'):
debug_out('Reading gzip compressed packets from file ' + cast(str, pcap_source))
source_file = open_gzip_file_to_tmp_file(cast(str, pcap_source))
delete_temp = True
else:
debug_out('Reading packets from file ' + cast(str, pcap_source))
source_file = pcap_source
#Try to process file first
if source_file:
if os.path.exists(source_file) and os.access(source_file, os.R_OK):
try:
if user_args['count']:
sniff(store=0, offline=source_file, filter=user_args['bpf'], count=user_args['count'], prn=lambda x: processpacket(x)) # pylint: disable=unnecessary-lambda
else:
sniff(store=0, offline=source_file, filter=user_args['bpf'], prn=lambda x: processpacket(x)) # pylint: disable=unnecessary-lambda
except (FileNotFoundError, IOError):
sys.stderr.write("Unable to open file " + str(pcap_source) + ', exiting.\n')
raise
else:
sys.stderr.write("Unable to open file " + str(source_file) + ', skipping.\n')
if close_temp:
tmp_packets.close()
if delete_temp and source_file != pcap_source and os.path.exists(cast(str, source_file)):
os.remove(cast(str, source_file))
nullbyte = binascii.unhexlify('00')
display_uuid_hosts = False
uuid_match = r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.local\.*'
hints = {'TCP_FLAGS_': 'Invalid/no_tcp_flags', 'TCP_FLAGS_SR': 'Invalid/syn_and_rst', 'TCP_FLAGS_FRA': 'Invalid/fin_and_rst', 'TCP_FLAGS_FSPEC': 'Invalid/fin_and_syn', 'TCP_FLAGS_FSPU': 'Invalid/fin_and_syn', 'TCP_FLAGS_FSRPAU': 'Invalid/fin_and_syn_and_rst', 'TCP_FLAGS_FSRPAUEN': 'Invalid/fin_and_syn_and_rst_christmas_tree',
'icmp_0.0': 'echo_reply',
'icmp_3.0': 'unreachable/net', 'icmp_3.1': 'unreachable/host', 'icmp_3.10': 'unreachable/host_admin_prohib', 'icmp_3.13': 'unreachable/communication_administratively_prohibited', 'icmp_3.2': 'unreachable/protocol', 'icmp_3.3': 'unreachable/port', 'icmp_3.4': 'unreachable/frag_needed_and_df_set',
'icmp_5.0': 'redirect/net', 'icmp_5.1': 'redirect/host', 'icmp_5.2': 'redirect/tos_and_net', 'icmp_5.3': 'redirect/tos_and_host',
'icmp_8.0': 'echo_request',
'icmp_9.0': 'router_advertisement/normal',
'icmp_11.0': 'time_exceeded/TTL', 'icmp_11.1': 'time_exceeded/frag_reassembly_time_exceeded',
'icmp_13.0': 'timestamp',
'icmp_14.0': 'timestamp_reply',
'icmp6_1.1': 'unreachable/communication_administratively_prohibited', 'icmp6_1.3': 'unreachable/address_unreachable', 'icmp6_1.4': 'unreachable/port_unreachable', 'icmp6_1.6': 'unreachable/reject_destination_route',
'icmp6_3.0': 'time_exceeded/hop_limit',
'icmp6_128.0': 'echo_request',
'icmp6_129.0': 'echo_reply',
'icmp6_133.0': 'router_solicitation/normal',
'icmp6_134.0': 'router_advertisement/normal',
'icmp6_135.0': 'neighbor_solicitation/normal',
'icmp6_136.0': 'neighbor_advertisement/normal',
'ip4_0.0.0.0': 'address_unspecified', 'ip4_1.1.1.1': 'public_dns/cloudflare', 'ip4_127.0.0.1': 'localhost', 'ip4_8.8.4.4': 'public_dns/google', 'ip4_8.8.8.8': 'public_dns/google', 'ip4_75.75.75.75': 'public_dns/cdns01.comcast.net', 'ip4_75.75.76.76': 'public_dns/cdns02.comcast.net', 'ip4_75.75.77.22': 'public_dns/doh.xfinity.com', 'ip4_75.75.77.98': 'public_dns/doh.xfinity.com', 'ip4_224.0.0.1': 'all_systems_on_this_subnet', 'ip4_224.0.0.2': 'all_routers_on_this_subnet', 'ip4_224.0.0.13': 'all_pim_routers', 'ip4_224.0.0.22': 'multicast/IGMP', 'ip4_224.0.0.251': 'multicast/mDNS', 'ip4_224.0.0.252': 'multicast/LLMNR', 'ip4_224.0.1.40': 'multicast/cisco_rp_discovery', 'ip4_224.0.1.60': 'multicast/hp_device_discovery', 'ip4_239.255.255.250': 'multicast/uPNP_or_SSDP', 'ip4_255.255.255.255': 'broadcast',
'ip6_2001:558:feed::1': 'public_dns/cdns01.comcast.net', 'ip6_2001:558:feed::2': 'public_dns/cdns02.comcast.net', 'ip6_2001:558:feed:443::98': 'public_dns/doh.xfinity.com',
'ip6_::': 'address_unspecified', 'ip6_::1': 'localhost', 'ip6_ff02::1': 'multicast/all_nodes', 'ip6_ff02::2': 'multicast/all_routers', 'ip6_ff02::c': 'multicast/ssdp', 'ip6_ff02::16': 'multicast/MLDv2_capable_routers', 'ip6_ff02::fb': 'multicast/mDNSv6', 'ip6_ff02::1:2': 'multicast/DHCP_Relay_Agents_and_Servers', 'ip6_ff02::1:3': 'multicast/LLMNR',
'proto_0': 'ip', 'proto_2': 'igmp', 'proto_47': 'gre', 'proto_47_35006': 'gre/erspan_type_II', 'proto_47_8939': 'gre/erspan_type_III', 'proto_50': 'esp', 'proto_51': 'ah', 'proto_58': 'ipv6_icmp', 'proto_103': 'pim',
'udp_7': 'echo', 'udp_17': 'qotd', 'udp_19': 'chargen', 'udp_53': 'dns', 'udp_67': 'bootp/dhcp', 'udp_69': 'tftp', 'udp_88': 'kerberos',
'udp_111': 'rpc', 'udp_123': 'ntp', 'udp_137': 'netbios/ns', 'udp_138': 'netbios/datagram', 'udp_161': 'snmp',
'udp_389': 'ldap',
'udp_443': 'https/quic',
'udp_500': 'isakmp/ike', 'udp_514': 'syslog', 'udp_520': 'rip', 'udp_546': 'dhcpv6_client', 'udp_547': 'dhcpv6',
'udp_1194': 'openvpn',
'udp_1434': 'mssql_monitor', 'udp_1853': 'videoconf/gotowebinar', 'udp_1900': 'ssdp/upnp',
'udp_1812': 'radius',
'udp_3389': 'remote_desktop_protocol', 'udp_3478': 'webrtc', 'udp_3479': 'webrtc', 'udp_3480': 'webrtc', 'udp_3481': 'webrtc', 'udp_3702': 'web_services_discovery',
'udp_4500': 'ipsec_nat_traversal', 'udp_4501': 'globalprotect_vpn', 'udp_4789': 'vxlan',
'udp_5002': 'drobo_discovery', 'udp_5060': 'sip', 'udp_5353': 'mDNS', 'udp_5355': 'LLMNR', 'udp_5938': 'teamviewer',
'udp_8200': 'videoconf/gotowebinar', 'udp_8801': 'videoconf/zoom', 'udp_8802': 'videoconf/zoom',
'udp_15509': 'videoconf/zoom',
'udp_17500': 'dropbox_lan_sync',
'udp_19305': 'google_meet',
'udp_33434': 'traceroute/udp', 'udp_33435': 'traceroute/udp', 'udp_33436': 'traceroute/udp', 'udp_33437': 'traceroute/udp', 'udp_33438': 'traceroute/udp', 'udp_33439': 'traceroute/udp',
'udp_33440': 'traceroute/udp', 'udp_33441': 'traceroute/udp', 'udp_33442': 'traceroute/udp', 'udp_33443': 'traceroute/udp', 'udp_33444': 'traceroute/udp', 'udp_33445': 'traceroute/udp', 'udp_33446': 'traceroute/udp', 'udp_33447': 'traceroute/udp', 'udp_33448': 'traceroute/udp', 'udp_33449': 'traceroute/udp',
'udp_33450': 'traceroute/udp', 'udp_33451': 'traceroute/udp', 'udp_33452': 'traceroute/udp', 'udp_33453': 'traceroute/udp', 'udp_33454': 'traceroute/udp', 'udp_33455': 'traceroute/udp', 'udp_33456': 'traceroute/udp', 'udp_33457': 'traceroute/udp', 'udp_33458': 'traceroute/udp', 'udp_33459': 'traceroute/udp',
'udp_33460': 'traceroute/udp', 'udp_33461': 'traceroute/udp', 'udp_33462': 'traceroute/udp', 'udp_33463': 'traceroute/udp', 'udp_33464': 'traceroute/udp', 'udp_33465': 'traceroute/udp', 'udp_33466': 'traceroute/udp', 'udp_33467': 'traceroute/udp', 'udp_33468': 'traceroute/udp', 'udp_33469': 'traceroute/udp',
'udp_33470': 'traceroute/udp', 'udp_33471': 'traceroute/udp', 'udp_33472': 'traceroute/udp', 'udp_33473': 'traceroute/udp', 'udp_33474': 'traceroute/udp', 'udp_33475': 'traceroute/udp', 'udp_33476': 'traceroute/udp', 'udp_33477': 'traceroute/udp', 'udp_33478': 'traceroute/udp', 'udp_33479': 'traceroute/udp',
'udp_33480': 'traceroute/udp', 'udp_33481': 'traceroute/udp', 'udp_33482': 'traceroute/udp', 'udp_33483': 'traceroute/udp', 'udp_33484': 'traceroute/udp', 'udp_33485': 'traceroute/udp', 'udp_33486': 'traceroute/udp', 'udp_33487': 'traceroute/udp', 'udp_33488': 'traceroute/udp', 'udp_33489': 'traceroute/udp',
'udp_33490': 'traceroute/udp', 'udp_33491': 'traceroute/udp', 'udp_33492': 'traceroute/udp', 'udp_33493': 'traceroute/udp', 'udp_33494': 'traceroute/udp', 'udp_33495': 'traceroute/udp', 'udp_33496': 'traceroute/udp', 'udp_33497': 'traceroute/udp', 'udp_33498': 'traceroute/udp', 'udp_33499': 'traceroute/udp',
'udp_33500': 'traceroute/udp', 'udp_33501': 'traceroute/udp', 'udp_33502': 'traceroute/udp', 'udp_33503': 'traceroute/udp', 'udp_33504': 'traceroute/udp', 'udp_33505': 'traceroute/udp', 'udp_33506': 'traceroute/udp', 'udp_33507': 'traceroute/udp', 'udp_33508': 'traceroute/udp', 'udp_33509': 'traceroute/udp',
'udp_33510': 'traceroute/udp', 'udp_33511': 'traceroute/udp', 'udp_33512': 'traceroute/udp', 'udp_33513': 'traceroute/udp', 'udp_33514': 'traceroute/udp', 'udp_33515': 'traceroute/udp', 'udp_33516': 'traceroute/udp', 'udp_33517': 'traceroute/udp', 'udp_33518': 'traceroute/udp', 'udp_33519': 'traceroute/udp',
'udp_33520': 'traceroute/udp', 'udp_33521': 'traceroute/udp', 'udp_33522': 'traceroute/udp', 'udp_33523': 'traceroute/udp',
'tcp_7': 'echo', 'tcp_11': 'systat', 'tcp_13': 'daytime', 'tcp_19': 'chargen', 'tcp_20': 'ftp-data', 'tcp_21': 'ftp', 'tcp_22': 'ssh', 'tcp_23': 'telnet', 'tcp_25': 'smtp', 'tcp_37': 'time', 'tcp_42': 'name', 'tcp_43': 'whois', 'tcp_53': 'dns', 'tcp_79': 'finger', 'tcp_80': 'http', 'tcp_88': 'kerberos',
'tcp_109': 'pop2', 'tcp_110': 'pop3', 'tcp_111': 'rpc', 'tcp_113': 'ident/auth', 'tcp_119': 'nntp', 'tcp_135': 'ms_rpc_endpoint_mapper', 'tcp_139': 'netbios/session', 'tcp_143': 'imap', 'tcp_179': 'bgp',
'tcp_389': 'ldap',
'tcp_443': 'https', 'tcp_445': 'microsoft-ds', 'tcp_465': 'smtps',
'tcp_512': 'r-commands/rexec', 'tcp_513': 'r-commands/rlogin', 'tcp_514': 'r-commands/rsh_rcp', 'tcp_587': 'smtp/msa',
'tcp_631': 'ipp',
'tcp_873': 'rsync',
'tcp_989': 'ftps-data', 'tcp_990': 'ftps', 'tcp_993': 'imaps',
'tcp_1119': 'bnetgame', 'tcp_1194': 'openvpn',
'tcp_1389': 'iclpv-dm_or_alt_jndi_ldap',
'tcp_1433': 'mssql', 'tcp_1434': 'mssql_monitor',
'tcp_1723': 'pptp',
'tcp_1935': 'rtmp', 'tcp_1984': 'bigbrother',
'tcp_3128': 'squid_proxy', 'tcp_3306': 'mysql', 'tcp_3389': 'remote_desktop_protocol', 'tcp_3478': 'webrtc', 'tcp_3724': 'blizwow',
'tcp_5001': 'drobo/nasd', 'tcp_5060': 'sip', 'tcp_5223': 'apple_push_notification', 'tcp_5228': 'google_talk', 'tcp_5555': 'monero', 'tcp_5601': 'kibana', 'tcp_5900': 'vnc/remote_framebuffer', 'tcp_5938': 'teamviewer',
'tcp_6379': 'redis',
'tcp_7680': 'windows/delivery_optimization',
'tcp_8008': 'apple_ical', 'tcp_8080': 'http-alt', 'tcp_8333': 'bitcoin',
'tcp_9200': 'elasticsearch'}
#The following are _sub_layers; additional details underneath a main layer, such as SNMPBulk under SNMP.
ignore_layers = ('DHCP options',
'DHCP6 Client Identifier Option', 'DHCP6 Elapsed Time Option', 'DHCP6 Identity Association for Non-temporary Addresses Option', 'DHCP6 Option - Client FQDN', 'DHCP6 Option Request Option', 'DHCP6 Vendor Class Option',
'DNS DNSKEY Resource Record', 'DNS DS Resource Record', 'DNS EDNS0 TLV', 'DNS NSEC Resource Record', 'DNS NSEC3 Resource Record', 'DNS OPT Resource Record', 'DNS Question Record', 'DNS RRSIG Resource Record', 'DNS Resource Record', 'DNS SRV Resource Record',
'Ethernet',
'ICMPv6 Neighbor Discovery Option - Prefix Information', 'ICMPv6 Neighbor Discovery Option - Recursive DNS Server Option', 'ICMPv6 Neighbor Discovery Option - Route Information Option', 'ICMPv6 Neighbor Discovery Option - Source Link-Layer Address',
'IP Option End of Options List', 'IP Option No Operation',
'ISAKMP Identification', 'ISAKMP Key Exchange', 'ISAKMP Nonce', 'ISAKMP SA', 'ISAKMP Vendor ID', 'ISAKMP payload',
'PadN', 'Padding',
'Raw',
'SCTPChunkInit', 'SCTPChunkParamCookiePreservative', 'SCTPChunkParamSupportedAddrTypes',
'SNMPbulk', 'SNMPget', 'SNMPnext', 'SNMPvarbind',
'vendor_class_data')
#tcp_ignore_ports = (123, 20547, 33046, 39882) #Was used for early troubleshooting, no longer needed.
#udp_ignore_ports = (10400, 10401, 16403, 38010)
cache_dir = os.environ["HOME"] + '/.cache/'
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser(description='pcap_stats version ' + str(__version__))
parser.add_argument('-i', '--interface', help='Interface from which to read packets', required=False, default=None)
parser.add_argument('-r', '--read', help='Pcap file(s) from which to read packets', required=False, default=[], nargs='*')
parser.add_argument('-d', '--devel', help='Enable development/debug statements', required=False, default=False, action='store_true')
parser.add_argument('-b', '--bpf', help='BPF to restrict which packets are processed', required=False, default='')
parser.add_argument('-c', '--count', help='Number of packets to sniff (if not specified, sniff forever/until end of pcap file)', type=int, required=False, default=None)
parser.add_argument('-m', '--mincount', help='Only show a record if we have seen it this many times (default: %(default)s)', type=int, required=False, default=0)
parser.add_argument('-s', '--minsize', help='Only show a record if have this many total bytes (default: %(default)s)', type=int, required=False, default=0)
parser.add_argument('-l', '--length', help='Which form of length to use (default: %(default)s)', choices=('ip',), required=False, default='ip') #Reinstate this when the length function accepts them: , choices=('ip', 'layer', 'payload')
parser.add_argument('-f', '--format', help='Output format (default: %(default)s)', choices=('ascii', 'html'), required=False, default='ascii')
parser.add_argument('--db_dir', help='Directory that holds sqlite databases for IP and hostname info', required=False, default=cache_dir + '/ip/')
parser.add_argument('--archive_dir', help='Directory that holds read-only older sqlite databases for IP and hostname info', required=False, default=cache_dir + '/ip_archive/')
(parsed, unparsed) = parser.parse_known_args()
cl_args = vars(parsed)
#We have to use libpcap instead of scapy's built-in code because the latter won't attach complex bpfs
try:
conf.use_pcap = True
except:
config.use_pcap = True
if cl_args['db_dir']:
mkdir_p(cl_args['db_dir'])
if cl_args['archive_dir']:
mkdir_p(cl_args['archive_dir'])
if cl_args['db_dir'] and cl_args['archive_dir']:
ip_netbios_db = [cl_args['db_dir'] + 'ip_names.sqlite3', cl_args['archive_dir'] + 'ip_names.sqlite3']
ip_hostnames_db = [cl_args['db_dir'] + 'ip_hostnames.sqlite3', cl_args['archive_dir'] + 'ip_hostnames.sqlite3']
elif cl_args['db_dir']:
ip_netbios_db = [cl_args['db_dir'] + 'ip_names.sqlite3']
ip_hostnames_db = [cl_args['db_dir'] + 'ip_hostnames.sqlite3']
else:
ip_netbios_db = []
ip_hostnames_db = []
read_from_stdin = False #If stdin requested, it needs to be processed last, so we remember it here. We also handle the case where the user enters '-' more than once by simply remembering it.
if cl_args['interface'] and cl_args['read']:
data_source = str(cl_args['interface']) + ' ' + str(cl_args['read'])
elif cl_args['interface']:
data_source = str(cl_args['interface'])
elif cl_args['read']:
data_source = str(cl_args['read'])
else:
#elif cl_args['interface'] is None and cl_args['read'] == []:
debug_out('No source specified, reading from stdin.')
read_from_stdin = True
data_source = 'stdin'
try:
if cl_args['read']:
#Process normal files first.
for one_source in cl_args['read']:
if one_source == '-':
read_from_stdin = True
else:
process_packet_source(None, one_source, cl_args)
#Now that normal files are out of the way process stdin and/or reading from an interface, either of which could be infinite.