Version 5.3 (#14)

# Backwards Compatible Interface Changes
- [x] Sponge plugin arguments: `pdml2flow +json '-0 -h'` -> `pdml2flow +json -0 -h` 
- [x] Add `Flow().id` property

# Improvements
- [x] Remove plugin submodules
- [x] Fix deprecated usage of resource_filename
- [x] Test for python version: `3.5-dev` `3.6` `3.6-dev` `'3.7-dev`

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "pdml2flow/plugins/pdml2flow-elasticsearch"]
-	path = pdml2flow/plugins/pdml2flow-elasticsearch
-	url = https://github.com/Enteee/pdml2flow-elasticsearch.git

README.md

@@ -98,8 +98,10 @@ $ tshark -i interface -Tpdml | pdml2flow +json | fluentflow rules.js
 
 ## Plugins
 
-* [Elasticsearch](https://github.com/Enteee/pdml2flow-elasticsearch)
-* see [pdml2flow/plugins/](pdml2flow/plugins/) for a full list of supported plugins
+* [elasticsearch](https://github.com/Enteee/pdml2flow-elasticsearch#readme): Saves pdml2flow output in Elasticsearch
+* [base64-strings](https://github.com/Enteee/pdml2flow-base64strings#readme): Extract strings encoded in base64
+* [frame-inter-arrival-time](https://github.com/Enteee/pdml2flow-frame-inter-arrival-time): Calculate frame inter arrival times
+* [pdml2flow/plugins/](pdml2flow/plugins/): Plugins shipped by default
 
 ### Interface
 

pdml2flow/autovivification.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # vim: set fenc=utf8 ts=4 sw=4 et :
 from functools import reduce
 

pdml2flow/cli.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # vim: set fenc=utf8 ts=4 sw=4 et :
 import sys
 import xml.sax
@@ -160,7 +159,7 @@ def add_arguments_cb(argparser):
             'plugin_name': plugin_name
         })
         copytree(
-            resource_filename(__name__, '/plugin-skeleton'),
+            resource_filename(__name__, 'plugin-skeleton'),
             dst,
             ignore=ignore_patterns('__pycache__')
         )

pdml2flow/conf.py

@@ -1,6 +1,6 @@
-#!/usr/bin/env python3
 # vim: set fenc=utf8 ts=4 sw=4 et :
 import sys
+import typing
 from os import path, environ
 from shlex import split
 from base64 import b32encode, b32decode
@@ -39,13 +39,13 @@ def get_version():
 
     FLOW_DEF_NESTCHAR = '.'
     FLOW_DEF_STR = [
-                'vlan{}id'.format(FLOW_DEF_NESTCHAR),
-                'ip{}src'.format(FLOW_DEF_NESTCHAR),
-                'ip{}dst'.format(FLOW_DEF_NESTCHAR),
-                'ipv6{}src'.format(FLOW_DEF_NESTCHAR),
-                'ipv6{}dst'.format(FLOW_DEF_NESTCHAR),
-                'udp{}stream'.format(FLOW_DEF_NESTCHAR),
-                'tcp{}stream'.format(FLOW_DEF_NESTCHAR),
+        'vlan{}id'.format(FLOW_DEF_NESTCHAR),
+        'ip{}src'.format(FLOW_DEF_NESTCHAR),
+        'ip{}dst'.format(FLOW_DEF_NESTCHAR),
+        'ipv6{}src'.format(FLOW_DEF_NESTCHAR),
+        'ipv6{}dst'.format(FLOW_DEF_NESTCHAR),
+        'udp{}stream'.format(FLOW_DEF_NESTCHAR),
+        'tcp{}stream'.format(FLOW_DEF_NESTCHAR),
     ]
     FLOW_DEF = get_real_paths.__func__(FLOW_DEF_STR, FLOW_DEF_NESTCHAR)
     DATA_MAXLEN = 200
@@ -61,6 +61,7 @@ def get_version():
     PARSE_SOURCE = sys.stdin
     SUPPORTED_PLUGIN_INTERFACES = [Plugin2]
     LOAD_PLUGINS = boolify(environ.get('LOAD_PLUGINS', 'True'))
+    LOAD_PLUGINS_CLI_PREFIX = '+'
     PLUGINS = []
     PLUGIN_GROUP_BASE = 'pdml2flow.plugins.base'
     PLUGIN_GROUP = 'pdml2flow.plugins'
@@ -143,12 +144,55 @@ def load_plugin_group(group):
         if Conf.LOAD_PLUGINS:
             load_plugin_group(Conf.PLUGIN_GROUP)
 
+        def escape_plugin_arguments(in_arguments: typing.List[str]) -> typing.List[str]:
+            """Sponge up plugin arguments and encode them as base32.
+
+            Note: Base32 was chosen because it does not
+            contain '-', '+'.
+
+            Example:
+                -a arg1 -b -c +plugin -h -d +plugin2 -k test
+            returns:
+                [
+                    '-a' ,'arg1', '-b'
+                    '+plugin', 'base32encode(-h -d)'
+                    '+plugin2', 'base32encode(-k test)
+                ]
+            """
+
+            arguments = []
+            plugin_args = ''
+            sponge_plugin_args = False
+            for v in in_arguments:
+                if v[0] == Conf.LOAD_PLUGINS_CLI_PREFIX:
+                    # plugin load detected, everything
+                    # from here are plugin args. Start
+                    # sponging them up.
+                    if plugin_args:
+                        # but first, store previous plugin args
+                        arguments.append(
+                            b32encode(plugin_args.encode()).decode()
+                        )
+                    plugin_args = ''
+                    sponge_plugin_args = True
+                    arguments.append(v)
+                elif sponge_plugin_args:
+                    plugin_args += ' ' + v
+                else:
+                    # normal argument
+                    arguments.append(v)
+            if plugin_args:
+                arguments.append(
+                    b32encode(plugin_args.encode()).decode()
+                )
+            return arguments
+
         conf = vars(
-            argparser.parse_args([
-                v if i == 0 or v[0] == '+' or Conf.ARGS[i-1][0] != '+'
-                else b32encode(v.encode()).decode()
-                for i, v in enumerate(Conf.ARGS)
-            ])
+            argparser.parse_args(
+                escape_plugin_arguments(
+                    Conf.ARGS
+                )
+            )
         )
 
         postprocess_conf_cb(conf)

pdml2flow/flow.py

@@ -47,6 +47,10 @@ def __hash__(self):
     def __eq__(self, other):
         return self.__id == other.__id
 
+    @property
+    def id(self):
+        return self.__id
+
     @property
     def frames(self):
         # clean the frame data

pdml2flow/logging.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # vim: set fenc=utf8 ts=4 sw=4 et :
 import sys
 

pdml2flow/pdmlhandler.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # vim: set fenc=utf8 ts=4 sw=4 et :
 import xml.sax
 import functools

pdml2flow/plugin-skeleton/.travis.yml

@@ -3,7 +3,10 @@ language: python
 python:
 - '3.4'
 - '3.5'
-- 3.5-dev
+- '3.5-dev'
+- '3.6'
+- '3.6-dev'
+- '3.7-dev'
 - nightly
 # VERSION END
 sudo: required

pdml2flow/utils.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # vim: set fenc=utf8 ts=4 sw=4 et :
 
 def boolify(string):

requirements.txt

@@ -1 +1,2 @@
+typing
 dict2xml

scripts/mkreadme.sh

@@ -4,7 +4,7 @@ set -exuo pipefail
 TOPLEVEL="$( cd "$(dirname "$0")" ; pwd -P )/../"
 
 # install pdml2flow
-sudo pip install --upgrade -e "${TOPLEVEL}"
+pip install --upgrade -e "${TOPLEVEL}"
 
 cat <<EOF > "${TOPLEVEL}/README.md"
 # pdml2flow [![PyPI version](https://badge.fury.io/py/pdml2flow.svg)](https://badge.fury.io/py/pdml2flow) 
@@ -76,8 +76,10 @@ $ tshark -i interface -Tpdml | pdml2flow +json | fluentflow rules.js
 
 ## Plugins
 
-* [Elasticsearch](https://github.com/Enteee/pdml2flow-elasticsearch)
-* see [pdml2flow/plugins/](pdml2flow/plugins/) for a full list of supported plugins
+* [elasticsearch](https://github.com/Enteee/pdml2flow-elasticsearch#readme): Saves pdml2flow output in Elasticsearch
+* [base64-strings](https://github.com/Enteee/pdml2flow-base64strings#readme): Extract strings encoded in base64
+* [frame-inter-arrival-time](https://github.com/Enteee/pdml2flow-frame-inter-arrival-time): Calculate frame inter arrival times
+* [pdml2flow/plugins/](pdml2flow/plugins/): Plugins shipped by default
 
 ### Interface
 

setup.py

@@ -22,9 +22,10 @@
 setup(
     name = 'pdml2flow',
     keywords = 'wireshark pdml flow aggregation plugins',
-    version = '5.2',
+    version = '5.3',
     packages = find_packages(exclude=['test']),
     install_requires = [
+        'typing',
         'dict2xml'
     ],
     # other arguments here...

test/autovivification_test.py → test/test_autovivification.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # vim: set fenc=utf8 ts=4 sw=4 et :
 from .testcase import TestCase
 
@@ -428,6 +427,3 @@ def test___getitem__(self):
             ),
             AutoVivification
         )
-
-if __name__ == '__main__':
-    unittest.main()

test/cli_test.py → test/test_cli.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # vim: set fenc=utf8 ts=4 sw=4 et :
 from os import path
 
@@ -12,11 +11,8 @@ class TestCli(TestCaseWithTestDir):
     def test_pdml2flow_new_plugin(self):
         Conf.ARGS = [
             path.join(
-                self.test_dir, 
+                self.test_dir,
                 'new-plugin'
             )
         ]
         pdml2flow_new_plugin()
-
-if __name__ == '__main__':
-    unittest.main()

test/conf_test.py → test/test_conf.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # vim: set fenc=utf8 ts=4 sw=4 et :
 import io
 
@@ -49,6 +48,3 @@ def test_get(self):
                 self.assertFalse(name.startswith("_"))
                 self.assertFalse(name.startswith("__"))
                 self.assertTrue(name.isupper())
-
-if __name__ == '__main__':
-    unittest.main()

test/flow_test.py → test/test_flow.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # vim: set fenc=utf8 ts=4 sw=4 et :
 from .testcase import TestCase
 
@@ -27,6 +26,28 @@ def test_get_flow_id(self):
         })
         self.assertEqual(Flow.get_flow_id(frame), None)
 
+    def test_id(self):
+        Conf.FRAME_TIME = [ 't' ]
+        Conf.FLOW_DEF = [ ['def1'] ]
+
+        frame = AutoVivification({
+            't' : 0,
+            'def1': 1,
+        })
+        self.assertEqual(
+            Flow(frame).id,
+            Flow.get_flow_id(frame),
+        )
+
+        frame2 = AutoVivification({
+            't' : 0,
+            'def1': 2,
+        })
+        self.assertNotEqual(
+            Flow(frame2).id,
+            Flow.get_flow_id(frame),
+        )
+
     def test__eq__(self):
         Conf.FRAME_TIME = [ 't' ]
         Conf.FLOW_DEF = [ ['def1'] ]
@@ -232,6 +253,3 @@ def test_not_expired(self):
         self.assertEqual(flow.not_expired(), True)
         Flow.newest_overall_frame_time = 123 + Conf.FLOW_BUFFER_TIME
         self.assertEqual(flow.not_expired(), False)
-
-if __name__ == '__main__':
-    unittest.main()

test/logging_test.py → test/test_logging.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # vim: set fenc=utf8 ts=4 sw=4 et :
 from .testcase import TestCase
 import io
@@ -33,6 +32,3 @@ def test_error(self):
 
             error('test')
             self.assertIn('test', out.getvalue())
-
-if __name__ == '__main__':
-    unittest.main()

test/plugin_test.py → test/test_plugin.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # vim: set fenc=utf8 ts=4 sw=4 et :
 from inspect import getmembers, ismethod, isfunction, signature
 
@@ -33,6 +32,3 @@ def test_functions(self):
                 ('frame_new', '(self, frame, flow)'),
             ])
         )
-
-if __name__ == '__main__':
-    unittest.main()

test/system_test.py → test/test_system.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # vim: set fenc=utf8 ts=4 sw=4 et :
 import os
 import io
@@ -118,6 +117,3 @@ def add_tests(run, directory):
 # Add tests
 add_tests(pdml2flow.pdml2flow, TEST_DIR_PDML2FLOW)
 add_tests(pdml2flow.pdml2frame, TEST_DIR_PDML2FRAME)
-
-if __name__ == '__main__':
-    unittest.main()

test/utils_test.py → test/test_utils.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # vim: set fenc=utf8 ts=4 sw=4 et :
 from unittest.mock import MagicMock
 
@@ -50,6 +49,3 @@ def test_call_plugin_function_not_implemented(self):
             ),
             None
         )
-
-if __name__ == '__main__':
-    unittest.main()