diff --git a/.collections.py b/.collections.py deleted file mode 100644 index f8dc44f299..0000000000 --- a/.collections.py +++ /dev/null @@ -1,315 +0,0 @@ -from _abcoll import * -from _collections import deque, defaultdict -from itertools import imap as _imap -from operator import eq as _eq, itemgetter as _itemgetter -from keyword import iskeyword as _iskeyword -import sys as _sys - -try: - from thread import get_ident as _get_ident -except ImportError: - from dummy_thread import get_ident as _get_ident - - -def namedtuple(typename, field_names, verbose=False): - """Returns a new subclass of tuple with named fields. - - >>> Point = namedtuple('Point', 'x y') - >>> Point.__doc__ # docstring for the new class - 'Point(x, y)' - >>> p = Point(11, y=22) # instantiate with positional args or keywords - >>> p[0] + p[1] # indexable like a plain tuple - 33 - >>> x, y = p # unpack like a regular tuple - >>> x, y - (11, 22) - >>> p.x + p.y # fields also accessable by name - 33 - >>> d = p._asdict() # convert to a dictionary - >>> d['x'] - 11 - >>> Point(**d) # convert from a dictionary - Point(x=11, y=22) - >>> p._replace(x=100) # _replace() is like str.replace() but targets named fields - Point(x=100, y=22) - - """ - - # Parse and validate the field names. Validation serves two purposes, - # generating informative error messages and preventing template injection attacks. - if isinstance(field_names, basestring): - field_names = field_names.replace(',', ' ').split() # names separated by whitespace and/or commas - field_names = tuple(map(str, field_names)) - for name in (typename,) + field_names: - if not all(c.isalnum() or c=='_' for c in name): - raise ValueError('Type names and field names can only contain alphanumeric characters and underscores: %r' % name) - if _iskeyword(name): - raise ValueError('Type names and field names cannot be a keyword: %r' % name) - if name[0].isdigit(): - raise ValueError('Type names and field names cannot start with a number: %r' % name) - seen_names = set() - for name in field_names: - if name.startswith('_'): - raise ValueError('Field names cannot start with an underscore: %r' % name) - if name in seen_names: - raise ValueError('Encountered duplicate field name: %r' % name) - seen_names.add(name) - - # Create and fill-in the class template - numfields = len(field_names) - argtxt = repr(field_names).replace("'", "")[1:-1] # tuple repr without parens or quotes - reprtxt = ', '.join('%s=%%r' % name for name in field_names) - dicttxt = ', '.join('%r: t[%d]' % (name, pos) for pos, name in enumerate(field_names)) - template = '''class %(typename)s(tuple): - '%(typename)s(%(argtxt)s)' \n - __slots__ = () \n - _fields = %(field_names)r \n - def __new__(_cls, %(argtxt)s): - return _tuple.__new__(_cls, (%(argtxt)s)) \n - @classmethod - def _make(cls, iterable, new=tuple.__new__, len=len): - 'Make a new %(typename)s object from a sequence or iterable' - result = new(cls, iterable) - if len(result) != %(numfields)d: - raise TypeError('Expected %(numfields)d arguments, got %%d' %% len(result)) - return result \n - def __repr__(self): - return '%(typename)s(%(reprtxt)s)' %% self \n - def _asdict(t): - 'Return a new dict which maps field names to their values' - return {%(dicttxt)s} \n - def _replace(_self, **kwds): - 'Return a new %(typename)s object replacing specified fields with new values' - result = _self._make(map(kwds.pop, %(field_names)r, _self)) - if kwds: - raise ValueError('Got unexpected field names: %%r' %% kwds.keys()) - return result \n - def __getnewargs__(self): - return tuple(self) \n\n''' % locals() - for i, name in enumerate(field_names): - template += ' %s = _property(_itemgetter(%d))\n' % (name, i) - if verbose: - print template - - # Execute the template string in a temporary namespace and - # support tracing utilities by setting a value for frame.f_globals['__name__'] - namespace = dict(_itemgetter=_itemgetter, __name__='namedtuple_%s' % typename, - _property=property, _tuple=tuple) - try: - exec template in namespace - except SyntaxError, e: - raise SyntaxError(e.message + ':\n' + template) - result = namespace[typename] - - # For pickling to work, the __module__ variable needs to be set to the frame - # where the named tuple is created. Bypass this step in enviroments where - # sys._getframe is not defined (Jython for example). - if hasattr(_sys, '_getframe'): - result.__module__ = _sys._getframe(1).f_globals.get('__name__', '__main__') - - return result - - -class OrderedDict(dict): - 'Dictionary that remembers insertion order' - # An inherited dict maps keys to values. - # The inherited dict provides __getitem__, __len__, __contains__, and get. - # The remaining methods are order-aware. - # Big-O running times for all methods are the same as regular dictionaries. - - # The internal self.__map dict maps keys to links in a doubly linked list. - # The circular doubly linked list starts and ends with a sentinel element. - # The sentinel element never gets deleted (this simplifies the algorithm). - # Each link is stored as a list of length three: [PREV, NEXT, KEY]. - - def __init__(self, *args, **kwds): - '''Initialize an ordered dictionary. The signature is the same as - regular dictionaries, but keyword arguments are not recommended because - their insertion order is arbitrary. - - ''' - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__root - except AttributeError: - self.__root = root = [] # sentinel node - root[:] = [root, root, None] - self.__map = {} - self.__update(*args, **kwds) - - def __setitem__(self, key, value, dict_setitem=dict.__setitem__): - 'od.__setitem__(i, y) <==> od[i]=y' - # Setting a new item creates a new link at the end of the linked list, - # and the inherited dictionary is updated with the new key/value pair. - if key not in self: - root = self.__root - last = root[0] - last[1] = root[0] = self.__map[key] = [last, root, key] - return dict_setitem(self, key, value) - - def __delitem__(self, key, dict_delitem=dict.__delitem__): - 'od.__delitem__(y) <==> del od[y]' - # Deleting an existing item uses self.__map to find the link which gets - # removed by updating the links in the predecessor and successor nodes. - dict_delitem(self, key) - link_prev, link_next, _ = self.__map.pop(key) - link_prev[1] = link_next # update link_prev[NEXT] - link_next[0] = link_prev # update link_next[PREV] - - def __iter__(self): - 'od.__iter__() <==> iter(od)' - # Traverse the linked list in order. - root = self.__root - curr = root[1] # start at the first node - while curr is not root: - yield curr[2] # yield the curr[KEY] - curr = curr[1] # move to next node - - def __reversed__(self): - 'od.__reversed__() <==> reversed(od)' - # Traverse the linked list in reverse order. - root = self.__root - curr = root[0] # start at the last node - while curr is not root: - yield curr[2] # yield the curr[KEY] - curr = curr[0] # move to previous node - - def clear(self): - 'od.clear() -> None. Remove all items from od.' - root = self.__root - root[:] = [root, root, None] - self.__map.clear() - dict.clear(self) - - # -- the following methods do not depend on the internal structure -- - - def keys(self): - 'od.keys() -> list of keys in od' - return list(self) - - def values(self): - 'od.values() -> list of values in od' - return [self[key] for key in self] - - def items(self): - 'od.items() -> list of (key, value) pairs in od' - return [(key, self[key]) for key in self] - - def iterkeys(self): - 'od.iterkeys() -> an iterator over the keys in od' - return iter(self) - - def itervalues(self): - 'od.itervalues -> an iterator over the values in od' - for k in self: - yield self[k] - - def iteritems(self): - 'od.iteritems -> an iterator over the (key, value) pairs in od' - for k in self: - yield (k, self[k]) - - update = MutableMapping.update - - __update = update # let subclasses override update without breaking __init__ - - __marker = object() - - def pop(self, key, default=__marker): - '''od.pop(k[,d]) -> v, remove specified key and return the corresponding - value. If key is not found, d is returned if given, otherwise KeyError - is raised. - - ''' - if key in self: - result = self[key] - del self[key] - return result - if default is self.__marker: - raise KeyError(key) - return default - - def setdefault(self, key, default=None): - 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' - if key in self: - return self[key] - self[key] = default - return default - - def popitem(self, last=True): - '''od.popitem() -> (k, v), return and remove a (key, value) pair. - Pairs are returned in LIFO order if last is true or FIFO order if false. - - ''' - if not self: - raise KeyError('dictionary is empty') - key = next(reversed(self) if last else iter(self)) - value = self.pop(key) - return key, value - - def __repr__(self, _repr_running={}): - 'od.__repr__() <==> repr(od)' - call_key = id(self), _get_ident() - if call_key in _repr_running: - return '...' - _repr_running[call_key] = 1 - try: - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - finally: - del _repr_running[call_key] - - def __reduce__(self): - 'Return state information for pickling' - items = [[k, self[k]] for k in self] - inst_dict = vars(self).copy() - for k in vars(OrderedDict()): - inst_dict.pop(k, None) - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def copy(self): - 'od.copy() -> a shallow copy of od' - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S. - If not specified, the value defaults to None. - - ''' - self = cls() - for key in iterable: - self[key] = value - return self - - def __eq__(self, other): - '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive - while comparison to a regular mapping is order-insensitive. - - ''' - if isinstance(other, OrderedDict): - return dict.__eq__(self, other) and all(_imap(_eq, self, other)) - return dict.__eq__(self, other) - - def __ne__(self, other): - 'od.__ne__(y) <==> od!=y' - return not self == other - - # -- the following methods support python 3.x style dictionary views -- - - def viewkeys(self): - "od.viewkeys() -> a set-like object providing a view on od's keys" - return KeysView(self) - - def viewvalues(self): - "od.viewvalues() -> an object providing a view on od's values" - return ValuesView(self) - - def viewitems(self): - "od.viewitems() -> a set-like object providing a view on od's items" - return ItemsView(self) - diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7ba53c1684..f1e1c72173 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -64,99 +64,6 @@ jobs: pip install -e .[testing] pytest - python26-test: - - runs-on: ubuntu-latest - container: - image: cronosmobi/python2.6:latest - env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - - steps: - - name: install system dependencies - run: | - apt-get install -y --no-install-recommends software-properties-common - apt-get install -y build-essential git curl unzip file tar - - name: clone the repo and checkout pull_request - run: | - git clone ${{ github.server_url }}/${{ github.repository }} - git -C insights-core fetch --no-tags --prune --no-recurse-submodules --depth=1 origin pull/${{ github.ref_name }}:${{ github.head_ref }} - git -C insights-core checkout ${{ github.head_ref }} - - name: build setuptools and pip - run: | - export PATH=$PATH:/github/home/.local/bin - CUR_DIR=$(pwd) - mkdir ../tools && cd ../tools - curl -L -O https://files.pythonhosted.org/packages/b8/04/be569e393006fa9a2c10ef72ea33133c2902baa115dd1d4279dae55c3b3b/setuptools-36.8.0.zip - unzip setuptools-36.8.0.zip && cd setuptools-36.8.0 - python setup.py install --user && cd .. - curl -L -O https://github.com/pypa/pip/archive/refs/tags/9.0.3.tar.gz - tar -xvzf 9.0.3.tar.gz && cd pip-9.0.3 - python setup.py install --user - cd ${CUR_DIR} - - name: install virtualenv depenencies - run: | - export PATH=$PATH:/github/home/.local/bin - CUR_DIR=$(pwd) - mkdir pip_packages && cd pip_packages - curl -L -O https://files.pythonhosted.org/packages/0c/5d/b077dbf309993d52c1d71e6bf6fe443a8029ea215135ebbe0b1b10e7aefc/pbr-3.1.1-py2.py3-none-any.whl - curl -L -O https://files.pythonhosted.org/packages/31/77/3781f65cafe55480b56914def99022a5d2965a4bb269655c89ef2f1de3cd/importlib-1.0.4.zip - curl -L -O https://files.pythonhosted.org/packages/37/aa/111610d8bf5b1bb7a295a048fc648cec346347a8b0be5881defd2d1b4a52/oyaml-1.0-py2.py3-none-any.whl - curl -L -O https://files.pythonhosted.org/packages/3b/f6/7a76333cf0b9251ecf49efff635015171843d9b977e4ffcf59f9c4428052/redis-2.10.6-py2.py3-none-any.whl - curl -L -O https://files.pythonhosted.org/packages/4b/2a/0276479a4b3caeb8a8c1af2f8e4355746a97fab05a372e4a2c6a6b876165/idna-2.7-py2.py3-none-any.whl - curl -L -O https://files.pythonhosted.org/packages/4d/de/32d741db316d8fdb7680822dd37001ef7a448255de9699ab4bfcbdf4172b/MarkupSafe-1.0.tar.gz - curl -L -O https://files.pythonhosted.org/packages/53/25/ef88e8e45db141faa9598fbf7ad0062df8f50f881a36ed6a0073e1572126/ordereddict-1.1.tar.gz - curl -L -O https://files.pythonhosted.org/packages/53/67/9620edf7803ab867b175e4fd23c7b8bd8eba11cb761514dcd2e726ef07da/py-1.4.34-py2.py3-none-any.whl - curl -L -O https://files.pythonhosted.org/packages/5a/4d/5a6bfb960738e503234f1d201eac5ed4eecbd6f66a8ed593097bd3306bec/coverage-4.3.4-cp26-cp26mu-manylinux1_x86_64.whl - curl -L -O https://files.pythonhosted.org/packages/5e/a0/5f06e1e1d463903cf0c0eebeb751791119ed7a4b3737fdc9a77f1cdfb51f/certifi-2020.12.5-py2.py3-none-any.whl - curl -L -O https://files.pythonhosted.org/packages/65/47/7e02164a2a3db50ed6d8a6ab1d6d60b69c4c3fdf57a284257925dfc12bda/requests-2.19.1-py2.py3-none-any.whl - curl -L -O https://files.pythonhosted.org/packages/65/e0/eb35e762802015cab1ccee04e8a277b03f1d8e53da3ec3106882ec42558b/Jinja2-2.10.3-py2.py3-none-any.whl - curl -L -O https://files.pythonhosted.org/packages/67/4b/141a581104b1f6397bfa78ac9d43d8ad29a7ca43ea90a2d863fe3056e86a/six-1.11.0-py2.py3-none-any.whl - curl -L -O https://files.pythonhosted.org/packages/69/be/9c322ed286263a93e5ee0ff575662d0709fc73ee906522e7cfa72b08b946/mccabe-0.5.3-py2.py3-none-any.whl - curl -L -O https://files.pythonhosted.org/packages/69/cb/f5be453359271714c01b9bd06126eaf2e368f1fddfff30818754b5ac2328/funcsigs-1.0.2-py2.py3-none-any.whl - curl -L -O https://files.pythonhosted.org/packages/6f/2c/a9386903ece2ea85e9807e0e062174dc26fdce8b05f216d00491be29fad5/enum34-1.1.10-py2-none-any.whl - curl -L -O https://files.pythonhosted.org/packages/70/a9/9b66f22d038de51e05f92d5e677fd89d8f9c980db0b8a130621baad052f5/flake8-2.6.2-py2.py3-none-any.whl - curl -L -O https://files.pythonhosted.org/packages/73/31/136a79364c1681a3c276796d1f5090833bd03461b78a1b037638d1a2c484/pycodestyle-2.0.0-py2.py3-none-any.whl - curl -L -O https://files.pythonhosted.org/packages/74/55/98f59358be6d7240ba546b8a74813cc21841a9145a0c1a3a7998f50acbe7/pyflakes-1.2.3-py2.py3-none-any.whl - curl -L -O https://files.pythonhosted.org/packages/75/5e/b84feba55e20f8da46ead76f14a3943c8cb722d40360702b2365b91dec00/PyYAML-3.11.tar.gz - curl -L -O https://files.pythonhosted.org/packages/85/e6/f041bcf77bcf7bf11ccb9b8a6cdb3a2ee70c1bd2ab49d87d2269cfd4f3e0/pytest_cov-2.4.0-py2.py3-none-any.whl - curl -L -O https://files.pythonhosted.org/packages/87/1c/17f3e3935a913dfe2a5ca85fa5ccbef366bfd82eb318b1f75dadbf0affca/defusedxml-0.5.0-py2.py3-none-any.whl - curl -L -O https://files.pythonhosted.org/packages/8a/20/6eca772d1a5830336f84aca1d8198e5a3f4715cd1c7fc36d3cc7f7185091/msgpack-python-0.5.6.tar.gz - curl -L -O https://files.pythonhosted.org/packages/8c/2d/aad7f16146f4197a11f8e91fb81df177adcc2073d36a17b1491fd09df6ed/pycparser-2.18.tar.gz - curl -L -O https://files.pythonhosted.org/packages/98/f5/76619a63f0e4a1d2f5a1792ebc233a395c648c63d3461dc0331479ef120a/CacheControl-0.12.4.tar.gz - curl -L -O https://files.pythonhosted.org/packages/ab/1a/ec151e5e703ac80041eaccef923611bbcec2b667c20383655a06962732e9/configparser-3.8.1-py2.py3-none-any.whl - curl -L -O https://files.pythonhosted.org/packages/ac/95/a05b56bb975efa78d3557efa36acaf9cf5d2fd0ee0062060493687432e03/pip-9.0.3-py2.py3-none-any.whl - curl -L -O https://files.pythonhosted.org/packages/bc/a9/01ffebfb562e4274b6487b4bb1ddec7ca55ec7510b22e4c51f14098443b8/chardet-3.0.4-py2.py3-none-any.whl - curl -L -O https://files.pythonhosted.org/packages/bd/c9/6fdd990019071a4a32a5e7cb78a1d92c53851ef4f56f62a3486e6a7d8ffb/urllib3-1.23-py2.py3-none-any.whl - curl -L -O https://files.pythonhosted.org/packages/c4/17/73e8eda8fbc18b8421a8b16be8bbbb2a461f4d2405f3628beb8e5d2ca567/pytest-3.0.6-py2.py3-none-any.whl - curl -L -O https://files.pythonhosted.org/packages/c8/22/9460e311f340cb62d26a38c419b1381b8593b0bb6b5d1f056938b086d362/lockfile-0.12.2-py2.py3-none-any.whl - curl -L -O https://files.pythonhosted.org/packages/cb/85/8a1588a04172e0853352ecfe214264c65a62ab35374d9ad9c569cf94c2a3/python_gnupg-0.4.6-py2.py3-none-any.whl - curl -L -O https://files.pythonhosted.org/packages/e6/35/f187bdf23be87092bd0f1200d43d23076cee4d0dec109f195173fd3ebc79/mock-2.0.0-py2.py3-none-any.whl - curl -L -O https://files.pythonhosted.org/packages/f2/94/3af39d34be01a24a6e65433d19e107099374224905f1e0cc6bbe1fd22a2f/argparse-1.4.0-py2.py3-none-any.whl - curl -L -O https://files.pythonhosted.org/packages/86/84/6bd1384196a6871a9108157ec934a1e1ee0078582cd208b43352566a86dc/pytest_catchlog-1.2.2-py2.py3-none-any.whl - cd ${CUR_DIR} - mkdir ../collections_module - curl -L -o ./../collections_module/collections.py https://raw.githubusercontent.com/RedHatInsights/insights-core/5c8ca0f2fb3de45908e8d931d40758af34a7997a/.collections.py - - name: flake8 - run: | - cd insights-core - # show the branch - git --no-pager branch - export PATH=$PATH:/github/home/.local/bin - pip install --user -e .[linting] --no-index -f ../pip_packages - flake8 . - cd - - - name: pytest - run: | - cd insights-core - # show the branch - git --no-pager branch - export PATH=$PATH:/github/home/.local/bin - pip install --user -e .[testing] --no-index -f ../pip_packages - export PYTHONPATH=${PYTHONPATH}:../../collections_module - pytest - cd - - docs-test: runs-on: ubuntu-latest diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index ea42da6d48..0000000000 --- a/Jenkinsfile +++ /dev/null @@ -1,115 +0,0 @@ -@Library('fh-pipeline-library')_ - -pipeline { - agent none - stages { - stage('Trust') { - steps { - enforceTrustedApproval('RedHatInsights') - } - } - stage('Build and Test Insights Core') { - parallel { - stage('Build RHEL6') { - agent { - node { - label 'python26' - } - } - steps { - echo "Testing with Pytest..." - sh """ - virtualenv .testenv - source .testenv/bin/activate - pip install /pip_packages/pip-9.0.3-py2.py3-none-any.whl - pip install -r /var/lib/jenkins/ci_requirements.txt -f /pip_packages - pip install -e .[testing] -f /pip_packages - pytest - """ - echo "Testing with Linter..." - sh """ - virtualenv .lintenv - source .lintenv/bin/activate - pip install /pip_packages/pip-9.0.3-py2.py3-none-any.whl - pip install -r /var/lib/jenkins/ci_requirements.txt -f /pip_packages - pip install -e .[linting] -f /pip_packages - flake8 - """ - } - } - stage('Build RHEL7 Python 2.7') { - agent { - node { - label 'python' - } - } - steps { - echo "Testing with Pytest..." - sh """ - virtualenv .testenv - source .testenv/bin/activate - pip install -e .[testing] - pytest - """ - echo "Testing with Linter..." - sh """ - virtualenv .lintenv - source .lintenv/bin/activate - pip install -e .[linting] - flake8 - """ - } - } - stage('Build RHEL7 Python 3.6') { - agent { - node { - label 'python3' - } - } - steps { - echo "Testing with Pytest..." - sh """ - /bin/python3 -m venv .testenv - source .testenv/bin/activate - pip install -e .[testing] - pytest - """ - echo "Testing with Linter..." - sh """ - /bin/python3 -m venv .lintenv - source .lintenv/bin/activate - pip install -e .[linting] - flake8 - """ - } - } - } - } - stage('Nofity Github - Code Check Passed') { - steps { - githubNotify description: 'Code Checks Passed', status: 'SUCCESS' - } - } - stage('Test Docs') { - agent { - node { - label 'python3' - } - } - steps { - echo "Building Docs..." - sh """ - /bin/python3 -m venv .docenv - source .docenv/bin/activate - pip install -e .[docs] - sphinx-build -W -b html -qa -E docs docs/_build/html - """ - } - } - stage('Nofity Github - Docs Check Passed') { - steps { - githubNotify description: 'Code Checks and Docs Generation Passed', status: 'SUCCESS' - } - } - } -} diff --git a/README.rst b/README.rst index a36ce2e443..9aa9dce45a 100644 --- a/README.rst +++ b/README.rst @@ -24,7 +24,7 @@ Features -------- * Over 200 Enterprise Linux data parsers -* Support for Python 2.6+ and 3.3+ +* Support for Python 2.7+ and 3.3+ * Built in support for local host collection * Data collection support for several archive formats diff --git a/build.sh b/build.sh index 339e9554c4..bd7481cc12 100755 --- a/build.sh +++ b/build.sh @@ -2,24 +2,18 @@ set -ev -if [ "`python -V 2>&1`" == "Python 2.6.9" ]; then - cp .collections.py /home/travis/virtualenv/python2.6.9/lib/python2.6/collections.py -fi - py.test -if [ "`python -V 2>&1`" != "Python 2.6.9" ]; then - flake8; - sphinx-build -W -b html -qa -E docs docs/_build/html; - if [ "${TRAVIS_PULL_REQUEST}" == "false" ] && [ -n "$DOCKER_USER" ] && [ -n "$DOCKER_PASS" ]; then - docker login -u $DOCKER_USER -p $DOCKER_PASS - export REPO=jhjaggars/insights-core - docker build -f Dockerfile -t $REPO:$COMMIT . - docker tag $REPO:$COMMIT $REPO:$TRAVIS_BRANCH - if [ "$TRAVIS_BRANCH" == "master" ]; then - docker tag $REPO:$COMMIT $REPO:latest - fi - docker push $REPO +flake8 + +sphinx-build -W -b html -qa -E docs docs/_build/html; +if [ "${TRAVIS_PULL_REQUEST}" == "false" ] && [ -n "$DOCKER_USER" ] && [ -n "$DOCKER_PASS" ]; then + docker login -u $DOCKER_USER -p $DOCKER_PASS + export REPO=jhjaggars/insights-core + docker build -f Dockerfile -t $REPO:$COMMIT . + docker tag $REPO:$COMMIT $REPO:$TRAVIS_BRANCH + if [ "$TRAVIS_BRANCH" == "master" ]; then + docker tag $REPO:$COMMIT $REPO:latest fi + docker push $REPO fi - diff --git a/insights/contrib/ElementPath.py b/insights/contrib/ElementPath.py deleted file mode 100644 index 310f8b6af7..0000000000 --- a/insights/contrib/ElementPath.py +++ /dev/null @@ -1,304 +0,0 @@ -# -# ElementTree -# $Id: ElementPath.py 3375 2008-02-13 08:05:08Z fredrik $ -# -# limited xpath support for element trees -# -# history: -# 2003-05-23 fl created -# 2003-05-28 fl added support for // etc -# 2003-08-27 fl fixed parsing of periods in element names -# 2007-09-10 fl new selection engine -# 2007-09-12 fl fixed parent selector -# 2007-09-13 fl added iterfind; changed findall to return a list -# 2007-11-30 fl added namespaces support -# 2009-10-30 fl added child element value filter -# -# Copyright (c) 2003-2009 by Fredrik Lundh. All rights reserved. -# -# fredrik@pythonware.com -# http://www.pythonware.com -# -# -------------------------------------------------------------------- -# The ElementTree toolkit is -# -# Copyright (c) 1999-2009 by Fredrik Lundh -# -# By obtaining, using, and/or copying this software and/or its -# associated documentation, you agree that you have read, understood, -# and will comply with the following terms and conditions: -# -# Permission to use, copy, modify, and distribute this software and -# its associated documentation for any purpose and without fee is -# hereby granted, provided that the above copyright notice appears in -# all copies, and that both that copyright notice and this permission -# notice appear in supporting documentation, and that the name of -# Secret Labs AB or the author not be used in advertising or publicity -# pertaining to distribution of the software without specific, written -# prior permission. -# -# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD -# TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT- -# ABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR -# BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY -# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, -# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS -# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE -# OF THIS SOFTWARE. -# -------------------------------------------------------------------- - -# Licensed to PSF under a Contributor Agreement. -# See http://www.python.org/psf/license for licensing details. - -## -# Implementation module for XPath support. There's usually no reason -# to import this module directly; the ElementTree does this for -# you, if needed. -## - -import re - -xpath_tokenizer_re = re.compile( - r"(" - r"'[^']*'|\"[^\"]*\"|" - r"::|" - r"//?|" - r"\.\.|" - r"\(\)|" - r"[/.*:\[\]\(\)@=])|" - r"((?:\{[^}]+\})?[^/\[\]\(\)@=\s]+)|" - r"\s+" -) - - -def xpath_tokenizer(pattern, namespaces=None): - for token in xpath_tokenizer_re.findall(pattern): - tag = token[1] - if tag and tag[0] != "{" and ":" in tag: - try: - prefix, uri = tag.split(":", 1) - if not namespaces: - raise KeyError - yield token[0], "{%s}%s" % (namespaces[prefix], uri) - except KeyError: - raise SyntaxError("prefix %r not found in prefix map" % prefix) - else: - yield token - -def get_parent_map(context): - parent_map = context.parent_map - if parent_map is None: - context.parent_map = parent_map = {} - for p in context.root.iter(): - for e in p: - parent_map[e] = p - return parent_map - -def prepare_child(next, token): - tag = token[1] - def select(context, result): - for elem in result: - for e in elem: - if e.tag == tag: - yield e - return select - -def prepare_star(next, token): - def select(context, result): - for elem in result: - for e in elem: - yield e - return select - -def prepare_self(next, token): - def select(context, result): - for elem in result: - yield elem - return select - -def prepare_descendant(next, token): - token = next() - if token[0] == "*": - tag = "*" - elif not token[0]: - tag = token[1] - else: - raise SyntaxError("invalid descendant") - def select(context, result): - for elem in result: - for e in elem.iter(tag): - if e is not elem: - yield e - return select - -def prepare_parent(next, token): - def select(context, result): - # FIXME: raise error if .. is applied at toplevel? - parent_map = get_parent_map(context) - result_map = {} - for elem in result: - if elem in parent_map: - parent = parent_map[elem] - if parent not in result_map: - result_map[parent] = None - yield parent - return select - -def prepare_predicate(next, token): - # FIXME: replace with real parser!!! refs: - # http://effbot.org/zone/simple-iterator-parser.htm - # http://javascript.crockford.com/tdop/tdop.html - signature = [] - predicate = [] - while 1: - token = next() - if token[0] == "]": - break - if token[0] and token[0][:1] in "'\"": - token = "'", token[0][1:-1] - signature.append(token[0] or "-") - predicate.append(token[1]) - signature = "".join(signature) - # use signature to determine predicate type - if signature == "@-": - # [@attribute] predicate - key = predicate[1] - def select(context, result): - for elem in result: - if elem.get(key) is not None: - yield elem - return select - if signature == "@-='": - # [@attribute='value'] - key = predicate[1] - value = predicate[-1] - def select(context, result): - for elem in result: - if elem.get(key) == value: - yield elem - return select - if signature == "-" and not re.match("\d+$", predicate[0]): - # [tag] - tag = predicate[0] - def select(context, result): - for elem in result: - if elem.find(tag) is not None: - yield elem - return select - if signature == "-='" and not re.match("\d+$", predicate[0]): - # [tag='value'] - tag = predicate[0] - value = predicate[-1] - def select(context, result): - for elem in result: - for e in elem.findall(tag): - if "".join(e.itertext()) == value: - yield elem - break - return select - if signature == "-" or signature == "-()" or signature == "-()-": - # [index] or [last()] or [last()-index] - if signature == "-": - index = int(predicate[0]) - 1 - else: - if predicate[0] != "last": - raise SyntaxError("unsupported function") - if signature == "-()-": - try: - index = int(predicate[2]) - 1 - except ValueError: - raise SyntaxError("unsupported expression") - else: - index = -1 - def select(context, result): - parent_map = get_parent_map(context) - for elem in result: - try: - parent = parent_map[elem] - # FIXME: what if the selector is "*" ? - elems = list(parent.findall(elem.tag)) - if elems[index] is elem: - yield elem - except (IndexError, KeyError): - pass - return select - raise SyntaxError("invalid predicate") - -ops = { - "": prepare_child, - "*": prepare_star, - ".": prepare_self, - "..": prepare_parent, - "//": prepare_descendant, - "[": prepare_predicate, - } - -_cache = {} - -class _SelectorContext: - parent_map = None - def __init__(self, root): - self.root = root - -# -------------------------------------------------------------------- - -## -# Generate all matching objects. - -def iterfind(elem, path, namespaces=None): - # compile selector pattern - if path[-1:] == "/": - path = path + "*" # implicit all (FIXME: keep this?) - try: - selector = _cache[path] - except KeyError: - if len(_cache) > 100: - _cache.clear() - if path[:1] == "/": - raise SyntaxError("cannot use absolute path on element") - next = iter(xpath_tokenizer(path, namespaces)).next - token = next() - selector = [] - while 1: - try: - selector.append(ops[token[0]](next, token)) - except StopIteration: - raise SyntaxError("invalid path") - try: - token = next() - if token[0] == "/": - token = next() - except StopIteration: - break - _cache[path] = selector - # execute selector pattern - result = [elem] - context = _SelectorContext(elem) - for select in selector: - result = select(context, result) - return result - -## -# Find first matching object. - -def find(elem, path, namespaces=None): - try: - return next(iterfind(elem, path, namespaces)) - except StopIteration: - return None - -## -# Find all matching objects. - -def findall(elem, path, namespaces=None): - return list(iterfind(elem, path, namespaces)) - -## -# Find text for first matching object. - -def findtext(elem, path, default=None, namespaces=None): - try: - elem = next(iterfind(elem, path, namespaces)) - return elem.text or "" - except StopIteration: - return default diff --git a/insights/contrib/ElementTree.py b/insights/contrib/ElementTree.py deleted file mode 100644 index d3ff2fb507..0000000000 --- a/insights/contrib/ElementTree.py +++ /dev/null @@ -1,1684 +0,0 @@ -# -# ElementTree -# $Id: ElementTree.py 3440 2008-07-18 14:45:01Z fredrik $ -# -# light-weight XML support for Python 2.3 and later. -# -# history (since 1.2.6): -# 2005-11-12 fl added tostringlist/fromstringlist helpers -# 2006-07-05 fl merged in selected changes from the 1.3 sandbox -# 2006-07-05 fl removed support for 2.1 and earlier -# 2007-06-21 fl added deprecation/future warnings -# 2007-08-25 fl added doctype hook, added parser version attribute etc -# 2007-08-26 fl added new serializer code (better namespace handling, etc) -# 2007-08-27 fl warn for broken /tag searches on tree level -# 2007-09-02 fl added html/text methods to serializer (experimental) -# 2007-09-05 fl added method argument to tostring/tostringlist -# 2007-09-06 fl improved error handling -# 2007-09-13 fl added itertext, iterfind; assorted cleanups -# 2007-12-15 fl added C14N hooks, copy method (experimental) -# -# Copyright (c) 1999-2008 by Fredrik Lundh. All rights reserved. -# -# fredrik@pythonware.com -# http://www.pythonware.com -# -# -------------------------------------------------------------------- -# The ElementTree toolkit is -# -# Copyright (c) 1999-2008 by Fredrik Lundh -# -# By obtaining, using, and/or copying this software and/or its -# associated documentation, you agree that you have read, understood, -# and will comply with the following terms and conditions: -# -# Permission to use, copy, modify, and distribute this software and -# its associated documentation for any purpose and without fee is -# hereby granted, provided that the above copyright notice appears in -# all copies, and that both that copyright notice and this permission -# notice appear in supporting documentation, and that the name of -# Secret Labs AB or the author not be used in advertising or publicity -# pertaining to distribution of the software without specific, written -# prior permission. -# -# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD -# TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT- -# ABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR -# BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY -# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, -# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS -# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE -# OF THIS SOFTWARE. -# -------------------------------------------------------------------- - -# Licensed to PSF under a Contributor Agreement. -# See http://www.python.org/psf/license for licensing details. - -__all__ = [ - # public symbols - "Comment", - "dump", - "Element", "ElementTree", - "fromstring", "fromstringlist", - "iselement", "iterparse", - "parse", "ParseError", - "PI", "ProcessingInstruction", - "QName", - "SubElement", - "tostring", "tostringlist", - "TreeBuilder", - "VERSION", - "XML", - "XMLParser", "XMLTreeBuilder", - ] - -VERSION = "1.3.0" - -## -# The Element type is a flexible container object, designed to -# store hierarchical data structures in memory. The type can be -# described as a cross between a list and a dictionary. -#

-# Each element has a number of properties associated with it: -#

-# -# To create an element instance, use the {@link #Element} constructor -# or the {@link #SubElement} factory function. -#

-# The {@link #ElementTree} class can be used to wrap an element -# structure, and convert it from and to XML. -## - -import sys -import re -import warnings - - -class _SimpleElementPath(object): - # emulate pre-1.2 find/findtext/findall behaviour - def find(self, element, tag, namespaces=None): - for elem in element: - if elem.tag == tag: - return elem - return None - def findtext(self, element, tag, default=None, namespaces=None): - elem = self.find(element, tag) - if elem is None: - return default - return elem.text or "" - def iterfind(self, element, tag, namespaces=None): - if tag[:3] == ".//": - for elem in element.iter(tag[3:]): - yield elem - for elem in element: - if elem.tag == tag: - yield elem - def findall(self, element, tag, namespaces=None): - return list(self.iterfind(element, tag, namespaces)) - -try: - from . import ElementPath -except ImportError: - ElementPath = _SimpleElementPath() - -## -# Parser error. This is a subclass of SyntaxError. -#

-# In addition to the exception value, an exception instance contains a -# specific exception code in the code attribute, and the line and -# column of the error in the position attribute. - -class ParseError(SyntaxError): - pass - -# -------------------------------------------------------------------- - -## -# Checks if an object appears to be a valid element object. -# -# @param An element instance. -# @return A true value if this is an element object. -# @defreturn flag - -def iselement(element): - # FIXME: not sure about this; might be a better idea to look - # for tag/attrib/text attributes - return isinstance(element, Element) or hasattr(element, "tag") - -## -# Element class. This class defines the Element interface, and -# provides a reference implementation of this interface. -#

-# The element name, attribute names, and attribute values can be -# either ASCII strings (ordinary Python strings containing only 7-bit -# ASCII characters) or Unicode strings. -# -# @param tag The element name. -# @param attrib An optional dictionary, containing element attributes. -# @param **extra Additional attributes, given as keyword arguments. -# @see Element -# @see SubElement -# @see Comment -# @see ProcessingInstruction - -class Element(object): - # text...tail - - ## - # (Attribute) Element tag. - - tag = None - - ## - # (Attribute) Element attribute dictionary. Where possible, use - # {@link #Element.get}, - # {@link #Element.set}, - # {@link #Element.keys}, and - # {@link #Element.items} to access - # element attributes. - - attrib = None - - ## - # (Attribute) Text before first subelement. This is either a - # string or the value None. Note that if there was no text, this - # attribute may be either None or an empty string, depending on - # the parser. - - text = None - - ## - # (Attribute) Text after this element's end tag, but before the - # next sibling element's start tag. This is either a string or - # the value None. Note that if there was no text, this attribute - # may be either None or an empty string, depending on the parser. - - tail = None # text after end tag, if any - - # constructor - - def __init__(self, tag, attrib={}, **extra): - attrib = attrib.copy() - attrib.update(extra) - self.tag = tag - self.attrib = attrib - self._children = [] - - def __repr__(self): - return "" % (repr(self.tag), id(self)) - - ## - # Creates a new element object of the same type as this element. - # - # @param tag Element tag. - # @param attrib Element attributes, given as a dictionary. - # @return A new element instance. - - def makeelement(self, tag, attrib): - return self.__class__(tag, attrib) - - ## - # (Experimental) Copies the current element. This creates a - # shallow copy; subelements will be shared with the original tree. - # - # @return A new element instance. - - def copy(self): - elem = self.makeelement(self.tag, self.attrib) - elem.text = self.text - elem.tail = self.tail - elem[:] = self - return elem - - ## - # Returns the number of subelements. Note that this only counts - # full elements; to check if there's any content in an element, you - # have to check both the length and the text attribute. - # - # @return The number of subelements. - - def __len__(self): - return len(self._children) - - def __nonzero__(self): - warnings.warn( - "The behavior of this method will change in future versions. " - "Use specific 'len(elem)' or 'elem is not None' test instead.", - FutureWarning, stacklevel=2 - ) - return len(self._children) != 0 # emulate old behaviour, for now - - ## - # Returns the given subelement, by index. - # - # @param index What subelement to return. - # @return The given subelement. - # @exception IndexError If the given element does not exist. - - def __getitem__(self, index): - return self._children[index] - - ## - # Replaces the given subelement, by index. - # - # @param index What subelement to replace. - # @param element The new element value. - # @exception IndexError If the given element does not exist. - - def __setitem__(self, index, element): - # if isinstance(index, slice): - # for elt in element: - # assert iselement(elt) - # else: - # assert iselement(element) - self._children[index] = element - - ## - # Deletes the given subelement, by index. - # - # @param index What subelement to delete. - # @exception IndexError If the given element does not exist. - - def __delitem__(self, index): - del self._children[index] - - ## - # Adds a subelement to the end of this element. In document order, - # the new element will appear after the last existing subelement (or - # directly after the text, if it's the first subelement), but before - # the end tag for this element. - # - # @param element The element to add. - - def append(self, element): - # assert iselement(element) - self._children.append(element) - - ## - # Appends subelements from a sequence. - # - # @param elements A sequence object with zero or more elements. - # @since 1.3 - - def extend(self, elements): - # for element in elements: - # assert iselement(element) - self._children.extend(elements) - - ## - # Inserts a subelement at the given position in this element. - # - # @param index Where to insert the new subelement. - - def insert(self, index, element): - # assert iselement(element) - self._children.insert(index, element) - - ## - # Removes a matching subelement. Unlike the find methods, - # this method compares elements based on identity, not on tag - # value or contents. To remove subelements by other means, the - # easiest way is often to use a list comprehension to select what - # elements to keep, and use slice assignment to update the parent - # element. - # - # @param element What element to remove. - # @exception ValueError If a matching element could not be found. - - def remove(self, element): - # assert iselement(element) - self._children.remove(element) - - ## - # (Deprecated) Returns all subelements. The elements are returned - # in document order. - # - # @return A list of subelements. - # @defreturn list of Element instances - - def getchildren(self): - warnings.warn( - "This method will be removed in future versions. " - "Use 'list(elem)' or iteration over elem instead.", - DeprecationWarning, stacklevel=2 - ) - return self._children - - ## - # Finds the first matching subelement, by tag name or path. - # - # @param path What element to look for. - # @keyparam namespaces Optional namespace prefix map. - # @return The first matching element, or None if no element was found. - # @defreturn Element or None - - def find(self, path, namespaces=None): - return ElementPath.find(self, path, namespaces) - - ## - # Finds text for the first matching subelement, by tag name or path. - # - # @param path What element to look for. - # @param default What to return if the element was not found. - # @keyparam namespaces Optional namespace prefix map. - # @return The text content of the first matching element, or the - # default value no element was found. Note that if the element - # is found, but has no text content, this method returns an - # empty string. - # @defreturn string - - def findtext(self, path, default=None, namespaces=None): - return ElementPath.findtext(self, path, default, namespaces) - - ## - # Finds all matching subelements, by tag name or path. - # - # @param path What element to look for. - # @keyparam namespaces Optional namespace prefix map. - # @return A list or other sequence containing all matching elements, - # in document order. - # @defreturn list of Element instances - - def findall(self, path, namespaces=None): - return ElementPath.findall(self, path, namespaces) - - ## - # Finds all matching subelements, by tag name or path. - # - # @param path What element to look for. - # @keyparam namespaces Optional namespace prefix map. - # @return An iterator or sequence containing all matching elements, - # in document order. - # @defreturn a generated sequence of Element instances - - def iterfind(self, path, namespaces=None): - return ElementPath.iterfind(self, path, namespaces) - - ## - # Resets an element. This function removes all subelements, clears - # all attributes, and sets the text and tail attributes - # to None. - - def clear(self): - self.attrib.clear() - self._children = [] - self.text = self.tail = None - - ## - # Gets an element attribute. Equivalent to attrib.get, but - # some implementations may handle this a bit more efficiently. - # - # @param key What attribute to look for. - # @param default What to return if the attribute was not found. - # @return The attribute value, or the default value, if the - # attribute was not found. - # @defreturn string or None - - def get(self, key, default=None): - return self.attrib.get(key, default) - - ## - # Sets an element attribute. Equivalent to attrib[key] = value, - # but some implementations may handle this a bit more efficiently. - # - # @param key What attribute to set. - # @param value The attribute value. - - def set(self, key, value): - self.attrib[key] = value - - ## - # Gets a list of attribute names. The names are returned in an - # arbitrary order (just like for an ordinary Python dictionary). - # Equivalent to attrib.keys(). - # - # @return A list of element attribute names. - # @defreturn list of strings - - def keys(self): - return self.attrib.keys() - - ## - # Gets element attributes, as a sequence. The attributes are - # returned in an arbitrary order. Equivalent to attrib.items(). - # - # @return A list of (name, value) tuples for all attributes. - # @defreturn list of (string, string) tuples - - def items(self): - return self.attrib.items() - - ## - # Creates a tree iterator. The iterator loops over this element - # and all subelements, in document order, and returns all elements - # with a matching tag. - #

- # If the tree structure is modified during iteration, new or removed - # elements may or may not be included. To get a stable set, use the - # list() function on the iterator, and loop over the resulting list. - # - # @param tag What tags to look for (default is to return all elements). - # @return An iterator containing all the matching elements. - # @defreturn iterator - - def iter(self, tag=None): - if tag == "*": - tag = None - if tag is None or self.tag == tag: - yield self - for e in self._children: - for e in e.iter(tag): - yield e - - # compatibility - def getiterator(self, tag=None): - # Change for a DeprecationWarning in 1.4 - warnings.warn( - "This method will be removed in future versions. " - "Use 'elem.iter()' or 'list(elem.iter())' instead.", - PendingDeprecationWarning, stacklevel=2 - ) - return list(self.iter(tag)) - - ## - # Creates a text iterator. The iterator loops over this element - # and all subelements, in document order, and returns all inner - # text. - # - # @return An iterator containing all inner text. - # @defreturn iterator - - def itertext(self): - tag = self.tag - if not isinstance(tag, basestring) and tag is not None: - return - if self.text: - yield self.text - for e in self: - for s in e.itertext(): - yield s - if e.tail: - yield e.tail - -# compatibility -_Element = _ElementInterface = Element - -## -# Subelement factory. This function creates an element instance, and -# appends it to an existing element. -#

-# The element name, attribute names, and attribute values can be -# either 8-bit ASCII strings or Unicode strings. -# -# @param parent The parent element. -# @param tag The subelement name. -# @param attrib An optional dictionary, containing element attributes. -# @param **extra Additional attributes, given as keyword arguments. -# @return An element instance. -# @defreturn Element - -def SubElement(parent, tag, attrib={}, **extra): - attrib = attrib.copy() - attrib.update(extra) - element = parent.makeelement(tag, attrib) - parent.append(element) - return element - -## -# Comment element factory. This factory function creates a special -# element that will be serialized as an XML comment by the standard -# serializer. -#

-# The comment string can be either an 8-bit ASCII string or a Unicode -# string. -# -# @param text A string containing the comment string. -# @return An element instance, representing a comment. -# @defreturn Element - -def Comment(text=None): - element = Element(Comment) - element.text = text - return element - -## -# PI element factory. This factory function creates a special element -# that will be serialized as an XML processing instruction by the standard -# serializer. -# -# @param target A string containing the PI target. -# @param text A string containing the PI contents, if any. -# @return An element instance, representing a PI. -# @defreturn Element - -def ProcessingInstruction(target, text=None): - element = Element(ProcessingInstruction) - element.text = target - if text: - element.text = element.text + " " + text - return element - -PI = ProcessingInstruction - -## -# QName wrapper. This can be used to wrap a QName attribute value, in -# order to get proper namespace handling on output. -# -# @param text A string containing the QName value, in the form {uri}local, -# or, if the tag argument is given, the URI part of a QName. -# @param tag Optional tag. If given, the first argument is interpreted as -# a URI, and this argument is interpreted as a local name. -# @return An opaque object, representing the QName. - -class QName(object): - def __init__(self, text_or_uri, tag=None): - if tag: - text_or_uri = "{%s}%s" % (text_or_uri, tag) - self.text = text_or_uri - def __str__(self): - return self.text - def __hash__(self): - return hash(self.text) - def __cmp__(self, other): - if isinstance(other, QName): - return cmp(self.text, other.text) - return cmp(self.text, other) - -# -------------------------------------------------------------------- - -## -# ElementTree wrapper class. This class represents an entire element -# hierarchy, and adds some extra support for serialization to and from -# standard XML. -# -# @param element Optional root element. -# @keyparam file Optional file handle or file name. If given, the -# tree is initialized with the contents of this XML file. - -class ElementTree(object): - - def __init__(self, element=None, file=None): - # assert element is None or iselement(element) - self._root = element # first node - if file: - self.parse(file) - - ## - # Gets the root element for this tree. - # - # @return An element instance. - # @defreturn Element - - def getroot(self): - return self._root - - ## - # Replaces the root element for this tree. This discards the - # current contents of the tree, and replaces it with the given - # element. Use with care. - # - # @param element An element instance. - - def _setroot(self, element): - # assert iselement(element) - self._root = element - - ## - # Loads an external XML document into this element tree. - # - # @param source A file name or file object. If a file object is - # given, it only has to implement a read(n) method. - # @keyparam parser An optional parser instance. If not given, the - # standard {@link XMLParser} parser is used. - # @return The document root element. - # @defreturn Element - # @exception ParseError If the parser fails to parse the document. - - def parse(self, source, parser=None): - close_source = False - if not hasattr(source, "read"): - source = open(source, "rb") - close_source = True - try: - if not parser: - parser = XMLParser(target=TreeBuilder()) - while 1: - data = source.read(65536) - if not data: - break - parser.feed(data) - self._root = parser.close() - return self._root - finally: - if close_source: - source.close() - - ## - # Creates a tree iterator for the root element. The iterator loops - # over all elements in this tree, in document order. - # - # @param tag What tags to look for (default is to return all elements) - # @return An iterator. - # @defreturn iterator - - def iter(self, tag=None): - # assert self._root is not None - return self._root.iter(tag) - - # compatibility - def getiterator(self, tag=None): - # Change for a DeprecationWarning in 1.4 - warnings.warn( - "This method will be removed in future versions. " - "Use 'tree.iter()' or 'list(tree.iter())' instead.", - PendingDeprecationWarning, stacklevel=2 - ) - return list(self.iter(tag)) - - ## - # Same as getroot().find(path), starting at the root of the - # tree. - # - # @param path What element to look for. - # @keyparam namespaces Optional namespace prefix map. - # @return The first matching element, or None if no element was found. - # @defreturn Element or None - - def find(self, path, namespaces=None): - # assert self._root is not None - if path[:1] == "/": - path = "." + path - warnings.warn( - "This search is broken in 1.3 and earlier, and will be " - "fixed in a future version. If you rely on the current " - "behaviour, change it to %r" % path, - FutureWarning, stacklevel=2 - ) - return self._root.find(path, namespaces) - - ## - # Same as getroot().findtext(path), starting at the root of the tree. - # - # @param path What element to look for. - # @param default What to return if the element was not found. - # @keyparam namespaces Optional namespace prefix map. - # @return The text content of the first matching element, or the - # default value no element was found. Note that if the element - # is found, but has no text content, this method returns an - # empty string. - # @defreturn string - - def findtext(self, path, default=None, namespaces=None): - # assert self._root is not None - if path[:1] == "/": - path = "." + path - warnings.warn( - "This search is broken in 1.3 and earlier, and will be " - "fixed in a future version. If you rely on the current " - "behaviour, change it to %r" % path, - FutureWarning, stacklevel=2 - ) - return self._root.findtext(path, default, namespaces) - - ## - # Same as getroot().findall(path), starting at the root of the tree. - # - # @param path What element to look for. - # @keyparam namespaces Optional namespace prefix map. - # @return A list or iterator containing all matching elements, - # in document order. - # @defreturn list of Element instances - - def findall(self, path, namespaces=None): - # assert self._root is not None - if path[:1] == "/": - path = "." + path - warnings.warn( - "This search is broken in 1.3 and earlier, and will be " - "fixed in a future version. If you rely on the current " - "behaviour, change it to %r" % path, - FutureWarning, stacklevel=2 - ) - return self._root.findall(path, namespaces) - - ## - # Finds all matching subelements, by tag name or path. - # Same as getroot().iterfind(path). - # - # @param path What element to look for. - # @keyparam namespaces Optional namespace prefix map. - # @return An iterator or sequence containing all matching elements, - # in document order. - # @defreturn a generated sequence of Element instances - - def iterfind(self, path, namespaces=None): - # assert self._root is not None - if path[:1] == "/": - path = "." + path - warnings.warn( - "This search is broken in 1.3 and earlier, and will be " - "fixed in a future version. If you rely on the current " - "behaviour, change it to %r" % path, - FutureWarning, stacklevel=2 - ) - return self._root.iterfind(path, namespaces) - - ## - # Writes the element tree to a file, as XML. - # - # @def write(file, **options) - # @param file A file name, or a file object opened for writing. - # @param **options Options, given as keyword arguments. - # @keyparam encoding Optional output encoding (default is US-ASCII). - # @keyparam xml_declaration Controls if an XML declaration should - # be added to the file. Use False for never, True for always, - # None for only if not US-ASCII or UTF-8. None is default. - # @keyparam default_namespace Sets the default XML namespace (for "xmlns"). - # @keyparam method Optional output method ("xml", "html", "text" or - # "c14n"; default is "xml"). - - def write(self, file_or_filename, - # keyword arguments - encoding=None, - xml_declaration=None, - default_namespace=None, - method=None): - # assert self._root is not None - if not method: - method = "xml" - elif method not in _serialize: - # FIXME: raise an ImportError for c14n if ElementC14N is missing? - raise ValueError("unknown method %r" % method) - if hasattr(file_or_filename, "write"): - file = file_or_filename - else: - file = open(file_or_filename, "wb") - write = file.write - if not encoding: - if method == "c14n": - encoding = "utf-8" - else: - encoding = "us-ascii" - elif xml_declaration or (xml_declaration is None and - encoding not in ("utf-8", "us-ascii")): - if method == "xml": - write("\n" % encoding) - if method == "text": - _serialize_text(write, self._root, encoding) - else: - qnames, namespaces = _namespaces( - self._root, encoding, default_namespace - ) - serialize = _serialize[method] - serialize(write, self._root, encoding, qnames, namespaces) - if file_or_filename is not file: - file.close() - - def write_c14n(self, file): - # lxml.etree compatibility. use output method instead - return self.write(file, method="c14n") - -# -------------------------------------------------------------------- -# serialization support - -def _namespaces(elem, encoding, default_namespace=None): - # identify namespaces used in this tree - - # maps qnames to *encoded* prefix:local names - qnames = {None: None} - - # maps uri:s to prefixes - namespaces = {} - if default_namespace: - namespaces[default_namespace] = "" - - def encode(text): - return text.encode(encoding) - - def add_qname(qname): - # calculate serialized qname representation - try: - if qname[:1] == "{": - uri, tag = qname[1:].rsplit("}", 1) - prefix = namespaces.get(uri) - if prefix is None: - prefix = _namespace_map.get(uri) - if prefix is None: - prefix = "ns%d" % len(namespaces) - if prefix != "xml": - namespaces[uri] = prefix - if prefix: - qnames[qname] = encode("%s:%s" % (prefix, tag)) - else: - qnames[qname] = encode(tag) # default element - else: - if default_namespace: - # FIXME: can this be handled in XML 1.0? - raise ValueError( - "cannot use non-qualified names with " - "default_namespace option" - ) - qnames[qname] = encode(qname) - except TypeError: - _raise_serialization_error(qname) - - # populate qname and namespaces table - try: - iterate = elem.iter - except AttributeError: - iterate = elem.getiterator # cET compatibility - for elem in iterate(): - tag = elem.tag - if isinstance(tag, QName): - if tag.text not in qnames: - add_qname(tag.text) - elif isinstance(tag, basestring): - if tag not in qnames: - add_qname(tag) - elif tag is not None and tag is not Comment and tag is not PI: - _raise_serialization_error(tag) - for key, value in elem.items(): - if isinstance(key, QName): - key = key.text - if key not in qnames: - add_qname(key) - if isinstance(value, QName) and value.text not in qnames: - add_qname(value.text) - text = elem.text - if isinstance(text, QName) and text.text not in qnames: - add_qname(text.text) - return qnames, namespaces - -def _serialize_xml(write, elem, encoding, qnames, namespaces): - tag = elem.tag - text = elem.text - if tag is Comment: - write("" % _encode(text, encoding)) - elif tag is ProcessingInstruction: - write("" % _encode(text, encoding)) - else: - tag = qnames[tag] - if tag is None: - if text: - write(_escape_cdata(text, encoding)) - for e in elem: - _serialize_xml(write, e, encoding, qnames, None) - else: - write("<" + tag) - items = elem.items() - if items or namespaces: - if namespaces: - for v, k in sorted(namespaces.items(), - key=lambda x: x[1]): # sort on prefix - if k: - k = ":" + k - write(" xmlns%s=\"%s\"" % ( - k.encode(encoding), - _escape_attrib(v, encoding) - )) - for k, v in sorted(items): # lexical order - if isinstance(k, QName): - k = k.text - if isinstance(v, QName): - v = qnames[v.text] - else: - v = _escape_attrib(v, encoding) - write(" %s=\"%s\"" % (qnames[k], v)) - if text or len(elem): - write(">") - if text: - write(_escape_cdata(text, encoding)) - for e in elem: - _serialize_xml(write, e, encoding, qnames, None) - write("") - else: - write(" />") - if elem.tail: - write(_escape_cdata(elem.tail, encoding)) - -HTML_EMPTY = ("area", "base", "basefont", "br", "col", "frame", "hr", - "img", "input", "isindex", "link", "meta", "param") - -try: - HTML_EMPTY = set(HTML_EMPTY) -except NameError: - pass - -def _serialize_html(write, elem, encoding, qnames, namespaces): - tag = elem.tag - text = elem.text - if tag is Comment: - write("" % _escape_cdata(text, encoding)) - elif tag is ProcessingInstruction: - write("" % _escape_cdata(text, encoding)) - else: - tag = qnames[tag] - if tag is None: - if text: - write(_escape_cdata(text, encoding)) - for e in elem: - _serialize_html(write, e, encoding, qnames, None) - else: - write("<" + tag) - items = elem.items() - if items or namespaces: - if namespaces: - for v, k in sorted(namespaces.items(), - key=lambda x: x[1]): # sort on prefix - if k: - k = ":" + k - write(" xmlns%s=\"%s\"" % ( - k.encode(encoding), - _escape_attrib(v, encoding) - )) - for k, v in sorted(items): # lexical order - if isinstance(k, QName): - k = k.text - if isinstance(v, QName): - v = qnames[v.text] - else: - v = _escape_attrib_html(v, encoding) - # FIXME: handle boolean attributes - write(" %s=\"%s\"" % (qnames[k], v)) - write(">") - ltag = tag.lower() - if text: - if ltag == "script" or ltag == "style": - write(_encode(text, encoding)) - else: - write(_escape_cdata(text, encoding)) - for e in elem: - _serialize_html(write, e, encoding, qnames, None) - if ltag not in HTML_EMPTY: - write("") - if elem.tail: - write(_escape_cdata(elem.tail, encoding)) - -def _serialize_text(write, elem, encoding): - for part in elem.itertext(): - write(part.encode(encoding)) - if elem.tail: - write(elem.tail.encode(encoding)) - -_serialize = { - "xml": _serialize_xml, - "html": _serialize_html, - "text": _serialize_text, -# this optional method is imported at the end of the module -# "c14n": _serialize_c14n, -} - -## -# Registers a namespace prefix. The registry is global, and any -# existing mapping for either the given prefix or the namespace URI -# will be removed. -# -# @param prefix Namespace prefix. -# @param uri Namespace uri. Tags and attributes in this namespace -# will be serialized with the given prefix, if at all possible. -# @exception ValueError If the prefix is reserved, or is otherwise -# invalid. - -def register_namespace(prefix, uri): - if re.match("ns\d+$", prefix): - raise ValueError("Prefix format reserved for internal use") - for k, v in _namespace_map.items(): - if k == uri or v == prefix: - del _namespace_map[k] - _namespace_map[uri] = prefix - -_namespace_map = { - # "well-known" namespace prefixes - "http://www.w3.org/XML/1998/namespace": "xml", - "http://www.w3.org/1999/xhtml": "html", - "http://www.w3.org/1999/02/22-rdf-syntax-ns#": "rdf", - "http://schemas.xmlsoap.org/wsdl/": "wsdl", - # xml schema - "http://www.w3.org/2001/XMLSchema": "xs", - "http://www.w3.org/2001/XMLSchema-instance": "xsi", - # dublin core - "http://purl.org/dc/elements/1.1/": "dc", -} - -def _raise_serialization_error(text): - raise TypeError( - "cannot serialize %r (type %s)" % (text, type(text).__name__) - ) - -def _encode(text, encoding): - try: - return text.encode(encoding, "xmlcharrefreplace") - except (TypeError, AttributeError): - _raise_serialization_error(text) - -def _escape_cdata(text, encoding): - # escape character data - try: - # it's worth avoiding do-nothing calls for strings that are - # shorter than 500 character, or so. assume that's, by far, - # the most common case in most applications. - if "&" in text: - text = text.replace("&", "&") - if "<" in text: - text = text.replace("<", "<") - if ">" in text: - text = text.replace(">", ">") - return text.encode(encoding, "xmlcharrefreplace") - except (TypeError, AttributeError): - _raise_serialization_error(text) - -def _escape_attrib(text, encoding): - # escape attribute value - try: - if "&" in text: - text = text.replace("&", "&") - if "<" in text: - text = text.replace("<", "<") - if ">" in text: - text = text.replace(">", ">") - if "\"" in text: - text = text.replace("\"", """) - if "\n" in text: - text = text.replace("\n", " ") - return text.encode(encoding, "xmlcharrefreplace") - except (TypeError, AttributeError): - _raise_serialization_error(text) - -def _escape_attrib_html(text, encoding): - # escape attribute value - try: - if "&" in text: - text = text.replace("&", "&") - if ">" in text: - text = text.replace(">", ">") - if "\"" in text: - text = text.replace("\"", """) - return text.encode(encoding, "xmlcharrefreplace") - except (TypeError, AttributeError): - _raise_serialization_error(text) - -# -------------------------------------------------------------------- - -## -# Generates a string representation of an XML element, including all -# subelements. -# -# @param element An Element instance. -# @keyparam encoding Optional output encoding (default is US-ASCII). -# @keyparam method Optional output method ("xml", "html", "text" or -# "c14n"; default is "xml"). -# @return An encoded string containing the XML data. -# @defreturn string - -def tostring(element, encoding=None, method=None): - class dummy: - pass - data = [] - file = dummy() - file.write = data.append - ElementTree(element).write(file, encoding, method=method) - return "".join(data) - -## -# Generates a string representation of an XML element, including all -# subelements. The string is returned as a sequence of string fragments. -# -# @param element An Element instance. -# @keyparam encoding Optional output encoding (default is US-ASCII). -# @keyparam method Optional output method ("xml", "html", "text" or -# "c14n"; default is "xml"). -# @return A sequence object containing the XML data. -# @defreturn sequence -# @since 1.3 - -def tostringlist(element, encoding=None, method=None): - class dummy: - pass - data = [] - file = dummy() - file.write = data.append - ElementTree(element).write(file, encoding, method=method) - # FIXME: merge small fragments into larger parts - return data - -## -# Writes an element tree or element structure to sys.stdout. This -# function should be used for debugging only. -#

-# The exact output format is implementation dependent. In this -# version, it's written as an ordinary XML file. -# -# @param elem An element tree or an individual element. - -def dump(elem): - # debugging - if not isinstance(elem, ElementTree): - elem = ElementTree(elem) - elem.write(sys.stdout) - tail = elem.getroot().tail - if not tail or tail[-1] != "\n": - sys.stdout.write("\n") - -# -------------------------------------------------------------------- -# parsing - -## -# Parses an XML document into an element tree. -# -# @param source A filename or file object containing XML data. -# @param parser An optional parser instance. If not given, the -# standard {@link XMLParser} parser is used. -# @return An ElementTree instance - -def parse(source, parser=None): - tree = ElementTree() - tree.parse(source, parser) - return tree - -## -# Parses an XML document into an element tree incrementally, and reports -# what's going on to the user. -# -# @param source A filename or file object containing XML data. -# @param events A list of events to report back. If omitted, only "end" -# events are reported. -# @param parser An optional parser instance. If not given, the -# standard {@link XMLParser} parser is used. -# @return A (event, elem) iterator. - -def iterparse(source, events=None, parser=None): - close_source = False - if not hasattr(source, "read"): - source = open(source, "rb") - close_source = True - try: - if not parser: - parser = XMLParser(target=TreeBuilder()) - return _IterParseIterator(source, events, parser, close_source) - except: - if close_source: - source.close() - raise - -class _IterParseIterator(object): - - def __init__(self, source, events, parser, close_source=False): - self._file = source - self._close_file = close_source - self._events = [] - self._index = 0 - self._error = None - self.root = self._root = None - self._parser = parser - # wire up the parser for event reporting - parser = self._parser._parser - append = self._events.append - if events is None: - events = ["end"] - for event in events: - if event == "start": - try: - parser.ordered_attributes = 1 - parser.specified_attributes = 1 - def handler(tag, attrib_in, event=event, append=append, - start=self._parser._start_list): - append((event, start(tag, attrib_in))) - parser.StartElementHandler = handler - except AttributeError: - def handler(tag, attrib_in, event=event, append=append, - start=self._parser._start): - append((event, start(tag, attrib_in))) - parser.StartElementHandler = handler - elif event == "end": - def handler(tag, event=event, append=append, - end=self._parser._end): - append((event, end(tag))) - parser.EndElementHandler = handler - elif event == "start-ns": - def handler(prefix, uri, event=event, append=append): - try: - uri = (uri or "").encode("ascii") - except UnicodeError: - pass - append((event, (prefix or "", uri or ""))) - parser.StartNamespaceDeclHandler = handler - elif event == "end-ns": - def handler(prefix, event=event, append=append): - append((event, None)) - parser.EndNamespaceDeclHandler = handler - else: - raise ValueError("unknown event %r" % event) - - def next(self): - try: - while 1: - try: - item = self._events[self._index] - self._index += 1 - return item - except IndexError: - pass - if self._error: - e = self._error - self._error = None - raise e - if self._parser is None: - self.root = self._root - break - # load event buffer - del self._events[:] - self._index = 0 - data = self._file.read(16384) - if data: - try: - self._parser.feed(data) - except SyntaxError as exc: - self._error = exc - else: - self._root = self._parser.close() - self._parser = None - except: - if self._close_file: - self._file.close() - raise - if self._close_file: - self._file.close() - raise StopIteration - - def __iter__(self): - return self - -## -# Parses an XML document from a string constant. This function can -# be used to embed "XML literals" in Python code. -# -# @param source A string containing XML data. -# @param parser An optional parser instance. If not given, the -# standard {@link XMLParser} parser is used. -# @return An Element instance. -# @defreturn Element - -def XML(text, parser=None): - if not parser: - parser = XMLParser(target=TreeBuilder()) - parser.feed(text) - return parser.close() - -## -# Parses an XML document from a string constant, and also returns -# a dictionary which maps from element id:s to elements. -# -# @param source A string containing XML data. -# @param parser An optional parser instance. If not given, the -# standard {@link XMLParser} parser is used. -# @return A tuple containing an Element instance and a dictionary. -# @defreturn (Element, dictionary) - -def XMLID(text, parser=None): - if not parser: - parser = XMLParser(target=TreeBuilder()) - parser.feed(text) - tree = parser.close() - ids = {} - for elem in tree.iter(): - id = elem.get("id") - if id: - ids[id] = elem - return tree, ids - -## -# Parses an XML document from a string constant. Same as {@link #XML}. -# -# @def fromstring(text) -# @param source A string containing XML data. -# @return An Element instance. -# @defreturn Element - -fromstring = XML - -## -# Parses an XML document from a sequence of string fragments. -# -# @param sequence A list or other sequence containing XML data fragments. -# @param parser An optional parser instance. If not given, the -# standard {@link XMLParser} parser is used. -# @return An Element instance. -# @defreturn Element -# @since 1.3 - -def fromstringlist(sequence, parser=None): - if not parser: - parser = XMLParser(target=TreeBuilder()) - for text in sequence: - parser.feed(text) - return parser.close() - -# -------------------------------------------------------------------- - -## -# Generic element structure builder. This builder converts a sequence -# of {@link #TreeBuilder.start}, {@link #TreeBuilder.data}, and {@link -# #TreeBuilder.end} method calls to a well-formed element structure. -#

-# You can use this class to build an element structure using a custom XML -# parser, or a parser for some other XML-like format. -# -# @param element_factory Optional element factory. This factory -# is called to create new Element instances, as necessary. - -class TreeBuilder(object): - - def __init__(self, element_factory=None): - self._data = [] # data collector - self._elem = [] # element stack - self._last = None # last element - self._tail = None # true if we're after an end tag - if element_factory is None: - element_factory = Element - self._factory = element_factory - - ## - # Flushes the builder buffers, and returns the toplevel document - # element. - # - # @return An Element instance. - # @defreturn Element - - def close(self): - assert len(self._elem) == 0, "missing end tags" - assert self._last is not None, "missing toplevel element" - return self._last - - def _flush(self): - if self._data: - if self._last is not None: - text = "".join(self._data) - if self._tail: - assert self._last.tail is None, "internal error (tail)" - self._last.tail = text - else: - assert self._last.text is None, "internal error (text)" - self._last.text = text - self._data = [] - - ## - # Adds text to the current element. - # - # @param data A string. This should be either an 8-bit string - # containing ASCII text, or a Unicode string. - - def data(self, data): - self._data.append(data) - - ## - # Opens a new element. - # - # @param tag The element name. - # @param attrib A dictionary containing element attributes. - # @return The opened element. - # @defreturn Element - - def start(self, tag, attrs): - self._flush() - self._last = elem = self._factory(tag, attrs) - if self._elem: - self._elem[-1].append(elem) - self._elem.append(elem) - self._tail = 0 - return elem - - ## - # Closes the current element. - # - # @param tag The element name. - # @return The closed element. - # @defreturn Element - - def end(self, tag): - self._flush() - self._last = self._elem.pop() - assert self._last.tag == tag,\ - "end tag mismatch (expected %s, got %s)" % ( - self._last.tag, tag) - self._tail = 1 - return self._last - -_sentinel = ['sentinel'] - -## -# Element structure builder for XML source data, based on the -# expat parser. -# -# @keyparam target Target object. If omitted, the builder uses an -# instance of the standard {@link #TreeBuilder} class. -# @keyparam html Predefine HTML entities. This flag is not supported -# by the current implementation. -# @keyparam encoding Optional encoding. If given, the value overrides -# the encoding specified in the XML file. -# @see #ElementTree -# @see #TreeBuilder - -class XMLParser(object): - - def __init__(self, html=_sentinel, target=None, encoding=None): - if html is not _sentinel: - warnings.warnpy3k( - "The html argument of XMLParser() is deprecated", - DeprecationWarning, stacklevel=2) - try: - from xml.parsers import expat - except ImportError: - try: - import pyexpat as expat - except ImportError: - raise ImportError( - "No module named expat; use SimpleXMLTreeBuilder instead" - ) - parser = expat.ParserCreate(encoding, "}") - if target is None: - target = TreeBuilder() - # underscored names are provided for compatibility only - self.parser = self._parser = parser - self.target = self._target = target - self._error = expat.error - self._names = {} # name memo cache - # callbacks - parser.DefaultHandlerExpand = self._default - parser.StartElementHandler = self._start - parser.EndElementHandler = self._end - parser.CharacterDataHandler = self._data - # optional callbacks - parser.CommentHandler = self._comment - parser.ProcessingInstructionHandler = self._pi - # let expat do the buffering, if supported - try: - self._parser.buffer_text = 1 - except AttributeError: - pass - # use new-style attribute handling, if supported - try: - self._parser.ordered_attributes = 1 - self._parser.specified_attributes = 1 - parser.StartElementHandler = self._start_list - except AttributeError: - pass - self._doctype = None - self.entity = {} - try: - self.version = "Expat %d.%d.%d" % expat.version_info - except AttributeError: - pass # unknown - - def _raiseerror(self, value): - err = ParseError(value) - err.code = value.code - err.position = value.lineno, value.offset - raise err - - def _fixtext(self, text): - # convert text string to ascii, if possible - try: - return text.encode("ascii") - except UnicodeError: - return text - - def _fixname(self, key): - # expand qname, and convert name string to ascii, if possible - try: - name = self._names[key] - except KeyError: - name = key - if "}" in name: - name = "{" + name - self._names[key] = name = self._fixtext(name) - return name - - def _start(self, tag, attrib_in): - fixname = self._fixname - fixtext = self._fixtext - tag = fixname(tag) - attrib = {} - for key, value in attrib_in.items(): - attrib[fixname(key)] = fixtext(value) - return self.target.start(tag, attrib) - - def _start_list(self, tag, attrib_in): - fixname = self._fixname - fixtext = self._fixtext - tag = fixname(tag) - attrib = {} - if attrib_in: - for i in range(0, len(attrib_in), 2): - attrib[fixname(attrib_in[i])] = fixtext(attrib_in[i+1]) - return self.target.start(tag, attrib) - - def _data(self, text): - return self.target.data(self._fixtext(text)) - - def _end(self, tag): - return self.target.end(self._fixname(tag)) - - def _comment(self, data): - try: - comment = self.target.comment - except AttributeError: - pass - else: - return comment(self._fixtext(data)) - - def _pi(self, target, data): - try: - pi = self.target.pi - except AttributeError: - pass - else: - return pi(self._fixtext(target), self._fixtext(data)) - - def _default(self, text): - prefix = text[:1] - if prefix == "&": - # deal with undefined entities - try: - self.target.data(self.entity[text[1:-1]]) - except KeyError: - from xml.parsers import expat - err = expat.error( - "undefined entity %s: line %d, column %d" % - (text, self._parser.ErrorLineNumber, - self._parser.ErrorColumnNumber) - ) - err.code = 11 # XML_ERROR_UNDEFINED_ENTITY - err.lineno = self._parser.ErrorLineNumber - err.offset = self._parser.ErrorColumnNumber - raise err - elif prefix == "<" and text[:9] == "": - self._doctype = None - return - text = text.strip() - if not text: - return - self._doctype.append(text) - n = len(self._doctype) - if n > 2: - type = self._doctype[1] - if type == "PUBLIC" and n == 4: - name, type, pubid, system = self._doctype - elif type == "SYSTEM" and n == 3: - name, type, system = self._doctype - pubid = None - else: - return - if pubid: - pubid = pubid[1:-1] - if hasattr(self.target, "doctype"): - self.target.doctype(name, pubid, system[1:-1]) - elif self.doctype != self._XMLParser__doctype: - # warn about deprecated call - self._XMLParser__doctype(name, pubid, system[1:-1]) - self.doctype(name, pubid, system[1:-1]) - self._doctype = None - - ## - # (Deprecated) Handles a doctype declaration. - # - # @param name Doctype name. - # @param pubid Public identifier. - # @param system System identifier. - - def doctype(self, name, pubid, system): - """This method of XMLParser is deprecated.""" - warnings.warn( - "This method of XMLParser is deprecated. Define doctype() " - "method on the TreeBuilder target.", - DeprecationWarning, - ) - - # sentinel, if doctype is redefined in a subclass - __doctype = doctype - - ## - # Feeds data to the parser. - # - # @param data Encoded data. - - def feed(self, data): - try: - self._parser.Parse(data, 0) - except self._error as v: - self._raiseerror(v) - - ## - # Finishes feeding data to the parser. - # - # @return An element structure. - # @defreturn Element - - def close(self): - try: - self._parser.Parse("", 1) # end of data - except self._error as v: - self._raiseerror(v) - tree = self.target.close() - del self.target, self._parser # get rid of circular references - return tree - -# compatibility -XMLTreeBuilder = XMLParser - -# workaround circular import. -try: - from ElementC14N import _serialize_c14n - _serialize["c14n"] = _serialize_c14n -except ImportError: - pass \ No newline at end of file diff --git a/insights/contrib/importlib.py b/insights/contrib/importlib.py deleted file mode 100644 index ad31a1ac47..0000000000 --- a/insights/contrib/importlib.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Backport of importlib.import_module from 3.x.""" -# While not critical (and in no way guaranteed!), it would be nice to keep this -# code compatible with Python 2.3. -import sys - -def _resolve_name(name, package, level): - """Return the absolute name of the module to be imported.""" - if not hasattr(package, 'rindex'): - raise ValueError("'package' not set to a string") - dot = len(package) - for x in xrange(level, 1, -1): - try: - dot = package.rindex('.', 0, dot) - except ValueError: - raise ValueError("attempted relative import beyond top-level " - "package") - return "%s.%s" % (package[:dot], name) - - -def import_module(name, package=None): - """Import a module. - - The 'package' argument is required when performing a relative import. It - specifies the package to use as the anchor point from which to resolve the - relative import to an absolute import. - - """ - if name.startswith('.'): - if not package: - raise TypeError("relative imports require the 'package' argument") - level = 0 - for character in name: - if character != '.': - break - level += 1 - name = _resolve_name(name[level:], package, level) - __import__(name) - return sys.modules[name] diff --git a/insights/core/__init__.py b/insights/core/__init__.py index 5a994883cc..beb3bdc4ec 100644 --- a/insights/core/__init__.py +++ b/insights/core/__init__.py @@ -26,21 +26,15 @@ try: from yaml import CSafeLoader as SafeLoader -except ImportError: +except ImportError: # pragma: no cover from yaml import SafeLoader -# Since XPath expression is not supported by the ElementTree in Python 2.6, -# import insights.contrib.ElementTree when running python is prior to 2.6 for compatibility. -# Script insights.contrib.ElementTree is the same with xml.etree.ElementTree in Python 2.7.14 -# Otherwise, import defusedxml.ElementTree to avoid XML vulnerabilities, +# Import defusedxml.ElementTree firstly to avoid XML vulnerabilities, # if dependency not installed import xml.etree.ElementTree instead. -if sys.version_info[0] == 2 and sys.version_info[1] <= 6: - import insights.contrib.ElementTree as ET -else: - try: - import defusedxml.ElementTree as ET - except: - import xml.etree.ElementTree as ET +try: + import defusedxml.ElementTree as ET +except: # pragma: no cover + import xml.etree.ElementTree as ET log = logging.getLogger(__name__) diff --git a/insights/core/dr.py b/insights/core/dr.py index fa8ce8c466..31f9400520 100644 --- a/insights/core/dr.py +++ b/insights/core/dr.py @@ -48,9 +48,10 @@ def add(a, b): """ from __future__ import print_function +import importlib import inspect -import logging import json +import logging import os import pkgutil import re @@ -62,7 +63,6 @@ def add(a, b): from collections import defaultdict from functools import reduce as _reduce -from insights.contrib import importlib from insights.contrib.toposort import toposort_flatten from insights.core.blacklist import BLACKLISTED_SPECS from insights.core.context import SerializedArchiveContext diff --git a/insights/parsers/nftables.py b/insights/parsers/nftables.py index 8b95703c3b..7510c75b6a 100644 --- a/insights/parsers/nftables.py +++ b/insights/parsers/nftables.py @@ -83,14 +83,14 @@ class NftListRuleSet(JSONParser): Examples: >>> type(nft_obj) - >>> [str(item) for item in nft_obj.tables('ip')] # change unicode to string to be compatible with python2.6/2.7 + >>> [str(item) for item in nft_obj.tables('ip')] # change unicode to string to be compatible with python2.7 ['table1'] >>> chains = nft_obj.chains('ip', 'table1') >>> len(chains) 1 - >>> str(chains[0]['name'].value) # change unicode to string to be compatible with python2.6/2.7 + >>> str(chains[0]['name'].value) # change unicode to string to be compatible with python2.7 'chain1' - >>> str(chains[0]['type'].value) # change unicode to string to be compatible with python2.6/2.7 + >>> str(chains[0]['type'].value) # change unicode to string to be compatible with python2.7 'filter' >>> rules = nft_obj.rules('ip', 'table1', 'chain1') >>> len(rules) diff --git a/insights/parsr/query/boolean.py b/insights/parsr/query/boolean.py index 6e9db5a7df..75a7906cb7 100644 --- a/insights/parsr/query/boolean.py +++ b/insights/parsr/query/boolean.py @@ -39,9 +39,9 @@ def is_positive(n): gt_five_and_lt_10 = gt(5) & lt(10) """ + from itertools import count import six -import sys class Boolean(object): @@ -61,10 +61,6 @@ def __call__(self, value): return self.test(value) def to_pyfunc(self): - ver = sys.version_info - if ver[0] == 2 and ver[1] == 6: - return self.test - env = {} ids = count() @@ -100,7 +96,9 @@ def predicate(value): return {body} except Exception as ex: return False - """.format(body=expr(self)) + """.format( + body=expr(self) + ) six.exec_(func, env, env) return env["predicate"] @@ -169,6 +167,7 @@ def inner(val): if ignore_case: return CaselessPredicate(func, val.lower()) return Predicate(func, val) + return inner diff --git a/insights/specs/datasources/yum_updates.py b/insights/specs/datasources/yum_updates.py index 13c1329ec2..ceb883d84c 100644 --- a/insights/specs/datasources/yum_updates.py +++ b/insights/specs/datasources/yum_updates.py @@ -1,23 +1,21 @@ """ Custom datasource for collecting yum updates +============================================ """ import json import logging import time +from distutils.version import LooseVersion as version +from functools import cmp_to_key + from insights import datasource, HostContext, SkipComponent from insights.components.rhel_version import IsRhel7, IsRhel8, IsRhel9 from insights.core.spec_factory import DatasourceProvider -from distutils.version import LooseVersion as version -try: - from functools import cmp_to_key - # cmp_to_key is not available in python 2.6, but it has sorted function which accepts cmp function - def sorted_cmp(it, cmp): - return sorted(it, key=cmp_to_key(cmp)) -except ImportError: - sorted_cmp = sorted +def sorted_cmp(it, cmp): + return sorted(it, key=cmp_to_key(cmp)) class DnfManager: diff --git a/insights/tests/client/apps/test_playbook_verifier.py b/insights/tests/client/apps/test_playbook_verifier.py index bd56240c2a..176052b3d8 100644 --- a/insights/tests/client/apps/test_playbook_verifier.py +++ b/insights/tests/client/apps/test_playbook_verifier.py @@ -12,20 +12,14 @@ from mock.mock import patch from pytest import raises +from insights.client.constants import InsightsConstants as constants +from insights.client.apps.ansible import playbook_verifier +from insights.client.apps.ansible.playbook_verifier import verify, PlaybookVerificationError, get_play_revocation_list, normalize_play_py2, load_playbook_yaml # noqa -# Because pytest 3 does not allow module-level skips, we have to import these -# modules conditionally and skip each individual module here. -if sys.version_info > (2, 6): - from insights.client.constants import InsightsConstants as constants - from insights.client.apps.ansible import playbook_verifier - from insights.client.apps.ansible.playbook_verifier import verify, PlaybookVerificationError, get_play_revocation_list, normalize_play_py2, load_playbook_yaml # noqa - -SKIP_BELOW_27 = pytest.mark.skipif(sys.version_info < (2, 7), reason="Unsupported; needs Python 2.7+ or 3.6+") SKIP_ON_3 = pytest.mark.skipif(sys.version_info[0] > 2, reason="Only required in Python 2") -@SKIP_BELOW_27 class TestErrors: @patch("insights.client.apps.ansible.playbook_verifier.get_play_revocation_list", return_value=[]) def test_vars_not_found_error(self, mock_method): @@ -206,7 +200,6 @@ def test_revoked_playbook(self, call_1, call_2): @SKIP_ON_3 -@SKIP_BELOW_27 def test_normalize_snippet(): playbook = '''task: when: @@ -227,7 +220,6 @@ def test_normalize_snippet(): assert normalize_play_py2(snippet) == want -@SKIP_BELOW_27 class TestExcludeDynamicElements: def test_ok_signature(self): source = { @@ -320,7 +312,6 @@ def test_invalid_exclusion_key(self): assert excinfo.value.message == expected -@SKIP_BELOW_27 class TestPlaybookSerializer: def test_list(self): @@ -374,7 +365,6 @@ def test_strings(self, source, expected): assert result == expected -@SKIP_BELOW_27 class TestSerializePlaybookSnippet: def test_serialize_dictionary(self): raw = "\n".join([ @@ -438,7 +428,6 @@ def test_real(self, filename): assert result == expected -@SKIP_BELOW_27 class TestGetPlaybookSnippetRevocationList: @mock.patch( "insights.client.apps.ansible.playbook_verifier.verify_play", @@ -494,7 +483,6 @@ def test_invalid_signature(self, mocked_verify_play): assert mocked_verify_play.call_count == 1 -@SKIP_BELOW_27 class TestHashPlaybookSnippets: @pytest.mark.parametrize("filename", ("insights_remove", "document-from-hell", "unicode")) def test_real(self, filename): diff --git a/insights/tests/client/test_utilities.py b/insights/tests/client/test_utilities.py index f8ede1ea96..30e4ec69ea 100644 --- a/insights/tests/client/test_utilities.py +++ b/insights/tests/client/test_utilities.py @@ -1,5 +1,4 @@ import os -import sys from tarfile import open as tar_open import tarfile import tempfile @@ -345,7 +344,6 @@ def test_get_tags_nonexist(): assert got is None -@pytest.mark.skipif(sys.version_info < (2, 7), reason='Playbook verifier uses oyaml library which is incompatable with this test') def test_write_tags(): tags = {'foo': 'bar'} fp = tempfile.NamedTemporaryFile() diff --git a/insights/tests/datasources/malware_detection/test_malware_detection.py b/insights/tests/datasources/malware_detection/test_malware_detection.py index f62b68f681..ebebcba771 100644 --- a/insights/tests/datasources/malware_detection/test_malware_detection.py +++ b/insights/tests/datasources/malware_detection/test_malware_detection.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import os import re -import sys import pytest import yaml @@ -71,9 +70,6 @@ TEST_GET_DISABLED_RULES_FAILURE = getenv_bool("TEST_GET_DISABLED_RULES_FAILURE", False) TEST_ALL = getenv_bool("TEST_ALL", False) -# Are we running on RHEL6? (well actually, with python 2.6) -IS_RHEL6 = sys.version_info < (2, 7) -SKIP_IF_RHEL6_REASON = "The malware-detection client isn't supported on RHEL6 / python 2.6" # Is the best way to determine if we are running in a container? IS_CONTAINER = os.path.exists("/.dockerenv") or os.path.exists("/run/.containerenv") or '1' not in call('pidof init systemd').strip().split() SKIP_IF_CONTAINER_REASON = "This test uses running process that may not exist when run in a container" @@ -111,7 +107,6 @@ def extract_tmp_files(): ####################################################################### # The following section tests functionality that does not require yara ####################################################################### -@pytest.mark.skipif(IS_RHEL6, reason=SKIP_IF_RHEL6_REASON) class TestsNotUtilizingYara: def test_default_spec(self): @@ -323,7 +318,6 @@ def test_process_include_exclude_tmp_files(self, extract_tmp_files): ################################################################################################### # The following section tests functionality that requires yara but can do with its execution faked ################################################################################################### -@pytest.mark.skipif(IS_RHEL6, reason=SKIP_IF_RHEL6_REASON) class TestsUtilizingFakeYara: @patch(BUILD_YARA_COMMAND_TARGET) @@ -2289,7 +2283,6 @@ def create_test_files_real_yara(): yield os.system('rm -rf %s' % TEMP_TEST_DIR) - @pytest.mark.skipif(IS_RHEL6, reason=SKIP_IF_RHEL6_REASON) class TestsUtilizingRealYara: @patch(CONFIG_FILE_TARGET, TEMP_CONFIG_FILE) diff --git a/insights/tests/test_formats.py b/insights/tests/test_formats.py index 90c03c872c..918919f4f2 100644 --- a/insights/tests/test_formats.py +++ b/insights/tests/test_formats.py @@ -1,4 +1,3 @@ -import sys import pytest from six import StringIO from insights import dr, make_fail, rule @@ -81,7 +80,6 @@ def test_syslog_format_archive(): assert SL_PATH in data -@pytest.mark.skipif(sys.version_info < (2, 7), reason='Playbook verifier code uses oyaml library which is incompatable with this test') def test_yaml_format(): broker = dr.Broker() output = StringIO() diff --git a/insights/tests/test_remote_resource.py b/insights/tests/test_remote_resource.py index ae66eb26c5..377d9a62c3 100644 --- a/insights/tests/test_remote_resource.py +++ b/insights/tests/test_remote_resource.py @@ -1,8 +1,6 @@ from cachecontrol.cache import DictCache from insights.core.remote_resource import RemoteResource, CachedRemoteResource from insights.tests.mock_web_server import TestMockServer -import sys -import pytest GOOD_PAYLOAD = b'Successful return from Mock Service' NOT_FOUND = b'{"error":{"code": "404", "message": "Not Found"}}' @@ -35,7 +33,6 @@ def test_get_cached_remote_resource(self): assert GOOD_PAYLOAD in rtn.content # Test CachedRemoteResource returns cached response - @pytest.mark.skipif(sys.version_info < (2, 7), reason="CacheControl requires python 2.7 or higher") def test_get_cached_remote_resource_cached(self): crr = CachedRemoteResource() diff --git a/insights/tests/tools/test_apply_spec_filters.py b/insights/tests/tools/test_apply_spec_filters.py index 7a325d3423..06126543a2 100644 --- a/insights/tests/tools/test_apply_spec_filters.py +++ b/insights/tests/tools/test_apply_spec_filters.py @@ -1,7 +1,5 @@ import json import os -import pytest -import sys import yaml from collections import defaultdict @@ -85,7 +83,6 @@ def teardown_function(): filters.FILTERS = defaultdict(dict) -@pytest.mark.skipif(sys.version_info < (2, 7), reason='Skip py26') def test_apply_specs_filters_json(): apply_spec_filters.apply_filters("json", 'insights.parsers', JSON_file) diff --git a/insights/tests/util/test_util.py b/insights/tests/util/test_util.py index 6181e0921e..f99feeae11 100644 --- a/insights/tests/util/test_util.py +++ b/insights/tests/util/test_util.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- import pytest -import sys import warnings from insights.tests import deep_compare @@ -256,7 +255,6 @@ def test_case_variants(): assert case_variants('hosts:') == ['hosts:', 'HOSTS:', 'Hosts:'] -@pytest.mark.skipif(sys.version_info < (2, 7), reason='Code with PYTEST_CURRENT_TEST is incompatible with py26') def test_deprecated(): def normal_fn(): return 1 diff --git a/setup.py b/setup.py index c8f10e3956..9edcc62613 100644 --- a/setup.py +++ b/setup.py @@ -35,8 +35,7 @@ 'lockfile', 'jinja2<=2.11.3; python_version <= "2.7"', 'jinja2; python_version > "2.7"', - 'pyyaml>=3.10,<=3.13; python_version < "2.7"', - 'pyyaml; python_version >= "2.7"', + 'pyyaml', 'setuptools; python_version >= "3.12"', ] ) @@ -82,17 +81,12 @@ def maybe_require(pkg): ] ) -# python 2.6 requires setuptools~=36.8.0 to support this syntax testing = set( [ - 'coverage==4.3.4; python_version < "2.7"', - 'coverage; python_version >= "2.7"', - 'pytest==3.0.6; python_version < "2.7"', - 'pytest_catchlog; python_version < "2.7"', + 'coverage', 'pytest~=4.6.0; python_version == "2.7"', 'pytest; python_version >= "3"', - 'pytest-cov==2.4.0; python_version < "2.7"', - 'pytest-cov; python_version >= "2.7"', + 'pytest-cov', 'mock==2.0.0', ] ) @@ -107,7 +101,7 @@ def maybe_require(pkg): openshift = set(['openshift']) -linting = set(['flake8==2.6.2; python_version < "2.7"', 'flake8; python_version >= "2.7"']) +linting = set(['flake8']) optional = set( [ @@ -136,7 +130,6 @@ def maybe_require(pkg): license='Apache 2.0', extras_require={ 'develop': list(runtime | develop | client | docs | linting | testing | cluster), - 'develop26': list(runtime | develop | client | linting | testing | cluster), 'client': list(runtime | client), 'client-develop': list(runtime | develop | client | linting | testing), 'cluster': list(runtime | cluster), @@ -152,7 +145,6 @@ def maybe_require(pkg): 'Natural Language :: English', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', diff --git a/shippable.yml b/shippable.yml deleted file mode 100644 index b3bcdccb3e..0000000000 --- a/shippable.yml +++ /dev/null @@ -1,17 +0,0 @@ -language: python - -python: - - 2.6 - - 2.7 - - 3.3 - -build: - ci: - - if [ $SHIPPABLE_PYTHON_VERSION == "2.6" ]; then cp .collections.py /usr/lib/python2.6/collections.py; fi - - pip install -e .[develop] - - pip install --upgrade setuptools - - mkdir -p shippable/testresults shippable/codecoverage - - py.test --cov=insights --cov-report=xml --junit-xml=shippable/testresults/results.xml - - mv coverage.xml shippable/codecoverage - - flake8 - - if [ $SHIPPABLE_PYTHON_VERSION == "2.7" ]; then apt-get -y install pandoc && sphinx-build -W -b html -qa -E docs docs/_build/html; fi