Add python packages cbor2 and pyhpke to third party tools

tools/localpaths.py

@@ -36,3 +36,5 @@
 sys.path.insert(0, os.path.join(here, "wptrunner"))
 sys.path.insert(0, os.path.join(here, "webtransport"))
 sys.path.insert(0, os.path.join(here, "third_party_modified", "mozlog"))
+sys.path.insert(0, os.path.join(here, "third_party", "cbor2"))
+sys.path.insert(0, os.path.join(here, "third_party", "pyhpke"))

tools/third_party/cbor2/LICENSE.txt

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Alex Grönholm
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

tools/third_party/cbor2/README.rst

@@ -0,0 +1,121 @@
+.. image:: https://github.com/agronholm/cbor2/actions/workflows/test.yml/badge.svg
+  :target: https://github.com/agronholm/cbor2/actions/workflows/test.yml
+  :alt: Testing Status
+.. image:: https://github.com/agronholm/cbor2/actions/workflows/publish.yml/badge.svg
+  :target: https://github.com/agronholm/cbor2/actions/workflows/publish.yml
+  :alt: Publish Status
+.. image:: https://coveralls.io/repos/github/agronholm/cbor2/badge.svg?branch=master
+  :target: https://coveralls.io/github/agronholm/cbor2?branch=master
+  :alt: Code Coverage
+.. image:: https://readthedocs.org/projects/cbor2/badge/?version=latest
+  :target: https://cbor2.readthedocs.io/en/latest/?badge=latest
+  :alt: Documentation Status
+
+About
+=====
+
+This library provides encoding and decoding for the Concise Binary Object Representation (CBOR)
+(`RFC 8949`_) serialization format. The specification is fully compatible with the original RFC 7049.
+`Read the docs <https://cbor2.readthedocs.io/>`_ to learn more.
+
+It is implemented in pure python with an optional C backend.
+
+On PyPy, cbor2 runs with almost identical performance to the C backend.
+
+.. _RFC 8949: https://www.rfc-editor.org/rfc/rfc8949.html
+
+Features
+--------
+
+* Simple api like ``json`` or ``pickle`` modules.
+* Support many `CBOR tags`_ with `stdlib objects`_.
+* Generic tag decoding.
+* `Shared value`_ references including cyclic references.
+* `String references`_ compact encoding with repeated strings replaced with indices.
+* Optional C module backend tested on big- and little-endian architectures.
+* Extensible `tagged value handling`_ using ``tag_hook`` and ``object_hook`` on decode and ``default`` on encode.
+* Command-line diagnostic tool, converting CBOR file or stream to JSON ``python -m cbor2.tool``
+  (This is a lossy conversion, for diagnostics only)
+* Thorough test suite.
+
+.. _CBOR tags: https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml
+.. _stdlib objects: https://cbor2.readthedocs.io/en/latest/usage.html#tag-support
+.. _Shared value: http://cbor.schmorp.de/value-sharing
+.. _String references: http://cbor.schmorp.de/stringref
+.. _tagged value handling: https://cbor2.readthedocs.io/en/latest/customizing.html#using-the-cbor-tags-for-custom-types
+
+Installation
+============
+
+::
+
+    pip install cbor2
+
+Requirements
+------------
+
+* Python >= 3.8 (or `PyPy3`_ 3.8+)
+* C-extension: Any C compiler that can build Python extensions.
+  Any modern libc with the exception of Glibc<2.9
+
+.. _PyPy3: https://www.pypy.org/
+
+Building the C-Extension
+------------------------
+
+To force building of the optional C-extension, set OS env ``CBOR2_BUILD_C_EXTENSION=1``.
+To disable building of the optional C-extension, set OS env ``CBOR2_BUILD_C_EXTENSION=0``.
+If this environment variable is unset, setup.py will default to auto detecting a compatible C library and
+attempt to compile the extension.
+
+
+Usage
+=====
+
+`Basic Usage <https://cbor2.readthedocs.io/en/latest/usage.html#basic-usage>`_
+
+Command-line Usage
+==================
+
+The provided command line tool (``cbor2``) converts CBOR data in raw binary or base64
+encoding into a representation that allows printing as JSON. This is a lossy
+transformation as each datatype is converted into something that can be represented as a
+JSON value.
+
+The tool can alternatively be invoked with ``python -m cbor2.tool``.
+
+Usage::
+
+    # Pass hexadecimal through xxd.
+    $ echo a16568656c6c6f65776f726c64 | xxd -r -ps | cbor2 --pretty
+    {
+        "hello": "world"
+    }
+    # Decode Base64 directly
+    $ echo ggEC | python -m cbor2.tool --decode
+    [1, 2]
+    # Read from a file encoded in Base64
+    $ python -m cbor2.tool -d tests/examples.cbor.b64
+    {...}
+
+It can be used in a pipeline with json processing tools like `jq`_ to allow syntax
+coloring, field extraction and more.
+
+CBOR data items concatenated into a sequence can be decoded also::
+
+    $ echo ggECggMEggUG | cbor2 -d --sequence
+    [1, 2]
+    [3, 4]
+    [5, 6]
+
+Multiple files can also be sent to a single output file::
+
+    $ cbor2 -o all_files.json file1.cbor file2.cbor ... fileN.cbor
+
+.. _jq: https://stedolan.github.io/jq/
+
+Security
+========
+
+This library has not been tested against malicious input. In theory it should be
+as safe as JSON, since unlike ``pickle`` the decoder does not execute any code.

tools/third_party/cbor2/benchmarks/README.rst

@@ -0,0 +1,8 @@
+cbor2 Benchmarks
+================
+
+To run the benchmarks::
+
+    pip install -e .[benchmarks]
+    cd benchmarks
+    python -m pytest

tools/third_party/cbor2/benchmarks/tests/conftest.py

@@ -0,0 +1,51 @@
+import json
+from collections import namedtuple
+from operator import attrgetter
+
+from cbor2.decoder import loads as pyloads
+from cbor2.encoder import dumps as pydumps
+
+try:
+    import _cbor2
+except ModuleNotFoundError as e:
+    if not str(e).startswith("No module"):
+        load_exc = str(e)
+    _cbor2 = None
+
+Contender = namedtuple("Contender", "name,dumps,loads")
+
+contenders = []
+
+
+contenders.append(Contender("json", json.dumps, json.loads))
+
+
+if _cbor2 is not None:
+    contenders.append(Contender("ccbor2", _cbor2.dumps, _cbor2.loads))
+else:
+    contenders.append(Contender("ppcbor2", pydumps, pyloads))
+
+
+# See https://github.com/pytest-dev/pytest-cov/issues/418
+def pytest_configure(config):
+    cov = config.pluginmanager.get_plugin("_cov")
+    cov.options.no_cov = True
+    if cov.cov_controller:
+        cov.cov_controller.pause()
+
+
+# See https://github.com/ionelmc/pytest-benchmark/issues/48
+
+
+def pytest_benchmark_group_stats(config, benchmarks, group_by):
+    result = {}
+    for bench in benchmarks:
+        engine, data_kind = bench["param"].split("-")
+        group = result.setdefault("{}: {}".format(data_kind, bench["group"]), [])
+        group.append(bench)
+    return sorted(result.items())
+
+
+def pytest_generate_tests(metafunc):
+    if "contender" in metafunc.fixturenames:
+        metafunc.parametrize("contender", contenders, ids=attrgetter("name"))

tools/third_party/cbor2/benchmarks/tests/test_vs_json.py

@@ -0,0 +1,104 @@
+from random import randint, random
+from sys import maxsize
+
+import pytest
+
+composite_object = {
+    "words": """
+        Lorem ipsum dolor sit amet, consectetur adipiscing
+        elit. Mauris adipiscing adipiscing placerat.
+        Vestibulum augue augue,
+        pellentesque quis sollicitudin id, adipiscing.
+        """,
+    "list": list(range(200)),
+    "dict": {str(i): "a" for i in list(range(200))},
+    "int": 100100100,
+    "float": 100999.123456,
+}
+
+user = {
+    "userId": 3381293,
+    "age": 213,
+    "username": "johndoe",
+    "fullname": "John Doe the Second",
+    "isAuthorized": True,
+    "liked": 31231.31231202,
+    "approval": 31.1471,
+    "jobs": [1, 2],
+    "currJob": None,
+}
+
+friends = [user, user, user, user, user, user, user, user]
+
+doubles = []
+numbers = []
+unicode_strings = []
+strings = []
+booleans = []
+datetimes = []
+list_dicts = []
+dict_lists = {}
+
+complex_object = [
+    [user, friends],
+    [user, friends],
+    [user, friends],
+    [user, friends],
+    [user, friends],
+    [user, friends],
+]
+
+for x in range(256):
+    doubles.append(maxsize * random())
+    numbers.append(maxsize * random())
+    numbers.append(randint(0, maxsize))
+    unicode_strings.append(
+        "نظام الحكم سلطاني وراثي في الذكو"
+        " ر من ذرية السيد تركي بن سعيد بن سلط"
+        " ان ويشترط فيمن يختار لولاية الحكم من بينه"
+        " م ان يكون مسلما رشيدا عاقلا ًوابنا شرعيا لابوين عمانيين "
+    )
+    strings.append("A pretty long string which is in a list")
+    booleans.append(True)
+
+for y in range(100):
+    arrays = []
+    list_dicts.append({str(random() * 20): int(random() * 1000000)})
+
+    for x in range(100):
+        arrays.append({str(random() * 20): int(random() * 1000000)})
+        dict_lists[str(random() * 20)] = arrays
+
+
+datasets = [
+    ("composite object", composite_object),
+    ("256 doubles array", doubles),
+    ("256 unicode array", unicode_strings),
+    ("256 ASCII array", strings),
+    ("256 Trues array", booleans),
+    ("100 dicts array", list_dicts),
+    ("100 arrays dict", dict_lists),
+    ("complex object", complex_object),
+]
+
+
+@pytest.mark.benchmark(group="serialize")
+@pytest.mark.parametrize("data", [d[1] for d in datasets], ids=[d[0] for d in datasets])
+def test_dumps(contender, data, benchmark):
+    benchmark(contender.dumps, data)
+
+
+@pytest.mark.benchmark(group="deserialize")
+@pytest.mark.parametrize("data", [d[1] for d in datasets], ids=[d[0] for d in datasets])
+def test_loads(contender, data, benchmark):
+    data = contender.dumps(data)
+    benchmark(contender.loads, data)
+
+
+@pytest.mark.benchmark(group="roundtrip")
+@pytest.mark.parametrize("data", [d[1] for d in datasets], ids=[d[0] for d in datasets])
+def test_roundtrip(contender, data, benchmark):
+    def roundtrip(d):
+        return contender.loads(contender.dumps(d))
+
+    benchmark(roundtrip, data)

tools/third_party/cbor2/cbor2/__init__.py

@@ -0,0 +1,83 @@
+from typing import Any
+
+from ._decoder import CBORDecoder as CBORDecoder
+from ._decoder import load as load
+from ._decoder import loads as loads
+from ._encoder import CBOREncoder as CBOREncoder
+from ._encoder import dump as dump
+from ._encoder import dumps as dumps
+from ._encoder import shareable_encoder as shareable_encoder
+from ._types import CBORDecodeEOF as CBORDecodeEOF
+from ._types import CBORDecodeError as CBORDecodeError
+from ._types import CBORDecodeValueError as CBORDecodeValueError
+from ._types import CBOREncodeError as CBOREncodeError
+from ._types import CBOREncodeTypeError as CBOREncodeTypeError
+from ._types import CBOREncodeValueError as CBOREncodeValueError
+from ._types import CBORError as CBORError
+from ._types import CBORSimpleValue as CBORSimpleValue
+from ._types import CBORTag as CBORTag
+from ._types import FrozenDict as FrozenDict
+from ._types import undefined as undefined
+
+try:
+    from _cbor2 import *  # noqa: F403
+except ImportError:
+    # Couldn't import the optimized C version; ignore the failure and leave the
+    # pure Python implementations in place.
+
+    # Re-export imports so they look like they live directly in this package
+    key: str
+    value: Any
+    for key, value in list(locals().items()):
+        if callable(value) and getattr(value, "__module__", "").startswith("cbor2."):
+            value.__module__ = __name__
+else:
+    # The pure Python implementations are replaced with the optimized C
+    # variants, but we still need to create the encoder dictionaries for the C
+    # variant here (this is much simpler than doing so in C, and doesn't affect
+    # overall performance as it's a one-off initialization cost).
+    def _init_cbor2() -> None:
+        from collections import OrderedDict
+
+        import _cbor2
+
+        from ._encoder import canonical_encoders, default_encoders
+        from ._types import CBORSimpleValue, CBORTag, undefined
+
+        _cbor2.default_encoders = OrderedDict(
+            [
+                (
+                    (
+                        _cbor2.CBORSimpleValue
+                        if type_ is CBORSimpleValue
+                        else _cbor2.CBORTag
+                        if type_ is CBORTag
+                        else type(_cbor2.undefined)
+                        if type_ is type(undefined)
+                        else type_
+                    ),
+                    getattr(_cbor2.CBOREncoder, method.__name__),
+                )
+                for type_, method in default_encoders.items()
+            ]
+        )
+        _cbor2.canonical_encoders = OrderedDict(
+            [
+                (
+                    (
+                        _cbor2.CBORSimpleValue
+                        if type_ is CBORSimpleValue
+                        else _cbor2.CBORTag
+                        if type_ is CBORTag
+                        else type(_cbor2.undefined)
+                        if type_ is type(undefined)
+                        else type_
+                    ),
+                    getattr(_cbor2.CBOREncoder, method.__name__),
+                )
+                for type_, method in canonical_encoders.items()
+            ]
+        )
+
+    _init_cbor2()
+    del _init_cbor2