From 0ac1d686cfb170e136d8ab99b3816a744bfddcb3 Mon Sep 17 00:00:00 2001 From: Eric Chen <67451029+cheqianh@users.noreply.github.com> Date: Wed, 9 Aug 2023 10:45:43 -0700 Subject: [PATCH 1/6] Improves the deserialization speed by reducing object initialization overhead (#39) --- amazon/ion/equivalence.py | 20 +- amazon/ion/ion_py_objects.py | 605 +++++++++++++++++++++++++++++++++++ amazon/ion/ioncmodule.c | 15 +- amazon/ion/simple_types.py | 53 ++- amazon/ion/simpleion.py | 2 + tests/test_core_multimap.py | 5 +- tests/test_events.py | 2 +- tests/test_simpleion.py | 21 +- 8 files changed, 690 insertions(+), 33 deletions(-) create mode 100644 amazon/ion/ion_py_objects.py diff --git a/amazon/ion/equivalence.py b/amazon/ion/equivalence.py index 56d41a344..bbe7f5db4 100644 --- a/amazon/ion/equivalence.py +++ b/amazon/ion/equivalence.py @@ -19,11 +19,21 @@ from math import isnan from amazon.ion.core import IonType, Timestamp, TimestampPrecision, MICROSECOND_PRECISION, OffsetTZInfo, Multimap +from amazon.ion.ion_py_objects import IonPyNull_new, IonPyDecimal_new, IonPyInt_new, IonPyFloat_new, IonPyText_new, \ + IonPyList_new, IonPyDict_new, IonPyBool_new, IonPySymbol_new from amazon.ion.simple_types import _IonNature, IonPyList, IonPyDict, IonPyTimestamp, IonPyNull, IonPySymbol, \ IonPyText, IonPyDecimal, IonPyFloat from amazon.ion.symbols import SymbolToken +def isinstance_of_ion_nature(obj): + return isinstance(obj, _IonNature) or isinstance(obj, IonPyNull_new) \ + or isinstance(obj, IonPyDecimal_new) or isinstance(obj, IonPyInt_new) \ + or isinstance(obj, IonPyFloat_new) or isinstance(obj, IonPyText_new) \ + or isinstance(obj, IonPyList_new) or isinstance(obj, IonPyDict_new) \ + or isinstance(obj, IonPyBool_new) or isinstance(obj, IonPySymbol_new) + + def ion_equals(a, b, timestamps_instants_only=False): """Tests two objects for equivalence under the Ion data model. @@ -60,8 +70,8 @@ def _ion_equals_timestamps_data_model(a, b): def _ion_equals(a, b, timestamp_comparison_func, recursive_comparison_func): """Compares a and b according to the description of the ion_equals method.""" for a, b in ((a, b), (b, a)): # Ensures that operand order does not matter. - if isinstance(a, _IonNature): - if isinstance(b, _IonNature): + if isinstance_of_ion_nature(a): + if isinstance_of_ion_nature(b): # Both operands have _IonNature. Their IonTypes and annotations must be equivalent. eq = a.ion_type is b.ion_type and _annotations_eq(a, b) else: @@ -120,8 +130,8 @@ def _sequences_eq(a, b, comparison_func): def _structs_eq(a, b, comparison_func): - assert isinstance(a, (dict, Multimap)) - if not isinstance(b, (dict, Multimap)): + assert isinstance(a, (dict, Multimap, IonPyDict_new)) + if not isinstance(b, (dict, Multimap, IonPyDict_new)): return False dict_len = len(a) if dict_len != len(b): @@ -135,7 +145,7 @@ def _structs_eq(a, b, comparison_func): break if key not in b: return False - if isinstance(a, Multimap) and isinstance(b, Multimap): + if isinstance(a, (IonPyDict_new, Multimap)) and isinstance(b, (IonPyDict_new, Multimap)): values_a = a.get_all_values(key) values_b = b.get_all_values(key) if len(values_a) != len(values_b): diff --git a/amazon/ion/ion_py_objects.py b/amazon/ion/ion_py_objects.py new file mode 100644 index 000000000..8501ea2d3 --- /dev/null +++ b/amazon/ion/ion_py_objects.py @@ -0,0 +1,605 @@ +from decimal import Decimal +from collections.abc import MutableMapping + +from amazon.ion.core import IonType, IonEvent +from amazon.ion.symbols import SymbolToken + + +class IonPyNull_new(object): + ion_annotations = () + ion_type = IonType.NULL # TODO initialized to NULL type first, what's the real type? + + __name__ = 'IonPyNull_new' + __qualname__ = 'IonPyNull_new' + + def __init__(self, ion_type=None, value=None, annotations=()): + self.ion_type = ion_type + self.ion_annotations = annotations + + def __bool__(self): + return False + + def _copy(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value + + @staticmethod + def _to_constructor_args(value): + return (None, value,), {} + + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, None, ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value + + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull_new() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value + + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = self + if isinstance(self, IonPyNull_new) or self.ion_type.is_container: + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) + + +class IonPyDecimal_new(Decimal): + ion_annotations = () + ion_type = IonType.DECIMAL + + __name__ = 'IonPyDecimal_new' + __qualname__ = 'IonPyDecimal_new' + + def __new__(cls, ion_type, value, annotations=()): + v = super().__new__(cls, value) + v.ion_annotations = annotations + return v + + def _copy(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value + + @staticmethod + def _to_constructor_args(value): + return (None, value,), {} + + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, None, ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value + + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull_new() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value + + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = self + if isinstance(self, IonPyNull_new) or self.ion_type.is_container: + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) + + +class IonPyInt_new(int): + ion_annotations = () + ion_type = IonType.INT + + __name__ = 'IonPyInt_new' + __qualname__ = 'IonPyInt_new' + + def __new__(cls, ion_type, value, annotations=()): + v = super().__new__(cls, value) + v.ion_annotations = annotations + return v + + def _copy(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value + + @staticmethod + def _to_constructor_args(value): + return (None, value,), {} + + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, None, ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value + + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull_new() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value + + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = self + if isinstance(self, IonPyNull_new) or self.ion_type.is_container: + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) + + +class IonPyBool_new(int): + ion_annotations = () + ion_type = IonType.BOOL + + __name__ = 'IonPyBool_new' + __qualname__ = 'IonPyBool_new' + + def __repr__(self): + return str(bool(self)) + + def __new__(cls, ion_type, value, annotations=()): + v = super().__new__(cls, value) + v.ion_annotations = annotations + return v + + def _copy(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value + + @staticmethod + def _to_constructor_args(value): + return (None, value,), {} + + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, None, ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value + + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull_new() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value + + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = self + if isinstance(self, IonPyNull_new) or self.ion_type.is_container: + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) + + +class IonPyFloat_new(float): + ion_annotations = () + ion_type = IonType.FLOAT + + __name__ = 'IonPyFloat_new' + __qualname__ = 'IonPyFloat_new' + + def __new__(cls, ion_type, value, annotations=()): + v = super().__new__(cls, value) + v.ion_annotations = annotations + return v + + def _copy(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value + + @staticmethod + def _to_constructor_args(value): + return (None, value,), {} + + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, None, ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value + + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull_new() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value + + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = self + if isinstance(self, IonPyNull_new) or self.ion_type.is_container: + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) + + +class IonPyText_new(str): + ion_annotations = () + ion_type = IonType.STRING + + __name__ = 'IonPyText_new' + __qualname__ = 'IonPyText_new' + + def __new__(cls, ion_type=None, value=None, annotations=()): + v = super().__new__(cls, value) + v.ion_annotations = annotations + return v + + def _copy(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value + + @staticmethod + def _to_constructor_args(value): + return (None, value,), {} + + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, None, ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value + + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull_new() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value + + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = self + if isinstance(self, IonPyNull_new) or self.ion_type.is_container: + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) + + +class IonPySymbol_new(SymbolToken): + ion_annotations = () + ion_type = IonType.SYMBOL + + __name__ = 'IonPySymbol_new' + __qualname__ = 'IonPySymbol_new' + + # a good signature: IonPySymbol(ion_type, symbol_token, annotation) + def __new__(cls, ion_type, value, annotations=()): + v = super().__new__(cls, *value) + v.ion_annotations = annotations + return v + + def _copy(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value + + @staticmethod + def _to_constructor_args(st): + try: + args = (None, (st.text, st.sid, st.location), ()) + except AttributeError: + args = (None, (st, None, None), ()) + kwargs = {} + return args, kwargs + + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, None, ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value + + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull_new() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value + + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = self + if isinstance(self, IonPyNull_new) or self.ion_type.is_container: + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) + + +class IonPyList_new(list): + ion_annotations = () + ion_type = IonType.LIST + + __name__ = 'IonPyList_new' + __qualname__ = 'IonPyList_new' + + def __init__(self, ion_type, value, annotations=()): + super().__init__(value) + self.ion_annotations = annotations + # it's possible to be Sexp + self.ion_type = ion_type + + def _copy(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value + + @staticmethod + def _to_constructor_args(value): + return (None, value,), {} + + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, [], ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value + + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull_new() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value + + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = self + if isinstance(self, IonPyNull_new) or self.ion_type.is_container: + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) + + +class IonPyDict_new(MutableMapping): + ion_annotations = () + ion_type = IonType.STRUCT + + __name__ = 'IonPyDict_new' + __qualname__ = 'IonPyDict_new' + + # one good initialization example: my_dict = IonPyDict(ion_type, value, annotations) + def __init__(self, *args, **kwargs): + super().__init__() + self.__store = {} + if args is not None: + if len(args) > 1 and args[1] is not None: + for key, value in iter(args[1].items()): + self.__store[key] = [value] + if len(args) > 2 and args[2] is not None: + self.ion_annotations = args[2] + + def __getitem__(self, key): + return self.__store[key][len(self.__store[key]) - 1] # Return only one in order not to break clients + + def __delitem__(self, key): + del self.__store[key] + + def __setitem__(self, key, value): + self.__store[key] = [value] + + def __len__(self): + return sum([len(values) for values in iter(self.__store.values())]) + + def __iter__(self): + for key in iter(self.__store.keys()): + yield key + + def __str__(self): + return repr(self) + + def __repr__(self): + return '{%s}' % ', '.join(['%r: %r' % (k, v) for k, v in self.items()]) + + def add_item(self, key, value): + if key in self.__store: + self.__store[key].append(value) + else: + self.__setitem__(key, value) + + def get_all_values(self, key): + return self.__store[key] + + def iteritems(self): + for key in self.__store: + for value in self.__store[key]: + yield key, value + + def items(self): + output = [] + for k, v in self.iteritems(): + output.append((k, v)) + return output + + def _copy(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value + + @staticmethod + def _to_constructor_args(value): + return (None, value,), {} + + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, None, ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value + + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull_new() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value + + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = self + if isinstance(self, IonPyNull_new) or self.ion_type.is_container: + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) diff --git a/amazon/ion/ioncmodule.c b/amazon/ion/ioncmodule.c index b118833bb..721755fab 100644 --- a/amazon/ion/ioncmodule.c +++ b/amazon/ion/ioncmodule.c @@ -1102,6 +1102,7 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s SUCCEED(); case tid_NULL_INT: { + // TODO double check the real null type, now it's initialized to IonType.NULL by default ION_TYPE null_type; // Hack for ion-c issue https://github.com/amazon-ion/ion-c/issues/223 if (original_t != tid_SYMBOL_INT) { @@ -1112,7 +1113,7 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s } ion_type = ION_TYPE_INT(null_type); - py_value = Py_BuildValue(""); // INCREFs and returns Python None. + py_value = Py_None; // INCREFs and returns Python None. emit_bare_values = emit_bare_values && (ion_type == tid_NULL_INT); ion_nature_constructor = _ionpynull_fromvalue; break; @@ -1577,6 +1578,18 @@ PyObject* ionc_init_module(void) { _ionpydict_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyDict"); _ionpydict_fromvalue = PyObject_GetAttrString(_ionpydict_cls, "from_value"); + // A temporary workaround to replace from_value() with class constructors + // TODO deprecate from_value later + _ionpyint_fromvalue = _ionpyint_cls; + _ionpynull_fromvalue = _ionpynull_cls; + _ionpyfloat_fromvalue = _ionpyfloat_cls; + _ionpydecimal_fromvalue = _ionpydecimal_cls; + _ionpytext_fromvalue = _ionpytext_cls; + _ionpylist_fromvalue = _ionpylist_cls; + _ionpydict_fromvalue = _ionpydict_cls; + _ionpybool_fromvalue = _ionpybool_cls; + _ionpysymbol_fromvalue = _ionpysymbol_cls; + _ion_core_module = PyImport_ImportModule("amazon.ion.core"); _py_timestamp_precision = PyObject_GetAttrString(_ion_core_module, "TimestampPrecision"); _py_timestamp_constructor = PyObject_GetAttrString(_ion_core_module, "timestamp"); diff --git a/amazon/ion/simple_types.py b/amazon/ion/simple_types.py index 3639cd2f8..4e8d3a65d 100644 --- a/amazon/ion/simple_types.py +++ b/amazon/ion/simple_types.py @@ -20,6 +20,9 @@ from decimal import Decimal +from amazon.ion.ion_py_objects import IonPyNull_new, IonPyDecimal_new, IonPyInt_new, IonPyFloat_new, IonPyText_new, \ + IonPyList_new, IonPyDict_new, IonPyBool_new, IonPySymbol_new + # in Python 3.10, abstract collections have moved into their own module # for compatibility with 3.10+, first try imports from the new location # if that fails, try from the pre-3.10 location @@ -144,27 +147,21 @@ def __init__(self, *args, **kwargs): return IonPyValueType -IonPyInt = _ion_type_for('IonPyInt', int, IonType.INT) -IonPyFloat = _ion_type_for('IonPyFloat', float, IonType.FLOAT) -IonPyDecimal = _ion_type_for('IonPyDecimal', Decimal, IonType.DECIMAL) -IonPyText = _ion_type_for('IonPyText', str) -IonPyBytes = _ion_type_for('IonPyBytes', bytes) - # All of our ion value types inherit from their python value types, for good or # ill. Python's builtin `bool` cannot be sub-classed and is itself a sub-class # of int so we just sub-class int directly for our Ion Boolean. # Considering that True == 1 and False == 0 this works as expected. -IonPyBool = _ion_type_for('IonPyBool', int, IonType.BOOL) +IonPyBool_original = _ion_type_for('IonPyBool', int, IonType.BOOL) # We override __repr__ so Booleans show up as True|False and not 1|0. # The representation is consistent with other primitives in that it returns a # string repr of the value but not annotations. -IonPyBool.__repr__ = lambda self: str(bool(self)) +IonPyBool_original.__repr__ = lambda self: str(bool(self)) -class IonPySymbol(SymbolToken, _IonNature): +class IonPySymbol_original(SymbolToken, _IonNature): def __init__(self, *args, **kwargs): - super(IonPySymbol, self).__init__(*args, **kwargs) + super(IonPySymbol_original, self).__init__(*args, **kwargs) self.ion_type = IonType.SYMBOL @staticmethod @@ -195,7 +192,7 @@ def _to_constructor_args(ts): return args, kwargs -class IonPyNull(_IonNature): +class IonPyNull_original(_IonNature): """Representation of ``null``. Notes: @@ -203,8 +200,9 @@ class IonPyNull(_IonNature): own value type for it. The function ``is_null`` is the best way to test for ``null``-ness or ``None``-ness. """ + def __init__(self, *args, **kwargs): - super(IonPyNull, self).__init__(*args, **kwargs) + super(IonPyNull_original, self).__init__(*args, **kwargs) def __nonzero__(self): return False @@ -222,5 +220,32 @@ def is_null(value): return value is None or isinstance(value, IonPyNull) -IonPyList = _ion_type_for('IonPyList', list) -IonPyDict = _ion_type_for('IonPyDict', Multimap, IonType.STRUCT) +IonPyInt_original = _ion_type_for('IonPyInt', int, IonType.INT) +IonPyFloat_original = _ion_type_for('IonPyFloat', float, IonType.FLOAT) +IonPyDecimal_original = _ion_type_for('IonPyDecimal', Decimal, IonType.DECIMAL) +IonPyText_original = _ion_type_for('IonPyText', str, IonType.STRING) +IonPyBytes = _ion_type_for('IonPyBytes', bytes) +IonPyList_original = _ion_type_for('IonPyList', list) +IonPyDict_original = _ion_type_for('IonPyDict', Multimap, IonType.STRUCT) + +# Using the original IonPyObjects by uncomment below +# IonPyDecimal = IonPyDecimal_original +# IonPyNull = IonPyNull_original +# IonPyInt = IonPyInt_original +# IonPyFloat = IonPyFloat_original +# IonPyText = IonPyText_original +# IonPyList = IonPyList_original +# IonPyDict = IonPyDict_original +# IonPyBool = IonPyBool_original +# IonPySymbol = IonPySymbol_original + +# Switch to the previous IonPyObjects by comment below lines +IonPyDecimal = IonPyDecimal_new +IonPyNull = IonPyNull_new +IonPyInt = IonPyInt_new +IonPyFloat = IonPyFloat_new +IonPyText = IonPyText_new +IonPyList = IonPyList_new +IonPyDict = IonPyDict_new +IonPyBool = IonPyBool_new +IonPySymbol = IonPySymbol_new diff --git a/amazon/ion/simpleion.py b/amazon/ion/simpleion.py index 557d27244..5946fb6e3 100644 --- a/amazon/ion/simpleion.py +++ b/amazon/ion/simpleion.py @@ -28,6 +28,7 @@ from .reader_managed import managed_reader from .simple_types import IonPyList, IonPyDict, IonPyNull, IonPyBool, IonPyInt, IonPyFloat, IonPyDecimal, \ IonPyTimestamp, IonPyText, IonPyBytes, IonPySymbol, is_null +from amazon.ion.ion_py_objects import IonPyNull_new from .symbols import SymbolToken from .writer import blocking_writer from .writer_binary import binary_writer @@ -165,6 +166,7 @@ def dump_python(obj, fp, imports=None, binary=True, sequence_as_stream=False, sk _FROM_TYPE = { + IonPyNull_new: IonType.NULL, type(None): IonType.NULL, type(True): IonType.BOOL, type(False): IonType.BOOL, diff --git a/tests/test_core_multimap.py b/tests/test_core_multimap.py index 7fdc0460f..226212aa7 100644 --- a/tests/test_core_multimap.py +++ b/tests/test_core_multimap.py @@ -1,6 +1,7 @@ from typing import Sequence, NamedTuple -from amazon.ion.core import Multimap +# from amazon.ion.core import Multimap +from amazon.ion.ion_py_objects import IonPyDict_new as Multimap # replace the original multiMap with the new IonPyDict from tests import parametrize @@ -63,7 +64,7 @@ def test_delete_item(item): {"a": 1, "b": 2, "c": [1, 2, {3: 4}]} ) def test_constructor(d): - m = Multimap(d) + m = Multimap(None, d) for k, v in iter(d.items()): assert m[k] == v assert len(m) == len(d) diff --git a/tests/test_events.py b/tests/test_events.py index bf8c6cbdc..1210753b7 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -34,7 +34,7 @@ def _to_event_parameters(): [loads('$0'), [IonEvent(IonEventType.SCALAR, IonType.SYMBOL, SymbolToken(None, 0), None, (), depth=0)]], [loads('{$0: $0}'), [ IonEvent(IonEventType.CONTAINER_START, IonType.STRUCT, depth=0), - IonEvent(IonEventType.SCALAR, IonType.SYMBOL, IonPySymbol(None, 0), SymbolToken(None, 0), (), depth=1), + IonEvent(IonEventType.SCALAR, IonType.SYMBOL, IonPySymbol(None, (None, 0)), SymbolToken(None, 0), (), depth=1), IonEvent(IonEventType.CONTAINER_END), ]], [loads('[1, 2, 3, [4, 5, 6], [7, 8, 9]]'), [ diff --git a/tests/test_simpleion.py b/tests/test_simpleion.py index 83dba80cd..664faf0a1 100644 --- a/tests/test_simpleion.py +++ b/tests/test_simpleion.py @@ -26,12 +26,13 @@ from amazon.ion import simpleion from amazon.ion.exceptions import IonException +from amazon.ion.ion_py_objects import IonPyDict_new from amazon.ion.symbols import SymbolToken, SYSTEM_SYMBOL_TABLE from amazon.ion.writer_binary import _IVM from amazon.ion.core import IonType, IonEvent, IonEventType, OffsetTZInfo, Multimap from amazon.ion.simple_types import IonPyDict, IonPyText, IonPyList, IonPyNull, IonPyBool, IonPyInt, IonPyFloat, \ IonPyDecimal, IonPyTimestamp, IonPyBytes, IonPySymbol, _IonNature -from amazon.ion.equivalence import ion_equals +from amazon.ion.equivalence import ion_equals, isinstance_of_ion_nature from amazon.ion.simpleion import dump, dumps, load, loads, _ion_type, _FROM_ION_TYPE, _FROM_TYPE_TUPLE_AS_SEXP, \ _FROM_TYPE from amazon.ion.writer_binary_raw import _serialize_symbol, _write_length @@ -231,7 +232,7 @@ def generate_scalars_binary(scalars_map, preceding_symbols=0): symbol_expected = _serialize_symbol( IonEvent(IonEventType.SCALAR, IonType.SYMBOL, SymbolToken(None, 10 + preceding_symbols))) yield _Parameter(IonType.SYMBOL.name + ' ' + native, - IonPyText.from_value(IonType.SYMBOL, native), symbol_expected, True) + IonPyText.from_value(IonType.SYMBOL, native, ()), symbol_expected, True) yield _Parameter('%s %s' % (ion_type.name, native), native, native_expected, has_symbols) wrapper = _FROM_ION_TYPE[ion_type].from_value(ion_type, native) yield _Parameter(repr(wrapper), wrapper, expected, has_symbols) @@ -248,7 +249,7 @@ def generate_containers_binary(container_map, preceding_symbols=0): for one_expected in expecteds: one_expected = one_expected.binary for elem in obj: - if isinstance(elem, (dict, Multimap)) and len(elem) > 0: + if isinstance(elem, (dict, Multimap, IonPyDict_new)) and len(elem) > 0: has_symbols = True elif ion_type is IonType.SEXP and isinstance(elem, tuple): tuple_as_sexp = True @@ -271,7 +272,7 @@ def generate_annotated_values_binary(scalars_map, container_map): for value_p in chain(generate_scalars_binary(scalars_map, preceding_symbols=2), generate_containers_binary(container_map, preceding_symbols=2)): obj = value_p.obj - if not isinstance(obj, _IonNature): + if not isinstance_of_ion_nature(obj): continue obj.ion_annotations = (_st(u'annot1'), _st(u'annot2'),) annot_length = 2 # 10 and 11 each fit in one VarUInt byte @@ -289,7 +290,7 @@ def generate_annotated_values_binary(scalars_map, container_map): _write_length(wrapper, length_field, 0xE0) if c_ext and obj.ion_type is IonType.SYMBOL and not isinstance(obj, IonPyNull) \ - and not (hasattr(obj, 'sid') and (obj.sid < 10 or obj.sid is None)): + and not (hasattr(obj, 'sid') and (obj.sid is None or obj.sid < 10)): wrapper.extend([ VARUINT_END_BYTE | annot_length, VARUINT_END_BYTE | 11, @@ -376,8 +377,8 @@ def _simple_loads(data, *args, **kw): @parametrize( *tuple(chain( generate_scalars_binary(SIMPLE_SCALARS_MAP_BINARY), - generate_containers_binary(_SIMPLE_CONTAINER_MAP), - generate_annotated_values_binary(SIMPLE_SCALARS_MAP_BINARY, _SIMPLE_CONTAINER_MAP), + # generate_containers_binary(_SIMPLE_CONTAINER_MAP), + # generate_annotated_values_binary(SIMPLE_SCALARS_MAP_BINARY, _SIMPLE_CONTAINER_MAP), )) ) def test_dump_load_binary(p): @@ -440,7 +441,7 @@ def generate_annotated_values_text(scalars_map, container_map): for value_p in chain(generate_scalars_text(scalars_map), generate_containers_text(container_map)): obj = value_p.obj - if not isinstance(obj, _IonNature): + if not isinstance_of_ion_nature(obj): continue obj.ion_annotations = (_st(u'annot1'), _st(u'annot2'),) @@ -570,7 +571,7 @@ def _to_obj(to_type=None, annotations=(), tuple_as_sexp=False): for obj in roundtrips: obj = _adjust_sids() yield obj, is_binary, indent, False - if not isinstance(obj, _IonNature): + if not isinstance_of_ion_nature(obj): ion_type = _ion_type(obj, _FROM_TYPE) yield _to_obj() else: @@ -591,7 +592,7 @@ def _assert_roundtrip(before, after, tuple_as_sexp): # wraps each input value in _IonNature for comparison against the output. def _to_ion_nature(obj): out = obj - if not isinstance(out, _IonNature): + if not isinstance_of_ion_nature(out): from_type = _FROM_TYPE_TUPLE_AS_SEXP if tuple_as_sexp else _FROM_TYPE ion_type = _ion_type(out, from_type) out = _FROM_ION_TYPE[ion_type].from_value(ion_type, out) From 9b737ea649f09222743a5c68095bdd77059c721a Mon Sep 17 00:00:00 2001 From: cheqianh Date: Tue, 22 Aug 2023 12:55:00 -0700 Subject: [PATCH 2/6] Addresses few comments - Refactors some class attributes and renames tests and methods - Removes _copy() - Refactors ionPyDict's constrcutor signature - Changes ion_type to be a class attribute --- amazon/ion/equivalence.py | 6 +-- amazon/ion/ion_py_objects.py | 87 +++++++++++++------------------- tests/test_core_multimap.py | 5 +- tests/test_multimap_IonPyDict.py | 69 +++++++++++++++++++++++++ tests/test_simpleion.py | 10 ++-- 5 files changed, 113 insertions(+), 64 deletions(-) create mode 100644 tests/test_multimap_IonPyDict.py diff --git a/amazon/ion/equivalence.py b/amazon/ion/equivalence.py index bbe7f5db4..b16143b19 100644 --- a/amazon/ion/equivalence.py +++ b/amazon/ion/equivalence.py @@ -26,7 +26,7 @@ from amazon.ion.symbols import SymbolToken -def isinstance_of_ion_nature(obj): +def isinstance_of_ion_nature_or_new_ion_py_objects(obj): return isinstance(obj, _IonNature) or isinstance(obj, IonPyNull_new) \ or isinstance(obj, IonPyDecimal_new) or isinstance(obj, IonPyInt_new) \ or isinstance(obj, IonPyFloat_new) or isinstance(obj, IonPyText_new) \ @@ -70,8 +70,8 @@ def _ion_equals_timestamps_data_model(a, b): def _ion_equals(a, b, timestamp_comparison_func, recursive_comparison_func): """Compares a and b according to the description of the ion_equals method.""" for a, b in ((a, b), (b, a)): # Ensures that operand order does not matter. - if isinstance_of_ion_nature(a): - if isinstance_of_ion_nature(b): + if isinstance_of_ion_nature_or_new_ion_py_objects(a): + if isinstance_of_ion_nature_or_new_ion_py_objects(b): # Both operands have _IonNature. Their IonTypes and annotations must be equivalent. eq = a.ion_type is b.ion_type and _annotations_eq(a, b) else: diff --git a/amazon/ion/ion_py_objects.py b/amazon/ion/ion_py_objects.py index 8501ea2d3..d81362ff0 100644 --- a/amazon/ion/ion_py_objects.py +++ b/amazon/ion/ion_py_objects.py @@ -6,20 +6,18 @@ class IonPyNull_new(object): - ion_annotations = () - ion_type = IonType.NULL # TODO initialized to NULL type first, what's the real type? - __name__ = 'IonPyNull_new' __qualname__ = 'IonPyNull_new' + ion_type = IonType.NULL - def __init__(self, ion_type=None, value=None, annotations=()): - self.ion_type = ion_type + def __init__(self, ion_type=IonType.NULL, value=None, annotations=()): + self.ion_type = ion_type # TODO initialized to NULL type first, what's the real type? self.ion_annotations = annotations def __bool__(self): return False - def _copy(self): + def __copy__(self): args, kwargs = self._to_constructor_args(self) value = self.__class__(*args, **kwargs) value.ion_type = self.ion_type @@ -68,18 +66,16 @@ def to_event(self, event_type, field_name=None, in_struct=False, depth=None): class IonPyDecimal_new(Decimal): - ion_annotations = () - ion_type = IonType.DECIMAL - __name__ = 'IonPyDecimal_new' __qualname__ = 'IonPyDecimal_new' + ion_type = IonType.DECIMAL - def __new__(cls, ion_type, value, annotations=()): + def __new__(cls, ion_type=IonType.DECIMAL, value=None, annotations=()): v = super().__new__(cls, value) v.ion_annotations = annotations return v - def _copy(self): + def __copy__(self): args, kwargs = self._to_constructor_args(self) value = self.__class__(*args, **kwargs) value.ion_type = self.ion_type @@ -128,18 +124,16 @@ def to_event(self, event_type, field_name=None, in_struct=False, depth=None): class IonPyInt_new(int): - ion_annotations = () - ion_type = IonType.INT - __name__ = 'IonPyInt_new' __qualname__ = 'IonPyInt_new' + ion_type = IonType.INT - def __new__(cls, ion_type, value, annotations=()): + def __new__(cls, ion_type=IonType.INT, value=None, annotations=()): v = super().__new__(cls, value) v.ion_annotations = annotations return v - def _copy(self): + def __copy__(self): args, kwargs = self._to_constructor_args(self) value = self.__class__(*args, **kwargs) value.ion_type = self.ion_type @@ -188,21 +182,19 @@ def to_event(self, event_type, field_name=None, in_struct=False, depth=None): class IonPyBool_new(int): - ion_annotations = () - ion_type = IonType.BOOL - __name__ = 'IonPyBool_new' __qualname__ = 'IonPyBool_new' + ion_type = IonType.BOOL def __repr__(self): return str(bool(self)) - def __new__(cls, ion_type, value, annotations=()): + def __new__(cls, ion_type=IonType.BOOL, value=None, annotations=()): v = super().__new__(cls, value) v.ion_annotations = annotations return v - def _copy(self): + def __copy__(self): args, kwargs = self._to_constructor_args(self) value = self.__class__(*args, **kwargs) value.ion_type = self.ion_type @@ -251,18 +243,16 @@ def to_event(self, event_type, field_name=None, in_struct=False, depth=None): class IonPyFloat_new(float): - ion_annotations = () - ion_type = IonType.FLOAT - __name__ = 'IonPyFloat_new' __qualname__ = 'IonPyFloat_new' + ion_type = IonType.FLOAT - def __new__(cls, ion_type, value, annotations=()): + def __new__(cls, ion_type=IonType.FLOAT, value=None, annotations=()): v = super().__new__(cls, value) v.ion_annotations = annotations return v - def _copy(self): + def __copy__(self): args, kwargs = self._to_constructor_args(self) value = self.__class__(*args, **kwargs) value.ion_type = self.ion_type @@ -311,18 +301,16 @@ def to_event(self, event_type, field_name=None, in_struct=False, depth=None): class IonPyText_new(str): - ion_annotations = () - ion_type = IonType.STRING - __name__ = 'IonPyText_new' __qualname__ = 'IonPyText_new' + ion_type = IonType.STRING - def __new__(cls, ion_type=None, value=None, annotations=()): + def __new__(cls, ion_type=IonType.STRING, value=None, annotations=()): v = super().__new__(cls, value) v.ion_annotations = annotations return v - def _copy(self): + def __copy__(self): args, kwargs = self._to_constructor_args(self) value = self.__class__(*args, **kwargs) value.ion_type = self.ion_type @@ -371,19 +359,17 @@ def to_event(self, event_type, field_name=None, in_struct=False, depth=None): class IonPySymbol_new(SymbolToken): - ion_annotations = () - ion_type = IonType.SYMBOL - __name__ = 'IonPySymbol_new' __qualname__ = 'IonPySymbol_new' + ion_type = IonType.SYMBOL # a good signature: IonPySymbol(ion_type, symbol_token, annotation) - def __new__(cls, ion_type, value, annotations=()): + def __new__(cls, ion_type=IonType.SYMBOL, value=None, annotations=()): v = super().__new__(cls, *value) v.ion_annotations = annotations return v - def _copy(self): + def __copy__(self): args, kwargs = self._to_constructor_args(self) value = self.__class__(*args, **kwargs) value.ion_type = self.ion_type @@ -437,19 +423,19 @@ def to_event(self, event_type, field_name=None, in_struct=False, depth=None): class IonPyList_new(list): - ion_annotations = () - ion_type = IonType.LIST - __name__ = 'IonPyList_new' __qualname__ = 'IonPyList_new' + ion_type = IonType.LIST - def __init__(self, ion_type, value, annotations=()): + def __init__(self, ion_type=IonType.LIST, value=None, annotations=()): + if value is None: + value = [] super().__init__(value) self.ion_annotations = annotations # it's possible to be Sexp self.ion_type = ion_type - def _copy(self): + def __copy__(self): args, kwargs = self._to_constructor_args(self) value = self.__class__(*args, **kwargs) value.ion_type = self.ion_type @@ -498,22 +484,17 @@ def to_event(self, event_type, field_name=None, in_struct=False, depth=None): class IonPyDict_new(MutableMapping): - ion_annotations = () - ion_type = IonType.STRUCT - __name__ = 'IonPyDict_new' __qualname__ = 'IonPyDict_new' + ion_type = IonType.STRUCT - # one good initialization example: my_dict = IonPyDict(ion_type, value, annotations) - def __init__(self, *args, **kwargs): + def __init__(self, ion_type=IonType.STRUCT, value=None, annotations=()): super().__init__() + self.ion_annotations = annotations self.__store = {} - if args is not None: - if len(args) > 1 and args[1] is not None: - for key, value in iter(args[1].items()): - self.__store[key] = [value] - if len(args) > 2 and args[2] is not None: - self.ion_annotations = args[2] + if value is not None: + for key, value in iter(value.items()): + self.__store[key] = [value] def __getitem__(self, key): return self.__store[key][len(self.__store[key]) - 1] # Return only one in order not to break clients @@ -557,7 +538,7 @@ def items(self): output.append((k, v)) return output - def _copy(self): + def __copy__(self): args, kwargs = self._to_constructor_args(self) value = self.__class__(*args, **kwargs) value.ion_type = self.ion_type diff --git a/tests/test_core_multimap.py b/tests/test_core_multimap.py index 226212aa7..7fdc0460f 100644 --- a/tests/test_core_multimap.py +++ b/tests/test_core_multimap.py @@ -1,7 +1,6 @@ from typing import Sequence, NamedTuple -# from amazon.ion.core import Multimap -from amazon.ion.ion_py_objects import IonPyDict_new as Multimap # replace the original multiMap with the new IonPyDict +from amazon.ion.core import Multimap from tests import parametrize @@ -64,7 +63,7 @@ def test_delete_item(item): {"a": 1, "b": 2, "c": [1, 2, {3: 4}]} ) def test_constructor(d): - m = Multimap(None, d) + m = Multimap(d) for k, v in iter(d.items()): assert m[k] == v assert len(m) == len(d) diff --git a/tests/test_multimap_IonPyDict.py b/tests/test_multimap_IonPyDict.py new file mode 100644 index 000000000..17c13b06e --- /dev/null +++ b/tests/test_multimap_IonPyDict.py @@ -0,0 +1,69 @@ +from typing import Sequence, NamedTuple + +from amazon.ion.ion_py_objects import IonPyDict_new as Multimap + +from tests import parametrize + + +class _P(NamedTuple): + pairs: Sequence + expected_all_values: Sequence + expected_single_value: Sequence + expected_total_len: int + + def __str__(self): + return '{name}'.format(name=self.pairs) + + +ALL_DATA = _P( + pairs=[('a', 1), ('a', 2), ('a', 3), ('a', [4, 5, 6]), ('b', 0), ('c', {'x': 'z', 'r': 's'})], + expected_all_values=[('a', [1, 2, 3, [4, 5, 6]]), ('b', [0]), ('c', [{'x': 'z', 'r': 's'}])], + expected_single_value=[('a', [4, 5, 6]), ('b', 0), ('c', {'x': 'z', 'r': 's'})], + expected_total_len=6 +) + + +def _create_multimap_with_items(pairs): + m = Multimap() + for pair in pairs: + m.add_item(pair[0], pair[1]) + return m + + +@parametrize( + ALL_DATA +) +def test_add_item(p): + m = _create_multimap_with_items(p.pairs) + for expected in p.expected_all_values: + assert list([x for x in m.get_all_values(expected[0])]) == expected[1] + for expected in p.expected_single_value: + assert m[expected[0]] == expected[1] + assert p.expected_total_len == len(m) + + +@parametrize( + (ALL_DATA, ["a"], 2), + (ALL_DATA, ["b"], 5), + (ALL_DATA, ["c"], 5), + (ALL_DATA, ["a", "b"], 1), + (ALL_DATA, ["a", "b", "c"], 0) +) +def test_delete_item(item): + m = _create_multimap_with_items(item[0].pairs) + p, items_to_remove, len_after_removal = item + for to_remove in items_to_remove: + del m[to_remove] + assert len(m) == item[2] + + +@parametrize( + {}, + {"a": 1}, + {"a": 1, "b": 2, "c": [1, 2, {3: 4}]} +) +def test_constructor(d): + m = Multimap(None, d) + for k, v in iter(d.items()): + assert m[k] == v + assert len(m) == len(d) diff --git a/tests/test_simpleion.py b/tests/test_simpleion.py index 664faf0a1..76542c772 100644 --- a/tests/test_simpleion.py +++ b/tests/test_simpleion.py @@ -32,7 +32,7 @@ from amazon.ion.core import IonType, IonEvent, IonEventType, OffsetTZInfo, Multimap from amazon.ion.simple_types import IonPyDict, IonPyText, IonPyList, IonPyNull, IonPyBool, IonPyInt, IonPyFloat, \ IonPyDecimal, IonPyTimestamp, IonPyBytes, IonPySymbol, _IonNature -from amazon.ion.equivalence import ion_equals, isinstance_of_ion_nature +from amazon.ion.equivalence import ion_equals, isinstance_of_ion_nature_or_new_ion_py_objects from amazon.ion.simpleion import dump, dumps, load, loads, _ion_type, _FROM_ION_TYPE, _FROM_TYPE_TUPLE_AS_SEXP, \ _FROM_TYPE from amazon.ion.writer_binary_raw import _serialize_symbol, _write_length @@ -272,7 +272,7 @@ def generate_annotated_values_binary(scalars_map, container_map): for value_p in chain(generate_scalars_binary(scalars_map, preceding_symbols=2), generate_containers_binary(container_map, preceding_symbols=2)): obj = value_p.obj - if not isinstance_of_ion_nature(obj): + if not isinstance_of_ion_nature_or_new_ion_py_objects(obj): continue obj.ion_annotations = (_st(u'annot1'), _st(u'annot2'),) annot_length = 2 # 10 and 11 each fit in one VarUInt byte @@ -441,7 +441,7 @@ def generate_annotated_values_text(scalars_map, container_map): for value_p in chain(generate_scalars_text(scalars_map), generate_containers_text(container_map)): obj = value_p.obj - if not isinstance_of_ion_nature(obj): + if not isinstance_of_ion_nature_or_new_ion_py_objects(obj): continue obj.ion_annotations = (_st(u'annot1'), _st(u'annot2'),) @@ -571,7 +571,7 @@ def _to_obj(to_type=None, annotations=(), tuple_as_sexp=False): for obj in roundtrips: obj = _adjust_sids() yield obj, is_binary, indent, False - if not isinstance_of_ion_nature(obj): + if not isinstance_of_ion_nature_or_new_ion_py_objects(obj): ion_type = _ion_type(obj, _FROM_TYPE) yield _to_obj() else: @@ -592,7 +592,7 @@ def _assert_roundtrip(before, after, tuple_as_sexp): # wraps each input value in _IonNature for comparison against the output. def _to_ion_nature(obj): out = obj - if not isinstance_of_ion_nature(out): + if not isinstance_of_ion_nature_or_new_ion_py_objects(out): from_type = _FROM_TYPE_TUPLE_AS_SEXP if tuple_as_sexp else _FROM_TYPE ion_type = _ion_type(out, from_type) out = _FROM_ION_TYPE[ion_type].from_value(ion_type, out) From 4c8317ad9b79226cf0d8d22cbdd162ee9c6092a9 Mon Sep 17 00:00:00 2001 From: cheqianh Date: Wed, 6 Sep 2023 13:03:29 -0700 Subject: [PATCH 3/6] Addressed feedback. --- amazon/ion/equivalence.py | 12 ++---- amazon/ion/ion_py_objects.py | 66 +++++++++++++++++++++++--------- tests/test_multimap_IonPyDict.py | 11 +++++- tests/test_simpleion.py | 10 ++--- 4 files changed, 67 insertions(+), 32 deletions(-) diff --git a/amazon/ion/equivalence.py b/amazon/ion/equivalence.py index b16143b19..19b1a1991 100644 --- a/amazon/ion/equivalence.py +++ b/amazon/ion/equivalence.py @@ -26,12 +26,8 @@ from amazon.ion.symbols import SymbolToken -def isinstance_of_ion_nature_or_new_ion_py_objects(obj): - return isinstance(obj, _IonNature) or isinstance(obj, IonPyNull_new) \ - or isinstance(obj, IonPyDecimal_new) or isinstance(obj, IonPyInt_new) \ - or isinstance(obj, IonPyFloat_new) or isinstance(obj, IonPyText_new) \ - or isinstance(obj, IonPyList_new) or isinstance(obj, IonPyDict_new) \ - or isinstance(obj, IonPyBool_new) or isinstance(obj, IonPySymbol_new) +def obj_has_ion_type_and_annotation(obj): + return hasattr(obj, 'ion_type') and hasattr(obj, 'ion_annotations') def ion_equals(a, b, timestamps_instants_only=False): @@ -70,8 +66,8 @@ def _ion_equals_timestamps_data_model(a, b): def _ion_equals(a, b, timestamp_comparison_func, recursive_comparison_func): """Compares a and b according to the description of the ion_equals method.""" for a, b in ((a, b), (b, a)): # Ensures that operand order does not matter. - if isinstance_of_ion_nature_or_new_ion_py_objects(a): - if isinstance_of_ion_nature_or_new_ion_py_objects(b): + if obj_has_ion_type_and_annotation(a): + if obj_has_ion_type_and_annotation(b): # Both operands have _IonNature. Their IonTypes and annotations must be equivalent. eq = a.ion_type is b.ion_type and _annotations_eq(a, b) else: diff --git a/amazon/ion/ion_py_objects.py b/amazon/ion/ion_py_objects.py index d81362ff0..e98e063e1 100644 --- a/amazon/ion/ion_py_objects.py +++ b/amazon/ion/ion_py_objects.py @@ -1,3 +1,4 @@ +from collections import OrderedDict from decimal import Decimal from collections.abc import MutableMapping @@ -8,7 +9,6 @@ class IonPyNull_new(object): __name__ = 'IonPyNull_new' __qualname__ = 'IonPyNull_new' - ion_type = IonType.NULL def __init__(self, ion_type=IonType.NULL, value=None, annotations=()): self.ion_type = ion_type # TODO initialized to NULL type first, what's the real type? @@ -52,7 +52,7 @@ def from_value(cls, ion_type, value, annotations=()): def to_event(self, event_type, field_name=None, in_struct=False, depth=None): value = self - if isinstance(self, IonPyNull_new) or self.ion_type.is_container: + if isinstance(self, IonPyNull_new): value = None if in_struct: @@ -110,7 +110,7 @@ def from_value(cls, ion_type, value, annotations=()): def to_event(self, event_type, field_name=None, in_struct=False, depth=None): value = self - if isinstance(self, IonPyNull_new) or self.ion_type.is_container: + if isinstance(self, IonPyNull_new): value = None if in_struct: @@ -168,7 +168,7 @@ def from_value(cls, ion_type, value, annotations=()): def to_event(self, event_type, field_name=None, in_struct=False, depth=None): value = self - if isinstance(self, IonPyNull_new) or self.ion_type.is_container: + if isinstance(self, IonPyNull_new): value = None if in_struct: @@ -229,7 +229,7 @@ def from_value(cls, ion_type, value, annotations=()): def to_event(self, event_type, field_name=None, in_struct=False, depth=None): value = self - if isinstance(self, IonPyNull_new) or self.ion_type.is_container: + if isinstance(self, IonPyNull_new): value = None if in_struct: @@ -287,7 +287,7 @@ def from_value(cls, ion_type, value, annotations=()): def to_event(self, event_type, field_name=None, in_struct=False, depth=None): value = self - if isinstance(self, IonPyNull_new) or self.ion_type.is_container: + if isinstance(self, IonPyNull_new): value = None if in_struct: @@ -345,7 +345,7 @@ def from_value(cls, ion_type, value, annotations=()): def to_event(self, event_type, field_name=None, in_struct=False, depth=None): value = self - if isinstance(self, IonPyNull_new) or self.ion_type.is_container: + if isinstance(self, IonPyNull_new): value = None if in_struct: @@ -409,7 +409,7 @@ def from_value(cls, ion_type, value, annotations=()): def to_event(self, event_type, field_name=None, in_struct=False, depth=None): value = self - if isinstance(self, IonPyNull_new) or self.ion_type.is_container: + if isinstance(self, IonPyNull_new): value = None if in_struct: @@ -425,7 +425,6 @@ def to_event(self, event_type, field_name=None, in_struct=False, depth=None): class IonPyList_new(list): __name__ = 'IonPyList_new' __qualname__ = 'IonPyList_new' - ion_type = IonType.LIST def __init__(self, ion_type=IonType.LIST, value=None, annotations=()): if value is None: @@ -469,9 +468,7 @@ def from_value(cls, ion_type, value, annotations=()): return value def to_event(self, event_type, field_name=None, in_struct=False, depth=None): - value = self - if isinstance(self, IonPyNull_new) or self.ion_type.is_container: - value = None + value = None if in_struct: if not isinstance(field_name, SymbolToken): @@ -484,6 +481,16 @@ def to_event(self, event_type, field_name=None, in_struct=False, depth=None): class IonPyDict_new(MutableMapping): + """ + Dictionary that can hold multiple values for the same key + + In order not to break existing customers, getting and inserting elements with ``[]`` keeps the same behaviour + as the built-in dict. If multiple elements are already mapped to the key, ``[]` will return + the newest one. + + To map multiple elements to a key, use the ``add_item`` operation. + To retrieve all the values map to a key, use ``get_all_values``. + """ __name__ = 'IonPyDict_new' __qualname__ = 'IonPyDict_new' ion_type = IonType.STRUCT @@ -491,18 +498,30 @@ class IonPyDict_new(MutableMapping): def __init__(self, ion_type=IonType.STRUCT, value=None, annotations=()): super().__init__() self.ion_annotations = annotations - self.__store = {} + self.__store = OrderedDict() if value is not None: for key, value in iter(value.items()): - self.__store[key] = [value] + if key in self.__store.keys(): + self.__store[key].append(value) + else: + self.__store[key] = [value] def __getitem__(self, key): + """ + Return the newest value for the given key. To retrieve all the values map to the key, use ``get_all_values``. + """ return self.__store[key][len(self.__store[key]) - 1] # Return only one in order not to break clients def __delitem__(self, key): + """ + Delete all values for the given key. + """ del self.__store[key] def __setitem__(self, key, value): + """ + Set the desired value to the given key. + """ self.__store[key] = [value] def __len__(self): @@ -519,20 +538,33 @@ def __repr__(self): return '{%s}' % ', '.join(['%r: %r' % (k, v) for k, v in self.items()]) def add_item(self, key, value): + """ + Add a value for the given key. This operation appends the value to the end of the value list instead of + overwriting the existing value. + """ if key in self.__store: self.__store[key].append(value) else: self.__setitem__(key, value) def get_all_values(self, key): + """ + Retrieve all the values mapped to the given key + """ return self.__store[key] def iteritems(self): + """ + Return an iterator over (key, value) tuple pairs. + """ for key in self.__store: for value in self.__store[key]: - yield key, value + yield (key, value) def items(self): + """ + Return a list of the IonPyDict's (key, value) tuple pairs. + """ output = [] for k, v in self.iteritems(): output.append((k, v)) @@ -572,9 +604,7 @@ def from_value(cls, ion_type, value, annotations=()): return value def to_event(self, event_type, field_name=None, in_struct=False, depth=None): - value = self - if isinstance(self, IonPyNull_new) or self.ion_type.is_container: - value = None + value = None if in_struct: if not isinstance(field_name, SymbolToken): diff --git a/tests/test_multimap_IonPyDict.py b/tests/test_multimap_IonPyDict.py index 17c13b06e..f9735a3f0 100644 --- a/tests/test_multimap_IonPyDict.py +++ b/tests/test_multimap_IonPyDict.py @@ -60,10 +60,19 @@ def test_delete_item(item): @parametrize( {}, {"a": 1}, - {"a": 1, "b": 2, "c": [1, 2, {3: 4}]} + {"a": 1, "b": 2, "c": [1, 2, {3: 4}]}, ) def test_constructor(d): m = Multimap(None, d) for k, v in iter(d.items()): assert m[k] == v assert len(m) == len(d) + + +def test_multimap_constructor(): + m = Multimap(None, {}) + m.add_item('a', 1) + m.add_item('a', 2) + m2 = Multimap(None, m) + + assert len(m2) == 2 diff --git a/tests/test_simpleion.py b/tests/test_simpleion.py index 76542c772..76514979f 100644 --- a/tests/test_simpleion.py +++ b/tests/test_simpleion.py @@ -32,7 +32,7 @@ from amazon.ion.core import IonType, IonEvent, IonEventType, OffsetTZInfo, Multimap from amazon.ion.simple_types import IonPyDict, IonPyText, IonPyList, IonPyNull, IonPyBool, IonPyInt, IonPyFloat, \ IonPyDecimal, IonPyTimestamp, IonPyBytes, IonPySymbol, _IonNature -from amazon.ion.equivalence import ion_equals, isinstance_of_ion_nature_or_new_ion_py_objects +from amazon.ion.equivalence import ion_equals, obj_has_ion_type_and_annotation from amazon.ion.simpleion import dump, dumps, load, loads, _ion_type, _FROM_ION_TYPE, _FROM_TYPE_TUPLE_AS_SEXP, \ _FROM_TYPE from amazon.ion.writer_binary_raw import _serialize_symbol, _write_length @@ -272,7 +272,7 @@ def generate_annotated_values_binary(scalars_map, container_map): for value_p in chain(generate_scalars_binary(scalars_map, preceding_symbols=2), generate_containers_binary(container_map, preceding_symbols=2)): obj = value_p.obj - if not isinstance_of_ion_nature_or_new_ion_py_objects(obj): + if not obj_has_ion_type_and_annotation(obj): continue obj.ion_annotations = (_st(u'annot1'), _st(u'annot2'),) annot_length = 2 # 10 and 11 each fit in one VarUInt byte @@ -441,7 +441,7 @@ def generate_annotated_values_text(scalars_map, container_map): for value_p in chain(generate_scalars_text(scalars_map), generate_containers_text(container_map)): obj = value_p.obj - if not isinstance_of_ion_nature_or_new_ion_py_objects(obj): + if not obj_has_ion_type_and_annotation(obj): continue obj.ion_annotations = (_st(u'annot1'), _st(u'annot2'),) @@ -571,7 +571,7 @@ def _to_obj(to_type=None, annotations=(), tuple_as_sexp=False): for obj in roundtrips: obj = _adjust_sids() yield obj, is_binary, indent, False - if not isinstance_of_ion_nature_or_new_ion_py_objects(obj): + if not obj_has_ion_type_and_annotation(obj): ion_type = _ion_type(obj, _FROM_TYPE) yield _to_obj() else: @@ -592,7 +592,7 @@ def _assert_roundtrip(before, after, tuple_as_sexp): # wraps each input value in _IonNature for comparison against the output. def _to_ion_nature(obj): out = obj - if not isinstance_of_ion_nature_or_new_ion_py_objects(out): + if not obj_has_ion_type_and_annotation(out): from_type = _FROM_TYPE_TUPLE_AS_SEXP if tuple_as_sexp else _FROM_TYPE ion_type = _ion_type(out, from_type) out = _FROM_ION_TYPE[ion_type].from_value(ion_type, out) From 79632129bdd2a756fd28c96a69c5ea3834f0b755 Mon Sep 17 00:00:00 2001 From: cheqianh Date: Fri, 8 Sep 2023 14:25:01 -0700 Subject: [PATCH 4/6] Reverts some commented tests. --- tests/test_simpleion.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_simpleion.py b/tests/test_simpleion.py index 76514979f..696f0e308 100644 --- a/tests/test_simpleion.py +++ b/tests/test_simpleion.py @@ -377,8 +377,8 @@ def _simple_loads(data, *args, **kw): @parametrize( *tuple(chain( generate_scalars_binary(SIMPLE_SCALARS_MAP_BINARY), - # generate_containers_binary(_SIMPLE_CONTAINER_MAP), - # generate_annotated_values_binary(SIMPLE_SCALARS_MAP_BINARY, _SIMPLE_CONTAINER_MAP), + generate_containers_binary(_SIMPLE_CONTAINER_MAP), + generate_annotated_values_binary(SIMPLE_SCALARS_MAP_BINARY, _SIMPLE_CONTAINER_MAP), )) ) def test_dump_load_binary(p): From b6fe2df6401c4bbb4d7b3172789c6d3285c12a12 Mon Sep 17 00:00:00 2001 From: cheqianh Date: Tue, 12 Sep 2023 10:34:30 -0700 Subject: [PATCH 5/6] Renames IonPyObject_new to IonPyObject --- amazon/ion/equivalence.py | 10 +++---- amazon/ion/ion_py_objects.py | 50 ++++++++++++++++---------------- amazon/ion/simple_types.py | 22 +++++++------- amazon/ion/simpleion.py | 4 +-- tests/test_multimap_IonPyDict.py | 2 +- tests/test_simpleion.py | 4 +-- 6 files changed, 46 insertions(+), 46 deletions(-) diff --git a/amazon/ion/equivalence.py b/amazon/ion/equivalence.py index 19b1a1991..c5e95e280 100644 --- a/amazon/ion/equivalence.py +++ b/amazon/ion/equivalence.py @@ -19,8 +19,8 @@ from math import isnan from amazon.ion.core import IonType, Timestamp, TimestampPrecision, MICROSECOND_PRECISION, OffsetTZInfo, Multimap -from amazon.ion.ion_py_objects import IonPyNull_new, IonPyDecimal_new, IonPyInt_new, IonPyFloat_new, IonPyText_new, \ - IonPyList_new, IonPyDict_new, IonPyBool_new, IonPySymbol_new +from amazon.ion.ion_py_objects import IonPyNull, IonPyDecimal, IonPyInt, IonPyFloat, IonPyText, \ + IonPyList, IonPyDict, IonPyBool, IonPySymbol from amazon.ion.simple_types import _IonNature, IonPyList, IonPyDict, IonPyTimestamp, IonPyNull, IonPySymbol, \ IonPyText, IonPyDecimal, IonPyFloat from amazon.ion.symbols import SymbolToken @@ -126,8 +126,8 @@ def _sequences_eq(a, b, comparison_func): def _structs_eq(a, b, comparison_func): - assert isinstance(a, (dict, Multimap, IonPyDict_new)) - if not isinstance(b, (dict, Multimap, IonPyDict_new)): + assert isinstance(a, (dict, Multimap, IonPyDict)) + if not isinstance(b, (dict, Multimap, IonPyDict)): return False dict_len = len(a) if dict_len != len(b): @@ -141,7 +141,7 @@ def _structs_eq(a, b, comparison_func): break if key not in b: return False - if isinstance(a, (IonPyDict_new, Multimap)) and isinstance(b, (IonPyDict_new, Multimap)): + if isinstance(a, (IonPyDict, Multimap)) and isinstance(b, (IonPyDict, Multimap)): values_a = a.get_all_values(key) values_b = b.get_all_values(key) if len(values_a) != len(values_b): diff --git a/amazon/ion/ion_py_objects.py b/amazon/ion/ion_py_objects.py index e98e063e1..3fd4d77f1 100644 --- a/amazon/ion/ion_py_objects.py +++ b/amazon/ion/ion_py_objects.py @@ -6,7 +6,7 @@ from amazon.ion.symbols import SymbolToken -class IonPyNull_new(object): +class IonPyNull(object): __name__ = 'IonPyNull_new' __qualname__ = 'IonPyNull_new' @@ -42,7 +42,7 @@ def from_event(cls, ion_event): @classmethod def from_value(cls, ion_type, value, annotations=()): if value is None: - value = IonPyNull_new() + value = IonPyNull() else: args, kwargs = cls._to_constructor_args(value) value = cls(*args, **kwargs) @@ -52,7 +52,7 @@ def from_value(cls, ion_type, value, annotations=()): def to_event(self, event_type, field_name=None, in_struct=False, depth=None): value = self - if isinstance(self, IonPyNull_new): + if isinstance(self, IonPyNull): value = None if in_struct: @@ -65,7 +65,7 @@ def to_event(self, event_type, field_name=None, in_struct=False, depth=None): annotations=self.ion_annotations, depth=depth) -class IonPyDecimal_new(Decimal): +class IonPyDecimal(Decimal): __name__ = 'IonPyDecimal_new' __qualname__ = 'IonPyDecimal_new' ion_type = IonType.DECIMAL @@ -100,7 +100,7 @@ def from_event(cls, ion_event): @classmethod def from_value(cls, ion_type, value, annotations=()): if value is None: - value = IonPyNull_new() + value = IonPyNull() else: args, kwargs = cls._to_constructor_args(value) value = cls(*args, **kwargs) @@ -110,7 +110,7 @@ def from_value(cls, ion_type, value, annotations=()): def to_event(self, event_type, field_name=None, in_struct=False, depth=None): value = self - if isinstance(self, IonPyNull_new): + if isinstance(self, IonPyNull): value = None if in_struct: @@ -123,7 +123,7 @@ def to_event(self, event_type, field_name=None, in_struct=False, depth=None): annotations=self.ion_annotations, depth=depth) -class IonPyInt_new(int): +class IonPyInt(int): __name__ = 'IonPyInt_new' __qualname__ = 'IonPyInt_new' ion_type = IonType.INT @@ -158,7 +158,7 @@ def from_event(cls, ion_event): @classmethod def from_value(cls, ion_type, value, annotations=()): if value is None: - value = IonPyNull_new() + value = IonPyNull() else: args, kwargs = cls._to_constructor_args(value) value = cls(*args, **kwargs) @@ -168,7 +168,7 @@ def from_value(cls, ion_type, value, annotations=()): def to_event(self, event_type, field_name=None, in_struct=False, depth=None): value = self - if isinstance(self, IonPyNull_new): + if isinstance(self, IonPyNull): value = None if in_struct: @@ -181,7 +181,7 @@ def to_event(self, event_type, field_name=None, in_struct=False, depth=None): annotations=self.ion_annotations, depth=depth) -class IonPyBool_new(int): +class IonPyBool(int): __name__ = 'IonPyBool_new' __qualname__ = 'IonPyBool_new' ion_type = IonType.BOOL @@ -219,7 +219,7 @@ def from_event(cls, ion_event): @classmethod def from_value(cls, ion_type, value, annotations=()): if value is None: - value = IonPyNull_new() + value = IonPyNull() else: args, kwargs = cls._to_constructor_args(value) value = cls(*args, **kwargs) @@ -229,7 +229,7 @@ def from_value(cls, ion_type, value, annotations=()): def to_event(self, event_type, field_name=None, in_struct=False, depth=None): value = self - if isinstance(self, IonPyNull_new): + if isinstance(self, IonPyNull): value = None if in_struct: @@ -242,7 +242,7 @@ def to_event(self, event_type, field_name=None, in_struct=False, depth=None): annotations=self.ion_annotations, depth=depth) -class IonPyFloat_new(float): +class IonPyFloat(float): __name__ = 'IonPyFloat_new' __qualname__ = 'IonPyFloat_new' ion_type = IonType.FLOAT @@ -277,7 +277,7 @@ def from_event(cls, ion_event): @classmethod def from_value(cls, ion_type, value, annotations=()): if value is None: - value = IonPyNull_new() + value = IonPyNull() else: args, kwargs = cls._to_constructor_args(value) value = cls(*args, **kwargs) @@ -287,7 +287,7 @@ def from_value(cls, ion_type, value, annotations=()): def to_event(self, event_type, field_name=None, in_struct=False, depth=None): value = self - if isinstance(self, IonPyNull_new): + if isinstance(self, IonPyNull): value = None if in_struct: @@ -300,7 +300,7 @@ def to_event(self, event_type, field_name=None, in_struct=False, depth=None): annotations=self.ion_annotations, depth=depth) -class IonPyText_new(str): +class IonPyText(str): __name__ = 'IonPyText_new' __qualname__ = 'IonPyText_new' ion_type = IonType.STRING @@ -335,7 +335,7 @@ def from_event(cls, ion_event): @classmethod def from_value(cls, ion_type, value, annotations=()): if value is None: - value = IonPyNull_new() + value = IonPyNull() else: args, kwargs = cls._to_constructor_args(value) value = cls(*args, **kwargs) @@ -345,7 +345,7 @@ def from_value(cls, ion_type, value, annotations=()): def to_event(self, event_type, field_name=None, in_struct=False, depth=None): value = self - if isinstance(self, IonPyNull_new): + if isinstance(self, IonPyNull): value = None if in_struct: @@ -358,7 +358,7 @@ def to_event(self, event_type, field_name=None, in_struct=False, depth=None): annotations=self.ion_annotations, depth=depth) -class IonPySymbol_new(SymbolToken): +class IonPySymbol(SymbolToken): __name__ = 'IonPySymbol_new' __qualname__ = 'IonPySymbol_new' ion_type = IonType.SYMBOL @@ -399,7 +399,7 @@ def from_event(cls, ion_event): @classmethod def from_value(cls, ion_type, value, annotations=()): if value is None: - value = IonPyNull_new() + value = IonPyNull() else: args, kwargs = cls._to_constructor_args(value) value = cls(*args, **kwargs) @@ -409,7 +409,7 @@ def from_value(cls, ion_type, value, annotations=()): def to_event(self, event_type, field_name=None, in_struct=False, depth=None): value = self - if isinstance(self, IonPyNull_new): + if isinstance(self, IonPyNull): value = None if in_struct: @@ -422,7 +422,7 @@ def to_event(self, event_type, field_name=None, in_struct=False, depth=None): annotations=self.ion_annotations, depth=depth) -class IonPyList_new(list): +class IonPyList(list): __name__ = 'IonPyList_new' __qualname__ = 'IonPyList_new' @@ -459,7 +459,7 @@ def from_event(cls, ion_event): @classmethod def from_value(cls, ion_type, value, annotations=()): if value is None: - value = IonPyNull_new() + value = IonPyNull() else: args, kwargs = cls._to_constructor_args(value) value = cls(*args, **kwargs) @@ -480,7 +480,7 @@ def to_event(self, event_type, field_name=None, in_struct=False, depth=None): annotations=self.ion_annotations, depth=depth) -class IonPyDict_new(MutableMapping): +class IonPyDict(MutableMapping): """ Dictionary that can hold multiple values for the same key @@ -595,7 +595,7 @@ def from_event(cls, ion_event): @classmethod def from_value(cls, ion_type, value, annotations=()): if value is None: - value = IonPyNull_new() + value = IonPyNull() else: args, kwargs = cls._to_constructor_args(value) value = cls(*args, **kwargs) diff --git a/amazon/ion/simple_types.py b/amazon/ion/simple_types.py index 4e8d3a65d..e54bec4a0 100644 --- a/amazon/ion/simple_types.py +++ b/amazon/ion/simple_types.py @@ -20,8 +20,8 @@ from decimal import Decimal -from amazon.ion.ion_py_objects import IonPyNull_new, IonPyDecimal_new, IonPyInt_new, IonPyFloat_new, IonPyText_new, \ - IonPyList_new, IonPyDict_new, IonPyBool_new, IonPySymbol_new +from amazon.ion.ion_py_objects import IonPyNull, IonPyDecimal, IonPyInt, IonPyFloat, IonPyText, \ + IonPyList, IonPyDict, IonPyBool, IonPySymbol # in Python 3.10, abstract collections have moved into their own module # for compatibility with 3.10+, first try imports from the new location @@ -240,12 +240,12 @@ def is_null(value): # IonPySymbol = IonPySymbol_original # Switch to the previous IonPyObjects by comment below lines -IonPyDecimal = IonPyDecimal_new -IonPyNull = IonPyNull_new -IonPyInt = IonPyInt_new -IonPyFloat = IonPyFloat_new -IonPyText = IonPyText_new -IonPyList = IonPyList_new -IonPyDict = IonPyDict_new -IonPyBool = IonPyBool_new -IonPySymbol = IonPySymbol_new +IonPyDecimal = IonPyDecimal +IonPyNull = IonPyNull +IonPyInt = IonPyInt +IonPyFloat = IonPyFloat +IonPyText = IonPyText +IonPyList = IonPyList +IonPyDict = IonPyDict +IonPyBool = IonPyBool +IonPySymbol = IonPySymbol diff --git a/amazon/ion/simpleion.py b/amazon/ion/simpleion.py index 5946fb6e3..46a0083fe 100644 --- a/amazon/ion/simpleion.py +++ b/amazon/ion/simpleion.py @@ -28,7 +28,7 @@ from .reader_managed import managed_reader from .simple_types import IonPyList, IonPyDict, IonPyNull, IonPyBool, IonPyInt, IonPyFloat, IonPyDecimal, \ IonPyTimestamp, IonPyText, IonPyBytes, IonPySymbol, is_null -from amazon.ion.ion_py_objects import IonPyNull_new +from amazon.ion.ion_py_objects import IonPyNull from .symbols import SymbolToken from .writer import blocking_writer from .writer_binary import binary_writer @@ -166,7 +166,7 @@ def dump_python(obj, fp, imports=None, binary=True, sequence_as_stream=False, sk _FROM_TYPE = { - IonPyNull_new: IonType.NULL, + IonPyNull: IonType.NULL, type(None): IonType.NULL, type(True): IonType.BOOL, type(False): IonType.BOOL, diff --git a/tests/test_multimap_IonPyDict.py b/tests/test_multimap_IonPyDict.py index f9735a3f0..6b3d2ab63 100644 --- a/tests/test_multimap_IonPyDict.py +++ b/tests/test_multimap_IonPyDict.py @@ -1,6 +1,6 @@ from typing import Sequence, NamedTuple -from amazon.ion.ion_py_objects import IonPyDict_new as Multimap +from amazon.ion.ion_py_objects import IonPyDict as Multimap from tests import parametrize diff --git a/tests/test_simpleion.py b/tests/test_simpleion.py index 696f0e308..fbbd88bc9 100644 --- a/tests/test_simpleion.py +++ b/tests/test_simpleion.py @@ -26,7 +26,7 @@ from amazon.ion import simpleion from amazon.ion.exceptions import IonException -from amazon.ion.ion_py_objects import IonPyDict_new +from amazon.ion.ion_py_objects import IonPyDict from amazon.ion.symbols import SymbolToken, SYSTEM_SYMBOL_TABLE from amazon.ion.writer_binary import _IVM from amazon.ion.core import IonType, IonEvent, IonEventType, OffsetTZInfo, Multimap @@ -249,7 +249,7 @@ def generate_containers_binary(container_map, preceding_symbols=0): for one_expected in expecteds: one_expected = one_expected.binary for elem in obj: - if isinstance(elem, (dict, Multimap, IonPyDict_new)) and len(elem) > 0: + if isinstance(elem, (dict, Multimap, IonPyDict)) and len(elem) > 0: has_symbols = True elif ion_type is IonType.SEXP and isinstance(elem, tuple): tuple_as_sexp = True From a8c5514025954d52ec939d64629c24afc1bcd1b5 Mon Sep 17 00:00:00 2001 From: cheqianh Date: Tue, 12 Sep 2023 12:38:08 -0700 Subject: [PATCH 6/6] Clean up works * Deprecated `ion_nature` * Migrated objects construction from `from_value()` to their constructors in C extension * Added IonPytimestamp and IonPyBytes * Replaced all the original IonPyObject and factory methods with the new IonPyObjects. * Worked on the IonPyTimestamp signature to make it compatible with the amazon.core.Timestamp * Updated the unit tests to accommodate the changes. --- amazon/ion/equivalence.py | 4 +- amazon/ion/ion_py_objects.py | 616 ------------------------ amazon/ion/ioncmodule.c | 57 +-- amazon/ion/simple_types.py | 800 ++++++++++++++++++++++++++----- amazon/ion/simpleion.py | 1 - tests/test_equivalence.py | 14 +- tests/test_multimap_IonPyDict.py | 78 --- tests/test_simple_types.py | 4 +- tests/test_simpleion.py | 27 +- 9 files changed, 693 insertions(+), 908 deletions(-) delete mode 100644 amazon/ion/ion_py_objects.py delete mode 100644 tests/test_multimap_IonPyDict.py diff --git a/amazon/ion/equivalence.py b/amazon/ion/equivalence.py index c5e95e280..01358c3ee 100644 --- a/amazon/ion/equivalence.py +++ b/amazon/ion/equivalence.py @@ -19,9 +19,7 @@ from math import isnan from amazon.ion.core import IonType, Timestamp, TimestampPrecision, MICROSECOND_PRECISION, OffsetTZInfo, Multimap -from amazon.ion.ion_py_objects import IonPyNull, IonPyDecimal, IonPyInt, IonPyFloat, IonPyText, \ - IonPyList, IonPyDict, IonPyBool, IonPySymbol -from amazon.ion.simple_types import _IonNature, IonPyList, IonPyDict, IonPyTimestamp, IonPyNull, IonPySymbol, \ +from amazon.ion.simple_types import IonPyList, IonPyDict, IonPyTimestamp, IonPyNull, IonPySymbol, \ IonPyText, IonPyDecimal, IonPyFloat from amazon.ion.symbols import SymbolToken diff --git a/amazon/ion/ion_py_objects.py b/amazon/ion/ion_py_objects.py deleted file mode 100644 index 3fd4d77f1..000000000 --- a/amazon/ion/ion_py_objects.py +++ /dev/null @@ -1,616 +0,0 @@ -from collections import OrderedDict -from decimal import Decimal -from collections.abc import MutableMapping - -from amazon.ion.core import IonType, IonEvent -from amazon.ion.symbols import SymbolToken - - -class IonPyNull(object): - __name__ = 'IonPyNull_new' - __qualname__ = 'IonPyNull_new' - - def __init__(self, ion_type=IonType.NULL, value=None, annotations=()): - self.ion_type = ion_type # TODO initialized to NULL type first, what's the real type? - self.ion_annotations = annotations - - def __bool__(self): - return False - - def __copy__(self): - args, kwargs = self._to_constructor_args(self) - value = self.__class__(*args, **kwargs) - value.ion_type = self.ion_type - value.ion_annotations = self.ion_annotations - return value - - @staticmethod - def _to_constructor_args(value): - return (None, value,), {} - - @classmethod - def from_event(cls, ion_event): - if ion_event.value is not None: - args, kwargs = cls._to_constructor_args(ion_event.value) - else: - args, kwargs = (None, None, ()), {} - value = cls(*args, **kwargs) - value.ion_type = ion_event.ion_type - value.ion_annotations = ion_event.annotations - return value - - @classmethod - def from_value(cls, ion_type, value, annotations=()): - if value is None: - value = IonPyNull() - else: - args, kwargs = cls._to_constructor_args(value) - value = cls(*args, **kwargs) - value.ion_type = ion_type - value.ion_annotations = annotations - return value - - def to_event(self, event_type, field_name=None, in_struct=False, depth=None): - value = self - if isinstance(self, IonPyNull): - value = None - - if in_struct: - if not isinstance(field_name, SymbolToken): - field_name = SymbolToken(field_name, 0 if field_name is None else None) - else: - field_name = None - - return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, - annotations=self.ion_annotations, depth=depth) - - -class IonPyDecimal(Decimal): - __name__ = 'IonPyDecimal_new' - __qualname__ = 'IonPyDecimal_new' - ion_type = IonType.DECIMAL - - def __new__(cls, ion_type=IonType.DECIMAL, value=None, annotations=()): - v = super().__new__(cls, value) - v.ion_annotations = annotations - return v - - def __copy__(self): - args, kwargs = self._to_constructor_args(self) - value = self.__class__(*args, **kwargs) - value.ion_type = self.ion_type - value.ion_annotations = self.ion_annotations - return value - - @staticmethod - def _to_constructor_args(value): - return (None, value,), {} - - @classmethod - def from_event(cls, ion_event): - if ion_event.value is not None: - args, kwargs = cls._to_constructor_args(ion_event.value) - else: - args, kwargs = (None, None, ()), {} - value = cls(*args, **kwargs) - value.ion_type = ion_event.ion_type - value.ion_annotations = ion_event.annotations - return value - - @classmethod - def from_value(cls, ion_type, value, annotations=()): - if value is None: - value = IonPyNull() - else: - args, kwargs = cls._to_constructor_args(value) - value = cls(*args, **kwargs) - value.ion_type = ion_type - value.ion_annotations = annotations - return value - - def to_event(self, event_type, field_name=None, in_struct=False, depth=None): - value = self - if isinstance(self, IonPyNull): - value = None - - if in_struct: - if not isinstance(field_name, SymbolToken): - field_name = SymbolToken(field_name, 0 if field_name is None else None) - else: - field_name = None - - return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, - annotations=self.ion_annotations, depth=depth) - - -class IonPyInt(int): - __name__ = 'IonPyInt_new' - __qualname__ = 'IonPyInt_new' - ion_type = IonType.INT - - def __new__(cls, ion_type=IonType.INT, value=None, annotations=()): - v = super().__new__(cls, value) - v.ion_annotations = annotations - return v - - def __copy__(self): - args, kwargs = self._to_constructor_args(self) - value = self.__class__(*args, **kwargs) - value.ion_type = self.ion_type - value.ion_annotations = self.ion_annotations - return value - - @staticmethod - def _to_constructor_args(value): - return (None, value,), {} - - @classmethod - def from_event(cls, ion_event): - if ion_event.value is not None: - args, kwargs = cls._to_constructor_args(ion_event.value) - else: - args, kwargs = (None, None, ()), {} - value = cls(*args, **kwargs) - value.ion_type = ion_event.ion_type - value.ion_annotations = ion_event.annotations - return value - - @classmethod - def from_value(cls, ion_type, value, annotations=()): - if value is None: - value = IonPyNull() - else: - args, kwargs = cls._to_constructor_args(value) - value = cls(*args, **kwargs) - value.ion_type = ion_type - value.ion_annotations = annotations - return value - - def to_event(self, event_type, field_name=None, in_struct=False, depth=None): - value = self - if isinstance(self, IonPyNull): - value = None - - if in_struct: - if not isinstance(field_name, SymbolToken): - field_name = SymbolToken(field_name, 0 if field_name is None else None) - else: - field_name = None - - return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, - annotations=self.ion_annotations, depth=depth) - - -class IonPyBool(int): - __name__ = 'IonPyBool_new' - __qualname__ = 'IonPyBool_new' - ion_type = IonType.BOOL - - def __repr__(self): - return str(bool(self)) - - def __new__(cls, ion_type=IonType.BOOL, value=None, annotations=()): - v = super().__new__(cls, value) - v.ion_annotations = annotations - return v - - def __copy__(self): - args, kwargs = self._to_constructor_args(self) - value = self.__class__(*args, **kwargs) - value.ion_type = self.ion_type - value.ion_annotations = self.ion_annotations - return value - - @staticmethod - def _to_constructor_args(value): - return (None, value,), {} - - @classmethod - def from_event(cls, ion_event): - if ion_event.value is not None: - args, kwargs = cls._to_constructor_args(ion_event.value) - else: - args, kwargs = (None, None, ()), {} - value = cls(*args, **kwargs) - value.ion_type = ion_event.ion_type - value.ion_annotations = ion_event.annotations - return value - - @classmethod - def from_value(cls, ion_type, value, annotations=()): - if value is None: - value = IonPyNull() - else: - args, kwargs = cls._to_constructor_args(value) - value = cls(*args, **kwargs) - value.ion_type = ion_type - value.ion_annotations = annotations - return value - - def to_event(self, event_type, field_name=None, in_struct=False, depth=None): - value = self - if isinstance(self, IonPyNull): - value = None - - if in_struct: - if not isinstance(field_name, SymbolToken): - field_name = SymbolToken(field_name, 0 if field_name is None else None) - else: - field_name = None - - return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, - annotations=self.ion_annotations, depth=depth) - - -class IonPyFloat(float): - __name__ = 'IonPyFloat_new' - __qualname__ = 'IonPyFloat_new' - ion_type = IonType.FLOAT - - def __new__(cls, ion_type=IonType.FLOAT, value=None, annotations=()): - v = super().__new__(cls, value) - v.ion_annotations = annotations - return v - - def __copy__(self): - args, kwargs = self._to_constructor_args(self) - value = self.__class__(*args, **kwargs) - value.ion_type = self.ion_type - value.ion_annotations = self.ion_annotations - return value - - @staticmethod - def _to_constructor_args(value): - return (None, value,), {} - - @classmethod - def from_event(cls, ion_event): - if ion_event.value is not None: - args, kwargs = cls._to_constructor_args(ion_event.value) - else: - args, kwargs = (None, None, ()), {} - value = cls(*args, **kwargs) - value.ion_type = ion_event.ion_type - value.ion_annotations = ion_event.annotations - return value - - @classmethod - def from_value(cls, ion_type, value, annotations=()): - if value is None: - value = IonPyNull() - else: - args, kwargs = cls._to_constructor_args(value) - value = cls(*args, **kwargs) - value.ion_type = ion_type - value.ion_annotations = annotations - return value - - def to_event(self, event_type, field_name=None, in_struct=False, depth=None): - value = self - if isinstance(self, IonPyNull): - value = None - - if in_struct: - if not isinstance(field_name, SymbolToken): - field_name = SymbolToken(field_name, 0 if field_name is None else None) - else: - field_name = None - - return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, - annotations=self.ion_annotations, depth=depth) - - -class IonPyText(str): - __name__ = 'IonPyText_new' - __qualname__ = 'IonPyText_new' - ion_type = IonType.STRING - - def __new__(cls, ion_type=IonType.STRING, value=None, annotations=()): - v = super().__new__(cls, value) - v.ion_annotations = annotations - return v - - def __copy__(self): - args, kwargs = self._to_constructor_args(self) - value = self.__class__(*args, **kwargs) - value.ion_type = self.ion_type - value.ion_annotations = self.ion_annotations - return value - - @staticmethod - def _to_constructor_args(value): - return (None, value,), {} - - @classmethod - def from_event(cls, ion_event): - if ion_event.value is not None: - args, kwargs = cls._to_constructor_args(ion_event.value) - else: - args, kwargs = (None, None, ()), {} - value = cls(*args, **kwargs) - value.ion_type = ion_event.ion_type - value.ion_annotations = ion_event.annotations - return value - - @classmethod - def from_value(cls, ion_type, value, annotations=()): - if value is None: - value = IonPyNull() - else: - args, kwargs = cls._to_constructor_args(value) - value = cls(*args, **kwargs) - value.ion_type = ion_type - value.ion_annotations = annotations - return value - - def to_event(self, event_type, field_name=None, in_struct=False, depth=None): - value = self - if isinstance(self, IonPyNull): - value = None - - if in_struct: - if not isinstance(field_name, SymbolToken): - field_name = SymbolToken(field_name, 0 if field_name is None else None) - else: - field_name = None - - return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, - annotations=self.ion_annotations, depth=depth) - - -class IonPySymbol(SymbolToken): - __name__ = 'IonPySymbol_new' - __qualname__ = 'IonPySymbol_new' - ion_type = IonType.SYMBOL - - # a good signature: IonPySymbol(ion_type, symbol_token, annotation) - def __new__(cls, ion_type=IonType.SYMBOL, value=None, annotations=()): - v = super().__new__(cls, *value) - v.ion_annotations = annotations - return v - - def __copy__(self): - args, kwargs = self._to_constructor_args(self) - value = self.__class__(*args, **kwargs) - value.ion_type = self.ion_type - value.ion_annotations = self.ion_annotations - return value - - @staticmethod - def _to_constructor_args(st): - try: - args = (None, (st.text, st.sid, st.location), ()) - except AttributeError: - args = (None, (st, None, None), ()) - kwargs = {} - return args, kwargs - - @classmethod - def from_event(cls, ion_event): - if ion_event.value is not None: - args, kwargs = cls._to_constructor_args(ion_event.value) - else: - args, kwargs = (None, None, ()), {} - value = cls(*args, **kwargs) - value.ion_type = ion_event.ion_type - value.ion_annotations = ion_event.annotations - return value - - @classmethod - def from_value(cls, ion_type, value, annotations=()): - if value is None: - value = IonPyNull() - else: - args, kwargs = cls._to_constructor_args(value) - value = cls(*args, **kwargs) - value.ion_type = ion_type - value.ion_annotations = annotations - return value - - def to_event(self, event_type, field_name=None, in_struct=False, depth=None): - value = self - if isinstance(self, IonPyNull): - value = None - - if in_struct: - if not isinstance(field_name, SymbolToken): - field_name = SymbolToken(field_name, 0 if field_name is None else None) - else: - field_name = None - - return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, - annotations=self.ion_annotations, depth=depth) - - -class IonPyList(list): - __name__ = 'IonPyList_new' - __qualname__ = 'IonPyList_new' - - def __init__(self, ion_type=IonType.LIST, value=None, annotations=()): - if value is None: - value = [] - super().__init__(value) - self.ion_annotations = annotations - # it's possible to be Sexp - self.ion_type = ion_type - - def __copy__(self): - args, kwargs = self._to_constructor_args(self) - value = self.__class__(*args, **kwargs) - value.ion_type = self.ion_type - value.ion_annotations = self.ion_annotations - return value - - @staticmethod - def _to_constructor_args(value): - return (None, value,), {} - - @classmethod - def from_event(cls, ion_event): - if ion_event.value is not None: - args, kwargs = cls._to_constructor_args(ion_event.value) - else: - args, kwargs = (None, [], ()), {} - value = cls(*args, **kwargs) - value.ion_type = ion_event.ion_type - value.ion_annotations = ion_event.annotations - return value - - @classmethod - def from_value(cls, ion_type, value, annotations=()): - if value is None: - value = IonPyNull() - else: - args, kwargs = cls._to_constructor_args(value) - value = cls(*args, **kwargs) - value.ion_type = ion_type - value.ion_annotations = annotations - return value - - def to_event(self, event_type, field_name=None, in_struct=False, depth=None): - value = None - - if in_struct: - if not isinstance(field_name, SymbolToken): - field_name = SymbolToken(field_name, 0 if field_name is None else None) - else: - field_name = None - - return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, - annotations=self.ion_annotations, depth=depth) - - -class IonPyDict(MutableMapping): - """ - Dictionary that can hold multiple values for the same key - - In order not to break existing customers, getting and inserting elements with ``[]`` keeps the same behaviour - as the built-in dict. If multiple elements are already mapped to the key, ``[]` will return - the newest one. - - To map multiple elements to a key, use the ``add_item`` operation. - To retrieve all the values map to a key, use ``get_all_values``. - """ - __name__ = 'IonPyDict_new' - __qualname__ = 'IonPyDict_new' - ion_type = IonType.STRUCT - - def __init__(self, ion_type=IonType.STRUCT, value=None, annotations=()): - super().__init__() - self.ion_annotations = annotations - self.__store = OrderedDict() - if value is not None: - for key, value in iter(value.items()): - if key in self.__store.keys(): - self.__store[key].append(value) - else: - self.__store[key] = [value] - - def __getitem__(self, key): - """ - Return the newest value for the given key. To retrieve all the values map to the key, use ``get_all_values``. - """ - return self.__store[key][len(self.__store[key]) - 1] # Return only one in order not to break clients - - def __delitem__(self, key): - """ - Delete all values for the given key. - """ - del self.__store[key] - - def __setitem__(self, key, value): - """ - Set the desired value to the given key. - """ - self.__store[key] = [value] - - def __len__(self): - return sum([len(values) for values in iter(self.__store.values())]) - - def __iter__(self): - for key in iter(self.__store.keys()): - yield key - - def __str__(self): - return repr(self) - - def __repr__(self): - return '{%s}' % ', '.join(['%r: %r' % (k, v) for k, v in self.items()]) - - def add_item(self, key, value): - """ - Add a value for the given key. This operation appends the value to the end of the value list instead of - overwriting the existing value. - """ - if key in self.__store: - self.__store[key].append(value) - else: - self.__setitem__(key, value) - - def get_all_values(self, key): - """ - Retrieve all the values mapped to the given key - """ - return self.__store[key] - - def iteritems(self): - """ - Return an iterator over (key, value) tuple pairs. - """ - for key in self.__store: - for value in self.__store[key]: - yield (key, value) - - def items(self): - """ - Return a list of the IonPyDict's (key, value) tuple pairs. - """ - output = [] - for k, v in self.iteritems(): - output.append((k, v)) - return output - - def __copy__(self): - args, kwargs = self._to_constructor_args(self) - value = self.__class__(*args, **kwargs) - value.ion_type = self.ion_type - value.ion_annotations = self.ion_annotations - return value - - @staticmethod - def _to_constructor_args(value): - return (None, value,), {} - - @classmethod - def from_event(cls, ion_event): - if ion_event.value is not None: - args, kwargs = cls._to_constructor_args(ion_event.value) - else: - args, kwargs = (None, None, ()), {} - value = cls(*args, **kwargs) - value.ion_type = ion_event.ion_type - value.ion_annotations = ion_event.annotations - return value - - @classmethod - def from_value(cls, ion_type, value, annotations=()): - if value is None: - value = IonPyNull() - else: - args, kwargs = cls._to_constructor_args(value) - value = cls(*args, **kwargs) - value.ion_type = ion_type - value.ion_annotations = annotations - return value - - def to_event(self, event_type, field_name=None, in_struct=False, depth=None): - value = None - - if in_struct: - if not isinstance(field_name, SymbolToken): - field_name = SymbolToken(field_name, 0 if field_name is None else None) - else: - field_name = None - - return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, - annotations=self.ion_annotations, depth=depth) diff --git a/amazon/ion/ioncmodule.c b/amazon/ion/ioncmodule.c index 721755fab..b258d5a08 100644 --- a/amazon/ion/ioncmodule.c +++ b/amazon/ion/ioncmodule.c @@ -38,27 +38,16 @@ static PyObject* _decimal_constructor; static PyObject* _py_timestamp_constructor; static PyObject* _simpletypes_module; static PyObject* _ionpynull_cls; -static PyObject* _ionpynull_fromvalue; static PyObject* _ionpybool_cls; -static PyObject* _ionpybool_fromvalue; static PyObject* _ionpyint_cls; -static PyObject* _ionpyint_fromvalue; static PyObject* _ionpyfloat_cls; -static PyObject* _ionpyfloat_fromvalue; static PyObject* _ionpydecimal_cls; -static PyObject* _ionpydecimal_fromvalue; static PyObject* _ionpytimestamp_cls; -static PyObject* _ionpytimestamp_fromvalue; static PyObject* _ionpytext_cls; -static PyObject* _ionpytext_fromvalue; static PyObject* _ionpysymbol_cls; -static PyObject* _ionpysymbol_fromvalue; static PyObject* _ionpybytes_cls; -static PyObject* _ionpybytes_fromvalue; static PyObject* _ionpylist_cls; -static PyObject* _ionpylist_fromvalue; static PyObject* _ionpydict_cls; -static PyObject* _ionpydict_fromvalue; static PyObject* _ion_core_module; static PyObject* _py_ion_type; static PyObject* py_ion_type_table[14]; @@ -1115,7 +1104,7 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s ion_type = ION_TYPE_INT(null_type); py_value = Py_None; // INCREFs and returns Python None. emit_bare_values = emit_bare_values && (ion_type == tid_NULL_INT); - ion_nature_constructor = _ionpynull_fromvalue; + ion_nature_constructor = _ionpynull_cls; break; } case tid_BOOL_INT: @@ -1123,7 +1112,7 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s BOOL bool_value; IONCHECK(ion_reader_read_bool(hreader, &bool_value)); py_value = PyBool_FromLong(bool_value); - ion_nature_constructor = _ionpybool_fromvalue; + ion_nature_constructor = _ionpybool_cls; break; } case tid_INT_INT: @@ -1146,7 +1135,7 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s py_value = PyLong_FromString(ion_int_str, NULL, 10); PyMem_Free(ion_int_str); - ion_nature_constructor = _ionpyint_fromvalue; + ion_nature_constructor = _ionpyint_cls; break; } case tid_FLOAT_INT: @@ -1154,7 +1143,7 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s double double_value; IONCHECK(ion_reader_read_double(hreader, &double_value)); py_value = Py_BuildValue("d", double_value); - ion_nature_constructor = _ionpyfloat_fromvalue; + ion_nature_constructor = _ionpyfloat_cls; break; } case tid_DECIMAL_INT: @@ -1209,13 +1198,13 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s } } - ion_nature_constructor = _ionpydecimal_fromvalue; + ion_nature_constructor = _ionpydecimal_cls; break; } case tid_TIMESTAMP_INT: { IONCHECK(ionc_read_timestamp(hreader, &py_value)); - ion_nature_constructor = _ionpytimestamp_fromvalue; + ion_nature_constructor = _ionpytimestamp_cls; break; } case tid_SYMBOL_INT: @@ -1223,7 +1212,7 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s emit_bare_values = FALSE; // Symbol values must always be emitted as IonNature because of ambiguity with string. ION_STRING string_value; IONCHECK(ion_reader_read_string(hreader, &string_value)); - ion_nature_constructor = _ionpysymbol_fromvalue; + ion_nature_constructor = _ionpysymbol_cls; py_value = ion_string_to_py_symboltoken(&string_value); break; } @@ -1232,7 +1221,7 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s ION_STRING string_value; IONCHECK(ion_reader_read_string(hreader, &string_value)); py_value = ion_build_py_string(&string_value); - ion_nature_constructor = _ionpytext_fromvalue; + ion_nature_constructor = _ionpytext_cls; break; } case tid_CLOB_INT: @@ -1264,12 +1253,12 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s if (length) { PyMem_Free(buf); } - ion_nature_constructor = _ionpybytes_fromvalue; + ion_nature_constructor = _ionpybytes_cls; break; } case tid_STRUCT_INT: { - ion_nature_constructor = _ionpydict_fromvalue; + ion_nature_constructor = _ionpydict_cls; //Init a IonPyDict PyObject* new_dict = PyDict_New(); py_value = PyObject_CallFunctionObjArgs( @@ -1294,7 +1283,7 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s { py_value = PyList_New(0); IONCHECK(ionc_read_into_container(hreader, py_value, /*is_struct=*/FALSE, emit_bare_values)); - ion_nature_constructor = _ionpylist_fromvalue; + ion_nature_constructor = _ionpylist_cls; break; } case tid_DATAGRAM_INT: @@ -1556,39 +1545,17 @@ PyObject* ionc_init_module(void) { _simpletypes_module = PyImport_ImportModule("amazon.ion.simple_types"); _ionpynull_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyNull"); - _ionpynull_fromvalue = PyObject_GetAttrString(_ionpynull_cls, "from_value"); _ionpybool_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyBool"); - _ionpybool_fromvalue = PyObject_GetAttrString(_ionpybool_cls, "from_value"); _ionpyint_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyInt"); - _ionpyint_fromvalue = PyObject_GetAttrString(_ionpyint_cls, "from_value"); _ionpyfloat_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyFloat"); - _ionpyfloat_fromvalue = PyObject_GetAttrString(_ionpyfloat_cls, "from_value"); _ionpydecimal_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyDecimal"); - _ionpydecimal_fromvalue = PyObject_GetAttrString(_ionpydecimal_cls, "from_value"); _ionpytimestamp_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyTimestamp"); - _ionpytimestamp_fromvalue = PyObject_GetAttrString(_ionpytimestamp_cls, "from_value"); _ionpybytes_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyBytes"); - _ionpybytes_fromvalue = PyObject_GetAttrString(_ionpybytes_cls, "from_value"); _ionpytext_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyText"); - _ionpytext_fromvalue = PyObject_GetAttrString(_ionpytext_cls, "from_value"); _ionpysymbol_cls = PyObject_GetAttrString(_simpletypes_module, "IonPySymbol"); - _ionpysymbol_fromvalue = PyObject_GetAttrString(_ionpysymbol_cls, "from_value"); _ionpylist_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyList"); - _ionpylist_fromvalue = PyObject_GetAttrString(_ionpylist_cls, "from_value"); _ionpydict_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyDict"); - _ionpydict_fromvalue = PyObject_GetAttrString(_ionpydict_cls, "from_value"); - - // A temporary workaround to replace from_value() with class constructors - // TODO deprecate from_value later - _ionpyint_fromvalue = _ionpyint_cls; - _ionpynull_fromvalue = _ionpynull_cls; - _ionpyfloat_fromvalue = _ionpyfloat_cls; - _ionpydecimal_fromvalue = _ionpydecimal_cls; - _ionpytext_fromvalue = _ionpytext_cls; - _ionpylist_fromvalue = _ionpylist_cls; - _ionpydict_fromvalue = _ionpydict_cls; - _ionpybool_fromvalue = _ionpybool_cls; - _ionpysymbol_fromvalue = _ionpysymbol_cls; + _ion_core_module = PyImport_ImportModule("amazon.ion.core"); _py_timestamp_precision = PyObject_GetAttrString(_ion_core_module, "TimestampPrecision"); diff --git a/amazon/ion/simple_types.py b/amazon/ion/simple_types.py index e54bec4a0..94989864c 100644 --- a/amazon/ion/simple_types.py +++ b/amazon/ion/simple_types.py @@ -20,9 +20,6 @@ from decimal import Decimal -from amazon.ion.ion_py_objects import IonPyNull, IonPyDecimal, IonPyInt, IonPyFloat, IonPyText, \ - IonPyList, IonPyDict, IonPyBool, IonPySymbol - # in Python 3.10, abstract collections have moved into their own module # for compatibility with 3.10+, first try imports from the new location # if that fails, try from the pre-3.10 location @@ -31,36 +28,84 @@ except: from collections import MutableMapping +from collections import OrderedDict +from decimal import Decimal + +from amazon.ion.core import IonType, IonEvent, Timestamp, TIMESTAMP_FRACTIONAL_SECONDS_FIELD, TIMESTAMP_PRECISION_FIELD, \ + TimestampPrecision from amazon.ion.symbols import SymbolToken -from .core import TIMESTAMP_PRECISION_FIELD -from .core import Multimap, Timestamp, IonEvent, IonType, TIMESTAMP_FRACTION_PRECISION_FIELD, TimestampPrecision, \ - MICROSECOND_PRECISION, TIMESTAMP_FRACTIONAL_SECONDS_FIELD -class _IonNature(object): - """Mix-in for Ion related properties. +class IonPyNull(object): + __name__ = 'IonPyNull' + __qualname__ = 'IonPyNull' - Attributes: - ion_event (Optional[IonEvent]): The event, if any associated with the value. - ion_type (IonType): The Ion type for the value. - ion_annotations (Sequence[unicode]): The annotations associated with the value. + def __init__(self, ion_type=IonType.NULL, value=None, annotations=()): + self.ion_type = ion_type + self.ion_annotations = annotations - Notes: - There is no ``field_name`` attribute as that is generally modeled as a property of the - container. + def __bool__(self): + return False - The ``ion_event`` field is only provided if the value was derived from a low-level event. - User constructed values will generally not set this field. - """ - def __init__(self, *args, **kwargs): - self.ion_type = None - self.ion_annotations = () + def __copy__(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value - def _copy(self): - """Copies this instance. Its IonEvent (if any) is not preserved. + @staticmethod + def _to_constructor_args(value): + return (None, value,), {} - Keeping this protected until/unless we decide there's use for it publicly. - """ + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, None, ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value + + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value + + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = self + if isinstance(self, IonPyNull): + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) + + +class IonPyDecimal(Decimal): + __name__ = 'IonPyDecimal' + __qualname__ = 'IonPyDecimal' + ion_type = IonType.DECIMAL + + def __new__(cls, ion_type=IonType.DECIMAL, value=None, annotations=()): + v = super().__new__(cls, value) + v.ion_annotations = annotations + return v + + def __copy__(self): args, kwargs = self._to_constructor_args(self) value = self.__class__(*args, **kwargs) value.ion_type = self.ion_type @@ -69,21 +114,72 @@ def _copy(self): @staticmethod def _to_constructor_args(value): - return (value, ), {} + return (None, value,), {} @classmethod def from_event(cls, ion_event): - """Constructs the given native extension from the properties of an event. + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, None, ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value - Args: - ion_event (IonEvent): The event to construct the native value from. - """ + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value + + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = self + if isinstance(self, IonPyNull): + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) + + +class IonPyBytes(bytes): + __name__ = 'IonPyBytes' + __qualname__ = 'IonPyBytes' + + def __new__(cls, ion_type=IonType.BLOB, value=None, annotations=()): + v = super().__new__(cls, value) + v.ion_annotations = annotations + v.ion_type = ion_type + return v + + def __copy__(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value + + @staticmethod + def _to_constructor_args(value): + return (None, value,), {} + + @classmethod + def from_event(cls, ion_event): if ion_event.value is not None: args, kwargs = cls._to_constructor_args(ion_event.value) else: - # if value is None (i.e. this is a container event), args must be empty or initialization of the - # underlying container will fail. - args, kwargs = (), {} + args, kwargs = (None, None, ()), {} value = cls(*args, **kwargs) value.ion_type = ion_event.ion_type value.ion_annotations = ion_event.annotations @@ -91,13 +187,64 @@ def from_event(cls, ion_event): @classmethod def from_value(cls, ion_type, value, annotations=()): - """Constructs a value as a copy with an associated Ion type and annotations. + if value is None: + value = IonPyNull() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value - Args: - ion_type (IonType): The associated Ion type. - value (Any): The value to construct from, generally of type ``cls``. - annotations (Sequence[unicode]): The sequence Unicode strings decorating this value. - """ + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = self + if isinstance(self, IonPyNull): + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) + + +class IonPyInt(int): + __name__ = 'IonPyInt' + __qualname__ = 'IonPyInt' + ion_type = IonType.INT + + def __new__(cls, ion_type=IonType.INT, value=None, annotations=()): + v = super().__new__(cls, value) + v.ion_annotations = annotations + return v + + def __copy__(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value + + @staticmethod + def _to_constructor_args(value): + return (None, value,), {} + + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, None, ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value + + @classmethod + def from_value(cls, ion_type, value, annotations=()): if value is None: value = IonPyNull() else: @@ -108,22 +255,8 @@ def from_value(cls, ion_type, value, annotations=()): return value def to_event(self, event_type, field_name=None, in_struct=False, depth=None): - """Constructs an IonEvent from this _IonNature value. - - Args: - event_type (IonEventType): The type of the resulting event. - field_name (Optional[text]): The field name associated with this value, if any. When ``None`` - is specified and ``in_struct`` is ``True``, the returned event's ``field_name`` will - represent symbol zero (a ``SymbolToken`` with text=None and sid=0). - in_struct (Optional[True|False]): When ``True``, indicates the returned event ``field_name`` - will be populated. When ``False``, ``field_name`` will be ``None``. - depth (Optional[int]): The depth of this value. - - Returns: - An IonEvent with the properties from this value. - """ value = self - if isinstance(self, IonPyNull) or self.ion_type.is_container: + if isinstance(self, IonPyNull): value = None if in_struct: @@ -133,51 +266,216 @@ def to_event(self, event_type, field_name=None, in_struct=False, depth=None): field_name = None return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, - annotations=self.ion_annotations, depth=depth) + annotations=self.ion_annotations, depth=depth) -def _ion_type_for(name, base_cls, ion_type=None): - class IonPyValueType(base_cls, _IonNature): - def __init__(self, *args, **kwargs): - super(IonPyValueType, self).__init__(*args, **kwargs) - self.ion_type = ion_type +class IonPyBool(int): + __name__ = 'IonPyBool' + __qualname__ = 'IonPyBool' + ion_type = IonType.BOOL - IonPyValueType.__name__ = name - IonPyValueType.__qualname__ = name - return IonPyValueType + def __repr__(self): + return str(bool(self)) + def __new__(cls, ion_type=IonType.BOOL, value=None, annotations=()): + v = super().__new__(cls, value) + v.ion_annotations = annotations + return v -# All of our ion value types inherit from their python value types, for good or -# ill. Python's builtin `bool` cannot be sub-classed and is itself a sub-class -# of int so we just sub-class int directly for our Ion Boolean. -# Considering that True == 1 and False == 0 this works as expected. -IonPyBool_original = _ion_type_for('IonPyBool', int, IonType.BOOL) + def __copy__(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value -# We override __repr__ so Booleans show up as True|False and not 1|0. -# The representation is consistent with other primitives in that it returns a -# string repr of the value but not annotations. -IonPyBool_original.__repr__ = lambda self: str(bool(self)) + @staticmethod + def _to_constructor_args(value): + return (None, value,), {} + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, None, ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value -class IonPySymbol_original(SymbolToken, _IonNature): - def __init__(self, *args, **kwargs): - super(IonPySymbol_original, self).__init__(*args, **kwargs) - self.ion_type = IonType.SYMBOL + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value + + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = self + if isinstance(self, IonPyNull): + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) + + +class IonPyFloat(float): + __name__ = 'IonPyFloat' + __qualname__ = 'IonPyFloat' + ion_type = IonType.FLOAT + + def __new__(cls, ion_type=IonType.FLOAT, value=None, annotations=()): + v = super().__new__(cls, value) + v.ion_annotations = annotations + return v + + def __copy__(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value @staticmethod - def _to_constructor_args(st): - try: - args = (st.text, st.sid, st.location) - except AttributeError: - args = (st, None, None) - kwargs = {} - return args, kwargs + def _to_constructor_args(value): + return (None, value,), {} + + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, None, ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value + + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = self + if isinstance(self, IonPyNull): + value = None -class IonPyTimestamp(Timestamp, _IonNature): - def __init__(self, *args, **kwargs): - super(IonPyTimestamp, self).__init__(*args, **kwargs) - self.ion_type = IonType.TIMESTAMP + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) + + +class IonPyText(str): + __name__ = 'IonPyText' + __qualname__ = 'IonPyText' + ion_type = IonType.STRING + + def __new__(cls, ion_type=IonType.STRING, value=None, annotations=()): + v = super().__new__(cls, value) + v.ion_annotations = annotations + return v + + def __copy__(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value + + @staticmethod + def _to_constructor_args(value): + return (None, value,), {} + + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, None, ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value + + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value + + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = self + if isinstance(self, IonPyNull): + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) + + +class IonPyTimestamp(Timestamp): + __name__ = 'IonPyTimestamp' + __qualname__ = 'IonPyTimestamp' + ion_type = IonType.TIMESTAMP + + def __new__(cls, *args, **kwargs): + # The from_value like signature + if isinstance(args[0], IonType): + value = args[1] + annotations = () + if len(args) > 2 and args[2] is not None: + annotations = args[2] + if value is not None: + args_new, kwargs = cls._to_constructor_args(value) + else: + args_new, kwargs = (None, None, ()), {} + v = super().__new__(cls, *args_new, **kwargs) + v.ion_type = args[0] + v.ion_annotations = annotations + # Regular timestamp constructor + else: + v = super().__new__(cls, *args, **kwargs) + return v + + def __copy__(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value @staticmethod def _to_constructor_args(ts): @@ -191,61 +489,301 @@ def _to_constructor_args(ts): kwargs = {TIMESTAMP_PRECISION_FIELD: TimestampPrecision.SECOND} return args, kwargs + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, None, ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value + + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value + + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = self + if isinstance(self, IonPyNull): + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) + + +class IonPySymbol(SymbolToken): + __name__ = 'IonPySymbol' + __qualname__ = 'IonPySymbol' + ion_type = IonType.SYMBOL + + # a good signature: IonPySymbol(ion_type, symbol_token, annotation) + def __new__(cls, ion_type=IonType.SYMBOL, value=None, annotations=()): + v = super().__new__(cls, *value) + v.ion_annotations = annotations + return v + + def __copy__(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value + + @staticmethod + def _to_constructor_args(st): + try: + args = (None, (st.text, st.sid, st.location), ()) + except AttributeError: + args = (None, (st, None, None), ()) + kwargs = {} + return args, kwargs + + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, None, ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value + + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value + + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = self + if isinstance(self, IonPyNull): + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) + + +class IonPyList(list): + __name__ = 'IonPyList' + __qualname__ = 'IonPyList' + + def __init__(self, ion_type=IonType.LIST, value=None, annotations=()): + if value is None: + value = [] + super().__init__(value) + self.ion_annotations = annotations + # it's possible to be Sexp + self.ion_type = ion_type + + def __copy__(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value + + @staticmethod + def _to_constructor_args(value): + return (None, value,), {} -class IonPyNull_original(_IonNature): - """Representation of ``null``. + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, [], ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value - Notes: - ``None`` is a singleton and cannot be sub-classed, so we have our - own value type for it. The function ``is_null`` is the best way - to test for ``null``-ness or ``None``-ness. + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value + + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) + + +class IonPyDict(MutableMapping): """ + Dictionary that can hold multiple values for the same key - def __init__(self, *args, **kwargs): - super(IonPyNull_original, self).__init__(*args, **kwargs) + In order not to break existing customers, getting and inserting elements with ``[]`` keeps the same behaviour + as the built-in dict. If multiple elements are already mapped to the key, ``[]` will return + the newest one. - def __nonzero__(self): - return False + To map multiple elements to a key, use the ``add_item`` operation. + To retrieve all the values map to a key, use ``get_all_values``. + """ + __name__ = 'IonPyDict' + __qualname__ = 'IonPyDict' + ion_type = IonType.STRUCT + + def __init__(self, ion_type=IonType.STRUCT, value=None, annotations=()): + super().__init__() + self.ion_annotations = annotations + self.__store = OrderedDict() + if value is not None: + for key, value in iter(value.items()): + if key in self.__store.keys(): + self.__store[key].append(value) + else: + self.__store[key] = [value] + + def __getitem__(self, key): + """ + Return the newest value for the given key. To retrieve all the values map to the key, use ``get_all_values``. + """ + return self.__store[key][len(self.__store[key]) - 1] # Return only one in order not to break clients - def __bool__(self): - return False + def __delitem__(self, key): + """ + Delete all values for the given key. + """ + del self.__store[key] + + def __setitem__(self, key, value): + """ + Set the desired value to the given key. + """ + self.__store[key] = [value] + + def __len__(self): + return sum([len(values) for values in iter(self.__store.values())]) + + def __iter__(self): + for key in iter(self.__store.keys()): + yield key + + def __str__(self): + return repr(self) + + def __repr__(self): + return '{%s}' % ', '.join(['%r: %r' % (k, v) for k, v in self.items()]) + + def add_item(self, key, value): + """ + Add a value for the given key. This operation appends the value to the end of the value list instead of + overwriting the existing value. + """ + if key in self.__store: + self.__store[key].append(value) + else: + self.__setitem__(key, value) + + def get_all_values(self, key): + """ + Retrieve all the values mapped to the given key + """ + return self.__store[key] + + def iteritems(self): + """ + Return an iterator over (key, value) tuple pairs. + """ + for key in self.__store: + for value in self.__store[key]: + yield (key, value) + + def items(self): + """ + Return a list of the IonPyDict's (key, value) tuple pairs. + """ + output = [] + for k, v in self.iteritems(): + output.append((k, v)) + return output + + def __copy__(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value @staticmethod def _to_constructor_args(value): - return (), {} + return (None, value,), {} + + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, None, ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value + + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value + + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) def is_null(value): """A mechanism to determine if a value is ``None`` or an Ion ``null``.""" return value is None or isinstance(value, IonPyNull) - - -IonPyInt_original = _ion_type_for('IonPyInt', int, IonType.INT) -IonPyFloat_original = _ion_type_for('IonPyFloat', float, IonType.FLOAT) -IonPyDecimal_original = _ion_type_for('IonPyDecimal', Decimal, IonType.DECIMAL) -IonPyText_original = _ion_type_for('IonPyText', str, IonType.STRING) -IonPyBytes = _ion_type_for('IonPyBytes', bytes) -IonPyList_original = _ion_type_for('IonPyList', list) -IonPyDict_original = _ion_type_for('IonPyDict', Multimap, IonType.STRUCT) - -# Using the original IonPyObjects by uncomment below -# IonPyDecimal = IonPyDecimal_original -# IonPyNull = IonPyNull_original -# IonPyInt = IonPyInt_original -# IonPyFloat = IonPyFloat_original -# IonPyText = IonPyText_original -# IonPyList = IonPyList_original -# IonPyDict = IonPyDict_original -# IonPyBool = IonPyBool_original -# IonPySymbol = IonPySymbol_original - -# Switch to the previous IonPyObjects by comment below lines -IonPyDecimal = IonPyDecimal -IonPyNull = IonPyNull -IonPyInt = IonPyInt -IonPyFloat = IonPyFloat -IonPyText = IonPyText -IonPyList = IonPyList -IonPyDict = IonPyDict -IonPyBool = IonPyBool -IonPySymbol = IonPySymbol diff --git a/amazon/ion/simpleion.py b/amazon/ion/simpleion.py index 46a0083fe..f2c8c02f7 100644 --- a/amazon/ion/simpleion.py +++ b/amazon/ion/simpleion.py @@ -28,7 +28,6 @@ from .reader_managed import managed_reader from .simple_types import IonPyList, IonPyDict, IonPyNull, IonPyBool, IonPyInt, IonPyFloat, IonPyDecimal, \ IonPyTimestamp, IonPyText, IonPyBytes, IonPySymbol, is_null -from amazon.ion.ion_py_objects import IonPyNull from .symbols import SymbolToken from .writer import blocking_writer from .writer_binary import binary_writer diff --git a/tests/test_equivalence.py b/tests/test_equivalence.py index ac2f27ee1..6cb7001e4 100644 --- a/tests/test_equivalence.py +++ b/tests/test_equivalence.py @@ -21,9 +21,9 @@ from datetime import datetime, timedelta from amazon.ion.core import IonType, timestamp, TimestampPrecision, OffsetTZInfo -from amazon.ion.equivalence import ion_equals +from amazon.ion.equivalence import ion_equals, obj_has_ion_type_and_annotation from amazon.ion.simple_types import IonPyNull, IonPyInt, IonPyBool, IonPyFloat, IonPyDecimal, IonPyText, IonPyBytes, \ - IonPyList, IonPyTimestamp, IonPySymbol, IonPyDict, _IonNature + IonPyList, IonPyTimestamp, IonPySymbol, IonPyDict from amazon.ion.symbols import SymbolToken, ImportLocation from tests import parametrize @@ -268,7 +268,7 @@ def __str__(self): def _desc(a, b, operator): def _str(val): - if isinstance(val, _IonNature): + if obj_has_ion_type_and_annotation(val): return '%s(%s, ion_type=%s, ion_annotations=%s)' % (type(val), val, val.ion_type, val.ion_annotations) return '%s(%s)' % (type(val), val) return 'assert %s %s %s' % (_str(a), operator, _str(b)) @@ -327,13 +327,13 @@ def _generate_equivs(equivs, param_func=_equivs_param): has_ion_nature = False a = seq[i] b = seq[j] - if isinstance(a, _IonNature): + if obj_has_ion_type_and_annotation(a): has_ion_nature = True - a = a._copy() + a = a.__copy__() _add_annotations(a) - if isinstance(b, _IonNature): + if obj_has_ion_type_and_annotation(b): has_ion_nature = True - b = b._copy() + b = b.__copy__() _add_annotations(b) if has_ion_nature: yield _nonequivs_param(a, b) diff --git a/tests/test_multimap_IonPyDict.py b/tests/test_multimap_IonPyDict.py deleted file mode 100644 index 6b3d2ab63..000000000 --- a/tests/test_multimap_IonPyDict.py +++ /dev/null @@ -1,78 +0,0 @@ -from typing import Sequence, NamedTuple - -from amazon.ion.ion_py_objects import IonPyDict as Multimap - -from tests import parametrize - - -class _P(NamedTuple): - pairs: Sequence - expected_all_values: Sequence - expected_single_value: Sequence - expected_total_len: int - - def __str__(self): - return '{name}'.format(name=self.pairs) - - -ALL_DATA = _P( - pairs=[('a', 1), ('a', 2), ('a', 3), ('a', [4, 5, 6]), ('b', 0), ('c', {'x': 'z', 'r': 's'})], - expected_all_values=[('a', [1, 2, 3, [4, 5, 6]]), ('b', [0]), ('c', [{'x': 'z', 'r': 's'}])], - expected_single_value=[('a', [4, 5, 6]), ('b', 0), ('c', {'x': 'z', 'r': 's'})], - expected_total_len=6 -) - - -def _create_multimap_with_items(pairs): - m = Multimap() - for pair in pairs: - m.add_item(pair[0], pair[1]) - return m - - -@parametrize( - ALL_DATA -) -def test_add_item(p): - m = _create_multimap_with_items(p.pairs) - for expected in p.expected_all_values: - assert list([x for x in m.get_all_values(expected[0])]) == expected[1] - for expected in p.expected_single_value: - assert m[expected[0]] == expected[1] - assert p.expected_total_len == len(m) - - -@parametrize( - (ALL_DATA, ["a"], 2), - (ALL_DATA, ["b"], 5), - (ALL_DATA, ["c"], 5), - (ALL_DATA, ["a", "b"], 1), - (ALL_DATA, ["a", "b", "c"], 0) -) -def test_delete_item(item): - m = _create_multimap_with_items(item[0].pairs) - p, items_to_remove, len_after_removal = item - for to_remove in items_to_remove: - del m[to_remove] - assert len(m) == item[2] - - -@parametrize( - {}, - {"a": 1}, - {"a": 1, "b": 2, "c": [1, 2, {3: 4}]}, -) -def test_constructor(d): - m = Multimap(None, d) - for k, v in iter(d.items()): - assert m[k] == v - assert len(m) == len(d) - - -def test_multimap_constructor(): - m = Multimap(None, {}) - m.add_item('a', 1) - m.add_item('a', 2) - m2 = Multimap(None, m) - - assert len(m2) == 2 diff --git a/tests/test_simple_types.py b/tests/test_simple_types.py index 5a82b29dd..e723ce7e4 100644 --- a/tests/test_simple_types.py +++ b/tests/test_simple_types.py @@ -23,7 +23,7 @@ from amazon.ion.symbols import SymbolToken from amazon.ion.simple_types import is_null, IonPyNull, IonPyBool, IonPyInt, IonPyFloat, \ IonPyDecimal, IonPyTimestamp, IonPyText, IonPyBytes, \ - IonPyList, IonPyDict, IonPySymbol, _IonNature + IonPyList, IonPyDict, IonPySymbol from amazon.ion.equivalence import ion_equals from amazon.ion.simpleion import _ion_type, _FROM_TYPE @@ -33,7 +33,7 @@ class _P(NamedTuple): desc: str - type: Type[_IonNature] + type: Type[object] event: IonEvent def __str__(self): diff --git a/tests/test_simpleion.py b/tests/test_simpleion.py index fbbd88bc9..2e4baf84f 100644 --- a/tests/test_simpleion.py +++ b/tests/test_simpleion.py @@ -26,12 +26,11 @@ from amazon.ion import simpleion from amazon.ion.exceptions import IonException -from amazon.ion.ion_py_objects import IonPyDict from amazon.ion.symbols import SymbolToken, SYSTEM_SYMBOL_TABLE from amazon.ion.writer_binary import _IVM from amazon.ion.core import IonType, IonEvent, IonEventType, OffsetTZInfo, Multimap from amazon.ion.simple_types import IonPyDict, IonPyText, IonPyList, IonPyNull, IonPyBool, IonPyInt, IonPyFloat, \ - IonPyDecimal, IonPyTimestamp, IonPyBytes, IonPySymbol, _IonNature + IonPyDecimal, IonPyTimestamp, IonPyBytes, IonPySymbol from amazon.ion.equivalence import ion_equals, obj_has_ion_type_and_annotation from amazon.ion.simpleion import dump, dumps, load, loads, _ion_type, _FROM_ION_TYPE, _FROM_TYPE_TUPLE_AS_SEXP, \ _FROM_TYPE @@ -588,29 +587,7 @@ def _to_obj(to_type=None, annotations=(), tuple_as_sexp=False): def _assert_roundtrip(before, after, tuple_as_sexp): - # All loaded Ion values extend _IonNature, even if they were dumped from primitives. This recursively - # wraps each input value in _IonNature for comparison against the output. - def _to_ion_nature(obj): - out = obj - if not obj_has_ion_type_and_annotation(out): - from_type = _FROM_TYPE_TUPLE_AS_SEXP if tuple_as_sexp else _FROM_TYPE - ion_type = _ion_type(out, from_type) - out = _FROM_ION_TYPE[ion_type].from_value(ion_type, out) - if isinstance(out, dict): - update = {} - for field, value in iter(out.items()): - update[field] = _to_ion_nature(value) - update = IonPyDict.from_value(out.ion_type, update, out.ion_annotations) - out = update - elif isinstance(out, list): - update = [] - for value in out: - update.append(_to_ion_nature(value)) - update = IonPyList.from_value(out.ion_type, update, out.ion_annotations) - out = update - - return out - assert ion_equals(_to_ion_nature(before), after) + assert ion_equals(before, after) @parametrize(