diff --git a/amazon/ion/equivalence.py b/amazon/ion/equivalence.py index 56d41a34..01358c3e 100644 --- a/amazon/ion/equivalence.py +++ b/amazon/ion/equivalence.py @@ -19,11 +19,15 @@ from math import isnan from amazon.ion.core import IonType, Timestamp, TimestampPrecision, MICROSECOND_PRECISION, OffsetTZInfo, Multimap -from amazon.ion.simple_types import _IonNature, IonPyList, IonPyDict, IonPyTimestamp, IonPyNull, IonPySymbol, \ +from amazon.ion.simple_types import IonPyList, IonPyDict, IonPyTimestamp, IonPyNull, IonPySymbol, \ IonPyText, IonPyDecimal, IonPyFloat from amazon.ion.symbols import SymbolToken +def obj_has_ion_type_and_annotation(obj): + return hasattr(obj, 'ion_type') and hasattr(obj, 'ion_annotations') + + def ion_equals(a, b, timestamps_instants_only=False): """Tests two objects for equivalence under the Ion data model. @@ -60,8 +64,8 @@ def _ion_equals_timestamps_data_model(a, b): def _ion_equals(a, b, timestamp_comparison_func, recursive_comparison_func): """Compares a and b according to the description of the ion_equals method.""" for a, b in ((a, b), (b, a)): # Ensures that operand order does not matter. - if isinstance(a, _IonNature): - if isinstance(b, _IonNature): + if obj_has_ion_type_and_annotation(a): + if obj_has_ion_type_and_annotation(b): # Both operands have _IonNature. Their IonTypes and annotations must be equivalent. eq = a.ion_type is b.ion_type and _annotations_eq(a, b) else: @@ -120,8 +124,8 @@ def _sequences_eq(a, b, comparison_func): def _structs_eq(a, b, comparison_func): - assert isinstance(a, (dict, Multimap)) - if not isinstance(b, (dict, Multimap)): + assert isinstance(a, (dict, Multimap, IonPyDict)) + if not isinstance(b, (dict, Multimap, IonPyDict)): return False dict_len = len(a) if dict_len != len(b): @@ -135,7 +139,7 @@ def _structs_eq(a, b, comparison_func): break if key not in b: return False - if isinstance(a, Multimap) and isinstance(b, Multimap): + if isinstance(a, (IonPyDict, Multimap)) and isinstance(b, (IonPyDict, Multimap)): values_a = a.get_all_values(key) values_b = b.get_all_values(key) if len(values_a) != len(values_b): diff --git a/amazon/ion/ioncmodule.c b/amazon/ion/ioncmodule.c index b118833b..b258d5a0 100644 --- a/amazon/ion/ioncmodule.c +++ b/amazon/ion/ioncmodule.c @@ -38,27 +38,16 @@ static PyObject* _decimal_constructor; static PyObject* _py_timestamp_constructor; static PyObject* _simpletypes_module; static PyObject* _ionpynull_cls; -static PyObject* _ionpynull_fromvalue; static PyObject* _ionpybool_cls; -static PyObject* _ionpybool_fromvalue; static PyObject* _ionpyint_cls; -static PyObject* _ionpyint_fromvalue; static PyObject* _ionpyfloat_cls; -static PyObject* _ionpyfloat_fromvalue; static PyObject* _ionpydecimal_cls; -static PyObject* _ionpydecimal_fromvalue; static PyObject* _ionpytimestamp_cls; -static PyObject* _ionpytimestamp_fromvalue; static PyObject* _ionpytext_cls; -static PyObject* _ionpytext_fromvalue; static PyObject* _ionpysymbol_cls; -static PyObject* _ionpysymbol_fromvalue; static PyObject* _ionpybytes_cls; -static PyObject* _ionpybytes_fromvalue; static PyObject* _ionpylist_cls; -static PyObject* _ionpylist_fromvalue; static PyObject* _ionpydict_cls; -static PyObject* _ionpydict_fromvalue; static PyObject* _ion_core_module; static PyObject* _py_ion_type; static PyObject* py_ion_type_table[14]; @@ -1102,6 +1091,7 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s SUCCEED(); case tid_NULL_INT: { + // TODO double check the real null type, now it's initialized to IonType.NULL by default ION_TYPE null_type; // Hack for ion-c issue https://github.com/amazon-ion/ion-c/issues/223 if (original_t != tid_SYMBOL_INT) { @@ -1112,9 +1102,9 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s } ion_type = ION_TYPE_INT(null_type); - py_value = Py_BuildValue(""); // INCREFs and returns Python None. + py_value = Py_None; // INCREFs and returns Python None. emit_bare_values = emit_bare_values && (ion_type == tid_NULL_INT); - ion_nature_constructor = _ionpynull_fromvalue; + ion_nature_constructor = _ionpynull_cls; break; } case tid_BOOL_INT: @@ -1122,7 +1112,7 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s BOOL bool_value; IONCHECK(ion_reader_read_bool(hreader, &bool_value)); py_value = PyBool_FromLong(bool_value); - ion_nature_constructor = _ionpybool_fromvalue; + ion_nature_constructor = _ionpybool_cls; break; } case tid_INT_INT: @@ -1145,7 +1135,7 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s py_value = PyLong_FromString(ion_int_str, NULL, 10); PyMem_Free(ion_int_str); - ion_nature_constructor = _ionpyint_fromvalue; + ion_nature_constructor = _ionpyint_cls; break; } case tid_FLOAT_INT: @@ -1153,7 +1143,7 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s double double_value; IONCHECK(ion_reader_read_double(hreader, &double_value)); py_value = Py_BuildValue("d", double_value); - ion_nature_constructor = _ionpyfloat_fromvalue; + ion_nature_constructor = _ionpyfloat_cls; break; } case tid_DECIMAL_INT: @@ -1208,13 +1198,13 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s } } - ion_nature_constructor = _ionpydecimal_fromvalue; + ion_nature_constructor = _ionpydecimal_cls; break; } case tid_TIMESTAMP_INT: { IONCHECK(ionc_read_timestamp(hreader, &py_value)); - ion_nature_constructor = _ionpytimestamp_fromvalue; + ion_nature_constructor = _ionpytimestamp_cls; break; } case tid_SYMBOL_INT: @@ -1222,7 +1212,7 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s emit_bare_values = FALSE; // Symbol values must always be emitted as IonNature because of ambiguity with string. ION_STRING string_value; IONCHECK(ion_reader_read_string(hreader, &string_value)); - ion_nature_constructor = _ionpysymbol_fromvalue; + ion_nature_constructor = _ionpysymbol_cls; py_value = ion_string_to_py_symboltoken(&string_value); break; } @@ -1231,7 +1221,7 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s ION_STRING string_value; IONCHECK(ion_reader_read_string(hreader, &string_value)); py_value = ion_build_py_string(&string_value); - ion_nature_constructor = _ionpytext_fromvalue; + ion_nature_constructor = _ionpytext_cls; break; } case tid_CLOB_INT: @@ -1263,12 +1253,12 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s if (length) { PyMem_Free(buf); } - ion_nature_constructor = _ionpybytes_fromvalue; + ion_nature_constructor = _ionpybytes_cls; break; } case tid_STRUCT_INT: { - ion_nature_constructor = _ionpydict_fromvalue; + ion_nature_constructor = _ionpydict_cls; //Init a IonPyDict PyObject* new_dict = PyDict_New(); py_value = PyObject_CallFunctionObjArgs( @@ -1293,7 +1283,7 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s { py_value = PyList_New(0); IONCHECK(ionc_read_into_container(hreader, py_value, /*is_struct=*/FALSE, emit_bare_values)); - ion_nature_constructor = _ionpylist_fromvalue; + ion_nature_constructor = _ionpylist_cls; break; } case tid_DATAGRAM_INT: @@ -1555,27 +1545,17 @@ PyObject* ionc_init_module(void) { _simpletypes_module = PyImport_ImportModule("amazon.ion.simple_types"); _ionpynull_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyNull"); - _ionpynull_fromvalue = PyObject_GetAttrString(_ionpynull_cls, "from_value"); _ionpybool_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyBool"); - _ionpybool_fromvalue = PyObject_GetAttrString(_ionpybool_cls, "from_value"); _ionpyint_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyInt"); - _ionpyint_fromvalue = PyObject_GetAttrString(_ionpyint_cls, "from_value"); _ionpyfloat_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyFloat"); - _ionpyfloat_fromvalue = PyObject_GetAttrString(_ionpyfloat_cls, "from_value"); _ionpydecimal_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyDecimal"); - _ionpydecimal_fromvalue = PyObject_GetAttrString(_ionpydecimal_cls, "from_value"); _ionpytimestamp_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyTimestamp"); - _ionpytimestamp_fromvalue = PyObject_GetAttrString(_ionpytimestamp_cls, "from_value"); _ionpybytes_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyBytes"); - _ionpybytes_fromvalue = PyObject_GetAttrString(_ionpybytes_cls, "from_value"); _ionpytext_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyText"); - _ionpytext_fromvalue = PyObject_GetAttrString(_ionpytext_cls, "from_value"); _ionpysymbol_cls = PyObject_GetAttrString(_simpletypes_module, "IonPySymbol"); - _ionpysymbol_fromvalue = PyObject_GetAttrString(_ionpysymbol_cls, "from_value"); _ionpylist_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyList"); - _ionpylist_fromvalue = PyObject_GetAttrString(_ionpylist_cls, "from_value"); _ionpydict_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyDict"); - _ionpydict_fromvalue = PyObject_GetAttrString(_ionpydict_cls, "from_value"); + _ion_core_module = PyImport_ImportModule("amazon.ion.core"); _py_timestamp_precision = PyObject_GetAttrString(_ion_core_module, "TimestampPrecision"); diff --git a/amazon/ion/simple_types.py b/amazon/ion/simple_types.py index 3639cd2f..94989864 100644 --- a/amazon/ion/simple_types.py +++ b/amazon/ion/simple_types.py @@ -28,36 +28,84 @@ except: from collections import MutableMapping +from collections import OrderedDict +from decimal import Decimal + +from amazon.ion.core import IonType, IonEvent, Timestamp, TIMESTAMP_FRACTIONAL_SECONDS_FIELD, TIMESTAMP_PRECISION_FIELD, \ + TimestampPrecision from amazon.ion.symbols import SymbolToken -from .core import TIMESTAMP_PRECISION_FIELD -from .core import Multimap, Timestamp, IonEvent, IonType, TIMESTAMP_FRACTION_PRECISION_FIELD, TimestampPrecision, \ - MICROSECOND_PRECISION, TIMESTAMP_FRACTIONAL_SECONDS_FIELD -class _IonNature(object): - """Mix-in for Ion related properties. +class IonPyNull(object): + __name__ = 'IonPyNull' + __qualname__ = 'IonPyNull' - Attributes: - ion_event (Optional[IonEvent]): The event, if any associated with the value. - ion_type (IonType): The Ion type for the value. - ion_annotations (Sequence[unicode]): The annotations associated with the value. + def __init__(self, ion_type=IonType.NULL, value=None, annotations=()): + self.ion_type = ion_type + self.ion_annotations = annotations - Notes: - There is no ``field_name`` attribute as that is generally modeled as a property of the - container. + def __bool__(self): + return False - The ``ion_event`` field is only provided if the value was derived from a low-level event. - User constructed values will generally not set this field. - """ - def __init__(self, *args, **kwargs): - self.ion_type = None - self.ion_annotations = () + def __copy__(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value - def _copy(self): - """Copies this instance. Its IonEvent (if any) is not preserved. + @staticmethod + def _to_constructor_args(value): + return (None, value,), {} - Keeping this protected until/unless we decide there's use for it publicly. - """ + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, None, ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value + + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value + + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = self + if isinstance(self, IonPyNull): + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) + + +class IonPyDecimal(Decimal): + __name__ = 'IonPyDecimal' + __qualname__ = 'IonPyDecimal' + ion_type = IonType.DECIMAL + + def __new__(cls, ion_type=IonType.DECIMAL, value=None, annotations=()): + v = super().__new__(cls, value) + v.ion_annotations = annotations + return v + + def __copy__(self): args, kwargs = self._to_constructor_args(self) value = self.__class__(*args, **kwargs) value.ion_type = self.ion_type @@ -66,21 +114,72 @@ def _copy(self): @staticmethod def _to_constructor_args(value): - return (value, ), {} + return (None, value,), {} @classmethod def from_event(cls, ion_event): - """Constructs the given native extension from the properties of an event. + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, None, ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value - Args: - ion_event (IonEvent): The event to construct the native value from. - """ + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value + + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = self + if isinstance(self, IonPyNull): + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) + + +class IonPyBytes(bytes): + __name__ = 'IonPyBytes' + __qualname__ = 'IonPyBytes' + + def __new__(cls, ion_type=IonType.BLOB, value=None, annotations=()): + v = super().__new__(cls, value) + v.ion_annotations = annotations + v.ion_type = ion_type + return v + + def __copy__(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value + + @staticmethod + def _to_constructor_args(value): + return (None, value,), {} + + @classmethod + def from_event(cls, ion_event): if ion_event.value is not None: args, kwargs = cls._to_constructor_args(ion_event.value) else: - # if value is None (i.e. this is a container event), args must be empty or initialization of the - # underlying container will fail. - args, kwargs = (), {} + args, kwargs = (None, None, ()), {} value = cls(*args, **kwargs) value.ion_type = ion_event.ion_type value.ion_annotations = ion_event.annotations @@ -88,13 +187,64 @@ def from_event(cls, ion_event): @classmethod def from_value(cls, ion_type, value, annotations=()): - """Constructs a value as a copy with an associated Ion type and annotations. + if value is None: + value = IonPyNull() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value - Args: - ion_type (IonType): The associated Ion type. - value (Any): The value to construct from, generally of type ``cls``. - annotations (Sequence[unicode]): The sequence Unicode strings decorating this value. - """ + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = self + if isinstance(self, IonPyNull): + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) + + +class IonPyInt(int): + __name__ = 'IonPyInt' + __qualname__ = 'IonPyInt' + ion_type = IonType.INT + + def __new__(cls, ion_type=IonType.INT, value=None, annotations=()): + v = super().__new__(cls, value) + v.ion_annotations = annotations + return v + + def __copy__(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value + + @staticmethod + def _to_constructor_args(value): + return (None, value,), {} + + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, None, ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value + + @classmethod + def from_value(cls, ion_type, value, annotations=()): if value is None: value = IonPyNull() else: @@ -105,22 +255,8 @@ def from_value(cls, ion_type, value, annotations=()): return value def to_event(self, event_type, field_name=None, in_struct=False, depth=None): - """Constructs an IonEvent from this _IonNature value. - - Args: - event_type (IonEventType): The type of the resulting event. - field_name (Optional[text]): The field name associated with this value, if any. When ``None`` - is specified and ``in_struct`` is ``True``, the returned event's ``field_name`` will - represent symbol zero (a ``SymbolToken`` with text=None and sid=0). - in_struct (Optional[True|False]): When ``True``, indicates the returned event ``field_name`` - will be populated. When ``False``, ``field_name`` will be ``None``. - depth (Optional[int]): The depth of this value. - - Returns: - An IonEvent with the properties from this value. - """ value = self - if isinstance(self, IonPyNull) or self.ion_type.is_container: + if isinstance(self, IonPyNull): value = None if in_struct: @@ -130,57 +266,216 @@ def to_event(self, event_type, field_name=None, in_struct=False, depth=None): field_name = None return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, - annotations=self.ion_annotations, depth=depth) + annotations=self.ion_annotations, depth=depth) -def _ion_type_for(name, base_cls, ion_type=None): - class IonPyValueType(base_cls, _IonNature): - def __init__(self, *args, **kwargs): - super(IonPyValueType, self).__init__(*args, **kwargs) - self.ion_type = ion_type +class IonPyBool(int): + __name__ = 'IonPyBool' + __qualname__ = 'IonPyBool' + ion_type = IonType.BOOL - IonPyValueType.__name__ = name - IonPyValueType.__qualname__ = name - return IonPyValueType + def __repr__(self): + return str(bool(self)) + def __new__(cls, ion_type=IonType.BOOL, value=None, annotations=()): + v = super().__new__(cls, value) + v.ion_annotations = annotations + return v -IonPyInt = _ion_type_for('IonPyInt', int, IonType.INT) -IonPyFloat = _ion_type_for('IonPyFloat', float, IonType.FLOAT) -IonPyDecimal = _ion_type_for('IonPyDecimal', Decimal, IonType.DECIMAL) -IonPyText = _ion_type_for('IonPyText', str) -IonPyBytes = _ion_type_for('IonPyBytes', bytes) + def __copy__(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value -# All of our ion value types inherit from their python value types, for good or -# ill. Python's builtin `bool` cannot be sub-classed and is itself a sub-class -# of int so we just sub-class int directly for our Ion Boolean. -# Considering that True == 1 and False == 0 this works as expected. -IonPyBool = _ion_type_for('IonPyBool', int, IonType.BOOL) + @staticmethod + def _to_constructor_args(value): + return (None, value,), {} -# We override __repr__ so Booleans show up as True|False and not 1|0. -# The representation is consistent with other primitives in that it returns a -# string repr of the value but not annotations. -IonPyBool.__repr__ = lambda self: str(bool(self)) + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, None, ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value + + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = self + if isinstance(self, IonPyNull): + value = None -class IonPySymbol(SymbolToken, _IonNature): - def __init__(self, *args, **kwargs): - super(IonPySymbol, self).__init__(*args, **kwargs) - self.ion_type = IonType.SYMBOL + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) + + +class IonPyFloat(float): + __name__ = 'IonPyFloat' + __qualname__ = 'IonPyFloat' + ion_type = IonType.FLOAT + + def __new__(cls, ion_type=IonType.FLOAT, value=None, annotations=()): + v = super().__new__(cls, value) + v.ion_annotations = annotations + return v + + def __copy__(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value @staticmethod - def _to_constructor_args(st): - try: - args = (st.text, st.sid, st.location) - except AttributeError: - args = (st, None, None) - kwargs = {} - return args, kwargs + def _to_constructor_args(value): + return (None, value,), {} + + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, None, ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value + + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value + + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = self + if isinstance(self, IonPyNull): + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) + + +class IonPyText(str): + __name__ = 'IonPyText' + __qualname__ = 'IonPyText' + ion_type = IonType.STRING + + def __new__(cls, ion_type=IonType.STRING, value=None, annotations=()): + v = super().__new__(cls, value) + v.ion_annotations = annotations + return v + + def __copy__(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value + @staticmethod + def _to_constructor_args(value): + return (None, value,), {} + + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, None, ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value + + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value -class IonPyTimestamp(Timestamp, _IonNature): - def __init__(self, *args, **kwargs): - super(IonPyTimestamp, self).__init__(*args, **kwargs) - self.ion_type = IonType.TIMESTAMP + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = self + if isinstance(self, IonPyNull): + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) + + +class IonPyTimestamp(Timestamp): + __name__ = 'IonPyTimestamp' + __qualname__ = 'IonPyTimestamp' + ion_type = IonType.TIMESTAMP + + def __new__(cls, *args, **kwargs): + # The from_value like signature + if isinstance(args[0], IonType): + value = args[1] + annotations = () + if len(args) > 2 and args[2] is not None: + annotations = args[2] + if value is not None: + args_new, kwargs = cls._to_constructor_args(value) + else: + args_new, kwargs = (None, None, ()), {} + v = super().__new__(cls, *args_new, **kwargs) + v.ion_type = args[0] + v.ion_annotations = annotations + # Regular timestamp constructor + else: + v = super().__new__(cls, *args, **kwargs) + return v + + def __copy__(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value @staticmethod def _to_constructor_args(ts): @@ -194,33 +489,301 @@ def _to_constructor_args(ts): kwargs = {TIMESTAMP_PRECISION_FIELD: TimestampPrecision.SECOND} return args, kwargs + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, None, ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value + + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value + + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = self + if isinstance(self, IonPyNull): + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) + + +class IonPySymbol(SymbolToken): + __name__ = 'IonPySymbol' + __qualname__ = 'IonPySymbol' + ion_type = IonType.SYMBOL + + # a good signature: IonPySymbol(ion_type, symbol_token, annotation) + def __new__(cls, ion_type=IonType.SYMBOL, value=None, annotations=()): + v = super().__new__(cls, *value) + v.ion_annotations = annotations + return v + + def __copy__(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value + + @staticmethod + def _to_constructor_args(st): + try: + args = (None, (st.text, st.sid, st.location), ()) + except AttributeError: + args = (None, (st, None, None), ()) + kwargs = {} + return args, kwargs + + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, None, ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value + + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value + + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = self + if isinstance(self, IonPyNull): + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) + + +class IonPyList(list): + __name__ = 'IonPyList' + __qualname__ = 'IonPyList' + + def __init__(self, ion_type=IonType.LIST, value=None, annotations=()): + if value is None: + value = [] + super().__init__(value) + self.ion_annotations = annotations + # it's possible to be Sexp + self.ion_type = ion_type + + def __copy__(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value + + @staticmethod + def _to_constructor_args(value): + return (None, value,), {} -class IonPyNull(_IonNature): - """Representation of ``null``. + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, [], ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value - Notes: - ``None`` is a singleton and cannot be sub-classed, so we have our - own value type for it. The function ``is_null`` is the best way - to test for ``null``-ness or ``None``-ness. + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value + + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) + + +class IonPyDict(MutableMapping): """ - def __init__(self, *args, **kwargs): - super(IonPyNull, self).__init__(*args, **kwargs) + Dictionary that can hold multiple values for the same key - def __nonzero__(self): - return False + In order not to break existing customers, getting and inserting elements with ``[]`` keeps the same behaviour + as the built-in dict. If multiple elements are already mapped to the key, ``[]` will return + the newest one. - def __bool__(self): - return False + To map multiple elements to a key, use the ``add_item`` operation. + To retrieve all the values map to a key, use ``get_all_values``. + """ + __name__ = 'IonPyDict' + __qualname__ = 'IonPyDict' + ion_type = IonType.STRUCT + + def __init__(self, ion_type=IonType.STRUCT, value=None, annotations=()): + super().__init__() + self.ion_annotations = annotations + self.__store = OrderedDict() + if value is not None: + for key, value in iter(value.items()): + if key in self.__store.keys(): + self.__store[key].append(value) + else: + self.__store[key] = [value] + + def __getitem__(self, key): + """ + Return the newest value for the given key. To retrieve all the values map to the key, use ``get_all_values``. + """ + return self.__store[key][len(self.__store[key]) - 1] # Return only one in order not to break clients + + def __delitem__(self, key): + """ + Delete all values for the given key. + """ + del self.__store[key] + + def __setitem__(self, key, value): + """ + Set the desired value to the given key. + """ + self.__store[key] = [value] + + def __len__(self): + return sum([len(values) for values in iter(self.__store.values())]) + + def __iter__(self): + for key in iter(self.__store.keys()): + yield key + + def __str__(self): + return repr(self) + + def __repr__(self): + return '{%s}' % ', '.join(['%r: %r' % (k, v) for k, v in self.items()]) + + def add_item(self, key, value): + """ + Add a value for the given key. This operation appends the value to the end of the value list instead of + overwriting the existing value. + """ + if key in self.__store: + self.__store[key].append(value) + else: + self.__setitem__(key, value) + + def get_all_values(self, key): + """ + Retrieve all the values mapped to the given key + """ + return self.__store[key] + + def iteritems(self): + """ + Return an iterator over (key, value) tuple pairs. + """ + for key in self.__store: + for value in self.__store[key]: + yield (key, value) + + def items(self): + """ + Return a list of the IonPyDict's (key, value) tuple pairs. + """ + output = [] + for k, v in self.iteritems(): + output.append((k, v)) + return output + + def __copy__(self): + args, kwargs = self._to_constructor_args(self) + value = self.__class__(*args, **kwargs) + value.ion_type = self.ion_type + value.ion_annotations = self.ion_annotations + return value @staticmethod def _to_constructor_args(value): - return (), {} + return (None, value,), {} + + @classmethod + def from_event(cls, ion_event): + if ion_event.value is not None: + args, kwargs = cls._to_constructor_args(ion_event.value) + else: + args, kwargs = (None, None, ()), {} + value = cls(*args, **kwargs) + value.ion_type = ion_event.ion_type + value.ion_annotations = ion_event.annotations + return value + + @classmethod + def from_value(cls, ion_type, value, annotations=()): + if value is None: + value = IonPyNull() + else: + args, kwargs = cls._to_constructor_args(value) + value = cls(*args, **kwargs) + value.ion_type = ion_type + value.ion_annotations = annotations + return value + + def to_event(self, event_type, field_name=None, in_struct=False, depth=None): + value = None + + if in_struct: + if not isinstance(field_name, SymbolToken): + field_name = SymbolToken(field_name, 0 if field_name is None else None) + else: + field_name = None + + return IonEvent(event_type, ion_type=self.ion_type, value=value, field_name=field_name, + annotations=self.ion_annotations, depth=depth) def is_null(value): """A mechanism to determine if a value is ``None`` or an Ion ``null``.""" return value is None or isinstance(value, IonPyNull) - - -IonPyList = _ion_type_for('IonPyList', list) -IonPyDict = _ion_type_for('IonPyDict', Multimap, IonType.STRUCT) diff --git a/amazon/ion/simpleion.py b/amazon/ion/simpleion.py index 557d2724..f2c8c02f 100644 --- a/amazon/ion/simpleion.py +++ b/amazon/ion/simpleion.py @@ -165,6 +165,7 @@ def dump_python(obj, fp, imports=None, binary=True, sequence_as_stream=False, sk _FROM_TYPE = { + IonPyNull: IonType.NULL, type(None): IonType.NULL, type(True): IonType.BOOL, type(False): IonType.BOOL, diff --git a/tests/test_equivalence.py b/tests/test_equivalence.py index ac2f27ee..6cb7001e 100644 --- a/tests/test_equivalence.py +++ b/tests/test_equivalence.py @@ -21,9 +21,9 @@ from datetime import datetime, timedelta from amazon.ion.core import IonType, timestamp, TimestampPrecision, OffsetTZInfo -from amazon.ion.equivalence import ion_equals +from amazon.ion.equivalence import ion_equals, obj_has_ion_type_and_annotation from amazon.ion.simple_types import IonPyNull, IonPyInt, IonPyBool, IonPyFloat, IonPyDecimal, IonPyText, IonPyBytes, \ - IonPyList, IonPyTimestamp, IonPySymbol, IonPyDict, _IonNature + IonPyList, IonPyTimestamp, IonPySymbol, IonPyDict from amazon.ion.symbols import SymbolToken, ImportLocation from tests import parametrize @@ -268,7 +268,7 @@ def __str__(self): def _desc(a, b, operator): def _str(val): - if isinstance(val, _IonNature): + if obj_has_ion_type_and_annotation(val): return '%s(%s, ion_type=%s, ion_annotations=%s)' % (type(val), val, val.ion_type, val.ion_annotations) return '%s(%s)' % (type(val), val) return 'assert %s %s %s' % (_str(a), operator, _str(b)) @@ -327,13 +327,13 @@ def _generate_equivs(equivs, param_func=_equivs_param): has_ion_nature = False a = seq[i] b = seq[j] - if isinstance(a, _IonNature): + if obj_has_ion_type_and_annotation(a): has_ion_nature = True - a = a._copy() + a = a.__copy__() _add_annotations(a) - if isinstance(b, _IonNature): + if obj_has_ion_type_and_annotation(b): has_ion_nature = True - b = b._copy() + b = b.__copy__() _add_annotations(b) if has_ion_nature: yield _nonequivs_param(a, b) diff --git a/tests/test_events.py b/tests/test_events.py index bf8c6cbd..1210753b 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -34,7 +34,7 @@ def _to_event_parameters(): [loads('$0'), [IonEvent(IonEventType.SCALAR, IonType.SYMBOL, SymbolToken(None, 0), None, (), depth=0)]], [loads('{$0: $0}'), [ IonEvent(IonEventType.CONTAINER_START, IonType.STRUCT, depth=0), - IonEvent(IonEventType.SCALAR, IonType.SYMBOL, IonPySymbol(None, 0), SymbolToken(None, 0), (), depth=1), + IonEvent(IonEventType.SCALAR, IonType.SYMBOL, IonPySymbol(None, (None, 0)), SymbolToken(None, 0), (), depth=1), IonEvent(IonEventType.CONTAINER_END), ]], [loads('[1, 2, 3, [4, 5, 6], [7, 8, 9]]'), [ diff --git a/tests/test_simple_types.py b/tests/test_simple_types.py index 5a82b29d..e723ce7e 100644 --- a/tests/test_simple_types.py +++ b/tests/test_simple_types.py @@ -23,7 +23,7 @@ from amazon.ion.symbols import SymbolToken from amazon.ion.simple_types import is_null, IonPyNull, IonPyBool, IonPyInt, IonPyFloat, \ IonPyDecimal, IonPyTimestamp, IonPyText, IonPyBytes, \ - IonPyList, IonPyDict, IonPySymbol, _IonNature + IonPyList, IonPyDict, IonPySymbol from amazon.ion.equivalence import ion_equals from amazon.ion.simpleion import _ion_type, _FROM_TYPE @@ -33,7 +33,7 @@ class _P(NamedTuple): desc: str - type: Type[_IonNature] + type: Type[object] event: IonEvent def __str__(self): diff --git a/tests/test_simpleion.py b/tests/test_simpleion.py index 83dba80c..2e4baf84 100644 --- a/tests/test_simpleion.py +++ b/tests/test_simpleion.py @@ -30,8 +30,8 @@ from amazon.ion.writer_binary import _IVM from amazon.ion.core import IonType, IonEvent, IonEventType, OffsetTZInfo, Multimap from amazon.ion.simple_types import IonPyDict, IonPyText, IonPyList, IonPyNull, IonPyBool, IonPyInt, IonPyFloat, \ - IonPyDecimal, IonPyTimestamp, IonPyBytes, IonPySymbol, _IonNature -from amazon.ion.equivalence import ion_equals + IonPyDecimal, IonPyTimestamp, IonPyBytes, IonPySymbol +from amazon.ion.equivalence import ion_equals, obj_has_ion_type_and_annotation from amazon.ion.simpleion import dump, dumps, load, loads, _ion_type, _FROM_ION_TYPE, _FROM_TYPE_TUPLE_AS_SEXP, \ _FROM_TYPE from amazon.ion.writer_binary_raw import _serialize_symbol, _write_length @@ -231,7 +231,7 @@ def generate_scalars_binary(scalars_map, preceding_symbols=0): symbol_expected = _serialize_symbol( IonEvent(IonEventType.SCALAR, IonType.SYMBOL, SymbolToken(None, 10 + preceding_symbols))) yield _Parameter(IonType.SYMBOL.name + ' ' + native, - IonPyText.from_value(IonType.SYMBOL, native), symbol_expected, True) + IonPyText.from_value(IonType.SYMBOL, native, ()), symbol_expected, True) yield _Parameter('%s %s' % (ion_type.name, native), native, native_expected, has_symbols) wrapper = _FROM_ION_TYPE[ion_type].from_value(ion_type, native) yield _Parameter(repr(wrapper), wrapper, expected, has_symbols) @@ -248,7 +248,7 @@ def generate_containers_binary(container_map, preceding_symbols=0): for one_expected in expecteds: one_expected = one_expected.binary for elem in obj: - if isinstance(elem, (dict, Multimap)) and len(elem) > 0: + if isinstance(elem, (dict, Multimap, IonPyDict)) and len(elem) > 0: has_symbols = True elif ion_type is IonType.SEXP and isinstance(elem, tuple): tuple_as_sexp = True @@ -271,7 +271,7 @@ def generate_annotated_values_binary(scalars_map, container_map): for value_p in chain(generate_scalars_binary(scalars_map, preceding_symbols=2), generate_containers_binary(container_map, preceding_symbols=2)): obj = value_p.obj - if not isinstance(obj, _IonNature): + if not obj_has_ion_type_and_annotation(obj): continue obj.ion_annotations = (_st(u'annot1'), _st(u'annot2'),) annot_length = 2 # 10 and 11 each fit in one VarUInt byte @@ -289,7 +289,7 @@ def generate_annotated_values_binary(scalars_map, container_map): _write_length(wrapper, length_field, 0xE0) if c_ext and obj.ion_type is IonType.SYMBOL and not isinstance(obj, IonPyNull) \ - and not (hasattr(obj, 'sid') and (obj.sid < 10 or obj.sid is None)): + and not (hasattr(obj, 'sid') and (obj.sid is None or obj.sid < 10)): wrapper.extend([ VARUINT_END_BYTE | annot_length, VARUINT_END_BYTE | 11, @@ -440,7 +440,7 @@ def generate_annotated_values_text(scalars_map, container_map): for value_p in chain(generate_scalars_text(scalars_map), generate_containers_text(container_map)): obj = value_p.obj - if not isinstance(obj, _IonNature): + if not obj_has_ion_type_and_annotation(obj): continue obj.ion_annotations = (_st(u'annot1'), _st(u'annot2'),) @@ -570,7 +570,7 @@ def _to_obj(to_type=None, annotations=(), tuple_as_sexp=False): for obj in roundtrips: obj = _adjust_sids() yield obj, is_binary, indent, False - if not isinstance(obj, _IonNature): + if not obj_has_ion_type_and_annotation(obj): ion_type = _ion_type(obj, _FROM_TYPE) yield _to_obj() else: @@ -587,29 +587,7 @@ def _to_obj(to_type=None, annotations=(), tuple_as_sexp=False): def _assert_roundtrip(before, after, tuple_as_sexp): - # All loaded Ion values extend _IonNature, even if they were dumped from primitives. This recursively - # wraps each input value in _IonNature for comparison against the output. - def _to_ion_nature(obj): - out = obj - if not isinstance(out, _IonNature): - from_type = _FROM_TYPE_TUPLE_AS_SEXP if tuple_as_sexp else _FROM_TYPE - ion_type = _ion_type(out, from_type) - out = _FROM_ION_TYPE[ion_type].from_value(ion_type, out) - if isinstance(out, dict): - update = {} - for field, value in iter(out.items()): - update[field] = _to_ion_nature(value) - update = IonPyDict.from_value(out.ion_type, update, out.ion_annotations) - out = update - elif isinstance(out, list): - update = [] - for value in out: - update.append(_to_ion_nature(value)) - update = IonPyList.from_value(out.ion_type, update, out.ion_annotations) - out = update - - return out - assert ion_equals(_to_ion_nature(before), after) + assert ion_equals(before, after) @parametrize(