Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improves the deserialization speed by reducing object initialization overhead #284

Merged
merged 6 commits into from
Sep 19, 2023
Merged

Improves the deserialization speed by reducing object initialization overhead #284

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
0ac1d68
Improves the deserialization speed by reducing object initialization …
cheqianh Aug 9, 2023
9b737ea
Addresses few comments
cheqianh Aug 22, 2023
4c8317a
Addressed feedback.
cheqianh Sep 6, 2023
7963212
Reverts some commented tests.
cheqianh Sep 8, 2023
b6fe2df
Renames IonPyObject_new to IonPyObject
cheqianh Sep 12, 2023
a8c5514
Clean up works
cheqianh Sep 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 10 additions & 6 deletions amazon/ion/equivalence.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,15 @@
from math import isnan

from amazon.ion.core import IonType, Timestamp, TimestampPrecision, MICROSECOND_PRECISION, OffsetTZInfo, Multimap
from amazon.ion.simple_types import _IonNature, IonPyList, IonPyDict, IonPyTimestamp, IonPyNull, IonPySymbol, \
from amazon.ion.simple_types import IonPyList, IonPyDict, IonPyTimestamp, IonPyNull, IonPySymbol, \
IonPyText, IonPyDecimal, IonPyFloat
from amazon.ion.symbols import SymbolToken


def obj_has_ion_type_and_annotation(obj):
return hasattr(obj, 'ion_type') and hasattr(obj, 'ion_annotations')


def ion_equals(a, b, timestamps_instants_only=False):
"""Tests two objects for equivalence under the Ion data model.

Expand Down Expand Up @@ -60,8 +64,8 @@ def _ion_equals_timestamps_data_model(a, b):
def _ion_equals(a, b, timestamp_comparison_func, recursive_comparison_func):
"""Compares a and b according to the description of the ion_equals method."""
for a, b in ((a, b), (b, a)): # Ensures that operand order does not matter.
if isinstance(a, _IonNature):
if isinstance(b, _IonNature):
if obj_has_ion_type_and_annotation(a):
if obj_has_ion_type_and_annotation(b):
# Both operands have _IonNature. Their IonTypes and annotations must be equivalent.
eq = a.ion_type is b.ion_type and _annotations_eq(a, b)
else:
Expand Down Expand Up @@ -120,8 +124,8 @@ def _sequences_eq(a, b, comparison_func):


def _structs_eq(a, b, comparison_func):
assert isinstance(a, (dict, Multimap))
if not isinstance(b, (dict, Multimap)):
assert isinstance(a, (dict, Multimap, IonPyDict))
if not isinstance(b, (dict, Multimap, IonPyDict)):
return False
dict_len = len(a)
if dict_len != len(b):
Expand All @@ -135,7 +139,7 @@ def _structs_eq(a, b, comparison_func):
break
if key not in b:
return False
if isinstance(a, Multimap) and isinstance(b, Multimap):
if isinstance(a, (IonPyDict, Multimap)) and isinstance(b, (IonPyDict, Multimap)):
values_a = a.get_all_values(key)
values_b = b.get_all_values(key)
if len(values_a) != len(values_b):
Expand Down
48 changes: 14 additions & 34 deletions amazon/ion/ioncmodule.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,27 +38,16 @@ static PyObject* _decimal_constructor;
static PyObject* _py_timestamp_constructor;
static PyObject* _simpletypes_module;
static PyObject* _ionpynull_cls;
static PyObject* _ionpynull_fromvalue;
static PyObject* _ionpybool_cls;
static PyObject* _ionpybool_fromvalue;
static PyObject* _ionpyint_cls;
static PyObject* _ionpyint_fromvalue;
static PyObject* _ionpyfloat_cls;
static PyObject* _ionpyfloat_fromvalue;
static PyObject* _ionpydecimal_cls;
static PyObject* _ionpydecimal_fromvalue;
static PyObject* _ionpytimestamp_cls;
static PyObject* _ionpytimestamp_fromvalue;
static PyObject* _ionpytext_cls;
static PyObject* _ionpytext_fromvalue;
static PyObject* _ionpysymbol_cls;
static PyObject* _ionpysymbol_fromvalue;
static PyObject* _ionpybytes_cls;
static PyObject* _ionpybytes_fromvalue;
static PyObject* _ionpylist_cls;
static PyObject* _ionpylist_fromvalue;
static PyObject* _ionpydict_cls;
static PyObject* _ionpydict_fromvalue;
static PyObject* _ion_core_module;
static PyObject* _py_ion_type;
static PyObject* py_ion_type_table[14];
Expand Down Expand Up @@ -1102,6 +1091,7 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s
SUCCEED();
case tid_NULL_INT:
{
// TODO double check the real null type, now it's initialized to IonType.NULL by default
ION_TYPE null_type;
// Hack for ion-c issue https://github.com/amazon-ion/ion-c/issues/223
if (original_t != tid_SYMBOL_INT) {
Expand All @@ -1112,17 +1102,17 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s
}

ion_type = ION_TYPE_INT(null_type);
py_value = Py_BuildValue(""); // INCREFs and returns Python None.
py_value = Py_None; // INCREFs and returns Python None.
cheqianh marked this conversation as resolved.
Show resolved Hide resolved
emit_bare_values = emit_bare_values && (ion_type == tid_NULL_INT);
ion_nature_constructor = _ionpynull_fromvalue;
ion_nature_constructor = _ionpynull_cls;
break;
}
case tid_BOOL_INT:
{
BOOL bool_value;
IONCHECK(ion_reader_read_bool(hreader, &bool_value));
py_value = PyBool_FromLong(bool_value);
ion_nature_constructor = _ionpybool_fromvalue;
ion_nature_constructor = _ionpybool_cls;
break;
}
case tid_INT_INT:
Expand All @@ -1145,15 +1135,15 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s
py_value = PyLong_FromString(ion_int_str, NULL, 10);
PyMem_Free(ion_int_str);

ion_nature_constructor = _ionpyint_fromvalue;
ion_nature_constructor = _ionpyint_cls;
break;
}
case tid_FLOAT_INT:
{
double double_value;
IONCHECK(ion_reader_read_double(hreader, &double_value));
py_value = Py_BuildValue("d", double_value);
ion_nature_constructor = _ionpyfloat_fromvalue;
ion_nature_constructor = _ionpyfloat_cls;
break;
}
case tid_DECIMAL_INT:
Expand Down Expand Up @@ -1208,21 +1198,21 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s
}
}

ion_nature_constructor = _ionpydecimal_fromvalue;
ion_nature_constructor = _ionpydecimal_cls;
break;
}
case tid_TIMESTAMP_INT:
{
IONCHECK(ionc_read_timestamp(hreader, &py_value));
ion_nature_constructor = _ionpytimestamp_fromvalue;
ion_nature_constructor = _ionpytimestamp_cls;
break;
}
case tid_SYMBOL_INT:
{
emit_bare_values = FALSE; // Symbol values must always be emitted as IonNature because of ambiguity with string.
ION_STRING string_value;
IONCHECK(ion_reader_read_string(hreader, &string_value));
ion_nature_constructor = _ionpysymbol_fromvalue;
ion_nature_constructor = _ionpysymbol_cls;
py_value = ion_string_to_py_symboltoken(&string_value);
break;
}
Expand All @@ -1231,7 +1221,7 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s
ION_STRING string_value;
IONCHECK(ion_reader_read_string(hreader, &string_value));
py_value = ion_build_py_string(&string_value);
ion_nature_constructor = _ionpytext_fromvalue;
ion_nature_constructor = _ionpytext_cls;
break;
}
case tid_CLOB_INT:
Expand Down Expand Up @@ -1263,12 +1253,12 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s
if (length) {
PyMem_Free(buf);
}
ion_nature_constructor = _ionpybytes_fromvalue;
ion_nature_constructor = _ionpybytes_cls;
break;
}
case tid_STRUCT_INT:
{
ion_nature_constructor = _ionpydict_fromvalue;
ion_nature_constructor = _ionpydict_cls;
//Init a IonPyDict
PyObject* new_dict = PyDict_New();
py_value = PyObject_CallFunctionObjArgs(
Expand All @@ -1293,7 +1283,7 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s
{
py_value = PyList_New(0);
IONCHECK(ionc_read_into_container(hreader, py_value, /*is_struct=*/FALSE, emit_bare_values));
ion_nature_constructor = _ionpylist_fromvalue;
ion_nature_constructor = _ionpylist_cls;
break;
}
case tid_DATAGRAM_INT:
Expand Down Expand Up @@ -1555,27 +1545,17 @@ PyObject* ionc_init_module(void) {
_simpletypes_module = PyImport_ImportModule("amazon.ion.simple_types");

_ionpynull_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyNull");
_ionpynull_fromvalue = PyObject_GetAttrString(_ionpynull_cls, "from_value");
_ionpybool_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyBool");
_ionpybool_fromvalue = PyObject_GetAttrString(_ionpybool_cls, "from_value");
_ionpyint_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyInt");
_ionpyint_fromvalue = PyObject_GetAttrString(_ionpyint_cls, "from_value");
_ionpyfloat_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyFloat");
_ionpyfloat_fromvalue = PyObject_GetAttrString(_ionpyfloat_cls, "from_value");
_ionpydecimal_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyDecimal");
_ionpydecimal_fromvalue = PyObject_GetAttrString(_ionpydecimal_cls, "from_value");
_ionpytimestamp_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyTimestamp");
_ionpytimestamp_fromvalue = PyObject_GetAttrString(_ionpytimestamp_cls, "from_value");
_ionpybytes_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyBytes");
_ionpybytes_fromvalue = PyObject_GetAttrString(_ionpybytes_cls, "from_value");
_ionpytext_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyText");
_ionpytext_fromvalue = PyObject_GetAttrString(_ionpytext_cls, "from_value");
_ionpysymbol_cls = PyObject_GetAttrString(_simpletypes_module, "IonPySymbol");
_ionpysymbol_fromvalue = PyObject_GetAttrString(_ionpysymbol_cls, "from_value");
_ionpylist_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyList");
_ionpylist_fromvalue = PyObject_GetAttrString(_ionpylist_cls, "from_value");
_ionpydict_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyDict");
_ionpydict_fromvalue = PyObject_GetAttrString(_ionpydict_cls, "from_value");


_ion_core_module = PyImport_ImportModule("amazon.ion.core");
_py_timestamp_precision = PyObject_GetAttrString(_ion_core_module, "TimestampPrecision");
Expand Down
Loading