diff --git a/libraries/CMakeLists.txt b/libraries/CMakeLists.txt index 9977656cf..9b1ba5a41 100644 --- a/libraries/CMakeLists.txt +++ b/libraries/CMakeLists.txt @@ -1,5 +1,7 @@ enable_testing() + add_subdirectory(abieos) +add_subdirectory(clio) if(DEFINED IS_WASM) add_subdirectory(eosiolib) diff --git a/libraries/clio/CMakeLists.txt b/libraries/clio/CMakeLists.txt new file mode 100644 index 000000000..0e5d4f2b3 --- /dev/null +++ b/libraries/clio/CMakeLists.txt @@ -0,0 +1,7 @@ +enable_testing() + +add_library(clio INTERFACE) +target_link_libraries(clio INTERFACE rapidjson) +target_include_directories(clio INTERFACE include ${Boost_INCLUDE_DIRS}) + +add_subdirectory(tests) diff --git a/libraries/clio/include/clio/bytes.hpp b/libraries/clio/include/clio/bytes.hpp new file mode 100644 index 000000000..cae397467 --- /dev/null +++ b/libraries/clio/include/clio/bytes.hpp @@ -0,0 +1,14 @@ +#pragma once +#include +#include + +namespace clio +{ + struct bytes + { + std::vector data; + }; + + CLIO_REFLECT(bytes, data); + +} // namespace clio diff --git a/libraries/clio/include/clio/bytes/from_json.hpp b/libraries/clio/include/clio/bytes/from_json.hpp new file mode 100644 index 000000000..8d03c6e85 --- /dev/null +++ b/libraries/clio/include/clio/bytes/from_json.hpp @@ -0,0 +1,13 @@ +#pragma once +#include +#include + +namespace clio +{ + template + void from_json(bytes& obj, S& stream) + { + clio::from_json_hex(obj.data, stream); + } + +} // namespace clio diff --git a/libraries/clio/include/clio/bytes/to_bin.hpp b/libraries/clio/include/clio/bytes/to_bin.hpp new file mode 100644 index 000000000..18e8962fb --- /dev/null +++ b/libraries/clio/include/clio/bytes/to_bin.hpp @@ -0,0 +1,6 @@ +#pragma once +#include + +namespace clio +{ +} diff --git a/libraries/clio/include/clio/bytes/to_json.hpp b/libraries/clio/include/clio/bytes/to_json.hpp new file mode 100644 index 000000000..7f23fa995 --- /dev/null +++ b/libraries/clio/include/clio/bytes/to_json.hpp @@ -0,0 +1,13 @@ +#pragma once +#include +#include + +namespace clio +{ + template + void to_json(const bytes& obj, S& stream) + { + clio::to_json_hex(obj.data.data(), obj.data.size(), stream); + } + +} // namespace clio diff --git a/libraries/clio/include/clio/compress.hpp b/libraries/clio/include/clio/compress.hpp new file mode 100644 index 000000000..95e1a0bd4 --- /dev/null +++ b/libraries/clio/include/clio/compress.hpp @@ -0,0 +1,180 @@ +#pragma once +#include + +namespace clio +{ + template + void capp_unpack(InStream& in, OutStream& out) + { + while (in.remaining()) + { + uint8_t word_bits; + in.read(&word_bits, 1); + + const uint64_t zero_word = 0; + + if (word_bits == 0x00) + { + out.write(&zero_word, 8); + + uint8_t zero_words; + in.read(&zero_words, 1); + + while (zero_words) + { + out.write(&zero_word, 8); + --zero_words; + } + if (in.pos >= in.end) + { + throw_error(clio::stream_error::overrun); + } + } + else if (word_bits == 0xff) + { + if (in.remaining() < 8) + { + out.write(in.pos, in.remaining()); + in.skip(in.remaining()); + return; + } + + uint64_t word; + in.read(&word, sizeof(word)); + out.write(&word, 8); + + uint8_t raw_words; + in.read(&raw_words, 1); + if (raw_words) + { + if (in.remaining() < uint32_t(raw_words * 8)) + throw_error(clio::stream_error::overrun); + out.write(in.pos, uint32_t(raw_words) * 8); + in.pos += uint32_t(raw_words) * 8; + } + } + else + { + uint64_t out_word = 0; + uint8_t* out_p = (uint8_t*)&out_word; + for (auto i = 7; i >= 0; --i) + { + if (word_bits & (1 << i)) + { + in.read(out_p, 1); + out_p++; + } + else + { + out_p++; + } + } + out.write(&out_word, sizeof(out_word)); + } + } + } + + template + void capp_pack(InStream& in, OutStream& out) + { + while (in.remaining() >= 8) + { + uint64_t next_word; + in.read_raw(next_word); + + if (not next_word) + { + out.write(char(0)); + + uint32_t num_zero = 0; + in.read_raw(next_word); + + while (not next_word && num_zero < 254) + { + in.read_raw(next_word); + ++num_zero; + } + out.write(char(num_zero)); + } + + uint8_t* inbyte = (uint8_t*)&next_word; + uint8_t* endin = inbyte + 8; + + uint8_t temp[9]; + uint8_t& zeros = temp[0]; + uint8_t* outbyte = &temp[1]; + zeros = 0; + + do + { + zeros <<= 1; + if (bool(*inbyte)) + { + zeros += 1; + *outbyte = *inbyte; + ++outbyte; + } + ++inbyte; + } while (inbyte != endin); + out.write(temp, outbyte - temp); + + if (zeros == 0xff) + { + uint8_t num_raw = 0; + auto* p = in.pos; + auto* e = in.end; + while (*p and p != e) + ++p; + + num_raw = (p - in.pos) / 8; + out.write(num_raw); + out.write(in.pos, num_raw * 8); + in.skip(num_raw * 8); + + // in.read_raw(next_word); + /// TODO: once we enter "raw" mode we shouldn't exit until + /// we have seen a certain percentage of zero bytes... for example, + /// one zero per word isn't worth encoding... + } + } + if (in.remaining()) + { + out.write(char(0xff)); /// indicate all bytes are present, receiver will truncate + out.write(in.pos, in.remaining()); + in.skip(in.remaining()); + } + return; + }; + + inline std::vector capn_compress(const std::vector& c) + { + clio::input_stream in(c.data(), c.size()); + /* + clio::size_stream sizess; + capp_unpack( in, sizess); + in.pos = c.data(); + */ + + std::vector out; + out.reserve(c.size()); + clio::vector_stream vout(out); + capp_pack(in, vout); + return out; + } + inline std::vector capn_uncompress(const std::vector& c) + { + clio::input_stream in(c.data(), c.size()); + /* + clio::size_stream sizess; + capp_unpack( in, sizess); + in.pos = c.data(); + */ + + std::vector out; + out.reserve(c.size() * 1.5); + clio::vector_stream vout(out); + capp_unpack(in, vout); + return out; + } + +} // namespace clio diff --git a/libraries/clio/include/clio/error.hpp b/libraries/clio/include/clio/error.hpp new file mode 100644 index 000000000..b6bcea931 --- /dev/null +++ b/libraries/clio/include/clio/error.hpp @@ -0,0 +1,19 @@ +#pragma once + +namespace clio +{ + /** + * On some platforms we need to disable + * exceptions and do something else, this + * wraps that. + */ + template + [[noreturn]] void throw_error(T&& e) + { +#ifdef __cpp_exceptions + throw e; +#else + abort(); +#endif + } +}; // namespace clio diff --git a/libraries/clio/include/clio/flatbuf.hpp b/libraries/clio/include/clio/flatbuf.hpp new file mode 100644 index 000000000..7851be0e3 --- /dev/null +++ b/libraries/clio/include/clio/flatbuf.hpp @@ -0,0 +1,953 @@ +#pragma once +#include +#include +#include + +namespace clio +{ + struct flat_view_proxy_impl; + + template + class flat_ptr; + + template + struct is_flat_ptr : std::false_type + { + }; + + template + struct is_flat_ptr> : std::true_type + { + using value_type = T; + }; + + template + class flat + { + using T::flat_type_not_defined; + }; + + template <> + class flat; + + template + class flat>; + + template + class flat>; + + template + auto get_view_type() + { + // if constexpr( is_flat_ptr::value ) { + // return get_view_type(); + if constexpr (reflect::is_struct) + { + using view_type = typename reflect::template proxy; + return (view_type*)(nullptr); + } + else if constexpr (std::is_trivially_copyable::value) + return (T*)(nullptr); + else + return (flat*)nullptr; + } + + template + using flat_view = std::remove_pointer_t())>; + + struct offset_ptr + { + uint32_t offset; + + template + auto get() const; + }; + + template + constexpr bool contains_offset_ptr(); + + template + struct tuple_contains_offset; + + template + constexpr uint32_t get_contains_offset_ptr() + { + if constexpr (sizeof...(Ts) == 0) + return contains_offset_ptr(); + else + { + return contains_offset_ptr() | get_contains_offset_ptr(); + } + } + + template + struct tuple_contains_offset> + { + static constexpr const auto value = get_contains_offset_ptr(); + }; + + /** + * Recursively checks the types for any field which requires dynamic allocation, + */ + template + constexpr bool contains_offset_ptr() + { + if constexpr (is_flat_ptr::value) + { + return true; + } + else if constexpr (is_std_tuple::value) + { + return tuple_contains_offset::value; + } + else if constexpr (is_std_variant::value) + { + return contains_offset_ptr::alts_as_tuple>(); + } + else if constexpr (std::is_same_v) + { + return true; + } + else if constexpr (is_std_vector::value) + { + return true; + } + else if constexpr (clio::reflect::is_struct) + { + bool is_flat = true; + clio::reflect::for_each([&](const clio::meta& ref, auto mptr) { + using member_type = std::decay_t; + is_flat &= not contains_offset_ptr(); + }); + return not is_flat; + } + else if constexpr (std::is_arithmetic_v) + { + return false; + } + else if constexpr (std::is_trivially_copyable::value) + { + return false; + } + else + { + T::contains_offset_ptr_not_defined; + } + } + + template + constexpr uint32_t flatpack_size() + { + if constexpr (is_flat_ptr::value) + { + return sizeof(offset_ptr); + } + else if constexpr (is_std_variant::value) + { + return 16; + } + else if constexpr (reflect::is_struct) + { + uint32_t size = 0; + reflect::for_each([&](const meta& ref, const auto& mptr) { + using member_type = decltype(result_of_member(mptr)); + if constexpr (contains_offset_ptr()) + { + size += sizeof(offset_ptr); + } + else + { + size += flatpack_size(); + } + }); + return size; + } + else if constexpr (std::is_same_v || is_std_vector::value) + { + return sizeof(offset_ptr); + } + else if constexpr (std::is_arithmetic_v) + { + return sizeof(T); + } + else if constexpr (std::is_trivially_copyable::value) + { + return sizeof(T); + } + else + { + T::flatpack_size_not_defined; + } + } + + template + struct get_tuple_offset; + + template + constexpr uint32_t get_offset() + { + static_assert(Idx < sizeof...(Ts) + 1, "index out of range"); + if constexpr (Idx == 0) + return 0; + else if constexpr (sizeof...(Ts) == 0) + if constexpr (contains_offset_ptr()) + return 4; + /// if contains ptr then 8 else flatpack_size... + else + return flatpack_size(); + // return get_flat_size( ((const First*)(nullptr)) ); + else + { + if constexpr (contains_offset_ptr()) + return get_offset() + 4; // flatpack_size(); + else + return get_offset() + flatpack_size(); + // return get_offset< Idx-1, Ts...>() + get_flat_size( ((const First*)(nullptr)) ); + } + } + + template + struct get_tuple_offset> + { + static constexpr const uint32_t value = get_offset(); + }; + + struct flat_view_proxy_impl + { + /** This method is called by the reflection library to get the field */ + template + constexpr auto get() + { + using class_type = decltype(clio::class_of_member(MemberPtr)); + using tuple_type = typename clio::reflect::struct_tuple_type; + using member_type = decltype(clio::result_of_member(MemberPtr)); + + constexpr uint32_t offset = clio::get_tuple_offset::value; + + char* out_ptr = reinterpret_cast(this) + offset; + + if constexpr (contains_offset_ptr()) + { + clio::offset_ptr* ptr = reinterpret_cast(out_ptr); + return ptr->get(); + } + else + { + return reinterpret_cast(out_ptr); + } + } + + template + constexpr const auto get() const + { + using class_type = decltype(clio::class_of_member(MemberPtr)); + using tuple_type = typename clio::reflect::struct_tuple_type; + using member_type = decltype(clio::result_of_member(MemberPtr)); + + constexpr uint32_t offset = clio::get_tuple_offset::value; + + auto out_ptr = reinterpret_cast(this) + offset; + + if constexpr (contains_offset_ptr()) + { + const clio::offset_ptr* ptr = reinterpret_cast(out_ptr); + return ptr->get(); + } + else + { + return reinterpret_cast(out_ptr); + } + } + }; + + /** + * A flat view of a flat pointer. + */ + template + class flat> + { + public: + auto get() { return reinterpret_cast())>(_data); } + + private: + uint32_t _size = 0; + char _data[]; + }; + + template <> + class flat + { + public: + uint32_t size() const { return _size; } + const char* c_str() const { return _size ? _data : (const char*)this; } + const char* data() const { return _size ? _data : nullptr; } + char* data() { return _size ? _data : nullptr; } + + operator std::string_view() const { return std::string_view(_data, _size); } + + template + friend S& operator<<(S& stream, const flat& str) + { + return stream << str.c_str(); + } + + private: + uint32_t _size = 0; + char _data[]; + }; + + template + class flat> + { + public: + uint64_t type = 0; + uint32_t flat_data = 0; + offset_ptr offset_data; + + flat() + { + static_assert(sizeof(flat) == 16); + static_assert(std::is_trivially_copyable::value); + } + + int64_t index_from_type() const { return get_index_from_type(); } + + void init_variant(std::variant& v) { _init_variant(v); } + + template + void visit(Visitor&& v) + { + _visit_variant(std::forward(v)); + } + + private: + template + int64_t get_index_from_type() const + { + if constexpr (sizeof...(Rest) == 0) + { + return get_type_hashname() != type; + } + else + { + if (get_type_hashname() == type) + return 0; + else + return 1 + get_index_from_type(); + } + } + + template + void _init_variant(std::variant& v) const + { + if (get_type_hashname() == type) + v = First(); + else if constexpr (sizeof...(Rest) > 0) + { + _init_variant(v); + } + } + template + void _visit_variant(Visitor&& v) const + { + if (get_type_hashname() == type) + { + if constexpr (not get_contains_offset_ptr() and flatpack_size() <= 8) + { + v(*((const flat_view*)&flat_data)); + } + else + { + v(*((const flat_view*)(((char*)(&offset_data)) + offset_data.offset))); + } + } + else if constexpr (sizeof...(Rest) > 0) + { + _visit_variant(std::forward(v)); + } + } + }; + + /// T == value of the array elements + template + class flat> + { + public: + auto& operator[](uint32_t index) + { + if (index >= _size) + throw_error(stream_error::overrun); + /** in this case the data is a series of offset_ptr<> */ + if constexpr (std::is_same::value) + { + auto ptr_array = reinterpret_cast(_data); + return *reinterpret_cast*>(ptr_array[index].get>()); + } + else if constexpr (contains_offset_ptr()) + { + auto ptr_array = reinterpret_cast(_data); + return *reinterpret_cast*>(ptr_array[index].get>()); + } + else if constexpr (reflect::is_struct) + { /// the data is a series of packed T + const auto offset = index * flatpack_size(); + return *reinterpret_cast*>(&_data[offset]); + } + else if constexpr (std::is_trivially_copyable::value) + { + auto T_array = reinterpret_cast(_data); + return T_array[index]; + } + else + { + T::is_not_a_known_flat_type; + } + } + const auto& operator[](uint32_t index) const + { + if (index >= _size) + throw_error(stream_error::overrun); + + /** in this case the data is a series of offset_ptr<> */ + if constexpr (std::is_same::value) + { + auto ptr_array = reinterpret_cast(_data); + return *reinterpret_cast*>( + ptr_array[index].get>()); + } + else if constexpr (contains_offset_ptr()) + { + auto ptr_array = + reinterpret_cast(_data + sizeof(offset_ptr) * index); + const auto& r = + *reinterpret_cast*>(ptr_array->get>()); + return r; + } + else if constexpr (reflect::is_struct) + { /// the data is a series of packed T + const auto offset = index * flatpack_size(); + return *reinterpret_cast*>(&_data[offset]); + } + else if constexpr (std::is_trivially_copyable::value) + { + auto T_array = reinterpret_cast(_data); + return T_array[index]; + } + else + { + T::is_not_a_known_flat_type; + } + } + + uint32_t size() const { return _size; } + + private: + uint32_t _size = 0; + char _data[]; + }; + + template + auto offset_ptr::get() const + { + const auto ptr = ((char*)this) + offset; + if constexpr (is_flat_ptr::value) + { + return reinterpret_cast())>(ptr + 4); + } + else if constexpr (reflect::is_struct) + { + return reinterpret_cast*>(ptr); + } + else if constexpr (std::is_same_v) + { + return reinterpret_cast*>(ptr); + } + else if constexpr (is_std_vector::value) + { + return reinterpret_cast*>(ptr); + } + else + { + T::is_not_reflected_for_offset_ptr; + } + } + + /** + * Verifies that the type pointed at could be unpacked without a buffer overflow, + * which means that all offset pointers are in bounds. It does this by unpacking + * the stream without allocating any memory. + */ + struct validate_input_stream : public input_stream + { + using input_stream::input_stream; + }; + + template + void flatcheck(InputStream& stream) + { + if constexpr (is_flat_ptr::value) + { + uint32_t size; + stream.read(&size, sizeof(size)); + stream.skip(size); + } + else if constexpr (is_std_variant::value) + { + flat fv; + stream.read(&fv, sizeof(fv)); + + T temp; /// this could do memory allocation for certain types... but hopefully won't, + /// this is used to do std::visit in the next line... in theory we could do + /// a dispatcher that only deals in types and not values. + fv.init_variant(temp); + std::visit( + [&](auto& iv) { + using item_type = std::decay_t; + if constexpr (get_contains_offset_ptr()) + { + /// the stream.pos is at the END of reading the variant, which + /// should be the same as the end of flat::offset_data + InputStream in(stream.pos + fv.offset_data.offset - sizeof(offset_ptr), + stream.end); + flatunpack(iv, in); + } + else if constexpr (flatpack_size() <= 8) + { + } + else + { + InputStream in(stream.pos + fv.offset_data.offset - sizeof(offset_ptr), + stream.end); + flatunpack(iv, in); + } + }, + temp); + /// maybe deref pointer.. + } + else if constexpr (std::is_same_v) + { + uint32_t size; + stream.read(&size, sizeof(size)); + stream.skip(size); + } + else if constexpr (is_std_vector::value) + { + uint32_t size; + stream.read(&size, sizeof(size)); + if constexpr (contains_offset_ptr::value_type>()) + { + auto start = stream.pos; + stream.skip(size * sizeof(offset_ptr)); + + offset_ptr* ptr = (offset_ptr*)(start); + for (uint32_t i = 0; i < size; ++i) + { + InputStream in(((char*)ptr) + ptr->offset, stream.end); + flatcheck(in); + } + } + else + { + stream.skip(size * flatpack_size()); + } + } + else if constexpr (reflect::is_struct) + { + if constexpr (contains_offset_ptr()) + { + reflect::for_each([&](const meta& ref, const auto& mptr) { + using member_type = decltype(result_of_member(mptr)); + + if constexpr (contains_offset_ptr()) + { + offset_ptr ptr; + stream.read(&ptr, sizeof(ptr)); + + InputStream substream(stream.pos + ptr.offset - sizeof(ptr), stream.end); + flatcheck(substream); + } + else + { + stream.skip(flatpack_size()); + } + }); + } + else + { + stream.skip(flatpack_size()); + } + } + else if constexpr (std::is_trivially_copyable::value) + { + stream.skip(sizeof(T)); + } + else + { + T::is_not_defined; + } + } + + template + void flatunpack(T& v, S& stream) + { + if constexpr (is_flat_ptr::value) + { + uint32_t size; + stream.read(&size, sizeof(size)); + v.reset(size); + stream.read(v.data(), size); + } + else if constexpr (is_std_variant::value) + { + flat fv; + stream.read(&fv, sizeof(fv)); + fv.init_variant(v); + std::visit( + [&](auto& iv) { + using item_type = std::decay_t; + if constexpr (get_contains_offset_ptr()) + { + /// the stream.pos is at the END of reading the variant, which + /// should be the same as the end of flat::offset_data + input_stream in(stream.pos + fv.offset_data.offset - sizeof(offset_ptr), + stream.end); + flatunpack(iv, in); + } + else if constexpr (flatpack_size() <= 8) + { + input_stream st((const char*)&fv.flat_data, 8); + flatunpack(iv, st); + } + else + { + input_stream in(stream.pos + fv.offset_data.offset - sizeof(offset_ptr), + stream.end); + flatunpack(iv, in); + } + }, + v); + } + else if constexpr (std::is_same_v) + { + uint32_t size; + stream.read(&size, sizeof(size)); + v.resize(size); + stream.read(v.data(), size); + stream.skip(1); // null + } + else if constexpr (is_std_vector::value) + { + uint32_t size; + stream.read(&size, sizeof(size)); + v.resize(size); + + if constexpr (contains_offset_ptr::value_type>()) + { + for (auto& item : v) + { + offset_ptr ptr; + stream.read(&ptr, sizeof(ptr)); + + /// TODO: we don't know the size of the buffer here...is it safe? + input_stream in(stream.pos + ptr.offset - sizeof(ptr), stream.end); + flatunpack(item, in); + } + } + else + { + for (auto& item : v) + { + flatunpack(item, stream); + } + } + } + else if constexpr (reflect::is_struct) + { + reflect::for_each([&](const meta& ref, const auto& mptr) { + auto& member = v.*mptr; + using member_type = decltype(result_of_member(mptr)); + + if constexpr (contains_offset_ptr()) + { + offset_ptr ptr; + stream.read(&ptr, sizeof(ptr)); + + input_stream substream(stream.pos + ptr.offset - sizeof(ptr), stream.end); + flatunpack(member, substream); + } + else + { + flatunpack(member, stream); + } + }); + } + else if constexpr (std::is_trivially_copyable::value) + { + stream.read((char*)&v, sizeof(v)); + } + else + { + T::is_not_defined; + } + } + + template + uint32_t flatpack(const T& v, S& stream) + { + uint32_t alloc_pos = 0; + uint32_t cur_pos = 0; + + if constexpr (is_flat_ptr::value) + { + uint32_t size = v.size(); + stream.write(&size, sizeof(size)); + cur_pos += sizeof(size); + + if (size) + { + stream.write(v.data(), v.size()); + } + } + else if constexpr (is_std_variant::value) + { + alloc_pos = flatpack_size(); + std::visit( + [&](const auto& iv) { + using item_type = std::decay_t; + flat fv; + fv.type = get_type_hashname(); + if constexpr (not get_contains_offset_ptr() and + flatpack_size() <= 8) + { + fixed_buf_stream st((char*)&fv.flat_data, 8); + flatpack(iv, st); + + static_assert(sizeof(fv) == 16); + stream.write(&fv, sizeof(fv)); + } + else + { + fv.offset_data.offset = + (alloc_pos - + (cur_pos + 12)); //+8 because we haven't written the type yet / pad + size_stream size_str; + flatpack(iv, size_str); + alloc_pos += size_str.size; + + static_assert(sizeof(fv) == 16); + stream.write(&fv, sizeof(fv)); + + if constexpr (std::is_same_v) + { + stream.skip(size_str.size); /// we already calculated this above + } + else + { + /// now pack the member into the allocated spot + fixed_buf_stream substream( + stream.pos + fv.offset_data.offset - sizeof(fv.offset_data), + size_str.size); + if (substream.end > stream.end) + throw_error(stream_error::overrun); + flatpack(iv, substream); + } + } + }, + v); + } + else if constexpr (std::is_same_v) + { + uint32_t size = v.size(); + stream.write(&size, sizeof(size)); + cur_pos += sizeof(size); + + if (size) + { + stream.write(v.data(), v.size()); + stream.write("\0", 1); /// null term + cur_pos += v.size() + 1; + } + } + else if constexpr (is_std_vector::value) + { + if constexpr (contains_offset_ptr()) + { + uint32_t size = v.size(); + stream.write(&size, sizeof(size)); + cur_pos += sizeof(size); + alloc_pos += sizeof(size) + size * sizeof(offset_ptr); + + for (const auto& member : v) + { + if constexpr (std::is_same_v || + is_std_vector::value) + { + if (member.size() == 0) + { + offset_ptr ptr = {.offset = 0}; + stream.write(&ptr, sizeof(ptr)); + cur_pos += sizeof(ptr); + continue; + } + } + + size_stream size_str; + flatpack(member, size_str); + + offset_ptr ptr = {.offset = (alloc_pos - cur_pos)}; + + alloc_pos += size_str.size; + + stream.write(&ptr, sizeof(ptr)); + + cur_pos += sizeof(ptr); + + //? cur_pos += size_str.size; + if constexpr (std::is_same_v) + { + stream.skip(size_str.size); /// we already calculated this above + } + else + { + /// now pack the member into the allocated spot + fixed_buf_stream substream(stream.pos + ptr.offset - sizeof(ptr), + size_str.size); // ptr.size ); + if (substream.end > stream.end) + throw_error(stream_error::overrun); + flatpack(member, substream); + } + } + } + else + { + // std::cout << "vector type, T, is flat types: " << boost::core::demangle(typeid(typename T::value_type).name()) <<"\n"; + uint32_t size = v.size(); + stream.write(&size, sizeof(size)); + cur_pos += sizeof(size); + for (const auto& item : v) + { + cur_pos += flatpack(item, stream); + } + } + } + else if constexpr (reflect::is_struct) + { + alloc_pos = flatpack_size(); //::struct_tuple_type>::value; + reflect::for_each([&](const meta& ref, const auto& mptr) { + const auto& member = v.*mptr; + + using member_type = decltype(result_of_member(mptr)); + if constexpr (contains_offset_ptr()) + { + if constexpr (std::is_same_v || + is_std_vector::value) + { + if (member.size() == 0) + { + offset_ptr ptr = {.offset = 0}; + stream.write(&ptr, sizeof(ptr)); + cur_pos += sizeof(ptr); + + return; + } + } + + size_stream size_str; + flatpack(member, size_str); + + offset_ptr ptr = {.offset = alloc_pos - cur_pos}; + + if (ptr.offset == 0) + exit(-1); + + alloc_pos += size_str.size; + stream.write(&ptr, sizeof(ptr)); + cur_pos += sizeof(ptr); + + if constexpr (std::is_same_v) + { + stream.skip(size_str.size); /// we already calculated this above + } + else + { + /// now pack the member into the allocated spot + fixed_buf_stream substream(stream.pos + ptr.offset - sizeof(ptr), size_str.size); + + if (substream.end > stream.end) + { + throw_error(stream_error::overrun); + } + flatpack(member, substream); + } + } + else + { + cur_pos += flatpack(member, stream); + } + }); + } + else if constexpr (std::is_trivially_copyable::value) + { + stream.write(&v, sizeof(v)); + cur_pos += sizeof(v); + } + else + { + T::flatpack_is_not_defined; + } + return cur_pos; + } + + /** + * Behaves like a shared_ptr, copies will all point to the same flat data array. + */ + template + class flat_ptr + { + public: + typedef T value_type; + + flat_ptr(const T& from) + { + clio::size_stream ss; + clio::flatpack(from, ss); + _size = ss.size; + if (_size) + { + _data = std::shared_ptr(new char[_size], [](char* c) { delete[] c; }); + fixed_buf_stream out(_data.get(), _size); + clio::flatpack(from, out); + } + } + flat_ptr(){}; + operator bool() const { return _size; }; + + auto operator->() { return reinterpret_cast*>(_data.get()); } + auto operator->() const { return reinterpret_cast*>(_data.get()); } + const char* data() const { return _data.get(); } + char* data() { return _data.get(); } + size_t size() const { return _size; } + + void reset(size_t s = 0) + { + _size = s; + if (_size) + _data = std::shared_ptr(new char[_size], [](char* c) { delete[] c; }); + else + _data.reset(); + } + + operator T() const + { + T tmp; + input_stream in(_data.get(), _size); + clio::flatunpack(tmp, in); + return tmp; + } + + private: + std::shared_ptr _data; + size_t _size = 0; + }; + +} // namespace clio diff --git a/libraries/clio/include/clio/fpconv.h b/libraries/clio/include/clio/fpconv.h new file mode 100644 index 000000000..d6620db80 --- /dev/null +++ b/libraries/clio/include/clio/fpconv.h @@ -0,0 +1,376 @@ + +/// From https://github.com/night-shift/fpconv +/// Boost Software License 1.0 +/// See accompanying license file + +#include +#include + +#include + +#define fracmask 0x000FFFFFFFFFFFFFU +#define expmask 0x7FF0000000000000U +#define hiddenbit 0x0010000000000000U +#define signmask 0x8000000000000000U +#define expbias (1023 + 52) + +#define absv(n) ((n) < 0 ? -(n) : (n)) +#define minv(a, b) ((a) < (b) ? (a) : (b)) + +static constexpr uint64_t tens[] = {10000000000000000000U, + 1000000000000000000U, + 100000000000000000U, + 10000000000000000U, + 1000000000000000U, + 100000000000000U, + 10000000000000U, + 1000000000000U, + 100000000000U, + 10000000000U, + 1000000000U, + 100000000U, + 10000000U, + 1000000U, + 100000U, + 10000U, + 1000U, + 100U, + 10U, + 1U}; + +static inline uint64_t get_dbits(double d) +{ + union + { + double dbl; + uint64_t i; + } dbl_bits = {d}; + + return dbl_bits.i; +} + +static inline Fp build_fp(double d) +{ + uint64_t bits = get_dbits(d); + + Fp fp; + fp.frac = bits & fracmask; + fp.exp = (bits & expmask) >> 52; + + if (fp.exp) + { + fp.frac += hiddenbit; + fp.exp -= expbias; + } + else + { + fp.exp = -expbias + 1; + } + + return fp; +} + +static inline void normalize(Fp* fp) +{ + while ((fp->frac & hiddenbit) == 0) + { + fp->frac <<= 1; + fp->exp--; + } + + int shift = 64 - 52 - 1; + fp->frac <<= shift; + fp->exp -= shift; +} + +static inline void get_normalized_boundaries(Fp* fp, Fp* lower, Fp* upper) +{ + upper->frac = (fp->frac << 1) + 1; + upper->exp = fp->exp - 1; + + while ((upper->frac & (hiddenbit << 1)) == 0) + { + upper->frac <<= 1; + upper->exp--; + } + + int u_shift = 64 - 52 - 2; + + upper->frac <<= u_shift; + upper->exp = upper->exp - u_shift; + + int l_shift = fp->frac == hiddenbit ? 2 : 1; + + lower->frac = (fp->frac << l_shift) - 1; + lower->exp = fp->exp - l_shift; + + lower->frac <<= lower->exp - upper->exp; + lower->exp = upper->exp; +} + +static inline Fp multiply(Fp* a, Fp* b) +{ + const uint64_t lomask = 0x00000000FFFFFFFF; + + uint64_t ah_bl = (a->frac >> 32) * (b->frac & lomask); + uint64_t al_bh = (a->frac & lomask) * (b->frac >> 32); + uint64_t al_bl = (a->frac & lomask) * (b->frac & lomask); + uint64_t ah_bh = (a->frac >> 32) * (b->frac >> 32); + + uint64_t tmp = (ah_bl & lomask) + (al_bh & lomask) + (al_bl >> 32); + /* round up */ + tmp += 1U << 31; + + Fp fp = {ah_bh + (ah_bl >> 32) + (al_bh >> 32) + (tmp >> 32), a->exp + b->exp + 64}; + + return fp; +} + +static inline void round_digit(char* digits, + int ndigits, + uint64_t delta, + uint64_t rem, + uint64_t kappa, + uint64_t frac) +{ + while (rem < frac && delta - rem >= kappa && + (rem + kappa < frac || frac - rem > rem + kappa - frac)) + { + digits[ndigits - 1]--; + rem += kappa; + } +} + +static inline int generate_digits(Fp* fp, Fp* upper, Fp* lower, char* digits, int* K) +{ + uint64_t wfrac = upper->frac - fp->frac; + uint64_t delta = upper->frac - lower->frac; + + Fp one; + one.frac = 1ULL << -upper->exp; + one.exp = upper->exp; + + uint64_t part1 = upper->frac >> -one.exp; + uint64_t part2 = upper->frac & (one.frac - 1); + + int idx = 0, kappa = 10; + const uint64_t* divp; + /* 1000000000 */ + for (divp = tens + 10; kappa > 0; divp++) + { + uint64_t div = *divp; + unsigned digit = part1 / div; + + if (digit || idx) + { + digits[idx++] = digit + '0'; + } + + part1 -= digit * div; + kappa--; + + uint64_t tmp = (part1 << -one.exp) + part2; + if (tmp <= delta) + { + *K += kappa; + round_digit(digits, idx, delta, tmp, div << -one.exp, wfrac); + + return idx; + } + } + + /* 10 */ + const uint64_t* unit = tens + 18; + + while (true) + { + part2 *= 10; + delta *= 10; + kappa--; + + unsigned digit = part2 >> -one.exp; + if (digit || idx) + { + digits[idx++] = digit + '0'; + } + + part2 &= one.frac - 1; + if (part2 < delta) + { + *K += kappa; + round_digit(digits, idx, delta, part2, one.frac, wfrac * *unit); + + return idx; + } + + unit--; + } +} + +static inline int grisu2(double d, char* digits, int* K) +{ + Fp w = build_fp(d); + + Fp lower, upper; + get_normalized_boundaries(&w, &lower, &upper); + + normalize(&w); + + int k; + Fp cp = find_cachedpow10(upper.exp, &k); + + w = multiply(&w, &cp); + upper = multiply(&upper, &cp); + lower = multiply(&lower, &cp); + + lower.frac++; + upper.frac--; + + *K = -k; + + return generate_digits(&w, &upper, &lower, digits, K); +} + +static inline int emit_digits(char* digits, int ndigits, char* dest, int K, bool neg) +{ + int exp = absv(K + ndigits - 1); + + /* write plain integer */ + if (K >= 0 && (exp < (ndigits + 7))) + { + memcpy(dest, digits, ndigits); + memset(dest + ndigits, '0', K); + + return ndigits + K; + } + + /* write decimal w/o scientific notation */ + if (K < 0 && (K > -7 || exp < 4)) + { + int offset = ndigits - absv(K); + /* fp < 1.0 -> write leading zero */ + if (offset <= 0) + { + offset = -offset; + dest[0] = '0'; + dest[1] = '.'; + memset(dest + 2, '0', offset); + memcpy(dest + offset + 2, digits, ndigits); + + return ndigits + 2 + offset; + + /* fp > 1.0 */ + } + else + { + memcpy(dest, digits, offset); + dest[offset] = '.'; + memcpy(dest + offset + 1, digits + offset, ndigits - offset); + + return ndigits + 1; + } + } + + /* write decimal w/ scientific notation */ + ndigits = minv(ndigits, 18 - neg); + + int idx = 0; + dest[idx++] = digits[0]; + + if (ndigits > 1) + { + dest[idx++] = '.'; + memcpy(dest + idx, digits + 1, ndigits - 1); + idx += ndigits - 1; + } + + dest[idx++] = 'e'; + + char sign = K + ndigits - 1 < 0 ? '-' : '+'; + dest[idx++] = sign; + + int cent = 0; + + if (exp > 99) + { + cent = exp / 100; + dest[idx++] = cent + '0'; + exp -= cent * 100; + } + if (exp > 9) + { + int dec = exp / 10; + dest[idx++] = dec + '0'; + exp -= dec * 10; + } + else if (cent) + { + dest[idx++] = '0'; + } + + dest[idx++] = exp % 10 + '0'; + + return idx; +} + +static inline int filter_special(double fp, char* dest) +{ + if (fp == 0.0) + { + dest[0] = '0'; + return 1; + } + + uint64_t bits = get_dbits(fp); + + bool nan = (bits & expmask) == expmask; + + if (!nan) + { + return 0; + } + + if (bits & fracmask) + { + dest[0] = 'n'; + dest[1] = 'a'; + dest[2] = 'n'; + } + else + { + dest[0] = 'i'; + dest[1] = 'n'; + dest[2] = 'f'; + } + + return 3; +} + +static inline int fpconv_dtoa(double d, char dest[24]) +{ + char digits[18]; + + int str_len = 0; + bool neg = false; + + if (get_dbits(d) & signmask) + { + dest[0] = '-'; + str_len++; + neg = true; + } + + int spec = filter_special(d, dest + str_len); + + if (spec) + { + return str_len + spec; + } + + int K = 0; + int ndigits = grisu2(d, digits, &K); + + str_len += emit_digits(digits, ndigits, dest + str_len, K, neg); + + return str_len; +} diff --git a/libraries/clio/include/clio/from_bin.hpp b/libraries/clio/include/clio/from_bin.hpp new file mode 100644 index 000000000..9a0f74246 --- /dev/null +++ b/libraries/clio/include/clio/from_bin.hpp @@ -0,0 +1,350 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace clio +{ + template + void from_bin(T& obj, S& stream); + + template + void varuint32_from_bin(uint32_t& dest, S& stream) + { + dest = 0; + int shift = 0; + uint8_t b = 0; + do + { + if (shift >= 35) + throw_error(stream_error::invalid_varuint_encoding); + from_bin(b, stream); + dest |= uint32_t(b & 0x7f) << shift; + shift += 7; + } while (b & 0x80); + } + + template + void varuint64_from_bin(uint64_t& dest, S& stream) + { + dest = 0; + int shift = 0; + uint8_t b = 0; + do + { + if (shift >= 70) + throw_error(stream_error::invalid_varuint_encoding); + from_bin(b, stream); + dest |= uint64_t(b & 0x7f) << shift; + shift += 7; + } while (b & 0x80); + } + + template + void varint32_from_bin(int32_t& result, S& stream) + { + uint32_t v; + varuint32_from_bin(v, stream); + if (v & 1) + result = ((~v) >> 1) | 0x8000'0000; + else + result = v >> 1; + } + + template + void from_bin_assoc(T& v, S& stream) + { + uint32_t size; + varuint32_from_bin(size, stream); + for (size_t i = 0; i < size; ++i) + { + typename T::value_type elem; + from_bin(elem, stream); + v.emplace(elem); + } + } + + template + void from_bin_sequence(T& v, S& stream) + { + uint32_t size; + varuint32_from_bin(size, stream); + for (size_t i = 0; i < size; ++i) + { + v.emplace_back(); + from_bin(v.back(), stream); + } + } + + template + void from_bin(T (&v)[N], S& stream) + { + uint32_t size; + varuint32_from_bin(size, stream); + if (size != N) + throw_error(stream_error::array_size_mismatch); + if constexpr (has_bitwise_serialization()) + { + stream.read(reinterpret_cast(v), size * sizeof(T)); + } + else + { + for (size_t i = 0; i < size; ++i) + { + OUTCOME_TRY(from_bin(v[i], stream)); + } + } + } + + template + void from_bin(std::vector& v, S& stream) + { + if constexpr (has_bitwise_serialization()) + { + if constexpr (sizeof(size_t) >= 8) + { + uint64_t size; + varuint64_from_bin(size, stream); + stream.check_available(size * sizeof(T)); + v.resize(size); + stream.read(reinterpret_cast(v.data()), size * sizeof(T)); + } + else + { + uint32_t size; + varuint32_from_bin(size, stream); + stream.check_available(size * sizeof(T)); + v.resize(size); + stream.read(reinterpret_cast(v.data()), size * sizeof(T)); + } + } + else + { + uint32_t size; + varuint32_from_bin(size, stream); + v.resize(size); + for (size_t i = 0; i < size; ++i) + { + from_bin(v[i], stream); + } + } + } + + template + void from_bin(std::set& v, S& stream) + { + from_bin_assoc(v, stream); + } + + template + void from_bin(std::map& v, S& stream) + { + uint32_t size; + varuint32_from_bin(size, stream); + for (size_t i = 0; i < size; ++i) + { + std::pair elem; + from_bin(elem, stream); + v.emplace(elem); + } + } + + template + void from_bin(std::deque& v, S& stream) + { + from_bin_sequence(v, stream); + } + + template + void from_bin(std::list& v, S& stream) + { + from_bin_sequence(v, stream); + } + + template + void from_bin(input_stream& obj, S& stream) + { + if constexpr (sizeof(size_t) >= 8) + { + uint64_t size; + varuint64_from_bin(size, stream); + stream.check_available(size); + stream.read_reuse_storage(obj.pos, size); + obj.end = obj.pos + size; + } + else + { + uint32_t size; + varuint32_from_bin(size, stream); + stream.check_available(size); + stream.read_reuse_storage(obj.pos, size); + obj.end = obj.pos + size; + } + } + + template + void from_bin(std::pair& obj, S& stream) + { + from_bin(obj.first, stream); + from_bin(obj.second, stream); + } + + template + inline void from_bin(std::string& obj, S& stream) + { + uint32_t size; + varuint32_from_bin(size, stream); + obj.resize(size); + stream.read(obj.data(), obj.size()); + } + + template + inline void from_bin(std::string_view& obj, S& stream) + { + uint32_t size; + varuint32_from_bin(size, stream); + obj = std::string_view(stream.get_pos(), size); + stream.skip(size); + } + + template + void from_bin(std::optional& obj, S& stream) + { + bool present; + from_bin(present, stream); + if (!present) + { + obj.reset(); + return; + } + obj.emplace(); + from_bin(*obj, stream); + } + + template + void variant_from_bin(std::variant& v, uint32_t i, S& stream) + { + if constexpr (I < std::variant_size_v>) + { + if (i == I) + { + auto& x = v.template emplace(); + from_bin(x, stream); + } + else + { + variant_from_bin(v, i, stream); + } + } + else + { + throw_error(stream_error::bad_variant_index); + } + } + + template + void from_bin(std::variant& obj, S& stream) + { + uint32_t u; + varuint32_from_bin(u, stream); + variant_from_bin<0>(obj, u, stream); + } + + template + void from_bin(std::array& obj, S& stream) + { + for (T& elem : obj) + { + from_bin(elem, stream); + } + } + + template + void from_bin_tuple(T& obj, S& stream) + { + if constexpr (N < std::tuple_size_v) + { + from_bin(std::get(obj), stream); + from_bin_tuple(obj, stream); + } + } + + template + void from_bin(std::tuple& obj, S& stream) + { + from_bin_tuple<0>(obj, stream); + } + + template + void from_bin(T& obj, S& stream) + { + if constexpr (has_bitwise_serialization()) + { + stream.read(reinterpret_cast(&obj), sizeof(T)); + } + else + { // if constexpr (std::is_same_v, void>) { + reflect::for_each([&](const clio::meta&, auto m) { + if constexpr (not std::is_member_function_pointer_v) + { + from_bin(obj.*m, stream); + } + }); + } /*else { + // TODO: This can operate in place for standard serializers + decltype(serialize_as(obj)) temp; + OUTCOME_TRY(from_bin(temp, stream)); + convert(temp, obj, choose_first); + return outcome::success(); + } */ + } + + template + void from_bin_vec(T& obj, const std::vector& bin) + { + input_stream stream{bin}; + from_bin(obj, stream); + } + template + void from_bin_vec(T& obj, std::vector& bin) + { + input_stream stream{bin}; + from_bin(obj, stream); + } + + template + void from_bin_vec(T& obj, const bytes& bin) + { + from_bin_vec(obj, bin.data); + } + template + void from_bin_vec(T& obj, bytes& bin) + { + from_bin_vec(obj, bin.data); + } + + template + T from_bin(const std::vector& bin) + { + T obj; + from_bin_vec(obj, bin); + return obj; + } + + template + T from_bin(S& stream) + { + T obj; + from_bin(obj, stream); + return obj; + } + +} // namespace clio diff --git a/libraries/clio/include/clio/from_bin/varint.hpp b/libraries/clio/include/clio/from_bin/varint.hpp new file mode 100644 index 000000000..29326e1e9 --- /dev/null +++ b/libraries/clio/include/clio/from_bin/varint.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include +#include + +namespace clio +{ + template + void from_bin(varuint32& obj, S& stream) + { + varuint32_from_bin(obj.value, stream); + } + + template + void from_bin(varint32& obj, S& stream) + { + varint32_from_bin(obj.value, stream); + } + +} // namespace clio diff --git a/libraries/clio/include/clio/from_json.hpp b/libraries/clio/include/clio/from_json.hpp new file mode 100644 index 000000000..c7b79f9dd --- /dev/null +++ b/libraries/clio/include/clio/from_json.hpp @@ -0,0 +1,855 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace clio +{ + enum class from_json_error + { + no_error, + + expected_end, + expected_null, + expected_bool, + expected_string, + expected_hex_string, + hex_string_incorrect_length, + invalid_signature, + invalid_name, + expected_start_object, + expected_key, + expected_end_object, + expected_start_array, + expected_end_array, + expected_positive_uint, + expected_field, + expected_variant, + expected_public_key, + expected_private_key, + expected_signature, + expected_number, + expected_int, + expected_time_point, + expected_symbol_code, + expected_symbol, + expected_asset, + invalid_type_for_variant, + unexpected_field, + number_out_of_range, + from_json_no_pair, + + // These are from rapidjson: + document_empty, + document_root_not_singular, + value_invalid, + object_miss_name, + object_miss_colon, + object_miss_comma_or_curly_bracket, + array_miss_comma_or_square_bracket, + string_unicode_escape_invalid_hex, + string_unicode_surrogate_invalid, + string_escape_invalid, + string_miss_quotation_mark, + string_invalid_encoding, + number_too_big, + number_miss_fraction, + number_miss_exponent, + terminated, + unspecific_syntax_error, + }; // from_json_error +} // namespace clio + +namespace std +{ + template <> + struct is_error_code_enum : true_type + { + }; +} // namespace std + +namespace clio +{ + class from_json_error_category_type : public std::error_category + { + public: + const char* name() const noexcept override final { return "ConversionError"; } + + std::string message(int c) const override final + { + switch (static_cast(c)) + { + // clang-format off + case from_json_error::no_error: return "No error"; + + case from_json_error::expected_end: return "Expected end of json"; + case from_json_error::expected_null: return "Expected null"; + case from_json_error::expected_bool: return "Expected true or false"; + case from_json_error::expected_string: return "Expected string"; + case from_json_error::expected_hex_string: return "Expected string containing hex"; + case from_json_error::hex_string_incorrect_length: return "Hex string has incorrect length"; + case from_json_error::invalid_signature: return "Invalid signature format"; + case from_json_error::invalid_name: return "Invalid name"; + case from_json_error::expected_start_object: return "Expected {"; + case from_json_error::expected_key: return "Expected key"; + case from_json_error::expected_end_object: return "Expected }"; + case from_json_error::expected_start_array: return "Expected ["; + case from_json_error::expected_end_array: return "Expected ]"; + case from_json_error::expected_positive_uint: return "Expected positive integer"; + case from_json_error::expected_field: return "Expected field"; + case from_json_error::expected_variant: return R"(Expected variant: ["type", value])"; + case from_json_error::expected_public_key: return "Expected public key"; + case from_json_error::expected_private_key: return "Expected private key"; + case from_json_error::expected_signature: return "Expected signature"; + case from_json_error::expected_number: return "Expected number or boolean"; + case from_json_error::expected_int: return "Expected integer"; + case from_json_error::expected_time_point: return "Expected time point"; + case from_json_error::expected_symbol_code: return "Expected symbol code"; + case from_json_error::expected_symbol: return "Expected symbol"; + case from_json_error::expected_asset: return "Expected asset"; + case from_json_error::invalid_type_for_variant: return "Invalid type for variant"; + case from_json_error::unexpected_field: return "Unexpected field"; + case from_json_error::number_out_of_range: return "number is out of range"; + case from_json_error::from_json_no_pair: return "from_json does not support std::pair"; + + case from_json_error::document_empty: return "The document is empty"; + case from_json_error::document_root_not_singular: return "The document root must not follow by other values"; + case from_json_error::value_invalid: return "Invalid value"; + case from_json_error::object_miss_name: return "Missing a name for object member"; + case from_json_error::object_miss_colon: return "Missing a colon after a name of object member"; + case from_json_error::object_miss_comma_or_curly_bracket: return "Missing a comma or '}' after an object member"; + case from_json_error::array_miss_comma_or_square_bracket: return "Missing a comma or ']' after an array element"; + case from_json_error::string_unicode_escape_invalid_hex: return "Incorrect hex digit after \\u escape in string"; + case from_json_error::string_unicode_surrogate_invalid: return "The surrogate pair in string is invalid"; + case from_json_error::string_escape_invalid: return "Invalid escape character in string"; + case from_json_error::string_miss_quotation_mark: return "Missing a closing quotation mark in string"; + case from_json_error::string_invalid_encoding: return "Invalid encoding in string"; + case from_json_error::number_too_big: return "Number too big to be stored in double"; + case from_json_error::number_miss_fraction: return "Miss fraction part in number"; + case from_json_error::number_miss_exponent: return "Miss exponent in number"; + case from_json_error::terminated: return "Parsing was terminated"; + case from_json_error::unspecific_syntax_error: return "Unspecific syntax error"; + // clang-format on + + default: + return "unknown"; + } + } + }; // from_json_error_category_type + + inline const from_json_error_category_type& from_json_error_category() + { + static from_json_error_category_type c; + return c; + } + + inline std::error_code make_error_code(from_json_error e) + { + return {static_cast(e), from_json_error_category()}; + } + + inline from_json_error convert_error(rapidjson::ParseErrorCode err) + { + switch (err) + { + // clang-format off + case rapidjson::kParseErrorNone: return from_json_error::no_error; + case rapidjson::kParseErrorDocumentEmpty: return from_json_error::document_empty; + case rapidjson::kParseErrorDocumentRootNotSingular: return from_json_error::document_root_not_singular; + case rapidjson::kParseErrorValueInvalid: return from_json_error::value_invalid; + case rapidjson::kParseErrorObjectMissName: return from_json_error::object_miss_name; + case rapidjson::kParseErrorObjectMissColon: return from_json_error::object_miss_colon; + case rapidjson::kParseErrorObjectMissCommaOrCurlyBracket: return from_json_error::object_miss_comma_or_curly_bracket; + case rapidjson::kParseErrorArrayMissCommaOrSquareBracket: return from_json_error::array_miss_comma_or_square_bracket; + case rapidjson::kParseErrorStringUnicodeEscapeInvalidHex: return from_json_error::string_unicode_escape_invalid_hex; + case rapidjson::kParseErrorStringUnicodeSurrogateInvalid: return from_json_error::string_unicode_surrogate_invalid; + case rapidjson::kParseErrorStringEscapeInvalid: return from_json_error::string_escape_invalid; + case rapidjson::kParseErrorStringMissQuotationMark: return from_json_error::string_miss_quotation_mark; + case rapidjson::kParseErrorStringInvalidEncoding: return from_json_error::string_invalid_encoding; + case rapidjson::kParseErrorNumberTooBig: return from_json_error::number_too_big; + case rapidjson::kParseErrorNumberMissFraction: return from_json_error::number_miss_fraction; + case rapidjson::kParseErrorNumberMissExponent: return from_json_error::number_miss_exponent; + case rapidjson::kParseErrorTermination: return from_json_error::terminated; + case rapidjson::kParseErrorUnspecificSyntaxError: return from_json_error::unspecific_syntax_error; + // clang-format on + + default: + return from_json_error::unspecific_syntax_error; + } + } + + enum class json_token_type + { + type_unread, + type_null, + type_bool, + type_string, + type_start_object, + type_key, + type_end_object, + type_start_array, + type_end_array, + }; + + struct json_token + { + json_token_type type = {}; + std::string_view key = {}; + bool value_bool = {}; + std::string_view value_string = {}; + }; + + class json_token_stream + : public rapidjson::BaseReaderHandler, json_token_stream> + { + private: + rapidjson::Reader reader; + rapidjson::InsituStringStream ss; + + public: + json_token current_token; + + // This modifies json (ok... why?) + json_token_stream(char* json) : ss{json} { reader.IterativeParseInit(); } + + bool complete() { return reader.IterativeParseComplete(); } + + std::reference_wrapper peek_token() + { + if (current_token.type != json_token_type::type_unread) + return current_token; + else if (reader.IterativeParseNext< + rapidjson::kParseInsituFlag | rapidjson::kParseValidateEncodingFlag | + rapidjson::kParseIterativeFlag | rapidjson::kParseNumbersAsStringsFlag>( + ss, *this)) + return current_token; + else + throw_error(convert_error(reader.GetParseErrorCode())); + } + + void eat_token() { current_token.type = json_token_type::type_unread; } + + void get_end() + { + if (current_token.type != json_token_type::type_unread || !complete()) + throw_error(from_json_error::expected_end); + } + + void get_null() + { + auto t = peek_token(); + if (t.get().type != json_token_type::type_null) + throw_error(from_json_error::expected_null); + eat_token(); + } + + bool get_bool() + { + auto t = peek_token(); + if (t.get().type != json_token_type::type_bool) + throw_error(from_json_error::expected_bool); + eat_token(); + return t.get().value_bool; + } + + std::string_view get_string() + { + auto t = peek_token(); + if (t.get().type != json_token_type::type_string) + throw_error(from_json_error::expected_string); + eat_token(); + return t.get().value_string; + } + + void get_start_object() + { + auto t = peek_token(); + if (t.get().type != json_token_type::type_start_object) + throw_error(from_json_error::expected_start_object); + eat_token(); + } + + std::string_view get_key() + { + auto t = peek_token(); + if (t.get().type != json_token_type::type_key) + throw_error(from_json_error::expected_key); + eat_token(); + return t.get().key; + } + + void get_end_object() + { + auto t = peek_token(); + if (t.get().type != json_token_type::type_end_object) + throw_error(from_json_error::expected_end_object); + eat_token(); + } + + void get_start_array() + { + auto t = peek_token(); + if (t.get().type != json_token_type::type_start_array) + throw_error(from_json_error::expected_start_array); + eat_token(); + } + + void get_end_array() + { + auto t = peek_token(); + if (t.get().type != json_token_type::type_end_array) + throw_error(from_json_error::expected_end_array); + eat_token(); + } + + // BaseReaderHandler methods + bool Null() + { + current_token.type = json_token_type::type_null; + return true; + } + bool Bool(bool v) + { + current_token.type = json_token_type::type_bool; + current_token.value_bool = v; + return true; + } + bool RawNumber(const char* v, rapidjson::SizeType length, bool copy) + { + return String(v, length, copy); + } + bool Int(int v) { return false; } + bool Uint(unsigned v) { return false; } + bool Int64(int64_t v) { return false; } + bool Uint64(uint64_t v) { return false; } + bool Double(double v) { return false; } + bool String(const char* v, rapidjson::SizeType length, bool) + { + current_token.type = json_token_type::type_string; + current_token.value_string = {v, length}; + return true; + } + bool StartObject() + { + current_token.type = json_token_type::type_start_object; + return true; + } + bool Key(const char* v, rapidjson::SizeType length, bool) + { + current_token.key = {v, length}; + current_token.type = json_token_type::type_key; + return true; + } + bool EndObject(rapidjson::SizeType) + { + current_token.type = json_token_type::type_end_object; + return true; + } + bool StartArray() + { + current_token.type = json_token_type::type_start_array; + return true; + } + bool EndArray(rapidjson::SizeType) + { + current_token.type = json_token_type::type_end_array; + return true; + } + }; // json_token_stream + + template + [[nodiscard]] bool unhex(DestIt dest, SrcIt begin, SrcIt end) + { + auto get_digit = [&](uint8_t& nibble) { + if (*begin >= '0' && *begin <= '9') + nibble = *begin++ - '0'; + else if (*begin >= 'a' && *begin <= 'f') + nibble = *begin++ - 'a' + 10; + else if (*begin >= 'A' && *begin <= 'F') + nibble = *begin++ - 'A' + 10; + else + return false; + return true; + }; + while (begin != end) + { + uint8_t h, l; + if (!get_digit(h) || !get_digit(l)) + return false; + *dest++ = (h << 4) | l; + } + return true; + } + + /// \exclude + template + void from_json(T& result, S& stream); + + /// \group from_json_explicit Parse JSON (Explicit Types) + /// Parse JSON and convert to `result`. These overloads handle specified types. + template + void from_json(std::string_view& result, S& stream) + { + result = stream.get_string(); + } + + /// \group from_json_explicit Parse JSON (Explicit Types) + /// Parse JSON and convert to `result`. These overloads handle specified types. + template + void from_json(std::string& result, S& stream) + { + result = stream.get_string(); + } + + /// \exclude + template + void from_json_int(T& result, S& stream) + { + auto r = stream.get_string(); + auto pos = r.data(); + auto end = pos + r.size(); + bool found = false; + result = 0; + T limit; + T sign; + if (std::is_signed_v && pos != end && *pos == '-') + { + ++pos; + sign = -1; + limit = std::numeric_limits::min(); + } + else + { + sign = 1; + limit = std::numeric_limits::max(); + } + while (pos != end && *pos >= '0' && *pos <= '9') + { + T digit = (*pos++ - '0'); + // abs(result) can overflow. Use -abs(result) instead. + if (std::is_signed_v && (-sign * limit + digit) / 10 > -sign * result) + throw_error(from_json_error::number_out_of_range); + if (!std::is_signed_v && (limit - digit) / 10 < result) + throw_error(from_json_error::number_out_of_range); + result = result * 10 + sign * digit; + found = true; + } + if (pos != end || !found) + throw_error(from_json_error::expected_int); + } + + /// \group from_json_explicit + template + void from_json(uint8_t& result, S& stream) + { + return from_json_int(result, stream); + } + + /// \group from_json_explicit + template + void from_json(uint16_t& result, S& stream) + { + return from_json_int(result, stream); + } + + /// \group from_json_explicit + template + void from_json(uint32_t& result, S& stream) + { + return from_json_int(result, stream); + } + + /// \group from_json_explicit + template + void from_json(uint64_t& result, S& stream) + { + return from_json_int(result, stream); + } + + /// \group from_json_explicit + template + void from_json(unsigned __int128& result, S& stream) + { + return from_json_int(result, stream); + } + + /// \group from_json_explicit + template + void from_json(int8_t& result, S& stream) + { + return from_json_int(result, stream); + } + + /// \group from_json_explicit + template + void from_json(int16_t& result, S& stream) + { + return from_json_int(result, stream); + } + + /// \group from_json_explicit + template + void from_json(int32_t& result, S& stream) + { + return from_json_int(result, stream); + } + + /// \group from_json_explicit + template + void from_json(int64_t& result, S& stream) + { + return from_json_int(result, stream); + } + + /// \group from_json_explicit + template + void from_json(__int128& result, S& stream) + { + return from_json_int(result, stream); + } + + template + void from_json(float& result, S& stream) + { + auto sv = stream.get_string(); + if (sv.empty()) + throw_error(from_json_error::expected_number); + std::string s(sv); // strtof expects a null-terminated string + errno = 0; + char* end; + result = std::strtof(s.c_str(), &end); + if (errno || end != s.c_str() + s.size()) + throw_error(from_json_error::expected_number); + } + + template + void from_json(double& result, S& stream) + { + auto sv = stream.get_string(); + if (sv.empty()) + throw_error(from_json_error::expected_number); + std::string s(sv); + errno = 0; + char* end; + result = std::strtod(s.c_str(), &end); + if (errno || end != s.c_str() + s.size()) + throw_error(from_json_error::expected_number); + } + + /* +/// \group from_json_explicit +template +void from_json(int32_t& result, S& stream) { + bool in_str = false; + if (pos != end && *pos == '"') { + in_str = true; + from_json_skip_space(pos, end); + } + bool neg = false; + if (pos != end && *pos == '-') { + neg = true; + ++pos; + } + bool found = false; + result = 0; + while (pos != end && *pos >= '0' && *pos <= '9') { + result = result * 10 + *pos++ - '0'; + found = true; + } + check(found, "expected integer"); + from_json_skip_space(pos, end); + if (in_str) { + from_json_expect(pos, end, '"', "expected integer"); + from_json_skip_space(pos, end); + } + if (neg) + result = -result; +} +*/ + /// \group from_json_explicit + template + void from_json(bool& result, S& stream) + { + result = stream.get_bool(); + } + + /// \group from_json_explicit + template + void from_json(std::vector& result, S& stream) + { + stream.get_start_array(); + result.clear(); + while (true) + { + auto t = stream.peek_token(); + if (t.get().type == json_token_type::type_end_array) + break; + result.emplace_back(); + from_json(result.back(), stream); + } + stream.get_end_array(); + } + + /// \group from_json_explicit + template + void from_json(std::optional& result, S& stream) + { + result = std::optional(); + if (stream.get_null()) + { + result = std::nullopt; + } + else + { + result.emplace(); + from_json(*result, stream); + } + } + + template + bool set_variant_impl(std::variant& result, uint32_t type) + { + if (type == N) + { + result.template emplace(); + return true; + } + else if constexpr (N + 1 < sizeof...(T)) + { + return set_variant_impl(result, type); + } + return false; + } + + /// TODO: this is attempting to parse variant as [ type, obj ], + /// but to_json is encoding it as { type: "", value: } + /// \group from_json_explicit + template + void from_json(std::variant& result, S& stream) + { + stream.get_start_array(); + std::string_view type; + from_json(type, stream); + const char* const type_names[] = {get_type_name((T*)nullptr)...}; + uint32_t type_idx = std::find(type_names, type_names + sizeof...(T), type) - type_names; + if (type_idx >= sizeof...(T)) + throw_error(from_json_error::invalid_type_for_variant); + if (set_variant_impl(result, type_idx)) + { + std::visit([&](auto& x) { return from_json(x, stream); }, result); + } + stream.get_end_array(); + } + + /** + * Accepts null as default value for all tuple elements + * Accepts [] with 0 to N as the first n elements and errors if there are too many elements + * accepts anything else as the presumed first element + * + * TODO: make robust against adding elements to the tuple by droping all remaining values + */ + template + void from_json(std::tuple& result, S& stream) + { + result = std::tuple(); + auto t = stream.peek_token(); + if (t.get().type == json_token_type::type_null) + { + return; + } + + if constexpr (sizeof...(T) > 0) + { + if (t.get().type != json_token_type::type_start_array) + { + from_json(std::get<0>(result), stream); + } + } + + stream.get_start_array(); + tuple_for_each(result, [&](int idx, auto& item) { + auto t = stream.peek_token(); + if (t.get().type == json_token_type::type_end_array) + return; + from_json(item, stream); + }); + stream.get_end_array(); + } + + /// \group from_json_explicit + template + void from_json_hex(std::vector& result, S& stream) + { + auto s = stream.get_string(); + if (s.size() & 1) + throw_error(from_json_error::expected_hex_string); + result.clear(); + result.reserve(s.size() / 2); + if (!unhex(std::back_inserter(result), s.begin(), s.end())) + throw_error(from_json_error::expected_hex_string); + } + + /// \exclude + template + inline void from_json_object(S& stream, F f) + { + stream.get_start_object(); + while (true) + { + auto t = stream.peek_token(); + if (t.get().type == json_token_type::type_end_object) + { + break; + } + auto k = stream.get_key(); + f(k); + } + stream.get_end_object(); + } + + template + void from_json_skip_value(S& stream) + { + uint64_t depth = 0; + do + { + auto t = stream.peek_token(); + auto type = t.get().type; + if (type == json_token_type::type_start_object || + type == json_token_type::type_start_array) + ++depth; + else if (type == json_token_type::type_end_object || + type == json_token_type::type_end_array) + --depth; + stream.eat_token(); + } while (depth); + } + + /// \output_section Parse JSON (Reflected Objects) + /// Parse JSON and convert to `obj`. This overload works with + /// [reflected objects](standardese://reflection/). + template + void from_json(T& obj, S& stream) + { + if constexpr (reflect::is_struct) + { + from_json_object(stream, [&](std::string_view key) -> void { + bool found = false; + reflect::get(key, [&](auto member) { + if constexpr (not std::is_member_function_pointer_v) + { + from_json(obj.*member, stream); + found = true; + } + else + { + } + }); + if (not found) + { + from_json_skip_value(stream); + } + }); + } + else + { + T::reflection_not_defined; + } + } + + /// not implimented, so don't link + template + void from_json(std::pair& obj, S& stream); + + /* +/// \output_section Convenience Wrappers +/// Parse JSON and return result. This overload wraps the other `to_json` overloads. +template +T from_json(const std::vector& v) { + const char* pos = v.data(); + const char* end = pos + v.size(); + from_json_skip_space(pos, end); + T result; + from_json(result, pos, end); + from_json_expect_end(pos, end); + return result; +} + +/// Parse JSON and return result. This overload wraps the other `to_json` overloads. +template +T from_json(std::string_view s) { + const char* pos = s.data(); + const char* end = pos + s.size(); + from_json_skip_space(pos, end); + T result; + from_json(result, pos, end); + from_json_expect_end(pos, end); + return result; +} +*/ + + /// Parse JSON and return result. This overload wraps the other `to_json` overloads. + template + T from_json(S& stream) + { + T x; + from_json(x, stream); + return x; + } + + template + T from_json(std::string json) + { + json_token_stream stream(json.data()); + return from_json(stream); + } + + /* +/// \exclude +template +__attribute__((noinline)) void parse_named_variant_impl(tagged_variant& v, size_t i, + const char*& pos, const char* end) { + if constexpr (I < sizeof...(NamedTypes)) { + if (i == I) { + auto& q = v.value; + auto& x = q.template emplace(); + if constexpr (!is_named_empty_type_v>) { + from_json_expect(pos, end, ',', "expected ,"); + from_json(x, pos, end); + } + } else { + return parse_named_variant_impl(v, i, pos, end); + } + } else { + check(false, "invalid variant index"); + } +} + +/// \group from_json_explicit +template +__attribute__((noinline)) void from_json(tagged_variant& result, const char*& pos, + const char* end) { + from_json_skip_space(pos, end); + from_json_expect(pos, end, '[', "expected array"); + + clio::name name; + from_json(name, pos, end); + + for (size_t i = 0; i < sizeof...(NamedTypes); ++i) { + if (name == tagged_variant::keys[i]) { + parse_named_variant_impl<0>(result, i, pos, end); + from_json_expect(pos, end, ']', "expected ]"); + return; + } + } + check(false, "invalid variant index name"); +} +*/ + +} // namespace clio diff --git a/libraries/clio/include/clio/from_json/varint.hpp b/libraries/clio/include/clio/from_json/varint.hpp new file mode 100644 index 000000000..756e23a4a --- /dev/null +++ b/libraries/clio/include/clio/from_json/varint.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include +#include + +namespace clio +{ + template + void from_json(varuint32& obj, S& stream) + { + from_json(obj.value, stream); + } + + template + void from_json(varint32& obj, S& stream) + { + from_json(obj.value, stream); + } + +} // namespace clio diff --git a/libraries/clio/include/clio/from_protobuf.hpp b/libraries/clio/include/clio/from_protobuf.hpp new file mode 100644 index 000000000..94d0b27b2 --- /dev/null +++ b/libraries/clio/include/clio/from_protobuf.hpp @@ -0,0 +1,226 @@ +#pragma once +#include +#include +#include + +namespace clio +{ + template + void from_protobuf_object(T& obj, S& stream); + + template + bool set_variant(std::variant& result, uint32_t type) + { + if (type == N) + { + result.template emplace(); + return true; + } + else if constexpr (N + 1 < sizeof...(T)) + { + return set_variant(result, type); + } + return false; + } + + template + void skip_member(uint16_t wire_type, S& stream) + { + uint32_t temp = 0; + switch (wire_type) + { + case 0: + varuint32_from_bin(temp, stream); + break; + case 1: + stream.skip(8); + break; + case 2: + varuint32_from_bin(temp, stream); + stream.skip(temp); + break; + case 5: + stream.skip(4); + break; + } + } + + template + void from_protobuf_object(std::variant& obj, S& stream) + { + uint32_t key = 0; + varuint32_from_bin(key, stream); + uint32_t field = key >> 3; + + if (set_variant(obj, field - 1)) + { + std::visit([&](auto& x) { return from_protobuf_member(x, stream); }, obj); + } + else + { + skip_member(uint8_t(key) & 0x07, stream); + obj = std::variant(); /// reset to default + } + } + template + void from_protobuf_object(std::tuple& obj, S& stream) + { + while (stream.remaining()) + { + uint32_t key = 0; + varuint32_from_bin(key, stream); + uint16_t wire_type = uint8_t(key) & 0x07; + uint32_t number = key >> 3; + + bool skip_it = true; + tuple_get(obj, number - 1, [&](auto& member) { + from_protobuf_member(member, stream); + skip_it = false; + }); + if (skip_it) + skip_member(wire_type, stream); + } + } + + template + void from_protobuf_object(std::vector& obj, S& stream) + { + obj.clear(); + if constexpr (std::is_arithmetic_v) + { + uint32_t key = 0; + varuint32_from_bin(key, stream); + uint16_t wire_type = uint8_t(key) & 0x07; + uint32_t number = key >> 3; + if (number != 1) + skip_member(wire_type, stream); + else + { + uint32_t size = 0; + varuint32_from_bin(size, stream); + if (size > stream.remaining()) + throw_error(stream_error::overrun); + obj.resize(size / sizeof(T)); + stream.read((char*)obj.data(), size); + } + } + else + { + while (stream.remaining()) + { + uint32_t key = 0; + varuint32_from_bin(key, stream); + uint16_t wire_type = uint8_t(key) & 0x07; + uint32_t number = key >> 3; + + bool skip_it = true; + if (number == 1) + { + skip_it = false; + obj.resize(obj.size() + 1); + from_protobuf_member(obj.back(), stream); + } + else + { + skip_member(wire_type, stream); + } + } + } + } + + template + void from_protobuf_member(T& obj, S& stream) + { + if constexpr (std::is_same_v) + { + uint32_t size = 0; + varuint32_from_bin(size, stream); + obj.resize(size); + stream.read(obj.data(), size); + } + else if constexpr (std::is_same_v) + { + uint32_t size = 0; + varuint32_from_bin(size, stream); + obj.data.resize(size); + stream.read(obj.data.data(), size); + } + else if constexpr (reflect::is_struct || is_std_variant::value || + is_std_vector::value || is_std_tuple::value) + { + uint32_t size = 0; + varuint32_from_bin(size, stream); + if (size > stream.remaining()) + throw_error(stream_error::overrun); + input_stream objstream(stream.pos, size); + stream.skip(size); + from_protobuf_object(obj, objstream); + } + else if constexpr (5 == wire_type::value or 1 == wire_type::value) + { + from_bin(obj, stream); + } + else if constexpr (0 == wire_type::value) + { + uint32_t v; + varuint32_from_bin(v, stream); + obj = v; + } + else + { + T::from_protobuf_is_not_defined; /// used to generate useful compile error + } + } + + template + void from_protobuf_object(T& obj, S& stream) + { + while (stream.remaining()) + { + uint32_t key = 0; + varuint32_from_bin(key, stream); + uint16_t wire_type = uint8_t(key) & 0x07; + uint32_t number = key >> 3; + + // bool skip_it = true; + if (not reflect::get(number, [&](auto m) { + if constexpr (std::is_member_function_pointer_v) + { + skip_member(wire_type, + stream); /// we cannot store return value of functions in T + } + else + { + // using member_type = std::decay_t; + // auto& member = obj.*m; + from_protobuf_member(obj.*m, stream); + } + })) + { // if not reflect::get(number...) + skip_member(wire_type, stream); + } + } + } + + template + T from_protobuf(S& stream) + { + T tmp; + from_protobuf_object(tmp, stream); + return tmp; + } + + template + T from_protobuf(const std::vector& in) + { + clio::input_stream stream(in.data(), in.size()); + return clio::from_protobuf(stream); + } + template + T from_protobuf(std::vector& in) + { + clio::input_stream stream(in.data(), in.size()); + return clio::from_protobuf(stream); + } + +} // namespace clio diff --git a/libraries/clio/include/clio/get_type_name.hpp b/libraries/clio/include/clio/get_type_name.hpp new file mode 100644 index 000000000..9225d7301 --- /dev/null +++ b/libraries/clio/include/clio/get_type_name.hpp @@ -0,0 +1,208 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace clio +{ +#define CLIO_REFLECT_TYPENAME(T) \ + constexpr const char* get_type_name(const T*) { return BOOST_PP_STRINGIZE(T); } + +#define CLIO_REFLECT_TYPENAME_CUSTOM(T, CUSTOM) \ + constexpr const char* get_type_name(const T*) { return BOOST_PP_STRINGIZE(CUSTOM); } + + constexpr const char* get_type_name(const bool*) { return "bool"; } + constexpr const char* get_type_name(const int8_t*) { return "int8"; } + constexpr const char* get_type_name(const uint8_t*) { return "uint8"; } + constexpr const char* get_type_name(const int16_t*) { return "int16"; } + constexpr const char* get_type_name(const uint16_t*) { return "uint16"; } + constexpr const char* get_type_name(const int32_t*) { return "int32"; } + constexpr const char* get_type_name(const uint32_t*) { return "uint32"; } + constexpr const char* get_type_name(const int64_t*) { return "int64"; } + constexpr const char* get_type_name(const uint64_t*) { return "uint64"; } + constexpr const char* get_type_name(const float*) { return "float32"; } + constexpr const char* get_type_name(const double*) { return "double"; } + constexpr const char* get_type_name(const char*) { return "char"; } + constexpr const char* get_type_name(const std::string*) { return "string"; } + constexpr const char* get_type_name(const __int128*) { return "int128"; } + constexpr const char* get_type_name(const unsigned __int128*) { return "uint128"; } + +#ifdef __eosio_cdt__ + constexpr const char* get_type_name(const long double*) { return "float128"; } +#endif + + template + constexpr std::array array_cat(std::array lhs, std::array rhs) + { + std::array result{}; + for (int i = 0; i < N; ++i) + { + result[i] = lhs[i]; + } + for (int i = 0; i < M; ++i) + { + result[i + N] = rhs[i]; + } + return result; + } + + template + constexpr std::array to_array(std::string_view s) + { + std::array result{}; + for (int i = 0; i < N; ++i) + { + result[i] = s[i]; + } + return result; + } + + template + constexpr auto append_type_name(const char (&suffix)[N]) + { + constexpr std::string_view name = get_type_name((T*)nullptr); + return array_cat(to_array(name), to_array({suffix, N})); + } + + template + constexpr auto vector_type_name = append_type_name("[]"); + + template + constexpr auto optional_type_name = append_type_name("?"); + + template + constexpr const char* get_type_name(const std::vector*) + { + return vector_type_name.data(); + } + + template + constexpr const char* get_type_name(const std::optional*) + { + return optional_type_name.data(); + } + + struct variant_type_appender + { + char* buf; + constexpr variant_type_appender operator+(std::string_view s) + { + *buf++ = '_'; + for (auto ch : s) + *buf++ = ch; + return *this; + } + }; + + template + constexpr auto get_variant_type_name() + { + constexpr std::size_t size = + sizeof("variant") + ((std::string_view(get_type_name((T*)nullptr)).size() + 1) + ...); + std::array buffer{'v', 'a', 'r', 'i', 'a', 'n', 't'}; + (variant_type_appender{buffer.data() + 7} + ... + + std::string_view(get_type_name((T*)nullptr))); + buffer[buffer.size() - 1] = '\0'; + return buffer; + } + + template + constexpr auto get_tuple_type_name() + { + constexpr std::size_t size = + sizeof("tuple") + ((std::string_view(get_type_name((T*)nullptr)).size() + 1) + ...); + std::array buffer{'t', 'u', 'p', 'l', 'e'}; + (variant_type_appender{buffer.data() + 5} + ... + + std::string_view(get_type_name((T*)nullptr))); + buffer[buffer.size() - 1] = '\0'; + return buffer; + } + + template + constexpr auto variant_type_name = get_variant_type_name(); + + template + constexpr const char* get_type_name(const std::variant*) + { + return variant_type_name.data(); + } + + template + constexpr const char* get_type_name() + { + return get_type_name((const T*)nullptr); + } + + [[nodiscard]] inline constexpr bool char_to_name_digit_strict(char c, uint64_t& result) + { + if (c >= 'a' && c <= 'z') + { + result = (c - 'a') + 6; + return true; + } + if (c >= '1' && c <= '5') + { + result = (c - '1') + 1; + return true; + } + if (c == '.') + { + result = 0; + return true; + } + return false; + } + + [[nodiscard]] inline constexpr bool string_to_name_strict(uint64_t& name, std::string_view str) + { + name = 0; + unsigned i = 0; + for (; i < str.size() && i < 12; ++i) + { + uint64_t x = 0; + // - this is not safe in const expression OUTCOME_TRY(char_to_name_digit_strict(str[i], x)); + auto r = char_to_name_digit_strict(str[i], x); + if (!r) + return false; + name |= (x & 0x1f) << (64 - 5 * (i + 1)); + } + if (i < str.size() && i == 12) + { + uint64_t x = 0; + // - this is not safe in const expression OUTCOME_TRY(char_to_name_digit_strict(str[i], x)); + auto r = char_to_name_digit_strict(str[i], x); + if (!r) + return false; + + if (x != (x & 0xf)) + return false; + name |= x; + ++i; + } + if (i < str.size()) + return false; + return true; + } + + inline constexpr uint64_t hash_name(std::string_view str) + { + uint64_t r = 0; + if (not string_to_name_strict(r, str)) + return murmur64(str.data(), str.size()); + return r; + } + + template + constexpr uint64_t get_type_hashname() + { + return hash_name(get_type_name()); + } + +} // namespace clio diff --git a/libraries/clio/include/clio/gql.hpp b/libraries/clio/include/clio/gql.hpp new file mode 100644 index 000000000..0e17afca7 --- /dev/null +++ b/libraries/clio/include/clio/gql.hpp @@ -0,0 +1,103 @@ +#pragma once +#include +#include + +#include +#include + +namespace clio +{ + /** + * The concept of our graph-ql like interface is that a graph-ql query is + * parsed and converted into a gql query object and then flattened into something + * that requires no unpacking inorder to dispatch. This means converting gql strings + * into base32 integer representations and/or hashes if he name does not fit in + * base32. + * + * We use base32 (eg eosio::name) based 64bit integers as the key values because they + * map to swtich(key) statements for high performance dispatch by our resolvers. + */ + namespace gql + { + struct null_t + { + }; + struct entry; + struct scalar; + using object = std::vector; + using array = std::vector; + + /** + * The scalar is the base value type, it is effectively a JSON object that uses + * base32 names for object keys and supports int64_t and uint64_t types in addition + * to double, string, object, array, and null. + * + * In principle you can convert from JSON to Scalar without any schema, and if you know + * the original keys for your objects, you can query the scalar for the value. In rare + * circumstances two keys may result in a hash collision, this should be detected at compile + * time and can be resolved by changing a field name. + * + * The flat view of a scalar is the same as a variant: + * [char_type][uint64_value] inline. If the type is a string,object,or array then the + * value field is a offset_ptr<> + * + */ + class scalar + { + public: + scalar() {} + + template + scalar(const T& v) : value(v) + { + } + + using value_type = + std::variant; + + value_type value; + }; + using scalar_value = scalar::value_type; + CLIO_REFLECT_TYPENAME_CUSTOM(null_t, null) + CLIO_REFLECT_TYPENAME_CUSTOM(scalar_value, scalar) + + /** + * The key/value pair of an object + */ + struct entry + { + uint64_t key; + scalar value; + }; + + struct query; + struct query_filter; + + /** + * Defines what fields to select from the result of a query, + * if type is non-zero then the filter only applies if the type + * of the return value matches the variant type. + */ + struct query_filter + { + std::vector filter; + /// the null type matches everything + uint64_t type = 0; + }; + + struct query + { + uint64_t key; + uint64_t alias; + object args; + /// for each type... define the filter... + std::vector filter; + }; + CLIO_REFLECT(scalar, value) + CLIO_REFLECT(entry, key, value) + CLIO_REFLECT(query_filter, filter, type) + CLIO_REFLECT(query, key, alias, args, filter) + + } // namespace gql + +} // namespace clio diff --git a/libraries/clio/include/clio/json/any.hpp b/libraries/clio/include/clio/json/any.hpp new file mode 100644 index 000000000..8f9f6587e --- /dev/null +++ b/libraries/clio/include/clio/json/any.hpp @@ -0,0 +1,232 @@ +#pragma once +#include +#include +#include + +namespace clio +{ + namespace json + { + struct any; + struct entry; + struct null_t + { + }; // uint8_t none = 0; }; + struct error_t + { + std::string what; + }; + + using any_object = std::vector; + using any_array = std::vector; + + struct any + { + public: + any(){}; + template + any(const T& v) : _value(v) + { + } + + template + const T* get_if() const + { + return std::get_if(&_value); + } + + template + const T& as() const + { + if (auto p = std::get_if(&_value)) + return *p; + + assert(!"invalid type cast from any"); + // static T dummy; /// used to prevent warnings about no return + // return dummy; + } + template + T& as() + { + if (auto p = std::get_if(&_value)) + return *p; + + assert(!"invalid type cast from any"); + __builtin_unreachable(); + } + + const auto& value() const { return _value; } + auto& value() { return _value; } + + inline any operator[](const std::string_view& key) const; + any operator[](uint32_t index) const + { + if (auto p = get_if()) + { + if (index >= p->size()) + return error_t{"index out of bounds"}; + return p->at(index); + } + return error_t{"not an array"}; + } + + template + void visit(Lambda&& l) const + { + std::visit(std::forward(l), _value); + } + + private: + std::variant + _value; + + template + friend void to_bin(const any& obj, S& stream) + { + to_bin(obj._value, stream); + } + template + friend void from_bin(any& obj, S& stream) + { + from_bin(obj._value, stream); + } + }; + + struct entry + { + std::string key; + any value; + }; + CLIO_REFLECT(entry, key, value) + CLIO_REFLECT_TYPENAME(null_t) + CLIO_REFLECT(error_t, what) + + inline any any::operator[](const std::string_view& key) const + { + if (auto p = get_if()) + { + for (const auto& i : *p) + { + if (i.key == key) + return i.value; + } + } + return error_t{"not an object"}; + } + + template + void to_json(const any& obj, S& stream) + { + std::visit( + [&](const auto& val) { + using T = std::decay_t; + if constexpr (std::is_same_v) + { + stream.write("null", 4); + } + else if constexpr (std::is_same_v) + { + stream.write('{'); + int size = val.size(); + for (const auto& e : val) + { + if (size == val.size()) + { + increase_indent(stream); + } + write_newline(stream); + to_json(e.key, stream); + write_colon(stream); + to_json(e.value, stream); + if (--size) + { + stream.write(','); + } + } + if (!val.empty()) + { + decrease_indent(stream); + write_newline(stream); + } + stream.write('}'); + } + else + { + to_json(val, stream); + } + }, + obj.value()); + } + + template + void from_json(any& obj, S& stream); + + template + void from_json(any_object& obj, S& stream) + { + stream.get_start_object(); + while (true) + { + auto t = stream.peek_token(); + if (t.get().type == json_token_type::type_end_object) + break; + auto k = stream.get_key(); + obj.push_back({.key = std::string(k)}); + from_json(obj.back().value, stream); + } + return stream.get_end_object(); + } + + template + void from_json(any& obj, S& stream) + { + while (true) + { + auto t = stream.peek_token(); + switch (t.get().type) + { + case json_token_type::type_null: + stream.eat_token(); + obj = null_t{}; + return; + case json_token_type::type_bool: + obj = t.get().value_bool; + stream.eat_token(); + return; + case json_token_type::type_string: + obj = std::string(t.get().value_string); + stream.eat_token(); + return; + case json_token_type::type_start_object: + { + obj = any_object(); + from_json(obj.as(), stream); + return; + } + case json_token_type::type_start_array: + { + obj = any_array(); + from_json(obj.as(), stream); + return; + } + case json_token_type::type_end_array: + case json_token_type::type_end_object: + case json_token_type::type_key: + default: + stream.eat_token(); + throw_error(clio::from_json_error::value_invalid); + }; + } + } + + } // namespace json + +} // namespace clio diff --git a/libraries/clio/include/clio/member_proxy.hpp b/libraries/clio/include/clio/member_proxy.hpp new file mode 100644 index 000000000..6e080daba --- /dev/null +++ b/libraries/clio/include/clio/member_proxy.hpp @@ -0,0 +1,107 @@ +#pragma once + +namespace clio +{ + template + struct member_proxy + { + /** + * This object is created on a type created by the macro, I represents the member number in + * the containing type. The containing type's first member is a ProxyObject. This function + * does the pointer math necessary to find the ProxyObject. + * + * Alternatively the macro code would have to initialize every member_proxy with this, which + * would bloat the size of the member_proxy object + * + * flat_view => reflect::member_proxy + * + * struct member_proxy { + * proxy_impl proxy___; + * member_proxy<0, ptr, proxy_impl> member_zero + * member_proxy<1, ptr, proxy_impl> member_one + * member_proxy<2, ptr, proxy_impl> member_two + * } + * a packed flat buffer. + * + * Let char* buf = point to a flat buffer; + * Let reinterpet buf as memper_proxy*, this makes the address of proxy__ equal + * to the address of member_proxy because it is the first element. + * + * because member_proxy has no values it takes 1 byte in member_proxy and the value of that + * byte is never read by member_proxy... member_proxy always gets the address of proxy___ and + * then does offset math. + * + */ + constexpr auto proxyptr() const + { + return (reinterpret_cast(reinterpret_cast(this) - + sizeof(*this) * (I + 1))); + } + constexpr auto proxyptr() + { + return (reinterpret_cast(reinterpret_cast(this) - + sizeof(*this) * (I + 1))); + } + constexpr const auto& get() const { return *(proxyptr()->template get()); } + constexpr auto& get() { return *(proxyptr()->template get()); } + + template + constexpr auto operator()(Ts&&... args) + { + return proxyptr()->template call(std::forward(args)...); + } + template + constexpr auto operator()(Ts&&... args) const + { + return proxyptr()->template call(std::forward(args)...); + } + + constexpr auto operator->() { return (proxyptr()->template get()); } + constexpr const auto operator->() const + { + return (proxyptr()->template get()); + } + + constexpr auto& operator*() { return get(); } + constexpr const auto& operator*() const { return get(); } + + template + constexpr auto& operator[](T&& k) + { + return get()[std::forward(k)]; + } + + template + constexpr const auto& operator[](T&& k) const + { + return get()[std::forward(k)]; + } + + template + friend S& operator<<(S& stream, const member_proxy& member) + { + return stream << member.get(); + } + + template + auto operator=(R&& r) + { + get() = std::forward(r); + return *this; + } + + template + operator R() const + { + return get(); + } + + /* + operator decltype( ((ProxyObject*)nullptr)->template get())() + { return get(); } + operator decltype( ((const ProxyObject*)nullptr)->template get()) ()const + { return get(); } + */ + }; + +} // namespace clio diff --git a/libraries/clio/include/clio/murmur.hpp b/libraries/clio/include/clio/murmur.hpp new file mode 100644 index 000000000..df273e19e --- /dev/null +++ b/libraries/clio/include/clio/murmur.hpp @@ -0,0 +1,58 @@ +#pragma once +namespace clio +{ + namespace + { + inline constexpr uint64_t unaligned_load(const char* p) + { + uint64_t r = 0; + for (uint32_t i = 0; i < 8; ++i) + { + r |= p[i]; + r <<= 8; + } + return r; + } + + // Loads n bytes, where 1 <= n < 8. + inline constexpr uint64_t load_bytes(const char* p, int n) + { + uint64_t result = 0; + --n; + do + result = (result << 8) + (unsigned char)(p[n]); + while (--n >= 0); + return result; + } + + inline constexpr uint64_t shift_mix(std::uint64_t v) { return v ^ (v >> 47); } + } // namespace + + // Implementation of Murmur hash for 64-bit size_t. + inline constexpr uint64_t murmur64(const char* ptr, uint64_t len, uint64_t seed = 0xbadd00d00) + { + const uint64_t mul = (((uint64_t)0xc6a4a793UL) << 32UL) + (uint64_t)0x5bd1e995UL; + const char* const buf = ptr; + + // Remove the bytes not divisible by the sizeof(uint64_t). This + // allows the main loop to process the data as 64-bit integers. + const uint64_t len_aligned = len & ~(uint64_t)0x7; + const char* const end = buf + len_aligned; + uint64_t hash = seed ^ (len * mul); + for (const char* p = buf; p != end; p += 8) + { + const uint64_t data = shift_mix(unaligned_load(p) * mul) * mul; + hash ^= data; + hash *= mul; + } + if ((len & 0x7) != 0) + { + const uint64_t data = load_bytes(end, len & 0x7); + hash ^= data; + hash *= mul; + } + hash = shift_mix(hash) * mul; + hash = shift_mix(hash); + return hash; + } +} // namespace clio diff --git a/libraries/clio/include/clio/name.hpp b/libraries/clio/include/clio/name.hpp new file mode 100644 index 000000000..724953c2f --- /dev/null +++ b/libraries/clio/include/clio/name.hpp @@ -0,0 +1,236 @@ +#pragma once +#include +#include +#include + +namespace clio +{ + inline std::string name_to_string(uint64_t name) + { + static const char* charmap = ".12345abcdefghijklmnopqrstuvwxyz"; + std::string str(13, '.'); + + uint64_t tmp = name; + for (uint32_t i = 0; i <= 12; ++i) + { + char c = charmap[tmp & (i == 0 ? 0x0f : 0x1f)]; + str[12 - i] = c; + tmp >>= (i == 0 ? 4 : 5); + } + + const auto last = str.find_last_not_of('.'); + return str.substr(0, last + 1); + } + + inline constexpr uint64_t char_to_name_digit(char c) + { + if (c >= 'a' && c <= 'z') + return (c - 'a') + 6; + if (c >= '1' && c <= '5') + return (c - '1') + 1; + return 0; + } + + inline constexpr uint64_t string_to_name(const char* str, int size) + { + uint64_t name = 0; + int i = 0; + for (; i < size && i < 12; ++i) + name |= (char_to_name_digit(str[i]) & 0x1f) << (64 - 5 * (i + 1)); + if (i < size) + name |= char_to_name_digit(str[i]) & 0x0F; + return name; + } + + inline constexpr uint64_t string_to_name(const char* str) + { + int len = 0; + while (str[len]) + ++len; + return string_to_name(str, len); + } + + inline constexpr uint64_t string_to_name(std::string_view str) + { + return string_to_name(str.data(), str.size()); + } + + struct name + { + enum class raw : uint64_t + { + }; + uint64_t value = 0; + + constexpr name() = default; + constexpr explicit name(uint64_t value) : value{value} {} + constexpr explicit name(name::raw value) : value{static_cast(value)} {} + constexpr explicit name(std::string_view str) + { + if (not string_to_name_strict(value, str)) + { + // TODO: throw_error + } + } + + constexpr name(const name&) = default; + + constexpr operator raw() const { return static_cast(value); } + explicit operator std::string() const { return name_to_string(value); } + std::string to_string() const { return std::string(*this); } + /** + * Explicit cast to bool of the uint64_t value of the name + * + * @return Returns true if the name is set to the default value of 0 else true. + */ + constexpr explicit operator bool() const { return value != 0; } + + /** + * Converts a %name Base32 symbol into its corresponding value + * + * @param c - Character to be converted + * @return constexpr char - Converted value + */ + static constexpr uint8_t char_to_value(char c) + { + if (c == '.') + return 0; + else if (c >= '1' && c <= '5') + return (c - '1') + 1; + else if (c >= 'a' && c <= 'z') + return (c - 'a') + 6; + return 0; // control flow will never reach here; just added to suppress warning + } + + /** + * Returns the length of the %name + */ + constexpr uint8_t length() const + { + constexpr uint64_t mask = 0xF800000000000000ull; + + if (value == 0) + return 0; + + uint8_t l = 0; + uint8_t i = 0; + for (auto v = value; i < 13; ++i, v <<= 5) + { + if ((v & mask) > 0) + { + l = i; + } + } + + return l + 1; + } + + /** + * Returns the suffix of the %name + */ + constexpr name suffix() const + { + uint32_t remaining_bits_after_last_actual_dot = 0; + uint32_t tmp = 0; + for (int32_t remaining_bits = 59; remaining_bits >= 4; remaining_bits -= 5) + { // Note: remaining_bits must remain signed integer + // Get characters one-by-one in name in order from left to right (not including the 13th + // character) + auto c = (value >> remaining_bits) & 0x1Full; + if (!c) + { // if this character is a dot + tmp = static_cast(remaining_bits); + } + else + { // if this character is not a dot + remaining_bits_after_last_actual_dot = tmp; + } + } + + uint64_t thirteenth_character = value & 0x0Full; + if (thirteenth_character) + { // if 13th character is not a dot + remaining_bits_after_last_actual_dot = tmp; + } + + if (remaining_bits_after_last_actual_dot == + 0) // there is no actual dot in the %name other than potentially leading dots + return name{value}; + + // At this point remaining_bits_after_last_actual_dot has to be within the range of 4 to 59 + // (and restricted to increments of 5). + + // Mask for remaining bits corresponding to characters after last actual dot, except for 4 + // least significant bits (corresponds to 13th character). + uint64_t mask = (1ull << remaining_bits_after_last_actual_dot) - 16; + uint32_t shift = 64 - remaining_bits_after_last_actual_dot; + + return name{((value & mask) << shift) + (thirteenth_character << (shift - 1))}; + } + + /** + * Returns the prefix of the %name + */ + constexpr name prefix() const + { + uint64_t result = value; + bool not_dot_character_seen = false; + uint64_t mask = 0xFull; + + // Get characters one-by-one in name in order from right to left + for (int32_t offset = 0; offset <= 59;) + { + auto c = (value >> offset) & mask; + + if (!c) + { // if this character is a dot + if (not_dot_character_seen) + { // we found the rightmost dot character + result = (value >> offset) << offset; + break; + } + } + else + { + not_dot_character_seen = true; + } + + if (offset == 0) + { + offset += 4; + mask = 0x1Full; + } + else + { + offset += 5; + } + } + + return name{result}; + } + }; + CLIO_REFLECT(name, value) + + template + void from_json(name& obj, S& stream) + { + auto r = stream.get_string(); + obj = name(hash_name(r)); + } + + template + void to_json(const name& obj, S& stream) + { + to_json(name_to_string(obj.value), stream); + } + + inline namespace literals + { + inline constexpr name operator""_n(const char* s, size_t) + { + return name(std::string_view(s)); + } + inline constexpr name operator""_h(const char* s, size_t) { return name(hash_name(s)); } + } // namespace literals + +} // namespace clio diff --git a/libraries/clio/include/clio/powers.h b/libraries/clio/include/clio/powers.h new file mode 100644 index 000000000..c4e249592 --- /dev/null +++ b/libraries/clio/include/clio/powers.h @@ -0,0 +1,81 @@ + +/// From https://github.com/night-shift/fpconv +/// Boost Software License 1.0 +/// See accompanying license file + +#pragma once + +#include + +#define npowers 87 +#define steppowers 8 +#define firstpower -348 /* 10 ^ -348 */ + +#define expmax -32 +#define expmin -60 + +typedef struct Fp +{ + uint64_t frac; + int exp; +} Fp; + +static constexpr Fp powers_ten[] = { + {18054884314459144840U, -1220}, {13451937075301367670U, -1193}, {10022474136428063862U, -1166}, + {14934650266808366570U, -1140}, {11127181549972568877U, -1113}, {16580792590934885855U, -1087}, + {12353653155963782858U, -1060}, {18408377700990114895U, -1034}, {13715310171984221708U, -1007}, + {10218702384817765436U, -980}, {15227053142812498563U, -954}, {11345038669416679861U, -927}, + {16905424996341287883U, -901}, {12595523146049147757U, -874}, {9384396036005875287U, -847}, + {13983839803942852151U, -821}, {10418772551374772303U, -794}, {15525180923007089351U, -768}, + {11567161174868858868U, -741}, {17236413322193710309U, -715}, {12842128665889583758U, -688}, + {9568131466127621947U, -661}, {14257626930069360058U, -635}, {10622759856335341974U, -608}, + {15829145694278690180U, -582}, {11793632577567316726U, -555}, {17573882009934360870U, -529}, + {13093562431584567480U, -502}, {9755464219737475723U, -475}, {14536774485912137811U, -449}, + {10830740992659433045U, -422}, {16139061738043178685U, -396}, {12024538023802026127U, -369}, + {17917957937422433684U, -343}, {13349918974505688015U, -316}, {9946464728195732843U, -289}, + {14821387422376473014U, -263}, {11042794154864902060U, -236}, {16455045573212060422U, -210}, + {12259964326927110867U, -183}, {18268770466636286478U, -157}, {13611294676837538539U, -130}, + {10141204801825835212U, -103}, {15111572745182864684U, -77}, {11258999068426240000U, -50}, + {16777216000000000000U, -24}, {12500000000000000000U, 3}, {9313225746154785156U, 30}, + {13877787807814456755U, 56}, {10339757656912845936U, 83}, {15407439555097886824U, 109}, + {11479437019748901445U, 136}, {17105694144590052135U, 162}, {12744735289059618216U, 189}, + {9495567745759798747U, 216}, {14149498560666738074U, 242}, {10542197943230523224U, 269}, + {15709099088952724970U, 295}, {11704190886730495818U, 322}, {17440603504673385349U, 348}, + {12994262207056124023U, 375}, {9681479787123295682U, 402}, {14426529090290212157U, 428}, + {10748601772107342003U, 455}, {16016664761464807395U, 481}, {11933345169920330789U, 508}, + {17782069995880619868U, 534}, {13248674568444952270U, 561}, {9871031767461413346U, 588}, + {14708983551653345445U, 614}, {10959046745042015199U, 641}, {16330252207878254650U, 667}, + {12166986024289022870U, 694}, {18130221999122236476U, 720}, {13508068024458167312U, 747}, + {10064294952495520794U, 774}, {14996968138956309548U, 800}, {11173611982879273257U, 827}, + {16649979327439178909U, 853}, {12405201291620119593U, 880}, {9242595204427927429U, 907}, + {13772540099066387757U, 933}, {10261342003245940623U, 960}, {15290591125556738113U, 986}, + {11392378155556871081U, 1013}, {16975966327722178521U, 1039}, {12648080533535911531U, 1066}}; + +static inline constexpr Fp find_cachedpow10(int exp, int* k) +{ + const double one_log_ten = 0.30102999566398114; + + int approx = -(exp + npowers) * one_log_ten; + int idx = (approx - firstpower) / steppowers; + + while (1) + { + int current = exp + powers_ten[idx].exp + 64; + + if (current < expmin) + { + idx++; + continue; + } + + if (current > expmax) + { + idx--; + continue; + } + + *k = (firstpower + idx * steppowers); + + return powers_ten[idx]; + } +} diff --git a/libraries/clio/include/clio/protobuf/any.hpp b/libraries/clio/include/clio/protobuf/any.hpp new file mode 100644 index 000000000..43b7a02c8 --- /dev/null +++ b/libraries/clio/include/clio/protobuf/any.hpp @@ -0,0 +1,149 @@ +#pragma once +#include +#include +#include + +namespace clio +{ + namespace protobuf + { + enum wire_type_enum + { + varint = 0, + fixed64 = 1, + buffer = 2, + fixed32 = 5 + }; + + struct entry + { + typedef std::variant value_type; + + entry() {} + template + entry(uint32_t n, T&& v) : number(n), value(std::forward(v)) + { + } + + uint32_t number; + value_type value; + }; + CLIO_REFLECT(entry, number, value) + + /** + * This class can be used to hold the + * deserialized contents of any protobuf stream + */ + struct any + { + std::vector members; + + void add(uint32_t field, const std::string& s) + { + members.push_back(entry{field, bytes{std::vector(s.begin(), s.end())}}); + } + template ::value> > + void add(uint32_t field, const std::vector& s) + { + members.push_back(entry{ + field, bytes{std::vector((char*)s.data(), (char*)(s.data() + s.size()))}}); + } + void add(uint32_t field, std::vector&& s) + { + members.push_back(entry{field, bytes{std::move(s)}}); + } + void add(uint32_t field, varuint32 s) { members.push_back(entry{field, s}); } + void add(uint32_t field, uint64_t s) + { + members.push_back(entry{field, static_cast(s)}); + } + void add(uint32_t field, int64_t s) { members.push_back(entry{field, s}); } + void add(uint32_t field, int32_t s) { members.push_back(entry{field, s}); } + void add(uint32_t field, int16_t s) { members.push_back(entry{field, varuint32(s)}); } + void add(uint32_t field, uint16_t s) { members.push_back(entry{field, varuint32(s)}); } + void add(uint32_t field, int8_t s) { members.push_back(entry{field, varuint32(s)}); } + void add(uint32_t field, uint8_t s) { members.push_back(entry{field, varuint32(s)}); } + void add(uint32_t field, bool s) { members.push_back(entry{field, varuint32(s)}); } + void add(uint32_t field, char s) { members.push_back(entry{field, varuint32(s)}); } + void add(uint32_t field, double s) + { + members.push_back(entry{field, *reinterpret_cast(&s)}); + } + void add(uint32_t field, float s) + { + members.push_back(entry{field, *reinterpret_cast(&s)}); + } + }; + CLIO_REFLECT(any, members) + + template + void to_bin(const any& a, Stream& s) + { + for (const auto& e : a.members) + { + uint32_t key = e.number << 3; + key |= e.value.index() == 3 ? wire_type_enum::fixed32 : e.value.index(); + varuint32_to_bin(key, s); + + std::visit([&](const auto& v) { to_bin(v, s); }, e.value); + } + } + + template + void from_bin(any& a, Stream& s) + { + while (s.remaining()) + { + uint32_t key = 0; + varuint32_from_bin(key, s); + wire_type_enum type = wire_type_enum(uint8_t(key) & 0x07); + uint32_t number = key >> 3; + + switch (type) + { + case wire_type_enum::varint: + { + varuint32 val; + from_bin(val, s); + a.members.push_back({number, val}); + } + break; + case wire_type_enum::fixed64: + { + int64_t val; + from_bin(val, s); + a.members.push_back({number, val}); + } + break; + case wire_type_enum::buffer: + { + bytes val; + from_bin(val, s); + a.members.emplace_back(number, std::move(val)); + } + break; + case wire_type_enum::fixed32: + { + int32_t val; + from_bin(val, s); + a.members.push_back({number, val}); + } + break; + } + } + } + + } // namespace protobuf + + template + void to_protobuf(const protobuf::any& a, Stream& s) + { + to_bin(a, s); + } + template + void from_protobuf(protobuf::any& a, Stream& s) + { + from_bin(a, s); + } + +} // namespace clio diff --git a/libraries/clio/include/clio/protobuf/json.hpp b/libraries/clio/include/clio/protobuf/json.hpp new file mode 100644 index 000000000..f9ca75051 --- /dev/null +++ b/libraries/clio/include/clio/protobuf/json.hpp @@ -0,0 +1,203 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace clio +{ + namespace protobuf + { + struct json_field_query; + struct json_query + { + std::vector fields; + }; + struct json_field_query + { + std::string name; + json::any args; /// args as json encoded tuple, or named object + json_query filter; /// apply to result + }; + CLIO_REFLECT(json_field_query, name, args, filter) + CLIO_REFLECT(json_query, fields) + + /** + * Given a json description of the query and an assumed reflected type (used to interpret + * names to numbers), this function will convert the JSON-query into a protobuf-style query. + */ + template + protobuf::query from_json_query(const json_query& jq) + { + protobuf::query result; + result.fields.reserve(jq.fields.size()); + + if constexpr (reflect::is_struct) + { + for (const auto& field : jq.fields) + { + reflect::for_each([&](const meta& ref, auto mptr) { + if constexpr (not std::is_member_function_pointer_v) + { + if (ref.name == field.name) + { + if (field.filter.fields.size()) + { + result.fields.push_back( + {.number = ref.number, + .filter = from_json_query< + std::decay_t(nullptr)->*mptr)>>( + field.filter)}); + } + else + { + result.fields.push_back(field_query{ref.number}); + } + } + } + else + { /// member function ptr + if (ref.name == field.name) + { + /// TODO: do a direct conversion from json::any to pb bytes instead of + /// any->json_string->native->pb + using args_tuple_type = decltype(args_as_tuple(mptr)); + std::string json = convert_to_json(field.args); + auto args_value = clio::from_json(std::move(json)); + + if (field.filter.fields.size()) + { + result.fields.push_back( + {.number = ref.number, + .args = to_protobuf(args_value), + .filter = + from_json_query(field.filter)}); + } + else + { + result.fields.push_back( + field_query{.number = ref.number, .args = to_protobuf(args_value)}); + } + } + } + }); + } + } + return result; + } + + /** + * Given a protobuf query and a type, convert it into a friendly JSON Query + */ + template + json_query to_json_query(const protobuf::query& pbuf) + { + json_query result; + + if constexpr (reflect::is_struct) + { + for (const auto& entry : pbuf.fields) + { + reflect::for_each([&](const meta& ref, auto mptr) { + if constexpr (not std::is_member_function_pointer_v) + { + if (ref.number == entry.number.value) + { + if (entry.filter.fields.size()) + { + result.fields.push_back( + {.name = ref.name, + .filter = to_json_query< + std::decay_t(nullptr)->*mptr)>>( + entry.filter)}); + } + else + { + result.fields.push_back({ref.name}); + } + } + } + else + { /// member function ptr + if (ref.number == entry.number.value) + { + /// TODO: do a direct conversion from PB to json::any instead of + /// pb->native->json_string->any + using args_tuple_type = decltype(args_as_tuple(mptr)); + auto pbargs = from_protobuf(entry.args.data); + auto jargs = to_json(pbargs); + + if (entry.filter.fields.size()) + { + result.fields.push_back( + {.name = ref.name, + .args = from_json(jargs), + .filter = to_json_query(entry.filter)}); + } + else + { + result.fields.push_back( + {.name = ref.name, .args = from_json(jargs)}); + } + } + } + }); + } + } + return result; + } + + } // namespace protobuf + template + void to_json(const protobuf::json_query& s, S& stream) + { + to_json(s.fields, stream); + } + template + void from_json(protobuf::json_query& s, S& stream) + { + from_json(s.fields, stream); + } + template + void to_json(const protobuf::json_field_query& s, S& stream) + { + stream.write('{'); + increase_indent(stream); + write_newline(stream); + stream.write("\"name\""); + write_colon(stream); + to_json(s.name, stream); + if (auto* a = s.args.get_if()) + { + if (a->size()) + { + stream.write(','); + write_newline(stream); + stream.write("\"args\""); + write_colon(stream); + to_json(s.args, stream); + } + } + else if (not s.args.get_if()) + { + stream.write(','); + write_newline(stream); + stream.write("\"args\""); + write_colon(stream); + to_json(s.args, stream); + } + if (s.filter.fields.size()) + { + stream.write(','); + write_newline(stream); + stream.write("\"filter\""); + write_colon(stream); + to_json(s.filter, stream); + } + decrease_indent(stream); + write_newline(stream); + stream.write('}'); + } + +} // namespace clio diff --git a/libraries/clio/include/clio/protobuf/query.hpp b/libraries/clio/include/clio/protobuf/query.hpp new file mode 100644 index 000000000..a26719a21 --- /dev/null +++ b/libraries/clio/include/clio/protobuf/query.hpp @@ -0,0 +1,232 @@ +#pragma once +#include +#include + +namespace clio +{ + namespace protobuf + { + struct field_query; + struct variant_query; + + struct variant_query + { + varuint32 type; + std::vector fields; + }; + + struct query + { + std::vector fields; + std::vector + variant_fields; /// only apply these fields when the type has them + }; + + struct field_query + { + varuint32 number; + bytes args; /// args as protobuf encoded tuple + query filter; /// apply to result + }; + + CLIO_REFLECT(variant_query, type, fields) + CLIO_REFLECT(field_query, number, args, filter) + CLIO_REFLECT(query, fields, variant_fields) + + template + any dispatch(T&& obj, const query& q) + { + any result; + result.members.reserve(q.fields.size()); + + for (const auto& field : q.fields) + { + reflect::get(int64_t(field.number), [&](auto mptr) { + using member_ptr_type = decltype(mptr); + if constexpr (std::is_member_function_pointer_v) + { + using return_type = decltype(result_of(mptr)); + using param_type = decltype(args_as_tuple(mptr)); + // std::cerr<< boost::core::demangle(typeid(param_type).name()) <<"\n"; + + param_type params; + input_stream in(field.args.data.data(), field.args.data.size()); + (void)from_protobuf_object(params, in); + std::apply( + [&](auto... args) { + if constexpr (is_std_variant::value) + { + /// TODO: + } + else if constexpr (not reflect::is_struct) + result.add(field.number, (obj.*mptr)(args...)); + else + { + any field_data = dispatch((obj.*mptr)(args...), field.filter); + result.add(field.number, clio::to_bin(field_data)); + } + }, + params); + } + else if constexpr (not std::is_member_function_pointer_v) + { + using member_type = std::decay_t; + + if constexpr (is_std_tuple::value) + { + /// TODO: + } + else if constexpr (is_std_variant::value) + { + /// TODO: + } + else if constexpr (is_std_vector::value) + { + using value_type = typename is_std_vector::value_type; + if constexpr (is_std_variant::value) + { + /// TODO: + } + else if constexpr (std::is_arithmetic_v) + { + result.add(field.number, obj.*mptr); + } + else if constexpr (reflect::is_struct) + { + for (const auto& item : obj.*mptr) + { + any field_data = dispatch(item, field.filter); + result.add(field.number, clio::to_bin(field_data)); + } + } + else + { + for (const auto& item : obj.*mptr) + { + result.add(field.number, item); + } + } + } + else if constexpr (not reflect::is_struct) + { + result.add(field.number, obj.*mptr); + } + else + { + any field_data = dispatch(obj.*mptr, field.filter); + result.add(field.number, clio::to_bin(field_data)); + } + } + }); + } + return result; + } + + struct query_proxy + { + protobuf::query q; + + template + query_proxy& call(Args&&... args) + { + return *this; + } + template + void get() + { + } + + template + void get() const + { + } + + template + query_proxy& call(const meta& ref, Mptr mptr, Filter&& filter) + { + using result_type = decltype(result_of(mptr)); + using args_tuple = decltype(args_as_tuple(mptr)); + + if constexpr (std::tuple_size::value == 1) + { + auto params = args_tuple(std::forward(filter)); + q.fields.push_back({ + .number = ref.number, + .args = to_protobuf(params), + }); + } + else + { + typename reflect::template proxy rp; + filter(rp); + q.fields.push_back({.number = ref.number, .filter = std::move(rp->q)}); + } + return *this; + } + + template + query_proxy& call(const meta& ref, Mptr mptr) + { + q.fields.push_back({ + .number = ref.number, + }); + return *this; + } + + template + query_proxy& call(const meta& ref, Mptr mptr, Filter&& filter, Args&&... args) + { + using result_type = decltype(result_of(mptr)); + using args_tuple = decltype(args_as_tuple(mptr)); + /** test to see whether the user passed a filter by comparing the number of args, + * if the number of var args equals the number of paramters then a filter was passed, + * otherwise the Filter is being interpreted as the first argument. This means that in + * order to use a filter you must pass all args, if you don't want a filter on the + * result then you can skip trailing args. + */ + if constexpr (std::tuple_size::value == sizeof...(Args)) + { + auto params = args_tuple(std::forward(args)...); + + if constexpr (reflect::is_struct) + { + typename reflect::template proxy rp; + filter(rp); + q.fields.push_back({.number = ref.number, + .args = to_protobuf(params), + .filter = std::move(rp->q)}); + } + else + { + q.fields.push_back({ + .number = ref.number, + .args = to_protobuf(params), + }); + } + } + else + { + auto params = args_tuple(std::forward(filter), std::forward(args)...); + + if constexpr (reflect::is_struct) + { + typename reflect::template proxy rp; + q.fields.push_back({.number = ref.number, + .args = to_protobuf(params), + .filter = std::move(rp->q)}); + } + else + { + q.fields.push_back({ + .number = ref.number, + .args = to_protobuf(params), + }); + } + } + return *this; + } + }; + + } // namespace protobuf + +} // namespace clio diff --git a/libraries/clio/include/clio/protobuf/schema.hpp b/libraries/clio/include/clio/protobuf/schema.hpp new file mode 100644 index 000000000..09a332f40 --- /dev/null +++ b/libraries/clio/include/clio/protobuf/schema.hpp @@ -0,0 +1,238 @@ +#pragma once +#include + +namespace clio +{ + /** + * Writes a protobuf schem file to stream given a schema object + */ + template + void to_protobuf_schema(const schema& sch, S& stream) + { + stream.write("syntax = \"proto3\";\n"); + + auto convert_variant_name = [](string name) { + for (auto& c : name) + { + if (c == '|') + c = '_'; + if (c == '?') + c = 'O'; + if (c == '[') + c = '_'; + if (c == ']') + c = 'a'; + } + return "variant_" + name.substr(0, name.size() - 1); + }; + auto convert_tuple_name = [](string name) { + for (auto& c : name) + { + if (c == '&') + c = '_'; + if (c == '?') + c = 'O'; + if (c == '[') + c = '_'; + if (c == ']') + c = 'a'; + } + return "tuple_" + name.substr(0, name.size() - 1); + }; + + auto convert_map_name = [](string name) { + /* //TODO... + for( auto& c : name ) { + if( c == '&' ) c = '_'; + if( c == '?' ) c = 'O'; + if( c == '[' ) c = '_'; + if( c == ']' ) c = 'a'; + } + */ + return name; + }; + + auto decay_type = [&](const std::string& str, bool& is_repeated) -> std::string { + if (str.back() == ']') + { + is_repeated = true; + return str.substr(0, str.size() - 2); + } + else if (str.back() == '?') + return str.substr(0, str.size() - 1); + else if (str.back() == '|') + return convert_variant_name(str); + else if (str.back() == '&') + return convert_tuple_name(str); + else if (str.back() == '>') + return convert_map_name(str); + else + return str; + }; + + auto convert_to_pb_type = [&](const string& str, bool& is_repeated, + bool& is_prim) -> std::string { + auto decay = decay_type(str, is_repeated); + is_prim = true; + if (decay == "int16_t") + return "int32"; + else if (decay == "char") + return "int32"; + else if (decay == "int8_t") + return "int32"; + else if (decay == "uint16_t") + return "int32"; + else if (decay == "uint8_t") + return "int32"; + else if (decay == "int32_t") + return "sfixed32"; + else if (decay == "uint32_t") + return "fixed32"; + else if (decay == "int64_t") + return "sfixed64"; + else if (decay == "uint64_t") + return "fixed64"; + else if (decay == "bool") + return "bool"; + else if (decay == "double") + return decay; + else if (decay == "float") + return decay; + is_prim = false; + return decay; + }; + + for (const auto& item : sch.types) + { + std::visit( + [&](auto i) { + if constexpr (std::is_same_v) + { + stream.write("message ", 8); + stream.write(item.first); + stream.write(" {\n"); + for (auto mem : i.members) + { + bool is_repeated = false; + bool is_prim = false; + auto pbt = convert_to_pb_type(mem.type, is_repeated, is_prim); + stream.write(" "); + if (is_repeated) + { + stream.write("repeated "); + } + stream.write(pbt); + stream.write(" "); + stream.write(mem.name.c_str()); + stream.write(" = "); + stream.write(std::to_string(mem.number).c_str()); + if (is_repeated && is_prim) + { + stream.write(" [packed=true]"); + } + stream.write(";\n"); + } + /* + for( auto mem : i.methods ) { + bool is_repeated = false; + bool is_prim = false; + auto pbt = convert_to_pb_type( mem.return_type, is_repeated, is_prim ); + stream.write( " " ); + stream.write( "repeated " ); + stream.write( pbt ); + stream.write( " " ); + stream.write( mem.name.c_str() ); + stream.write( " = " ); + stream.write( std::to_string( mem.number ).c_str() ); + stream.write( ";\n" ); + }*/ + stream.write("}\n"); + } + else if constexpr (std::is_same_v) + { + stream.write("message "); + if (item.first.back() == '|') + { + stream.write(convert_variant_name(item.first)); + } + else + { + stream.write(item.first); + } + + stream.write(" {\n"); + stream.write(" oneof which {\n"); + + for (uint32_t idx = 0; idx < i.types.size(); ++idx) + { + bool is_repeated = false; + bool is_prim = false; + auto pbt = convert_to_pb_type(i.types[idx], is_repeated, is_prim); + + stream.write(" "); + stream.write(pbt); + stream.write(" "); + stream.write(pbt); + stream.write("_value = "); + stream.write(std::to_string(idx + 1)); + stream.write(";\n"); + } + stream.write(" }\n"); + stream.write("}\n"); + } + else if constexpr (std::is_same_v) + { + stream.write("message "); + + if (item.first.back() == '&') + { + stream.write(convert_tuple_name(item.first)); + } + else + { + stream.write(item.first); + } + + stream.write(" {\n"); + for (uint32_t idx = 0; idx < i.types.size(); ++idx) + { + stream.write(" "); + bool is_repeated = false; + bool is_prim = false; + auto pbt = convert_to_pb_type(i.types[idx], is_repeated, is_prim); + if (is_repeated) + { + stream.write("repeated "); + } + stream.write(pbt); + stream.write(" _"); + stream.write(std::to_string(idx + 1)); + stream.write(" = "); + stream.write(std::to_string(idx + 1)); + if (is_repeated && is_prim) + { + stream.write(" [packed=true]"); + } + stream.write(";\n"); + } + stream.write("}\n"); + } + }, + item.second); + } + } + inline std::string to_protobuf_schema(const schema& s) + { + size_stream ss; + to_protobuf_schema(s, ss); + + std::string result(ss.size, 0); + fixed_buf_stream fbs(result.data(), result.size()); + to_protobuf_schema(s, fbs); + + if (fbs.pos != fbs.end) + throw_error(stream_error::underrun); + + return result; + } +} // namespace clio diff --git a/libraries/clio/include/clio/reflect.hpp b/libraries/clio/include/clio/reflect.hpp new file mode 100644 index 000000000..cc7892526 --- /dev/null +++ b/libraries/clio/include/clio/reflect.hpp @@ -0,0 +1,443 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace clio +{ + template + std::tuple...> args_as_tuple(R (C::*)(Args...)); + template + std::tuple...> args_as_tuple(R (C::*)(Args...) const); + + template + R result_of(R (C::*)(Args...) const); + template + R result_of(R (C::*)(Args...)); + + template + constexpr R result_of_member(R(C::*)); + template + constexpr C class_of_member(R(C::*)); + + template + void result_of_member(R (C::*)(Args...) const); + template + void result_of_member(R (C::*)(Args...)); + + struct meta + { + const char* name; + int32_t number; + std::initializer_list param_names; + }; + +#define CLIO_REFLECT_ARGS_INTERNAL(r, OP, i, PARAM) BOOST_PP_COMMA_IF(i) BOOST_PP_STRINGIZE(PARAM) + +#define CLIO_REFLECT_ARGS_HELPER(METHOD, PARAM_NAMES) \ + BOOST_PP_SEQ_FOR_EACH_I(CLIO_REFLECT_ARGS_INTERNAL, METHOD, PARAM_NAMES) + +#define CLIO_REFLECT_FILTER_PARAMS(NAME, IDX, ...) \ + { \ + CLIO_REFLECT_ARGS_HELPER(METHOD, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + } +#define CLIO_REFLECT_FILTER_NAME(NAME, IDX, ...) NAME +#define CLIO_REFLECT_FILTER_NAME_STR(NAME, IDX, ...) BOOST_PP_STRINGIZE(NAME) +#define CLIO_REFLECT_FILTER_IDX(NAME, IDX, ...) IDX + +#define CLIO_REFLECT_FOREACH_PB_INTERNAL(r, OP, member) \ + lambda(clio::meta{.number = CLIO_REFLECT_FILTER_IDX member, \ + .name = CLIO_REFLECT_FILTER_NAME_STR member, \ + .param_names = CLIO_REFLECT_FILTER_PARAMS member}, \ + &OP::CLIO_REFLECT_FILTER_NAME member); + +#define CLIO_REFLECT_FOREACH_INTERNAL(r, OP, i, member) \ + (void)lambda(clio::meta{ \ + .name = BOOST_PP_STRINGIZE(member), .number = i + 1, \ + }, \ + &OP::member); + +#define CLIO_REFLECT_MEMBER_BY_STR_INTERNAL(r, OP, member) \ + if (BOOST_PP_STRINGIZE(member) == m) \ + { \ + (void)lambda(&OP::member); \ + return true; \ + } + +#define CLIO_REFLECT_MEMBER_BY_STR_PB_INTERNAL(r, OP, member) \ + if (CLIO_REFLECT_FILTER_NAME_STR member == m) \ + { \ + (void)lambda(&OP::CLIO_REFLECT_FILTER_NAME member); \ + return true; \ + } + +#define CLIO_REFLECT_MEMBER_BY_IDX_PB_INTERNAL(r, OP, member) \ + case CLIO_REFLECT_FILTER_IDX member: \ + (void)lambda(&OP::CLIO_REFLECT_FILTER_NAME member); \ + return true; + +#define CLIO_REFLECT_PROXY_MEMBER_BY_IDX_INTERNAL(r, OP, I, member) \ + template \ + auto member(Args&&... args) \ + { \ + return proxy___.call(clio::meta{.number = I + 1, .name = BOOST_PP_STRINGIZE(member)}, \ + &OP::member, \ + std::forward(args)...); \ + } + +#define CLIO_REFLECT_MPROXY_MEMBER_BY_IDX_INTERNAL(r, OP, I, member) \ + clio::member_proxy \ + member; + +#define CLIO_REFLECT_MPROXY_MEMBER_BY_IDX_HELPER(QUERY_CLASS, MEMBER_IDXS) \ + BOOST_PP_SEQ_FOR_EACH_I(CLIO_REFLECT_MPROXY_MEMBER_BY_IDX_INTERNAL, QUERY_CLASS, MEMBER_IDXS) + +#define CLIO_REFLECT_PROXY_MEMBER_BY_PB_INTERNAL(r, OP, member) \ + template \ + auto CLIO_REFLECT_FILTER_NAME member(Args&&... args) \ + { \ + return proxy___.call(clio::meta{.number = CLIO_REFLECT_FILTER_IDX member, \ + .name = CLIO_REFLECT_FILTER_NAME_STR member, \ + .param_names = CLIO_REFLECT_FILTER_PARAMS member}, \ + &OP::CLIO_REFLECT_FILTER_NAME member, std::forward(args)...); \ + } + +#define CLIO_REFLECT_FOREACH_MEMBER_HELPER(QUERY_CLASS, MEMBERS) \ + BOOST_PP_SEQ_FOR_EACH_I(CLIO_REFLECT_FOREACH_INTERNAL, QUERY_CLASS, MEMBERS) + +#define CLIO_REFLECT_MEMBER_BY_STR_HELPER(QUERY_CLASS, MEMBERS) \ + BOOST_PP_SEQ_FOR_EACH(CLIO_REFLECT_MEMBER_BY_STR_INTERNAL, QUERY_CLASS, MEMBERS) + +#define CLIO_REFLECT_MEMBER_BY_IDX_I_INTERNAL(r, OP, I, member) \ + case I + 1: \ + (void)lambda(&OP::member); \ + return true; + +#define CLIO_REFLECT_MEMBER_BY_IDX_HELPER(QUERY_CLASS, MEMBER_IDXS) \ + BOOST_PP_SEQ_FOR_EACH_I(CLIO_REFLECT_MEMBER_BY_IDX_I_INTERNAL, QUERY_CLASS, MEMBER_IDXS) + +#define CLIO_REFLECT_MEMBER_BY_NAME_I_INTERNAL(r, OP, I, member) \ + case clio::hash_name(BOOST_PP_STRINGIZE(member)): \ + (void)lambda(&OP::member); \ + return true; + +#define CLIO_REFLECT_MEMBER_BY_NAME_HELPER(QUERY_CLASS, MEMBER_NAMES) \ + BOOST_PP_SEQ_FOR_EACH_I(CLIO_REFLECT_MEMBER_BY_NAME_I_INTERNAL, QUERY_CLASS, MEMBER_NAMES) + +#define CLIO_REFLECT_MEMBER_TYPE_BY_IDX_INTERNAL(r, OP, I, member) \ + BOOST_PP_COMMA_IF(I) std::decay_t + +#define CLIO_REFLECT_MEMBER_TYPE_IDX_HELPER(QUERY_CLASS, MEMBER_IDXS) \ + BOOST_PP_SEQ_FOR_EACH_I(CLIO_REFLECT_MEMBER_TYPE_BY_IDX_INTERNAL, QUERY_CLASS, MEMBER_IDXS) + +#define CLIO_REFLECT_FOREACH_MEMBER_PB_HELPER(QUERY_CLASS, MEMBERS) \ + BOOST_PP_SEQ_FOR_EACH(CLIO_REFLECT_FOREACH_PB_INTERNAL, QUERY_CLASS, MEMBERS) + +#define CLIO_REFLECT_MEMBER_INDEX_HELPER(QUERY_CLASS, MEMBER_IDXS) \ + BOOST_PP_SEQ_FOR_EACH_I(CLIO_REFLECT_MEMBER_PB_INTERNAL, QUERY_CLASS, MEMBER_IDXS) + +#define CLIO_REFLECT_MEMBER_BY_STR_PB_HELPER(QUERY_CLASS, MEMBER_IDXS) \ + BOOST_PP_SEQ_FOR_EACH(CLIO_REFLECT_MEMBER_BY_STR_PB_INTERNAL, QUERY_CLASS, MEMBER_IDXS) + +#define CLIO_REFLECT_MEMBER_BY_IDX_PB_HELPER(QUERY_CLASS, MEMBER_IDXS) \ + BOOST_PP_SEQ_FOR_EACH(CLIO_REFLECT_MEMBER_BY_IDX_PB_INTERNAL, QUERY_CLASS, MEMBER_IDXS) + +#define CLIO_REFLECT_PROXY_MEMBER_BY_IDX_HELPER(QUERY_CLASS, MEMBER_IDXS) \ + BOOST_PP_SEQ_FOR_EACH_I(CLIO_REFLECT_PROXY_MEMBER_BY_IDX_INTERNAL, QUERY_CLASS, MEMBER_IDXS) + +#define CLIO_REFLECT_PROXY_MEMBER_BY_PB_HELPER(QUERY_CLASS, MEMBER_IDXS) \ + BOOST_PP_SEQ_FOR_EACH(CLIO_REFLECT_PROXY_MEMBER_BY_PB_INTERNAL, QUERY_CLASS, MEMBER_IDXS) + +#define CLIO_REFLECT_PARAMS_BY_IDX_PB_INTERNAL(r, OP, i, member) \ + if constexpr (std::is_member_function_pointer_v) \ + return OP::BOOST_PP_CAT(CLIO_REFLECT_FILTER_NAME member, ___PARAM_NAMES); + +#define CLIO_REFLECT_PARAMS_BY_IDX_PB_HELPER(QUERY_CLASS, MEMBER_IDXS) \ + BOOST_PP_SEQ_FOR_EACH_I(CLIO_REFLECT_PARAMS_BY_IDX_PB_INTERNAL, QUERY_CLASS, MEMBER_IDXS) + + /* + template \ + struct proxy { \ + template \ + proxy( Args&&... args ):proxy___( std::forward(args)... ){}\ + ProxyObject* operator->(){ return &proxy___; } \ + const ProxyObject* operator->()const{ return &proxy___; } \ + ProxyObject& operator*(){ return proxy___; } \ + const ProxyObject& operator*()const{ return proxy___; } \ + CLIO_REFLECT_PROXY_MEMBER_BY_IDX_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + private: \ + ProxyObject proxy___; \ + }; \ + */ + +#define CLIO_REFLECT(QUERY_CLASS, ...) \ + CLIO_REFLECT_TYPENAME(QUERY_CLASS) \ + struct reflect_impl_##QUERY_CLASS \ + { \ + static constexpr bool is_defined = true; \ + static constexpr bool is_struct = true; \ + static inline constexpr const char* name() { return BOOST_PP_STRINGIZE(QUERY_CLASS); } \ + typedef std::tuple< \ + CLIO_REFLECT_MEMBER_TYPE_IDX_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__))> \ + struct_tuple_type; \ + template \ + constexpr inline static void for_each(L&& lambda) \ + { \ + CLIO_REFLECT_FOREACH_MEMBER_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + } \ + template \ + inline static bool get(const std::string_view& m, L&& lambda) \ + { \ + CLIO_REFLECT_MEMBER_BY_STR_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + return false; \ + } \ + template \ + inline static bool get(int64_t m, L&& lambda) \ + { \ + switch (m) \ + { \ + CLIO_REFLECT_MEMBER_BY_IDX_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + } \ + return false; \ + } \ + template \ + inline static bool get_by_name(uint64_t n, L&& lambda) \ + { \ + switch (n) \ + { \ + CLIO_REFLECT_MEMBER_BY_NAME_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + } \ + return false; \ + } \ + template \ + struct proxy \ + { \ + private: \ + ProxyObject proxy___; \ + \ + public: \ + template \ + proxy(Args&&... args) : proxy___(std::forward(args)...) \ + { \ + } \ + ProxyObject* operator->() { return &proxy___; } \ + const ProxyObject* operator->() const { return &proxy___; } \ + ProxyObject& operator*() { return proxy___; } \ + const ProxyObject& operator*() const { return proxy___; } \ + CLIO_REFLECT_MPROXY_MEMBER_BY_IDX_HELPER(QUERY_CLASS, \ + BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + }; \ + }; \ + reflect_impl_##QUERY_CLASS get_reflect_impl(const QUERY_CLASS&); + +#define CLIO_REFLECT_PB(QUERY_CLASS, ...) \ + CLIO_REFLECT_TYPENAME(QUERY_CLASS) \ + struct reflect_impl_##QUERY_CLASS \ + { \ + static constexpr bool is_defined = true; \ + static constexpr bool is_struct = true; \ + static inline constexpr const char* name() { return BOOST_PP_STRINGIZE(QUERY_CLASS); } \ + template \ + inline static void for_each(L&& lambda) \ + { \ + CLIO_REFLECT_FOREACH_MEMBER_PB_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + } \ + template \ + inline static bool get(const std::string_view& m, L&& lambda) \ + { \ + CLIO_REFLECT_MEMBER_BY_STR_PB_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + return false; \ + } \ + template \ + inline static bool get(int64_t m, L&& lambda) \ + { \ + switch (m) \ + { \ + CLIO_REFLECT_MEMBER_BY_IDX_PB_HELPER(QUERY_CLASS, \ + BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + } \ + return false; \ + } \ + }; \ + reflect_impl_##QUERY_CLASS get_reflect_impl(const QUERY_CLASS&); + +#define CLIO_REFLECT_TEMPLATE_OBJECT(QUERY_CLASS, TPARAM, ...) \ + template \ + struct reflect_impl_##QUERY_CLASS \ + { \ + static constexpr bool is_defined = true; \ + static constexpr bool is_struct = true; \ + static inline const char* name() \ + { \ + return BOOST_PP_STRINGIZE(QUERY_CLASS) "<" BOOST_PP_STRINGIZE(TPARAM) ">"; \ + } \ + template \ + inline static void for_each(L&& lambda) \ + { \ + CLIO_REFLECT_FOREACH_MEMBER_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + } \ + template \ + inline static void get(const std::string_view& m, L&& lambda) \ + { \ + CLIO_REFLECT_MEMBER_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + return false; \ + } \ + template \ + inline static void get(int64_t m, L&& lambda) \ + { \ + switch (m) \ + { \ + CLIO_REFLECT_MEMBER_INDEX_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + } \ + return false; \ + } \ + using tuple_type = \ + std::tuple \ + }; \ + template \ + reflect_impl_##QUERY_CLASS get_reflect_impl(const QUERY_CLASS&); \ + constexpr const char* get_type_name(QUERY_CLASS*) \ + { \ + return reflect_impl_##QUERY_CLASS ::name(); \ + } + + template + struct reflect_undefined + { + static constexpr bool is_defined = false; + static constexpr bool is_struct = false; + template + static void get(const std::string_view& m, L&& lambda); + }; + + /* + using std::string; + CLIO_REFLECT_TYPENAME( int32_t ) + CLIO_REFLECT_TYPENAME( int64_t ) + CLIO_REFLECT_TYPENAME( int16_t ) + CLIO_REFLECT_TYPENAME( int8_t ) + CLIO_REFLECT_TYPENAME( uint32_t ) + CLIO_REFLECT_TYPENAME( uint64_t ) + CLIO_REFLECT_TYPENAME( uint16_t ) + CLIO_REFLECT_TYPENAME( uint8_t ) + CLIO_REFLECT_TYPENAME( float ) + CLIO_REFLECT_TYPENAME( double ) + CLIO_REFLECT_TYPENAME( char ) + CLIO_REFLECT_TYPENAME( bool ) + CLIO_REFLECT_TYPENAME( string ) + */ + + template + reflect_undefined get_reflect_impl(const QueryClass&); + + template + using reflect = std::decay_t()))>; + + template + struct is_std_vector : std::false_type + { + }; + + template + struct is_std_vector> : std::true_type + { + using value_type = T; + }; + + template + struct is_std_optional : std::false_type + { + }; + + template + struct is_std_optional> : std::true_type + { + using value_type = T; + }; + + template + struct is_std_variant : std::false_type + { + }; + + template + struct is_std_variant> : std::true_type + { + static std::string name() { return get_variant_typename(); } + template + static std::string get_variant_typename() + { + if constexpr (sizeof...(Rest) > 0) + return std::string(get_type_name()) + "|" + get_variant_typename(); + else + return std::string(get_type_name()) + "|"; + } + using alts_as_tuple = std::tuple; + }; + + template + struct is_std_tuple : std::false_type + { + }; + + template + struct is_std_tuple> : std::true_type + { + static std::string name() { return get_tuple_typename(); } + template + static std::string get_tuple_typename() + { + if constexpr (sizeof...(Rest) > 0) + return std::string(get_type_name()) + "&" + get_tuple_typename(); + else + return std::string(get_type_name()) + "&"; + } + }; + + template + struct is_std_map : std::false_type + { + }; + + template + struct is_std_map> : std::true_type + { + static const std::string& name() + { + static std::string n = + std::string("map<") + get_type_name() + "," + get_type_name() + ">"; + return n; + } + }; + +} // namespace clio + +namespace std +{ + namespace + { + CLIO_REFLECT_TYPENAME(string) + } +} // namespace std diff --git a/libraries/clio/include/clio/schema.hpp b/libraries/clio/include/clio/schema.hpp new file mode 100644 index 000000000..73e892e9d --- /dev/null +++ b/libraries/clio/include/clio/schema.hpp @@ -0,0 +1,327 @@ +#pragma once +#include +#include +#include +#include +#include +#include + +namespace clio +{ + using std::string; + using std::vector; + + enum scalar_type + { + int64_type, + int32_type, + int16_type, + int8_type, + uint64_type, + uint32_type, + uint16_type, + uint8_type, + varuint32_type, + double_type, + float_type, + string_type, + bool_type + }; + + using type_name = std::string; + + struct variant_type + { + vector types; + }; + + CLIO_REFLECT(variant_type, types) + + struct tuple_type + { + vector types; + }; + CLIO_REFLECT(tuple_type, types) + + struct vector_type + { + type_name type; + }; + CLIO_REFLECT(vector_type, type) + + struct enum_type + { + scalar_type value_type; + struct enum_value + { + string name; + int64_t value; + }; + vector values; + }; + using enum_value = enum_type::enum_value; + CLIO_REFLECT(enum_value, name, value) + CLIO_REFLECT(enum_type, values) + + /* + struct function_type { + struct parameter { + type_name type; + string name; + }; + vector args; + type_name result; + }; + using function_parameter = function_type::parameter; + CLIO_REFLECT( function_parameter, type, name ) + CLIO_REFLECT( function_type, args, result ) + */ + + struct object_type + { + struct member + { + string name; + type_name type; + int32_t number; + + template + void set_params(std::initializer_list names, R (T::*method)(Args...)) + { + if constexpr (sizeof...(Args) > 0) + push_param(names.begin(), names.end()); + } + template + void set_params(std::initializer_list names, R (T::*method)(Args...) const) + { + if constexpr (sizeof...(Args) > 0) + push_param(names.begin(), names.end()); + } + + struct param + { + string name; + type_name type; + }; + std::vector params; + + private: + template + void push_param(std::initializer_list::iterator begin, + std::initializer_list::iterator end) + { + params.push_back( + {.name = begin != end ? *begin : "", .type = get_type_name>()}); + if constexpr (sizeof...(Args) > 0) + push_param(begin != end ? ++begin : end, end); + } + }; + + const member* get_member_by_name(const std::string_view& n) const + { + for (const auto& m : members) + if (m.name == n) + return &m; + return nullptr; + } + + const member* get_member_by_number(uint32_t n) const + { + for (const auto& m : members) + if (m.number == n) + return &m; + return nullptr; + } + + vector members; + }; + + using object_member = object_type::member; + using object_method_param = object_type::member::param; + CLIO_REFLECT(object_method_param, name, type) + CLIO_REFLECT(object_member, name, type, number, params) + CLIO_REFLECT(object_type, members) + + struct typedef_type + { + type_name type; + }; + CLIO_REFLECT(typedef_type, type) + + using schema_type = + std::variant; + CLIO_REFLECT_TYPENAME(schema_type) + + struct schema + { + std::map types; + + std::optional get_type(const std::string& name) const + { + auto itr = types.find(name); + if (itr == types.end()) + return {}; + return itr->second; + } + std::optional get_object(const string& name) const + { + auto itr = types.find(name); + if (itr == types.end()) + return {}; + auto objptr = std::get_if(&itr->second); + if (objptr) + return *objptr; + return {}; + } + std::optional get_vector(const string& name) const + { + auto itr = types.find(name); + if (itr == types.end()) + return {}; + auto objptr = std::get_if(&itr->second); + if (objptr) + return *objptr; + return {}; + } + template + bool visit_type(const std::string& type, Visitor&& v) + { + auto itr = types.find(type); + if (itr == types.end()) + return false; + std::visit(std::forward(v), itr->second); + return true; + } + + template + bool add_type(schema_type t, L&& on_generate) + { + auto tn = get_type_name(); + if (types.find(tn) == types.end()) + { + on_generate(static_cast(nullptr)); + types[tn] = t; + return true; + } + return false; + } + + template + void generate_map(const std::map*, L&& on_generate) + { + generate(on_generate); + } + + template + void generate_variant(const std::tuple* v, L&& on_generate) + { + if (add_type>(tuple_type(), on_generate)) + { + tuple_type vt; + generate_helper(vt, on_generate); + types[get_type_name>()] = vt; + } + } + + template + void generate_variant(const std::variant* v, L&& on_generate) + { + auto tn = get_type_name>(); + if (add_type>(variant_type(), on_generate)) + { + variant_type vt; + generate_helper(vt, on_generate); + types[tn] = vt; + } + } + + template + void generate_helper(OT& vt, L&& on_generate) + { + generate(on_generate); + vt.types.push_back(get_type_name()); + if constexpr (sizeof...(Args) > 0) + { + generate_helper(vt, on_generate); + } + } + + template + void generate() + { + generate([](auto) {}); + } + + template + void generate(L on_generate) + { + if constexpr (std::is_same_v>) + { + } + else if constexpr (is_std_map::value) + { + generate_map((const T*)nullptr, on_generate); + } + else if constexpr (is_std_tuple::value) + { + generate_variant((const T*)nullptr, on_generate); + } + else if constexpr (is_std_variant::value) + { + generate_variant((const T*)nullptr, on_generate); + } + else if constexpr (is_std_optional::value) + { + generate::value_type>(on_generate); + } + else if constexpr (is_std_vector::value) + { + using value_type = typename is_std_vector::value_type; + if (add_type(vector_type{get_type_name()}, on_generate)) + { + generate(on_generate); + } + } + else if constexpr (reflect::is_struct) + { + auto n = get_type_name(); + if (add_type(object_type(), on_generate)) + { + object_type ot; + reflect::for_each([&](const meta& r, auto m) { + if constexpr (not std::is_member_function_pointer_v) + { + using member_type = + std::decay_t*>(nullptr)->*m)>; + generate(on_generate); + + auto tn = get_type_name(); + ot.members.push_back({.name = r.name, .type = tn, .number = r.number}); + } + else + { + using member_type = decltype(result_of(m)); + generate(on_generate); + auto tn = get_type_name(); + ot.members.push_back({.name = r.name, .type = tn, .number = r.number}); + ot.members.back().set_params(r.param_names, m); + } + }); + types[n] = ot; + } + } + } /// generate + }; + + CLIO_REFLECT(schema, types) + +} // namespace clio + +namespace std +{ + namespace + { + using clio::schema_type; + CLIO_REFLECT_TYPENAME(schema_type) + } // namespace +} // namespace std diff --git a/libraries/clio/include/clio/stream.hpp b/libraries/clio/include/clio/stream.hpp new file mode 100644 index 000000000..b76eedfe8 --- /dev/null +++ b/libraries/clio/include/clio/stream.hpp @@ -0,0 +1,360 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace clio +{ + enum class stream_error + { + no_error, + overrun, + underrun, + float_error, + varuint_too_big, + invalid_varuint_encoding, + bad_variant_index, + invalid_asset_format, + array_size_mismatch, + invalid_name_char, + invalid_name_char13, + name_too_long, + json_writer_error, // !!! + }; // stream_error +} // namespace clio + +namespace std +{ + template <> + struct is_error_code_enum : true_type + { + }; +} // namespace std + +namespace clio +{ + class stream_error_category_type : public std::error_category + { + public: + const char* name() const noexcept override final { return "ConversionError"; } + + std::string message(int c) const override final + { + switch (static_cast(c)) + { + // clang-format off + case stream_error::no_error: return "No error"; + case stream_error::overrun: return "Stream overrun"; + case stream_error::underrun: return "Stream underrun"; + case stream_error::float_error: return "Float error"; + case stream_error::varuint_too_big: return "Varuint too big"; + case stream_error::invalid_varuint_encoding: return "Invalid varuint encoding"; + case stream_error::bad_variant_index: return "Bad variant index"; + case stream_error::invalid_asset_format: return "Invalid asset format"; + case stream_error::array_size_mismatch: return "T[] size and unpacked size don't match"; + case stream_error::invalid_name_char: return "character is not in allowed character set for names"; + case stream_error::invalid_name_char13: return "thirteenth character in name cannot be a letter that comes after j"; + case stream_error::name_too_long: return "string is too long to be a valid name"; + case stream_error::json_writer_error: return "Error writing json"; + // clang-format on + + default: + return "unknown"; + } + } + }; // stream_error_category_type + + inline const stream_error_category_type& stream_error_category() + { + static stream_error_category_type c; + return c; + } + + inline std::error_code make_error_code(stream_error e) + { + return {static_cast(e), stream_error_category()}; + } + + template + constexpr bool has_bitwise_serialization() + { + if constexpr (std::is_arithmetic_v) + { + return true; + } + else if constexpr (std::is_enum_v) + { + static_assert(!std::is_convertible_v>, + "Serializing unscoped enum"); + return true; + } + else + { + return false; + } + } + + template + struct small_buffer + { + char data[max_size]; + char* pos{data}; + + void reverse() { std::reverse(data, pos); } + }; + + struct string_stream + { + std::string& data; + string_stream(std::string& data) : data(data) {} + + void write(char ch) { data += ch; } + + void write(const void* src, size_t size) + { + auto s = reinterpret_cast(src); + data.insert(data.end(), s, s + size); + } + + template + void write(const char (&src)[size]) + { + write(src, size - 1); + } + + template + void write_raw(const T& v) + { + write(&v, sizeof(v)); + } + + void write(const std::string& v) { write(v.c_str(), v.size()); } + }; + + struct vector_stream + { + std::vector& data; + vector_stream(std::vector& data) : data(data) {} + + void write(char ch) { data.push_back(ch); } + + void write(const void* src, size_t size) + { + auto s = reinterpret_cast(src); + data.insert(data.end(), s, s + size); + } + + template + void write(const char (&src)[size]) + { + write(src, size - 1); + } + + template + void write_raw(const T& v) + { + write(&v, sizeof(v)); + } + + void write(const std::string& v) { write(v.c_str(), v.size()); } + }; + + struct fixed_buf_stream + { + char* begin; + char* pos; + char* end; + + fixed_buf_stream(char* pos, size_t size) : begin(pos), pos{pos}, end{pos + size} {} + + void write(char ch) + { + if (pos >= end) + throw_error(stream_error::overrun); + *pos++ = ch; + } + + void write(const void* src, size_t size) + { + if (pos + size > end) + throw_error(stream_error::overrun); + memcpy(pos, src, size); + pos += size; + } + + template + void write(const char (&src)[size]) + { + write(src, size - 1); + } + + template + void write_raw(const T& v) + { + write(&v, sizeof(v)); + } + + void write(const std::string& v) { write(v.c_str(), v.size()); } + + void skip(int32_t s) + { + if ((pos + s > end) or (pos + s < begin)) + throw_error(stream_error::overrun); + pos += s; + } + auto get_pos() const { return pos; } + + size_t remaining() { return end - pos; } + }; + + struct size_stream + { + size_t size = 0; + + size_t get_pos() const { return size; } + void write(char ch) { ++size; } + + void write(const void* src, size_t size) { this->size += size; } + + template + void write(const char (&src)[size]) + { + this->size += size - 1; + } + + template + void write_raw(const T& v) + { + size += sizeof(v); + } + + void write(const std::string& v) { write(v.c_str(), v.size()); } + + void skip(int32_t s) { size += s; } + }; + + template + void increase_indent(S&) + { + } + + template + void decrease_indent(S&) + { + } + + template + void write_colon(S& s) + { + s.write(':'); + } + + template + void write_newline(S&) + { + } + + template + struct pretty_stream : Base + { + using Base::Base; + int indent_size = 4; + std::vector current_indent; + }; + + template + void increase_indent(pretty_stream& s) + { + s.current_indent.resize(s.current_indent.size() + s.indent_size, ' '); + } + + template + void decrease_indent(pretty_stream& s) + { + if (s.current_indent.size() < s.indent_size) + throw_error(stream_error::overrun); + s.current_indent.resize(s.current_indent.size() - s.indent_size); + } + + template + void write_colon(pretty_stream& s) + { + s.write(": ", 2); + } + + template + void write_newline(pretty_stream& s) + { + s.write('\n'); + s.write(s.current_indent.data(), s.current_indent.size()); + } + + struct input_stream + { + const char* pos; + const char* end; + + input_stream() : pos{nullptr}, end{nullptr} {} + input_stream(const char* pos, size_t size) : pos{pos}, end{pos + size} + { + if (size < 0) + throw_error(stream_error::overrun); + } + input_stream(const char* pos, const char* end) : pos{pos}, end{end} + { + if (end < pos) + throw_error(stream_error::overrun); + } + input_stream(const std::vector& v) : pos{v.data()}, end{v.data() + v.size()} {} + input_stream(std::string_view v) : pos{v.data()}, end{v.data() + v.size()} {} + input_stream(const input_stream&) = default; + + input_stream& operator=(const input_stream&) = default; + + size_t remaining() { return end - pos; } + + void check_available(size_t size) + { + if (size > size_t(end - pos)) + throw_error(stream_error::overrun); + } + + auto get_pos() const { return pos; } + + void read(void* dest, size_t size) + { + if (size > size_t(end - pos)) + throw_error(stream_error::overrun); + memcpy(dest, pos, size); + pos += size; + } + + template + void read_raw(T& dest) + { + read(&dest, sizeof(dest)); + } + + void skip(size_t size) + { + if (size > size_t(end - pos)) + throw_error(stream_error::overrun); + pos += size; + } + + void read_reuse_storage(const char*& result, size_t size) + { + if (size > size_t(end - pos)) + throw_error(stream_error::overrun); + result = pos; + pos += size; + } + }; + +} // namespace clio diff --git a/libraries/clio/include/clio/to_bin.hpp b/libraries/clio/include/clio/to_bin.hpp new file mode 100644 index 000000000..75989feec --- /dev/null +++ b/libraries/clio/include/clio/to_bin.hpp @@ -0,0 +1,211 @@ +#pragma once +#include +#include +#include +#include + +namespace clio +{ + template + std::vector to_bin(const T& t); + + template + void to_bin(std::string_view sv, S& stream); + + template + void to_bin(const std::string& s, S& stream); + + template + void to_bin(const std::vector& obj, S& stream); + + template + void to_bin(const std::optional& obj, S& stream); + + template + void to_bin(const std::variant& obj, S& stream); + + template + void to_bin(const std::tuple& obj, S& stream); + + template + void to_bin(const T& obj, S& stream); + + template + void varuint32_to_bin(uint64_t val, S& stream) + { + if (val >> 32) + { + /// TODO throw error return stream_error::varuint_too_big; + return; + } + do + { + uint8_t b = val & 0x7f; + val >>= 7; + b |= ((val > 0) << 7); + stream.write(b); + } while (val); + } + + // !!! temp + inline void push_varuint32(std::vector& bin, uint32_t v) + { + vector_stream st{bin}; + varuint32_to_bin(v, st); + } + + template + void to_bin(std::string_view sv, S& stream) + { + varuint32_to_bin(sv.size(), stream); + stream.write(sv.data(), sv.size()); + } + + template + void to_bin(const std::string& s, S& stream) + { + to_bin(std::string_view{s}, stream); + } + + template + void to_bin_range(const T& obj, S& stream) + { + varuint32_to_bin(obj.size(), stream); + for (auto& x : obj) + { + to_bin(x, stream); + } + } + + template + void to_bin(const T (&obj)[N], S& stream) + { + varuint32_to_bin(N, stream); + if constexpr (has_bitwise_serialization()) + { + stream.write(reinterpret_cast(&obj), N * sizeof(T)); + } + else + { + for (auto& x : obj) + { + to_bin(x, stream); + } + } + } + + template + void to_bin(const std::vector& obj, S& stream) + { + varuint32_to_bin(obj.size(), stream); + if constexpr (has_bitwise_serialization()) + { + stream.write(reinterpret_cast(obj.data()), obj.size() * sizeof(T)); + } + else + { + for (auto& x : obj) + { + to_bin(x, stream); + } + } + } + + template + void to_bin(const std::variant& obj, S& stream) + { + varuint32_to_bin(obj.index(), stream); + std::visit([&](auto& x) { to_bin(x, stream); }, obj); + } + + template + void to_bin(const input_stream& obj, S& stream) + { + varuint32_to_bin(obj.end - obj.pos, stream); + stream.write(obj.pos, obj.end - obj.pos); + } + + template + void to_bin(const std::pair& obj, S& stream) + { + to_bin(obj.first, stream); + return to_bin(obj.second, stream); + } + + template + void to_bin(const std::optional& obj, S& stream) + { + to_bin(obj.has_value(), stream); + if (obj) + to_bin(*obj, stream); + } + + template + void to_bin_tuple(const T& obj, S& stream) + { + if constexpr (i < std::tuple_size_v) + { + to_bin(std::get(obj), stream); + to_bin_tuple(obj, stream); + } + } + + template + void to_bin(const std::tuple& obj, S& stream) + { + return to_bin_tuple<0>(obj, stream); + } + + template + void to_bin(const std::array& obj, S& stream) + { + for (const T& elem : obj) + { + to_bin(elem, stream); + } + } + + template + void to_bin(const T& obj, S& stream) + { + if constexpr (has_bitwise_serialization()) + { + stream.write(reinterpret_cast(&obj), sizeof(obj)); + } + else + { + reflect::for_each([&](const clio::meta&, auto m) { + if constexpr (not std::is_member_function_pointer_v) + { + to_bin(obj.*m, stream); + } + }); + } + } + + template + void convert_to_bin(const T& t, std::vector& bin) + { + size_stream ss; + to_bin(t, ss); + + auto orig_size = bin.size(); + bin.resize(orig_size + ss.size); + fixed_buf_stream fbs(bin.data() + orig_size, ss.size); + to_bin(t, fbs); + + /** TODO maybe throw + if (fbs.pos != fbs.end) + return stream_error::underrun; + */ + } + + template + std::vector to_bin(const T& t) + { + std::vector result; + convert_to_bin(t, result); + return result; + } + +} // namespace clio diff --git a/libraries/clio/include/clio/to_bin/deque.hpp b/libraries/clio/include/clio/to_bin/deque.hpp new file mode 100644 index 000000000..df66b0337 --- /dev/null +++ b/libraries/clio/include/clio/to_bin/deque.hpp @@ -0,0 +1,13 @@ +#pragma once +#include +#include + +namespace clio +{ + template + result to_bin(const std::deque& obj, S& stream) + { + return to_bin_range(obj, stream); + } + +} // namespace clio diff --git a/libraries/clio/include/clio/to_bin/list.hpp b/libraries/clio/include/clio/to_bin/list.hpp new file mode 100644 index 000000000..a8dcde1e4 --- /dev/null +++ b/libraries/clio/include/clio/to_bin/list.hpp @@ -0,0 +1,13 @@ +#pragma once +#include +#include + +namespace clio +{ + template + result to_bin(const std::list& obj, S& stream) + { + return to_bin_range(obj, stream); + } + +} // namespace clio diff --git a/libraries/clio/include/clio/to_bin/map.hpp b/libraries/clio/include/clio/to_bin/map.hpp new file mode 100644 index 000000000..2c00f934b --- /dev/null +++ b/libraries/clio/include/clio/to_bin/map.hpp @@ -0,0 +1,13 @@ +#pragma once +#include +#include + +namespace clio +{ + template + result to_bin(const std::map& obj, S& stream) + { + return to_bin_range(obj, stream); + } + +} // namespace clio diff --git a/libraries/clio/include/clio/to_bin/set.hpp b/libraries/clio/include/clio/to_bin/set.hpp new file mode 100644 index 000000000..b1d04f239 --- /dev/null +++ b/libraries/clio/include/clio/to_bin/set.hpp @@ -0,0 +1,13 @@ +#pragma once +#include +#include + +namespace clio +{ + template + result to_bin(const std::set& obj, S& stream) + { + return to_bin_range(obj, stream); + } + +} // namespace clio diff --git a/libraries/clio/include/clio/to_bin/varint.hpp b/libraries/clio/include/clio/to_bin/varint.hpp new file mode 100644 index 000000000..beefbd2bb --- /dev/null +++ b/libraries/clio/include/clio/to_bin/varint.hpp @@ -0,0 +1,19 @@ +#pragma once +#include +#include + +namespace clio +{ + template + void to_bin(const varuint32& obj, S& stream) + { + varuint32_to_bin(obj.value, stream); + } + + template + void to_bin(const varint32& obj, S& stream) + { + varuint32_to_bin((uint32_t(obj.value) << 1) ^ uint32_t(obj.value >> 31), stream); + } + +} // namespace clio diff --git a/libraries/clio/include/clio/to_json.hpp b/libraries/clio/include/clio/to_json.hpp new file mode 100644 index 000000000..bbf5e09a2 --- /dev/null +++ b/libraries/clio/include/clio/to_json.hpp @@ -0,0 +1,338 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace clio +{ + inline constexpr char hex_digits[] = "0123456789ABCDEF"; + + // Adaptors for rapidjson + struct stream_adaptor + { + stream_adaptor(const char* src, int sz) + { + int chars = std::min(sz, 4); + memcpy(buf, src, chars); + memset(buf + chars, 0, 4 - chars); + } + void Put(char ch) {} + char Take() { return buf[idx++]; } + char buf[4]; + int idx = 0; + }; + + // Replaces any invalid utf-8 bytes with ? + template + void to_json(std::string_view sv, S& stream) + { + stream.write('"'); + auto begin = sv.begin(); + auto end = sv.end(); + while (begin != end) + { + auto pos = begin; + while (pos != end && *pos != '"' && *pos != '\\' && (unsigned char)(*pos) >= 32 && + *pos != 127) + ++pos; + while (begin != pos) + { + stream_adaptor s2(begin, static_cast(pos - begin)); + if (rapidjson::UTF8<>::Validate(s2, s2)) + { + stream.write(begin, s2.idx); + begin += s2.idx; + } + else + { + ++begin; + stream.write('?'); + } + } + if (begin != end) + { + if (*begin == '"') + { + stream.write("\\\"", 2); + } + else if (*begin == '\\') + { + stream.write("\\\\", 2); + } + else + { + stream.write("\\u00", 4); + stream.write(hex_digits[(unsigned char)(*begin) >> 4]); + stream.write(hex_digits[(unsigned char)(*begin) & 15]); + } + ++begin; + } + } + stream.write('"'); + } + + template + void to_json(const std::string& s, S& stream) + { + return to_json(std::string_view{s}, stream); + } + + template + void to_json(const char* s, S& stream) + { + return to_json(std::string_view{s}, stream); + } + + template + void to_json(bool value, S& stream) + { + if (value) + return stream.write("true", 4); + else + return stream.write("false", 5); + } + + template + void int_to_json(T value, S& stream) + { + auto uvalue = std::make_unsigned_t(value); + small_buffer::digits10 + 4> b; + bool neg = value < 0; + if (neg) + uvalue = -uvalue; + if (sizeof(T) > 4) + *b.pos++ = '"'; + do + { + *b.pos++ = '0' + (uvalue % 10); + uvalue /= 10; + } while (uvalue); + if (neg) + *b.pos++ = '-'; + if (sizeof(T) > 4) + *b.pos++ = '"'; + b.reverse(); + return stream.write(b.data, b.pos - b.data); + } + + template + void fp_to_json(double value, S& stream) + { + // fpconv is not quite consistent with javascript for nans and infinities + if (value == std::numeric_limits::infinity()) + { + return stream.write("\"Infinity\"", 10); + } + else if (value == -std::numeric_limits::infinity()) + { + return stream.write("\"-Infinity\"", 11); + } + else if (std::isnan(value)) + { + return stream.write("\"NaN\"", 5); + } + small_buffer<24> b; // fpconv_dtoa generates at most 24 characters + int n = fpconv_dtoa(value, b.pos); + if (n <= 0) + throw_error(stream_error::float_error); + b.pos += n; + stream.write(b.data, b.pos - b.data); + } + + // clang-format off +template void to_json(uint8_t value, S& stream) { return int_to_json(value, stream); } +template void to_json(uint16_t value, S& stream) { return int_to_json(value, stream); } +template void to_json(uint32_t value, S& stream) { return int_to_json(value, stream); } +template void to_json(uint64_t value, S& stream) { return int_to_json(value, stream); } +template void to_json(unsigned __int128 value, S& stream) { return int_to_json(value, stream); } +template void to_json(int8_t value, S& stream) { return int_to_json(value, stream); } +template void to_json(int16_t value, S& stream) { return int_to_json(value, stream); } +template void to_json(int32_t value, S& stream) { return int_to_json(value, stream); } +template void to_json(int64_t value, S& stream) { return int_to_json(value, stream); } +template void to_json(__int128 value, S& stream) { return int_to_json(value, stream); } +template void to_json(double value, S& stream) { return fp_to_json(value, stream); } +template void to_json(float value, S& stream) { return fp_to_json(value, stream); } + // clang-format on + + template + void to_json(const std::vector& obj, S& stream) + { + stream.write('['); + bool first = true; + for (auto& v : obj) + { + if (first) + { + increase_indent(stream); + } + else + { + stream.write(','); + } + write_newline(stream); + first = false; + to_json(v, stream); + } + if (!first) + { + decrease_indent(stream); + write_newline(stream); + } + stream.write(']'); + } + + template + void to_json(const std::optional& obj, S& stream) + { + if (obj) + { + return to_json(*obj, stream); + } + else + { + return stream.write("null", 4); + } + } + + template + void to_json(const std::variant& obj, S& stream) + { + stream.write('['); + std::visit( + [&](const auto& t) { to_json(get_type_name>(), stream); }, obj); + stream.write(','); + std::visit([&](auto& x) { return to_json(x, stream); }, obj); + stream.write(']'); + } + + template + void to_json(const std::tuple& obj, S& stream) + { + stream.write('['); + if (sizeof...(T) == 0) + return stream.write(']'); + increase_indent(stream); + + tuple_for_each(obj, [&](int idx, auto& item) { + write_newline(stream); + if (idx) + { + stream.write(','); + } + to_json(item, stream); + }); + decrease_indent(stream); + write_newline(stream); + return stream.write(']'); + } + + template + void to_json(const T& t, S& stream) + { + if constexpr (not reflect::is_defined) + { + stream.write('"'); + std::string str(t); + stream.write(str.data(), str.size()); + stream.write('"'); + } + else + { + bool first = true; + stream.write('{'); + reflect::for_each([&](const clio::meta& ref, auto&& member) { + if constexpr (not std::is_member_function_pointer_v>) + { + auto addfield = [&]() { + if (first) + { + increase_indent(stream); + first = false; + } + else + { + stream.write(','); + } + write_newline(stream); + to_json(ref.name, stream); + write_colon(stream); + to_json(t.*member, stream); + }; + + using member_type = std::decay_t; + if constexpr (not is_std_optional::value) + { + addfield(); + } + else + { + if (!!(t.*member)) + addfield(); + } + } + }); + if (!first) + { + decrease_indent(stream); + write_newline(stream); + } + stream.write('}'); + } + } + + template + void to_json_hex(const char* data, size_t size, S& stream) + { + stream.write('"'); + for (size_t i = 0; i < size; ++i) + { + unsigned char byte = data[i]; + stream.write(hex_digits[byte >> 4]); + stream.write(hex_digits[byte & 15]); + } + stream.write('"'); + } + + template + std::string convert_to_json(const T& t) + { + size_stream ss; + to_json(t, ss); + + std::string result(ss.size, 0); + fixed_buf_stream fbs(result.data(), result.size()); + to_json(t, fbs); + + if (fbs.pos == fbs.end) + return result; + else + throw_error(stream_error::underrun); + } + + template + std::string to_json(const T& t) + { + return convert_to_json(t); + } + + template + std::string format_json(const T& t) + { + pretty_stream ss; + to_json(t, ss); + std::string result(ss.size, 0); + pretty_stream fbs(result.data(), result.size()); + to_json(t, fbs); + if (fbs.pos != fbs.end) + throw_error(stream_error::underrun); + return result; + } + +} // namespace clio diff --git a/libraries/clio/include/clio/to_json/map.hpp b/libraries/clio/include/clio/to_json/map.hpp new file mode 100644 index 000000000..3e965de9c --- /dev/null +++ b/libraries/clio/include/clio/to_json/map.hpp @@ -0,0 +1,33 @@ +#pragma once +#include +#include + +namespace clio +{ + template + void to_json(const std::map& m, S& stream) + { + stream.write('{'); + if (m.size() == 0) + { + return stream.write('}'); + } + increase_indent(stream); + bool not_first = false; + for (const auto& p : m) + { + if (not_first) + { + stream.write(','); + } + write_newline(stream); + to_json(p.first, stream); + write_colon(stream); + to_json(p.second, stream); + not_first = true; + } + decrease_indent(stream); + write_newline(stream); + return stream.write('}'); + } +} // namespace clio diff --git a/libraries/clio/include/clio/to_json/varint.hpp b/libraries/clio/include/clio/to_json/varint.hpp new file mode 100644 index 000000000..5066bf9f1 --- /dev/null +++ b/libraries/clio/include/clio/to_json/varint.hpp @@ -0,0 +1,19 @@ +#pragma once +#include +#include + +namespace clio +{ + template + void to_json(const varuint32& obj, S& stream) + { + to_json(obj.value, stream); + } + + template + void to_json(const varint32& obj, S& stream) + { + to_json(obj.value, stream); + } + +} // namespace clio diff --git a/libraries/clio/include/clio/to_key.hpp b/libraries/clio/include/clio/to_key.hpp new file mode 100644 index 000000000..7a01f92b4 --- /dev/null +++ b/libraries/clio/include/clio/to_key.hpp @@ -0,0 +1,381 @@ +#pragma once + +//#include +//#include +//#include +//#include +#include +#include +#include +#include +#include +#include + +#include + +namespace clio +{ + template + void to_key(const std::tuple& obj, S& stream); + + // to_key defines a conversion from a type to a sequence of bytes whose lexicograpical + // ordering is the same as the ordering of the original type. + // + // For any two objects of type T, a and b: + // + // - key(a) < key(b) iff a < b + // - key(a) is not a prefix of key(b) + // + // Overloads of to_key for user-defined types can be found by Koenig lookup. + // + // Abieos provides specializations of to_key for the following types + // - std::string and std::string_view + // - std::vector, std::list, std::deque + // - std::tuple + // - std::array + // - std::optional + // - std::variant + // - Arithmetic types + // - Scoped enumeration types + // - Reflected structs + // - All smart-contract related types defined by abieos + template + void to_key(const T& obj, S& stream); + + template + void to_key_tuple(const T& obj, S& stream) + { + if constexpr (i < std::tuple_size_v) + { + to_key(std::get(obj), stream); + to_key_tuple(obj, stream); + } + } + + template + void to_key(const std::tuple& obj, S& stream) + { + return to_key_tuple<0>(obj, stream); + } + + template + void to_key(const std::array& obj, S& stream) + { + for (const T& elem : obj) + { + to_key(elem, stream); + } + } + + template + void to_key_optional(const bool* obj, S& stream) + { + if (obj == nullptr) + return stream.write('\0'); + else if (!*obj) + return stream.write('\1'); + else + return stream.write('\2'); + } + + template + void to_key_optional(const T* obj, S& stream) + { + if constexpr (has_bitwise_serialization() && sizeof(T) == 1) + { + if (obj == nullptr) + return stream.write("\0", 2); + else + { + char buf[1]; + fixed_buf_stream tmp_stream(buf, 1); + to_key(*obj, tmp_stream); + stream.write(buf[0]); + if (buf[0] == '\0') + stream.write('\1'); + } + } + else + { + if (obj) + { + stream.write('\1'); + return to_key(*obj, stream); + } + else + { + return stream.write('\0'); + } + } + } + + template + void to_key(const std::pair& obj, S& stream) + { + to_key(obj.first, stream); + return to_key(obj.second, stream); + } + + template + void to_key_range(const T& obj, S& stream) + { + for (const auto& elem : obj) + { + to_key_optional(&elem, stream); + } + return to_key_optional((decltype(&*std::begin(obj))) nullptr, stream); + } + + template + void to_key(const std::vector& obj, S& stream) + { + for (const T& elem : obj) + { + to_key_optional(&elem, stream); + } + return to_key_optional((const T*)nullptr, stream); + } + + /* +template +void to_key(const std::list& obj, S& stream) { + return to_key_range(obj, stream); +} + +template +void to_key(const std::deque& obj, S& stream) { + return to_key_range(obj, stream); +} + +template +void to_key(const std::set& obj, S& stream) { + return to_key_range(obj, stream); +} + +template +void to_key(const std::map& obj, S& stream) { + return to_key_range(obj, stream); +} +*/ + + template + void to_key(const std::optional& obj, S& stream) + { + return to_key_optional(obj ? &*obj : nullptr, stream); + } + + // The first byte holds: + // 0-4 1's (number of additional bytes) 0 (terminator) bits + // + // The number is represented as big-endian using the low order + // bits of the first byte and all of the remaining bytes. + // + // Notes: + // - values must be encoded using the minimum number of bytes, + // as non-canonical representations will break the sort order. + template + void to_key_varuint32(std::uint32_t obj, S& stream) + { + int num_bytes; + if (obj < 0x80u) + { + num_bytes = 1; + } + else if (obj < 0x4000u) + { + num_bytes = 2; + } + else if (obj < 0x200000u) + { + num_bytes = 3; + } + else if (obj < 0x10000000u) + { + num_bytes = 4; + } + else + { + num_bytes = 5; + } + + stream.write(static_cast(~(0xFFu >> (num_bytes - 1)) | + (num_bytes == 5 ? 0 : (obj >> ((num_bytes - 1) * 8))))); + for (int i = num_bytes - 2; i >= 0; --i) + { + stream.write(static_cast((obj >> i * 8) & 0xFFu)); + } + } + + // for non-negative values + // The first byte holds: + // 1 (signbit) 0-4 1's (number of additional bytes) 0 (terminator) bits + // The value is represented as big endian + // for negative values + // The first byte holds: + // 0 (signbit) 0-4 0's (number of additional bytes) 1 (terminator) bits + // The value is adjusted to be positive based on the range that can + // be represented with this number of bytes and then encoded as big endian. + // + // Notes: + // - negative values must sort before positive values + // - For negative value, numbers that need more bytes are smaller, hence + // the encoding of the width must be opposite the encoding used for + // non-negative values. + // - A 5-byte varint can represent values in $[-2^34, 2^34)$. In this case, + // the argument will be sign-extended. + template + void to_key_varint32(std::int32_t obj, S& stream) + { + static_assert(std::is_same_v, "to_key for varint32 has been temporarily disabled"); + int num_bytes; + bool sign = (obj < 0); + if (obj < 0x40 && obj >= -0x40) + { + num_bytes = 1; + } + else if (obj < 0x2000 && obj >= -0x2000) + { + num_bytes = 2; + } + else if (obj < 0x100000 && obj >= -0x100000) + { + num_bytes = 3; + } + else if (obj < 0x08000000 && obj >= -0x08000000) + { + num_bytes = 4; + } + else + { + num_bytes = 5; + } + + unsigned char width_field; + if (sign) + { + width_field = 0x80u >> num_bytes; + } + else + { + width_field = 0x80u | ~(0xFFu >> num_bytes); + } + auto uobj = static_cast(obj); + unsigned char value_mask = (0xFFu >> (num_bytes + 1)); + unsigned char high_byte = + (num_bytes == 5 ? (sign ? 0xFF : 0) : (uobj >> ((num_bytes - 1) * 8))); + stream.write(width_field | (high_byte & value_mask)); + for (int i = num_bytes - 2; i >= 0; --i) + { + stream.write(static_cast((uobj >> i * 8) & 0xFFu)); + } + } + + template + void to_key(const std::variant& obj, S& stream) + { + to_key_varuint32(static_cast(obj.index()), stream); + std::visit([&](const auto& item) { return to_key(item, stream); }, obj); + } + + template + void to_key(std::string_view obj, S& stream) + { + for (char ch : obj) + { + stream.write(ch); + if (ch == '\0') + { + stream.write('\1'); + } + } + return stream.write("\0", 2); + } + + template + void to_key(const std::string& obj, S& stream) + { + return to_key(std::string_view(obj), stream); + } + + template + void to_key(bool obj, S& stream) + { + return stream.write(static_cast(obj ? 1 : 0)); + } + + template + UInt float_to_key(T value) + { + static_assert(sizeof(T) == sizeof(UInt), "Expected unsigned int of the same size"); + UInt result; + std::memcpy(&result, &value, sizeof(T)); + UInt signbit = (static_cast(1) << (std::numeric_limits::digits - 1)); + UInt mask = 0; + if (result == signbit) + result = 0; + if (result & signbit) + mask = ~mask; + return result ^ (mask | signbit); + } + + template + void to_key(const T& obj, S& stream) + { + if constexpr (std::is_floating_point_v) + { + if constexpr (sizeof(T) == 4) + { + return to_key(float_to_key(obj), stream); + } + else + { + static_assert(sizeof(T) == 8, "Unknown floating point type"); + return to_key(float_to_key(obj), stream); + } + } + else if constexpr (std::is_integral_v) + { + auto v = static_cast>(obj); + v -= static_cast>(std::numeric_limits::min()); + std::reverse(reinterpret_cast(&v), reinterpret_cast(&v + 1)); + return stream.write_raw(v); + } + else if constexpr (std::is_enum_v) + { + static_assert(!std::is_convertible_v>, + "Serializing unscoped enum"); + return to_key(static_cast>(obj), stream); + } + else + { + clio::reflect::for_each([&](const clio::meta&, auto member) { + if constexpr (std::is_member_pointer_v) + { + to_key(obj.*member, stream); + } + }); + } + } + + template + void convert_to_key(const T& t, std::vector& bin) + { + size_stream ss; + to_key(t, ss); + auto orig_size = bin.size(); + bin.resize(orig_size + ss.size); + fixed_buf_stream fbs(bin.data() + orig_size, ss.size); + to_key(t, fbs); + if (fbs.pos != fbs.end) + throw_error(stream_error::underrun); + } + + template + std::vector convert_to_key(const T& t) + { + std::vector result; + convert_to_key(t, result); + return result; + } + +} // namespace clio diff --git a/libraries/clio/include/clio/to_protobuf.hpp b/libraries/clio/include/clio/to_protobuf.hpp new file mode 100644 index 000000000..301f2292e --- /dev/null +++ b/libraries/clio/include/clio/to_protobuf.hpp @@ -0,0 +1,177 @@ +#pragma once +#include +#include +#include + +namespace clio +{ + template + void to_protobuf_object(const std::tuple& obj, S& stream); + + template + void to_protobuf_object(const std::variant& obj, S& stream); + + template + void to_protobuf_object(const std::vector& obj, S& stream); + + template + void to_protobuf_object(const T& obj, S& stream); + + template + struct wire_type + { + static constexpr const auto value = 2; + }; + +#define WIRE_TYPE(X, T) \ + template <> \ + struct wire_type \ + { \ + static constexpr const auto value = T; \ + }; + + template + struct wire_type> + { + static constexpr const auto value = 2; + }; + + WIRE_TYPE(uint16_t, 0) + WIRE_TYPE(uint8_t, 0) + WIRE_TYPE(int16_t, 0) + WIRE_TYPE(int8_t, 0) + WIRE_TYPE(char, 0) + WIRE_TYPE(bool, 0) + WIRE_TYPE(varuint32, 0) + WIRE_TYPE(uint64_t, 1) + WIRE_TYPE(int64_t, 1) + WIRE_TYPE(double, 1) + WIRE_TYPE(uint32_t, 5) + WIRE_TYPE(int32_t, 5) + WIRE_TYPE(float, 5) + WIRE_TYPE(std::string, 2) + WIRE_TYPE(bytes, 2) + + /** + * Assumes key has been writen, then writes the rest... + */ + template + void to_protobuf_member(const T& obj, S& stream) + { + if constexpr (std::is_same_v) + { + varuint32_to_bin(obj.size(), stream); + stream.write(obj.data(), obj.size()); + } + else if constexpr (std::is_same_v) + { + varuint32_to_bin(obj.data.size(), stream); + stream.write(obj.data.data(), obj.data.size()); + } + else if constexpr (5 == wire_type::value or 1 == wire_type::value) + { + to_bin(obj, stream); + } + else if constexpr (0 == wire_type::value) + { + varuint32_to_bin(obj, stream); + } + else if constexpr (reflect::is_struct || is_std_vector::value || + is_std_tuple::value || is_std_variant::value) + { + size_stream ss; + to_protobuf_object(obj, ss); + varuint32_to_bin(ss.size, stream); + to_protobuf_object(obj, stream); + } + else + { + T::to_protobuf_is_not_defined; /// used to generate useful compile error + } + } + + template + void write_protobuf_field(int field, const Member& member, Stream& stream) + { + if constexpr (is_std_vector>::value) + { + if (member.size() == 0) + return; + } + uint32_t key = (field << 3) | wire_type::value; + varuint32_to_bin(key, stream); + to_protobuf_member(member, stream); + } + + template + void to_protobuf_object(const std::variant& obj, S& stream) + { + std::visit([&](auto m) { write_protobuf_field(obj.index() + 1, m, stream); }, obj); + } + + /** + * A vector is protobuf object that is either packed in a single field or + * listed as N fields of the same type. + */ + template + void to_protobuf_object(const std::vector& vec, S& stream) + { + uint32_t key = (1 << 3) | wire_type>::value; + if constexpr (std::is_arithmetic_v) + { /// [packed=true] + varuint32_to_bin(key, stream); + auto size = uint32_t(vec.size() * sizeof(T)); + varuint32_to_bin(size, stream); + stream.write(vec.data(), vec.size() * sizeof(T)); + } + else + { + for (const auto& item : vec) + { + varuint32_to_bin(key, stream); + to_protobuf_member(item, stream); + } + } + } + + template + void to_protobuf_object(const T& obj, S& stream) + { + reflect::for_each([&](const clio::meta& ref, auto m) { + if constexpr (not std::is_member_function_pointer_v) + { + write_protobuf_field(ref.number, obj.*m, stream); + } + }); + } + + template + void to_protobuf_object(const std::tuple& obj, S& stream) + { + tuple_for_each(obj, + [&](int idx, const auto& m) { write_protobuf_field(idx + 1, m, stream); }); + } + + template + void to_protobuf(const T& obj, S& stream) + { + to_protobuf_object(obj, stream); + } + + template + std::vector to_protobuf(const T& t) + { + size_stream ss; + to_protobuf(t, ss); + + std::vector result(ss.size, 0); + fixed_buf_stream fbs(result.data(), result.size()); + + to_protobuf(t, fbs); + + if (fbs.pos != fbs.end) + throw_error(stream_error::underrun); + return result; + } + +} // namespace clio diff --git a/libraries/clio/include/clio/translator.hpp b/libraries/clio/include/clio/translator.hpp new file mode 100644 index 000000000..5ca6d12d4 --- /dev/null +++ b/libraries/clio/include/clio/translator.hpp @@ -0,0 +1,484 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace clio +{ + class meta_type_base + { + public: + meta_type_base(uint32_t n) : _number(n) {} + + virtual ~meta_type_base() {} + virtual const char* name() const = 0; + + virtual std::vector json_to_protobuf(std::string json) const = 0; + virtual std::vector json_to_bin(std::string json) const = 0; + + virtual std::string protobuf_to_json(const std::vector& b) const = 0; + virtual std::vector protobuf_to_bin(const std::vector& b) const = 0; + + virtual std::vector bin_to_protobuf(const std::vector& b) const = 0; + virtual std::string bin_to_json(const std::vector& b) const = 0; + + uint32_t number() const { return _number; } + + private: + uint32_t _number; + }; + + template + class meta_type : public meta_type_base + { + public: + meta_type(uint32_t n) : meta_type_base(n) {} + virtual ~meta_type(){}; + + virtual const char* name() const override + { + return get_type_name(); // reflect::name(); + } + + virtual std::vector json_to_protobuf(std::string json) const override + { + auto t = from_json(json); + return to_protobuf(t); + } + + virtual std::vector json_to_bin(std::string json) const override + { + auto t = from_json(json); + return to_bin(t); + } + + virtual std::string protobuf_to_json(const std::vector& b) const override + { + auto t = from_protobuf(b); + return to_json(t); + } + + virtual std::vector protobuf_to_bin(const std::vector& b) const override + { + auto t = from_protobuf(b); + return to_bin(t); + } + + virtual std::vector bin_to_protobuf(const std::vector& b) const override + { + auto t = from_bin(b); + return to_protobuf(t); + } + + virtual std::string bin_to_json(const std::vector& b) const override + { + auto t = from_bin(b); + return to_json(t); + } + }; + + template + class translator + { + public: + translator() + { + _schema.generate([&](auto* p) { + _types.push_back(new meta_type>(_types.size())); + }); + } + string get_json_schema() const { return format_json(_schema); } + string get_protobuf_schema() { return to_protobuf_schema(_schema); } + // string get_gql_schema(); + + uint32_t get_type_num(const string& type_name) const + { + for (const auto& t : _types) + if (t->name() == type_name) + return t->number(); + return -1; + } + + string get_type_name(uint32_t type_num) const + { + if (type_num < _types.size()) + return _types[type_num]->name(); + return string(); + } + + std::vector json_to_bin(uint32_t type_num, std::string json) const + { + if (type_num < _types.size()) + return _types[type_num]->json_to_bin(json); + return std::vector(); + } + + std::vector json_to_protobuf(int type_num, std::string json) const + { + if (type_num < _types.size()) + return _types[type_num]->json_to_protobuf(json); + return std::vector(); + } + + string bin_to_json(int type_num, const std::vector& bin) const + { + if (type_num < _types.size()) + return _types[type_num]->bin_to_json(bin); + return string(); + } + std::vector bin_to_protobuf(int type_num, const std::vector& bin) const + { + if (type_num < _types.size()) + return _types[type_num]->bin_to_protobuf(bin); + return std::vector(); + } + + string protobuf_to_json(int type_num, const std::vector& pbuf) const + { + if (type_num < _types.size()) + return _types[type_num]->protobuf_to_json(pbuf); + return string(); + } + std::vector protobuf_to_bin(int type_num, const std::vector& pbuf) const + { + if (type_num < _types.size()) + return _types[type_num]->protobuf_to_bin(pbuf); + return std::vector(); + } + + string query_protobuf_to_json(const std::vector& pbuf_query) const + { + clio::input_stream in(pbuf_query.data(), pbuf_query.size()); + auto pbquery = clio::from_protobuf(in); + auto jquery = protobuf::to_json_query(pbquery); + return to_json(jquery); + } + + std::vector query_json_to_protobuf(string json_query) const + { + auto jq = from_json(std::move(json_query)); + auto pbq = clio::protobuf::from_json_query(jq); + return to_protobuf(pbq); + } + + // string query_protobuf_to_gql( const byte_view& pbuf_query ); + // string query_json_to_gql( const string_view& json_query ); + // string query_gql_to_protobuf( const string_view& gql_query ); + // string query_gql_to_josn( const string_view& gql_query ); + private: + schema _schema; + std::vector _types; + }; + + template + void translate_json_to_protobuf(uint32_t number, InStream& in, OutStream& out) + { + varuint32 key; + key.value = number << 3; + if constexpr (std::is_same_v) + { + // wiretype 0 + } + else if constexpr (sizeof(T) == 8) + key.value |= uint8_t(protobuf::fixed64); + else if constexpr (sizeof(T) == 4) + key.value |= uint8_t(protobuf::fixed32); + T r; + from_json(r, in); + to_bin(key, out); + to_bin(r, out); + } + + template + bool translate_json_to_protobuf(const schema& sch, + const std::string& pbuf_type, + InStream& in, + OutStream& out) + { + auto otype = sch.get_object(pbuf_type); + if (not otype) + { + auto vtype = sch.get_vector(pbuf_type); + if (vtype) + { + } + return false; + } + in.get_start_object(); + auto t = in.peek_token(); + while (t.type != json_token_type::type_end_object) + { + std::string_view key = in.get_key(); + auto member = otype->get_member_by_name(key); + if (member) + { + if (member->type == "double") + translate_json_to_protobuf(member->number, in, out); + else if (member->type == "int64_t") + translate_json_to_protobuf(member->number, in, out); + else if (member->type == "uint64_t") + translate_json_to_protobuf(member->number, in, out); + else if (member->type == "int32_t") + translate_json_to_protobuf(member->number, in, out); + else if (member->type == "uint32_t") + translate_json_to_protobuf(member->number, in, out); + else if (member->type == "int16_t") + translate_json_to_protobuf(member->number, in, out); + else if (member->type == "uint16_t") + translate_json_to_protobuf(member->number, in, out); + else if (member->type == "int8_t") + translate_json_to_protobuf(member->number, in, out); + else if (member->type == "uint8_t") + translate_json_to_protobuf(member->number, in, out); + else if (member->type == "bool") + translate_json_to_protobuf(member->number, in, out); + else if (member->type == "char") + translate_json_to_protobuf(member->number, in, out); + else if (member->type == "varuint32") + translate_json_to_protobuf(member->number, in, out); + else + { + auto st = sch.get_type(member->type); + if (not st) + from_json_skip_value(in); + else if (object_type* obj = std::get_if(&*st)) + { + /// wire_type == wire_type_enum::buffer + } + else if (vector_type* obj = std::get_if(&*st)) + { + /// wire_type == wire_type_enum::buffer + } + else if (variant_type* obj = std::get_if(&*st)) + { + /// wire_type == wire_type_enum::buffer + } + else if (tuple_type* obj = std::get_if(&*st)) + { + /// wire_type == wire_type_enum::buffer + } + } + /* + translate_json_to_protobuf( sch, member->type, in, out ); + auto t = in.peek_token(); + switch( t.type ) { + case json_token_type::type_string: + case json_token_type::type_null: + case json_token_type::type_bool: + case json_token_type::type_start_object: + case json_token_type::type_start_array: + } + */ + } + else + { + from_json_skip_value(in); + } + } + } + + template + bool translate_protobuf_to_json(const schema& sch, + const std::string& pbuf_type, + InStream& in, + OutStream& out) + { + using namespace clio::protobuf; + auto otype = sch.get_object(pbuf_type); + if (not otype) + { + auto vtype = sch.get_vector(pbuf_type); + if (vtype) + { + out.write('['); + bool first = true; + + while (in.remaining()) + { + uint32_t key = 0; + varuint32_from_bin(key, in); + wire_type_enum type = wire_type_enum(uint8_t(key) & 0x07); + // uint32_t number = key >> 3; + + auto contained_type = vtype->type; + + if (first) + { + first = false; + } + else + { + out.write(','); + } + + switch (type) + { + case wire_type_enum::varint: + { + varuint32 val; + from_bin(val, in); + if (contained_type == "bool") + to_json(bool(val.value), out); + else + to_json(val.value, out); + } + break; + case wire_type_enum::fixed64: + { + int64_t val; + from_bin(val, in); + if (contained_type == "double") + to_json(*((double*)&val), out); + else + to_json(val, out); + } + break; + case wire_type_enum::buffer: + { + if (contained_type == "string") + { + std::string val; + from_bin(val, in); + to_json(val, out); + } + else if (contained_type == "bytes") + { + bytes val; + from_bin(val, in); + to_json(val, out); + } + else + { + varuint32 size; + from_bin(size, in); + if (size.value > in.remaining()) + return false; + input_stream obj_in(in.pos, in.pos + size); + translate_protobuf_to_json(sch, contained_type, obj_in, out); + in.skip(size); + } + } + break; + case wire_type_enum::fixed32: + { + int32_t val; + from_bin(val, in); + if (contained_type == "float") + to_json(*((float*)&val), out); + else + to_json(val, out); + } + break; + }; + } + + out.write(']'); + return true; + } + return false; + } + + out.write('{'); + bool first = true; + while (in.remaining()) + { + uint32_t key = 0; + varuint32_from_bin(key, in); + wire_type_enum type = wire_type_enum(uint8_t(key) & 0x07); + uint32_t number = key >> 3; + + // std::cout <<"field number: " << number <<" type: " << int64_t(type)<<" "; + auto member_meta = otype->get_member_by_number(number); + + if (first) + { + first = false; + } + else + { + out.write(','); + } + + if (member_meta) + { + to_json(member_meta->name, out); + } + else + { + to_json(std::to_string(number), out); + } + + out.write(':'); + + switch (type) + { + case wire_type_enum::varint: + { + varuint32 val; + from_bin(val, in); + if (member_meta->type == "bool") + to_json(bool(val.value), out); + else + to_json(val.value, out); + } + break; + case wire_type_enum::fixed64: + { + int64_t val; + from_bin(val, in); + if (member_meta->type == "double") + to_json(*((double*)&val), out); + else + to_json(val, out); + } + break; + case wire_type_enum::buffer: + { + if (member_meta->type == "string") + { + std::string val; + from_bin(val, in); + to_json(val, out); + } + else if (member_meta->type == "bytes") + { + bytes val; + from_bin(val, in); + to_json(val, out); + } + else + { + varuint32 size; + from_bin(size, in); + if (size.value > in.remaining()) + return false; + input_stream obj_in(in.pos, in.pos + size); + translate_protobuf_to_json(sch, member_meta->type, obj_in, out); + in.skip(size); + } + // a.members.emplace_back( number, std::move(val) ); + } + break; + case wire_type_enum::fixed32: + { + int32_t val; + from_bin(val, in); + if (member_meta->type == "float") + to_json(*((float*)&val), out); + else + to_json(val, out); + } + break; + }; + } + out.write('}'); + return true; + } + +}; // namespace clio diff --git a/libraries/clio/include/clio/tuple.hpp b/libraries/clio/include/clio/tuple.hpp new file mode 100644 index 000000000..ff1316d6c --- /dev/null +++ b/libraries/clio/include/clio/tuple.hpp @@ -0,0 +1,92 @@ +#pragma once +#include + +namespace clio +{ + enum class tuple_error + { + no_error, + invalid_tuple_index + }; // tuple_error +} // namespace clio + +namespace std +{ + template <> + struct is_error_code_enum : true_type + { + }; +} // namespace std + +namespace clio +{ + class tuple_error_category_type : public std::error_category + { + public: + const char* name() const noexcept override final { return "ConversionError"; } + + std::string message(int c) const override final + { + switch (static_cast(c)) + { + // clang-format off + case tuple_error::no_error: return "No error"; + case tuple_error::invalid_tuple_index: return "invalid tuple index"; + default: return "unknown"; + }; + } +}; + +inline const tuple_error_category_type& tuple_error_category() { + static tuple_error_category_type c; + return c; +} + +inline std::error_code make_error_code(tuple_error e) { return { static_cast(e), tuple_error_category() }; } + + + template + void tuple_get(T& obj, int pos, L&& lambda) { + if constexpr (N < std::tuple_size_v) { + if( N == pos ) { + lambda( std::get(obj) ); + } + else tuple_get(obj, pos, std::forward(lambda) ); + } else { + throw_error(tuple_error::invalid_tuple_index); + } + } + + template + void tuple_get(std::tuple& obj, int pos, L&& lambda) { + tuple_get<0>(obj, pos, std::forward(lambda) ); + } + + template + void tuple_for_each(T& obj, L&& lambda) { + if constexpr (N < std::tuple_size_v) { + lambda( N, std::get(obj) ); + tuple_for_each(obj, std::forward(lambda) ); + } + + } + + template + void tuple_for_each(std::tuple& obj, L&& lambda) { + tuple_for_each<0>(obj, std::forward(lambda) ); + } + + template + void tuple_for_each(const T& obj, L&& lambda) { + if constexpr (N < std::tuple_size_v) { + lambda( N, std::get(obj) ); + tuple_for_each(obj, std::forward(lambda) ); + } + } + + template + void tuple_for_each(const std::tuple& obj, L&& lambda) { + tuple_for_each<0>(obj, std::forward(lambda) ); + } + +} diff --git a/libraries/clio/include/clio/varint.hpp b/libraries/clio/include/clio/varint.hpp new file mode 100644 index 000000000..53ee369bb --- /dev/null +++ b/libraries/clio/include/clio/varint.hpp @@ -0,0 +1,470 @@ +/** + * @file + * @copyright defined in clio/LICENSE + */ +#pragma once +#include + +namespace clio +{ + /** + * @defgroup varint Variable Length Integer Type + * @ingroup core + * @ingroup types + * @brief Defines variable length integer type which provides more efficient serialization + */ + + /** + * Variable Length Unsigned Integer. This provides more efficient serialization of 32-bit + * unsigned int. It serialuzes a 32-bit unsigned integer in as few bytes as possible `varuint32` + * is unsigned and uses [VLQ or Base-128 + * encoding](https://en.wikipedia.org/wiki/Variable-length_quantity) + * + * @ingroup varint + */ + struct unsigned_int + { + /** + * Construct a new unsigned int object + * + * @param v - Source + */ + unsigned_int(uint32_t v = 0) : value(v) {} + + /** + * Construct a new unsigned int object from a type that is convertible to uint32_t + * + * @tparam T - Type of the source + * @param v - Source + * @pre T must be convertible to uint32_t + */ + template + unsigned_int(T v) : value(v) + { + } + + // operator uint32_t()const { return value; } + // operator uint64_t()const { return value; } + + /** + * Convert unsigned_int as T + * + * @tparam T - Target type of conversion + * @return T - Converted target + */ + template + operator T() const + { + return static_cast(value); + } + + /// @cond OPERATORS + + /** + * Assign 32-bit unsigned integer + * + * @param v - Soruce + * @return unsigned_int& - Reference to this object + */ + unsigned_int& operator=(uint32_t v) + { + value = v; + return *this; + } + + /// @endcond + + /** + * Contained value + */ + uint32_t value; + + /// @cond OPERATORS + + /** + * Check equality between a unsigned_int object and 32-bit unsigned integer + * + * @param i - unsigned_int object to compare + * @param v - 32-bit unsigned integer to compare + * @return true - if equal + * @return false - otherwise + */ + friend bool operator==(const unsigned_int& i, const uint32_t& v) { return i.value == v; } + + /** + * Check equality between 32-bit unsigned integer and a unsigned_int object + * + * @param i - 32-bit unsigned integer to compare + * @param v - unsigned_int object to compare + * @return true - if equal + * @return false - otherwise + */ + friend bool operator==(const uint32_t& i, const unsigned_int& v) { return i == v.value; } + + /** + * Check equality between two unsigned_int objects + * + * @param i - First unsigned_int object to compare + * @param v - Second unsigned_int object to compare + * @return true - if equal + * @return false - otherwise + */ + friend bool operator==(const unsigned_int& i, const unsigned_int& v) + { + return i.value == v.value; + } + + /** + * Check inequality between a unsigned_int object and 32-bit unsigned integer + * + * @param i - unsigned_int object to compare + * @param v - 32-bit unsigned integer to compare + * @return true - if inequal + * @return false - otherwise + */ + friend bool operator!=(const unsigned_int& i, const uint32_t& v) { return i.value != v; } + + /** + * Check inequality between 32-bit unsigned integer and a unsigned_int object + * + * @param i - 32-bit unsigned integer to compare + * @param v - unsigned_int object to compare + * @return true - if unequal + * @return false - otherwise + */ + friend bool operator!=(const uint32_t& i, const unsigned_int& v) { return i != v.value; } + + /** + * Check inequality between two unsigned_int objects + * + * @param i - First unsigned_int object to compare + * @param v - Second unsigned_int object to compare + * @return true - if inequal + * @return false - otherwise + */ + friend bool operator!=(const unsigned_int& i, const unsigned_int& v) + { + return i.value != v.value; + } + + /** + * Check if the given unsigned_int object is less than the given 32-bit unsigned integer + * + * @param i - unsigned_int object to compare + * @param v - 32-bit unsigned integer to compare + * @return true - if i less than v + * @return false - otherwise + */ + friend bool operator<(const unsigned_int& i, const uint32_t& v) { return i.value < v; } + + /** + * Check if the given 32-bit unsigned integer is less than the given unsigned_int object + * + * @param i - 32-bit unsigned integer to compare + * @param v - unsigned_int object to compare + * @return true - if i less than v + * @return false - otherwise + */ + friend bool operator<(const uint32_t& i, const unsigned_int& v) { return i < v.value; } + + /** + * Check if the first given unsigned_int is less than the second given unsigned_int object + * + * @param i - First unsigned_int object to compare + * @param v - Second unsigned_int object to compare + * @return true - if i less than v + * @return false - otherwise + */ + friend bool operator<(const unsigned_int& i, const unsigned_int& v) + { + return i.value < v.value; + } + + /** + * Check if the given unsigned_int object is greater or equal to the given 32-bit unsigned + * integer + * + * @param i - unsigned_int object to compare + * @param v - 32-bit unsigned integer to compare + * @return true - if i is greater or equal to v + * @return false - otherwise + */ + friend bool operator>=(const unsigned_int& i, const uint32_t& v) { return i.value >= v; } + + /** + * Check if the given 32-bit unsigned integer is greater or equal to the given unsigned_int + * object + * + * @param i - 32-bit unsigned integer to compare + * @param v - unsigned_int object to compare + * @return true - if i is greater or equal to v + * @return false - otherwise + */ + friend bool operator>=(const uint32_t& i, const unsigned_int& v) { return i >= v.value; } + + /** + * Check if the first given unsigned_int is greater or equal to the second given unsigned_int + * object + * + * @param i - First unsigned_int object to compare + * @param v - Second unsigned_int object to compare + * @return true - if i is greater or equal to v + * @return false - otherwise + */ + friend bool operator>=(const unsigned_int& i, const unsigned_int& v) + { + return i.value >= v.value; + } + + /// @endcond + }; + + using varuint32 = unsigned_int; + CLIO_REFLECT(varuint32, value); + + template + void convert(const varuint32& src, uint32_t& dst, F&& chooser) + { + dst = src.value; + } + + template + void from_bin(varuint32& obj, S& stream); + + template + void to_bin(const varuint32& obj, S& stream); + + template + void from_json(varuint32& obj, S& stream); + + template + void to_json(const varuint32& obj, S& stream); + + template + void to_key(const varuint32& obj, S& stream) + { + return to_key_varuint32(obj.value, stream); + } + + /** + * Variable Length Signed Integer. This provides more efficient serialization of 32-bit signed + * int. It serializes a 32-bit signed integer in as few bytes as possible. + * + * @ingroup varint + * @note `varint32' is signed and uses [Zig-Zag + * encoding](https://developers.google.com/protocol-buffers/docs/encoding#signed-integers) + */ + struct signed_int + { + /** + * Construct a new signed int object + * + * @param v - Source + */ + signed_int(int32_t v = 0) : value(v) {} + + /// @cond OPERATORS + + /** + * Convert signed_int to primitive 32-bit signed integer + * + * @return int32_t - The converted result + */ + operator int32_t() const { return value; } + + /** + * Assign an object that is convertible to int32_t + * + * @tparam T - Type of the assignment object + * @param v - Source + * @return unsigned_int& - Reference to this object + */ + template + signed_int& operator=(const T& v) + { + value = v; + return *this; + } + + /** + * Increment operator + * + * @return signed_int - New signed_int with value incremented from the current object's value + */ + signed_int operator++(int) { return value++; } + + /** + * Increment operator + * + * @return signed_int - Reference to current object + */ + signed_int& operator++() + { + ++value; + return *this; + } + + /// @endcond + + /** + * Contained value + */ + int32_t value; + + /// @cond OPERATORS + + /** + * Check equality between a signed_int object and 32-bit integer + * + * @param i - signed_int object to compare + * @param v - 32-bit integer to compare + * @return true - if equal + * @return false - otherwise + */ + friend bool operator==(const signed_int& i, const int32_t& v) { return i.value == v; } + + /** + * Check equality between 32-bit integer and a signed_int object + * + * @param i - 32-bit integer to compare + * @param v - signed_int object to compare + * @return true - if equal + * @return false - otherwise + */ + friend bool operator==(const int32_t& i, const signed_int& v) { return i == v.value; } + + /** + * Check equality between two signed_int objects + * + * @param i - First signed_int object to compare + * @param v - Second signed_int object to compare + * @return true - if equal + * @return false - otherwise + */ + friend bool operator==(const signed_int& i, const signed_int& v) + { + return i.value == v.value; + } + + /** + * Check inequality between a signed_int object and 32-bit integer + * + * @param i - signed_int object to compare + * @param v - 32-bit integer to compare + * @return true - if inequal + * @return false - otherwise + */ + friend bool operator!=(const signed_int& i, const int32_t& v) { return i.value != v; } + + /** + * Check inequality between 32-bit integer and a signed_int object + * + * @param i - 32-bit integer to compare + * @param v - signed_int object to compare + * @return true - if unequal + * @return false - otherwise + */ + friend bool operator!=(const int32_t& i, const signed_int& v) { return i != v.value; } + + /** + * Check inequality between two signed_int objects + * + * @param i - First signed_int object to compare + * @param v - Second signed_int object to compare + * @return true - if inequal + * @return false - otherwise + */ + friend bool operator!=(const signed_int& i, const signed_int& v) + { + return i.value != v.value; + } + + /** + * Check if the given signed_int object is less than the given 32-bit integer + * + * @param i - signed_int object to compare + * @param v - 32-bit integer to compare + * @return true - if i less than v + * @return false - otherwise + */ + friend bool operator<(const signed_int& i, const int32_t& v) { return i.value < v; } + + /** + * Check if the given 32-bit integer is less than the given signed_int object + * + * @param i - 32-bit integer to compare + * @param v - signed_int object to compare + * @return true - if i less than v + * @return false - otherwise + */ + friend bool operator<(const int32_t& i, const signed_int& v) { return i < v.value; } + + /** + * Check if the first given signed_int is less than the second given signed_int object + * + * @param i - First signed_int object to compare + * @param v - Second signed_int object to compare + * @return true - if i less than v + * @return false - otherwise + */ + friend bool operator<(const signed_int& i, const signed_int& v) { return i.value < v.value; } + + /** + * Check if the given signed_int object is greater or equal to the given 32-bit integer + * + * @param i - signed_int object to compare + * @param v - 32-bit integer to compare + * @return true - if i is greater or equal to v + * @return false - otherwise + */ + friend bool operator>=(const signed_int& i, const int32_t& v) { return i.value >= v; } + + /** + * Check if the given 32-bit integer is greater or equal to the given signed_int object + * + * @param i - 32-bit integer to compare + * @param v - signed_int object to compare + * @return true - if i is greater or equal to v + * @return false - otherwise + */ + friend bool operator>=(const int32_t& i, const signed_int& v) { return i >= v.value; } + + /** + * Check if the first given signed_int is greater or equal to the second given signed_int + * object + * + * @param i - First signed_int object to compare + * @param v - Second signed_int object to compare + * @return true - if i is greater or equal to v + * @return false - otherwise + */ + friend bool operator>=(const signed_int& i, const signed_int& v) + { + return i.value >= v.value; + } + + /// @endcond + }; + + using varint32 = signed_int; + CLIO_REFLECT(varint32, value); + + template + void from_bin(varint32& obj, S& stream); + + template + void to_bin(const varint32& obj, S& stream); + + template + void from_json(varint32& obj, S& stream); + + template + void to_json(const varint32& obj, S& stream); + + template + void to_key(const varint32& obj, S& stream) + { + to_key_varint32(obj.value, stream); + } + +} // namespace clio diff --git a/libraries/clio/tests/CMakeLists.txt b/libraries/clio/tests/CMakeLists.txt new file mode 100644 index 000000000..b7265bb7f --- /dev/null +++ b/libraries/clio/tests/CMakeLists.txt @@ -0,0 +1,15 @@ +enable_testing() + +add_executable(test-clio + clio_tests.cpp + flat_views.cpp + benchmark.cpp +) +target_link_libraries(test-clio clio catch2) +set_target_properties(test-clio PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${ROOT_BINARY_DIR}) + +if(DEFINED IS_WASM) + target_link_libraries(test-clio basic-polyfill boost) +endif() + +native_and_wasm_test(test-clio) diff --git a/libraries/clio/tests/benchmark.cpp b/libraries/clio/tests/benchmark.cpp new file mode 100644 index 000000000..e22fa533c --- /dev/null +++ b/libraries/clio/tests/benchmark.cpp @@ -0,0 +1,404 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +namespace benchmark +{ + struct no_sub + { + double myd = 3.14; + uint16_t my16 = 22; + ; + }; + CLIO_REFLECT(no_sub, myd, my16); + + struct sub_obj + { + int a; + int b; + std::string substr; + no_sub ns; + }; + CLIO_REFLECT(sub_obj, a, b, ns, substr); + + struct flat_object + { + uint32_t x; + double y; + std::string z; + std::vector veci; + std::vector vecstr = {"aaaaaaaaa", "bbbbbbbbbbb", "ccccccccccccc"}; + std::vector vecns; + std::vector nested; + sub_obj sub; + }; + CLIO_REFLECT(flat_object, x, y, z, veci, vecstr, vecns, nested, sub); +} // namespace benchmark + +TEST_CASE("benchmark") +{ + using namespace benchmark; + + flat_object tester; + tester.z = "my oh my"; + tester.x = 99; + tester.y = 99.99; + tester.veci = {1, 2, 3, 4, 6}; + tester.vecns = {{.myd = 33.33}, {}}; + tester.nested = {{.x = 11, .y = 3.21, .nested = {{.x = 88}}}, {.x = 33, .y = .123}}; + tester.sub.substr = "sub str"; + tester.sub.a = 3; + tester.sub.b = 4; + + SECTION("unpacking from flat") + { + clio::size_stream ss; + clio::flatpack(tester, ss); + + std::vector buf(ss.size); + clio::fixed_buf_stream ps(buf.data(), buf.size()); + clio::flatpack(tester, ps); + auto start = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < 10000; ++i) + { + flat_object temp; + clio::input_stream in(buf.data(), buf.size()); + clio::flatunpack(temp, in); + } + auto end = std::chrono::steady_clock::now(); + auto delta = end - start; + + std::cout << "unpack flat: " << std::chrono::duration(delta).count() + << " ms size: " << ss.size << " "; + + auto compressed = clio::capn_compress(buf); + auto uncompressed = clio::capn_uncompress(compressed); + + clio::size_stream capsize; + clio::input_stream capin(buf.data(), buf.size()); + clio::capp_pack(capin, capsize); + + // std::cout <<"compressed.size: " << compressed.size() <<"\n"; + std::cout << "capsize: " << capsize.size << "\n"; + // std::cout <<"uncompressed.size: " << uncompressed.size() <<"\n"; + } + SECTION("validating flat without unpacking") + { + clio::size_stream ss; + clio::flatpack(tester, ss); + + std::vector buf(ss.size); + clio::fixed_buf_stream ps(buf.data(), buf.size()); + clio::flatpack(tester, ps); + + auto start = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < 10000; ++i) + { + clio::input_stream in(buf.data(), buf.size()); + clio::flatcheck(in); + } + auto end = std::chrono::steady_clock::now(); + auto delta = end - start; + + std::cout << "check flat: " << std::chrono::duration(delta).count() + << " ms size: " << ss.size << " "; + + auto compressed = clio::capn_compress(buf); + auto uncompressed = clio::capn_uncompress(compressed); + + clio::size_stream capsize; + clio::input_stream capin(buf.data(), buf.size()); + clio::capp_pack(capin, capsize); + + // std::cout <<"compressed.size: " << compressed.size() <<"\n"; + std::cout << "capsize: " << capsize.size << "\n"; + // std::cout <<"uncompressed.size: " << uncompressed.size() <<"\n"; + } + + SECTION("flat unpack and uncompress") + { + clio::size_stream ss; + clio::flatpack(tester, ss); + + std::vector buf(ss.size); + clio::fixed_buf_stream ps(buf.data(), buf.size()); + clio::flatpack(tester, ps); + + auto compressed = clio::capn_compress(buf); + + auto start = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < 10000; ++i) + { + flat_object temp; + auto uncompressed = clio::capn_uncompress(compressed); + clio::input_stream in(uncompressed.data(), uncompressed.size()); + clio::flatunpack(temp, in); + } + auto end = std::chrono::steady_clock::now(); + auto delta = end - start; + + std::cout << "unpack flat&cap: " << std::chrono::duration(delta).count() + << " ms size: " << ss.size << " "; + clio::size_stream capsize; + clio::input_stream capin(buf.data(), buf.size()); + clio::capp_pack(capin, capsize); + + std::cout << "capsize: " << capsize.size << "\n"; + } + + SECTION("protbuf unpack") + { + clio::size_stream ss; + clio::to_protobuf(tester, ss); + + std::vector buf(ss.size); + clio::fixed_buf_stream ps(buf.data(), buf.size()); + clio::to_protobuf(tester, ps); + + auto start = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < 10000; ++i) + { + flat_object temp; + + clio::input_stream in(buf.data(), buf.size()); + clio::from_protobuf_object(temp, in); + } + auto end = std::chrono::steady_clock::now(); + auto delta = end - start; + std::cout << "unpack pb: " << std::chrono::duration(delta).count() + << " ms size: " << ss.size << " "; + clio::size_stream capsize; + clio::input_stream capin(buf.data(), buf.size()); + clio::capp_pack(capin, capsize); + + std::cout << "capsize: " << capsize.size << "\n"; + } + + SECTION("protbuf unpack and uncompress") + { + clio::size_stream ss; + clio::to_protobuf(tester, ss); + + std::vector buf(ss.size); + clio::fixed_buf_stream ps(buf.data(), buf.size()); + clio::to_protobuf(tester, ps); + + auto compressed = clio::capn_compress(buf); + + auto start = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < 10000; ++i) + { + flat_object temp; + auto uncompressed = clio::capn_uncompress(compressed); + + clio::input_stream in(uncompressed.data(), uncompressed.size()); + clio::from_protobuf_object(temp, in); + } + auto end = std::chrono::steady_clock::now(); + auto delta = end - start; + std::cout << "unpack pb&cap: " << std::chrono::duration(delta).count() + << " ms size: " << ss.size << " "; + clio::size_stream capsize; + clio::input_stream capin(buf.data(), buf.size()); + clio::capp_pack(capin, capsize); + + std::cout << "capsize: " << capsize.size << "\n"; + } + + SECTION("bin unpack") + { + clio::size_stream ss; + clio::to_bin(tester, ss); + + std::vector buf(ss.size); + clio::fixed_buf_stream ps(buf.data(), buf.size()); + clio::to_bin(tester, ps); + auto start = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < 10000; ++i) + { + flat_object temp; + clio::input_stream in(buf.data(), buf.size()); + clio::from_bin(temp, in); + } + auto end = std::chrono::steady_clock::now(); + auto delta = end - start; + std::cout << "unpack bin: " << std::chrono::duration(delta).count() + << " ms size: " << ss.size << " "; + clio::size_stream capsize; + clio::input_stream capin(buf.data(), buf.size()); + clio::capp_pack(capin, capsize); + + std::cout << "capsize: " << capsize.size << "\n"; + } + SECTION("bin unpack and uncompress") + { + clio::size_stream ss; + clio::to_bin(tester, ss); + + std::vector buf(ss.size); + clio::fixed_buf_stream ps(buf.data(), buf.size()); + clio::to_bin(tester, ps); + + auto compressed = clio::capn_compress(buf); + + auto start = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < 10000; ++i) + { + flat_object temp; + auto uncompressed = clio::capn_uncompress(compressed); + + clio::input_stream in(uncompressed.data(), uncompressed.size()); + clio::from_bin(temp, in); + } + auto end = std::chrono::steady_clock::now(); + auto delta = end - start; + std::cout << "unpack bin&cap: " << std::chrono::duration(delta).count() + << " ms size: " << ss.size << " "; + clio::size_stream capsize; + clio::input_stream capin(buf.data(), buf.size()); + clio::capp_pack(capin, capsize); + + std::cout << "capsize: " << capsize.size << "\n"; + } + + SECTION("json unpack") + { + clio::size_stream ss; + clio::to_json(tester, ss); + + std::vector buf(ss.size + 1); + std::vector buf2(ss.size + 1); + buf.back() = 0; + clio::fixed_buf_stream ps(buf.data(), buf.size()); + clio::to_json(tester, ps); + buf2 = buf; + auto start = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < 10000; ++i) + { + flat_object temp; + clio::json_token_stream in(buf.data()); + clio::from_json(temp, in); + buf = buf2; + } + auto end = std::chrono::steady_clock::now(); + auto delta = end - start; + std::cout << "unpack json: " << std::chrono::duration(delta).count() + << " ms size: " << ss.size << " capsize: "; + clio::size_stream capsize; + clio::input_stream capin(buf.data(), buf.size()); + clio::capp_pack(capin, capsize); + + std::cout << "capsize: " << capsize.size << "\n"; + } + + SECTION("packing") + { + size_t s = 0; + auto start = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < 10000; ++i) + { + clio::size_stream ss; + clio::flatpack(tester, ss); + s = ss.size; + + std::vector buf(ss.size); + clio::fixed_buf_stream ps(buf.data(), buf.size()); + clio::flatpack(tester, ps); + // std::cout <<"capsize: " << capsize.size <<"\n"; + } + auto end = std::chrono::steady_clock::now(); + auto delta = end - start; + std::cout << "pack flat: " << std::chrono::duration(delta).count() + << " ms size: " << s << "\n"; + + s = 0; + start = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < 10000; ++i) + { + clio::size_stream ss; + clio::flatpack(tester, ss); + s = ss.size; + + std::vector buf(ss.size); + clio::fixed_buf_stream ps(buf.data(), buf.size()); + clio::flatpack(tester, ps); + + // clio::size_stream capsize; + + std::vector buf2(ss.size); + clio::fixed_buf_stream capout(buf2.data(), buf2.size()); + clio::input_stream capin(buf.data(), buf.size()); + clio::capp_pack(capin, capout); + s = buf2.size() - capout.remaining(); // capsize.size; + + // std::cout <<"capsize: " << capsize.size <<"\n"; + } + end = std::chrono::steady_clock::now(); + delta = end - start; + std::cout << "pack flat&cap: " << std::chrono::duration(delta).count() + << " ms size: " << s << "\n"; + + start = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < 10000; ++i) + { + clio::size_stream ss; + clio::to_protobuf(tester, ss); + s = ss.size; + + std::vector buf(ss.size); + clio::fixed_buf_stream ps(buf.data(), buf.size()); + clio::to_protobuf(tester, ps); + } + end = std::chrono::steady_clock::now(); + delta = end - start; + std::cout << "pack pb: " << std::chrono::duration(delta).count() + << " ms size: " << s << "\n"; + + start = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < 10000; ++i) + { + clio::size_stream ss; + clio::to_bin(tester, ss); + s = ss.size; + + std::vector buf(ss.size); + clio::fixed_buf_stream ps(buf.data(), buf.size()); + clio::to_bin(tester, ps); + } + end = std::chrono::steady_clock::now(); + delta = end - start; + std::cout << "pack bin: " << std::chrono::duration(delta).count() + << " ms size: " << s << "\n"; + + start = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < 10000; ++i) + { + clio::size_stream ss; + clio::to_json(tester, ss); + s = ss.size; + + std::vector buf(ss.size); + clio::fixed_buf_stream ps(buf.data(), buf.size()); + clio::to_json(tester, ps); + } + end = std::chrono::steady_clock::now(); + delta = end - start; + std::cout << "pack json: " << std::chrono::duration(delta).count() + << " ms size: " << s << "\n"; + } +} diff --git a/libraries/clio/tests/clio_tests.cpp b/libraries/clio/tests/clio_tests.cpp new file mode 100644 index 000000000..4ed06df1f --- /dev/null +++ b/libraries/clio/tests/clio_tests.cpp @@ -0,0 +1,2 @@ +#define CATCH_CONFIG_MAIN +#include diff --git a/libraries/clio/tests/crypto.cpp b/libraries/clio/tests/crypto.cpp new file mode 100644 index 000000000..5740afd01 --- /dev/null +++ b/libraries/clio/tests/crypto.cpp @@ -0,0 +1,66 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +struct identity +{ + clcrypto::private_key priv; + clcrypto::public_key pub; + clcrypto::signature sig; + std::string hello; + clcrypto::sha256 digest; +}; + +CLIO_REFLECT(identity, hello, digest, priv, pub, sig) + +TEST_CASE("keygen") +{ + std::cout << "KEYGEN\n"; + auto Priv = fc::crypto::private_key::generate_r1(); + auto Pub = Priv.get_public_key(); + + std::cout << Priv.to_string() << "\n"; + std::cout << Pub.to_string() << "\n"; + + clcrypto::k1pri privk = clcrypto::k1pri::generate(); + clcrypto::k1pub pubk = privk.get_public_key(); // clcrypto::k1pub(); + + // auto hashhello = fc::sha256::hash( "hello world", 11 ); + auto hashhello = fc::sha256::hash(std::string("hello world")); + auto sig = privk.sign(hashhello); + + std::cout << std::string(pubk) << "\n"; + std::cout << std::string(privk) << "\n"; + std::cout << hashhello.str() << "\n"; + std::cout << std::string(sig) << "\n"; + + auto rek = sig.recover(hashhello); + std::cout << std::string(rek) << "\n"; + + identity id; + id.priv = clcrypto::r1pri::generate(); + id.pub = clcrypto::get_public_key(id.priv); + id.sig = clcrypto::sign(id.priv, id.digest); + std::cout << clio::to_json(id) << "\n"; +} diff --git a/libraries/clio/tests/database.cpp b/libraries/clio/tests/database.cpp new file mode 100644 index 000000000..4fdbf48a2 --- /dev/null +++ b/libraries/clio/tests/database.cpp @@ -0,0 +1,727 @@ +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +struct db_handle +{ + uint32_t id; +}; + +/** + * Defines an global API that represents an abstract database that is able to + * create, fetch, and free records. + */ +class intrinsic +{ + public: + static intrinsic& instance() + { + static intrinsic i; + return i; + } + + enum error_code + { + ok = 0, /// everything is ok + invalid_handle = 1, /// the handle is unknwon to us + freed_handle = 2, /// the handle requested has already been freed + invalid_buffer_pram = 3, + invalid_size_param = 4, + invalid_handle_param = 5 /// the user provided an invalid handle paramter ref + }; + + error_code db_create(size_t size, db_handle* new_handle) + { + if (size <= 0) + return invalid_size_param; + if (new_handle == nullptr) + return invalid_handle_param; + + _database.emplace_back(std::vector(size)); + + auto& rec = _database.back(); + rec.resize(size); + + new_handle->id = _database.size() - 1; + return ok; + } + + error_code db_set(db_handle h, const char* buffer, size_t size) + { + if (size <= 0) + return invalid_size_param; + if (h.id < _database.size()) + { + auto& rec = _database[h.id]; + if (rec.size() == 0) + return freed_handle; + rec.resize(size); + memcpy(rec.data(), buffer, size); + return ok; + } + else + return invalid_handle; + } + + size_t db_get_size(db_handle h) + { + if (h.id < _database.size()) + { + return _database[h.id].size(); + } + else + return invalid_handle; + } + + error_code db_get(db_handle h, char* buffer, size_t size_in) + { + if (h.id < _database.size()) + { + auto& rec = _database[h.id]; + if (buffer && size_in) + { + auto read_size = std::min(size_in, rec.size()); + memcpy(buffer, rec.data(), read_size); + } + else + { + return invalid_buffer_pram; + } + } + else + return invalid_handle; + return ok; + } + + error_code db_free(db_handle h) + { + if (h.id >= _database.size()) + return invalid_handle_param; + _free_list.emplace_back(h.id, std::move(_database[h.id])); + return ok; + } + + private: + std::vector>> _free_list; + std::vector> _database; +}; + +/** + * This serves as a wasm-side cache of records already feteched from the intrinsic interface, + * it has a singleton interface and all fetches from the database should check with the cache + * instead of going to the intrinsic API. The cache will fetch from the database if not found. + */ +class cache +{ + public: + /** + * This object holds the one-true copy of the buffer for a given db handle. + * It is movable but not copyable. + */ + class cache_record + { + public: + size_t size() const { return _buffer.size(); } + const char* data() const { return _buffer.data(); } + bool is_dirty() const { return _dirty; } + db_handle handle() const { return _handle; } + + void resize(uint32_t s) { _buffer.resize(s); } + char* data() { return _buffer.data(); } + + /** + * @return a non-const reference to this after setting the dirty bit and notifying + * the cache + */ + cache_record& modify() const + { + if (not _dirty) + { + _dirty = true; + _cache._dirty.push_back(this); + } + return *const_cast(this); + } + + cache_record(cache_record&& mv) + : _cache(mv._cache), + _handle(mv._handle), + _buffer(std::move(mv._buffer)), + _dirty(mv._dirty) + { + mv._handle.id = -1; + } + + cache_record& operator=(cache_record&& mv) + { + /// TODO assert &_cache == &mv._cache... this may be redundant + /// since the cache is a global singleton, there may be no good + /// reason to store _cache on the record at all and we can save + /// some memory. + + if (&mv != this) + { + _handle = mv._handle; + mv._handle.id = -1; + _buffer = std::move(mv._buffer); + _dirty = mv._dirty; + } + return *this; + } + + private: + cache_record(const cache_record&) = delete; + cache_record(cache& c, db_handle h) : _cache(c), _handle(h) {} + + cache& _cache; + db_handle _handle; + std::vector _buffer; + mutable bool _dirty = false; + + friend class cache; + }; + + cache_record& create(size_t new_size) + { + db_handle new_handle; + intrinsic::instance().db_create(new_size, &new_handle); + auto& r = _db_cache + .emplace(std::pair(new_handle.id, + cache_record(*this, new_handle))) + .first->second; + r.modify(); // pushes it to cache + return r; + } + + const cache_record& get(db_handle h) + { + auto itr = _db_cache.find(h.id); + if (itr != _db_cache.end()) + return itr->second; + + size_t s = intrinsic::instance().db_get_size(h); + + if (s <= 0) + { + /// some kind of error happened, abort! + } + + auto& r = _db_cache.emplace(h.id, cache_record(*this, h)).first->second; + + r._buffer.resize(s); + if (auto err = intrinsic::instance().db_get(h, r._buffer.data(), s)) + { + /// some kind of error happened, abort! + } + return r; + } + + static cache& instance() + { + static cache c; + return c; + } + + private: + std::vector _dirty; + std::map _db_cache; +}; + +template +struct db_ptr; + +template +struct db_ref +{ + db_ref(const cache::cache_record& c) : _cr(c) {} + + const T* operator->() const { return reinterpret_cast(_cr.data()); } + + T* operator->() { return reinterpret_cast(_cr.modify().data()); } + + operator db_ptr() const { return db_ptr(_cr.handle()); } + operator db_handle() const { return _cr.handle(); } + + private: + const cache::cache_record& _cr; +}; + +template +struct db_ptr +{ + db_ptr() {} + db_ptr(db_handle h) : id(h.id) {} + + uint32_t id = -1; + + bool operator!() const { return id == -1; } + + friend bool operator==(const db_ptr& a, const db_ptr& b) { return a.id == b.id; } + friend bool operator!=(const db_ptr& a, const db_ptr& b) { return a.id != b.id; } + + const db_ref operator->() const { return cache::instance().get({id}); } + const db_ref operator*() const { return cache::instance().get({id}); } +}; + +template +db_ref make_db_ptr(Args&&... args) +{ + auto& cr = cache::instance().create(sizeof(T)); + cr.resize(sizeof(T)); + new (cr.data()) T(std::forward(args)...); + return cr; +} + +/** + * This is the data that is actually stored in the database, it should + * be trivially copyable. + */ +template +struct node +{ + static_assert(std::is_trivially_copyable::value); + + template + node(Args&&... args) : data(std::forward(args)...) + { + } + + db_ptr> left; + db_ptr> right; + db_ptr> parent; + int height = 0; + T data; +}; + +/** + * The underlying node object only has db_ptr<> for left/right, which + * means that every time you dereference one of them you have to go to + * the cache and do a map/hash table lookup. This would dramatically hurt + * the performance of the log(n) lookup required. The node_cache object + * keeps a cache of the left/right db_ref that will only fectch if they + * have not yet been loaded from cache/db. + * + * When performing tree operations you must modify both self->left/right and + * this->left/right or the cache will break its mirror to the underlying. + */ +template +class node_cache : public std::enable_shared_from_this> +{ + public: + using node_cptr = std::shared_ptr>; + using node_db_ptr = db_ptr>; + + node_cache(db_ref> s) : self(s) {} + + const T& value() const { return self->data; } + int height() const { return self->height; } + + const node_cptr& left() const + { + if (not _left and !!self->left) + { + _left = std::make_shared>(*self->left); + } + return _left; + } + const node_cptr& right() const + { + if (not _right and !!self->right) + { + _right = std::make_shared>(*self->right); + } + return _right; + } + const node_cptr& parent() const + { + if (not _parent and !!self->parent) + { + _parent = std::make_shared>(*self->parent); + } + return _parent; + } + + void set_parent(node_cptr new_parent) const + { + _parent = new_parent; + + node_db_ptr new_parent_id = _parent ? _parent->ptr() : node_db_ptr(); + if (self->parent != new_parent_id) + { + const_cast>&>(self)->parent = new_parent_id; + } + } + + void set_right(node_cptr new_right) const + { + _right = new_right; + + node_db_ptr new_right_id = _right ? _right->ptr() : node_db_ptr(); + if (self->right != new_right_id) + { + const_cast>&>(self)->right = new_right_id; + } + if (_right) + _right->set_parent(this->shared_from_this()); + } + + void set_left(node_cptr new_left) const + { + _left = new_left; + node_db_ptr new_left_id = _left ? _left->ptr() : node_db_ptr(); + if (self->left != new_left_id) + { + const_cast>&>(self)->left = new_left_id; + } + if (_left) + _left->set_parent(this->shared_from_this()); + } + + void set_height(int h) const + { + if (self->height != h) + { + const_cast>&>(self)->height = h; + } + } + + /* + operator db_ref>& () { return self; } + operator const db_ref>& ()const { return self; } + operator db_ptr>()const { return self; } + operator db_handle()const { return self; } + db_handle handle()const { return self; } + */ + + db_ptr> ptr() const { return self; } + + private: + db_ref> self; + + mutable node_cptr _left; + mutable node_cptr _right; + mutable node_cptr _parent; +}; + +/** + * This is the datatype that stores the tree in the DB, it only + * stores the root. + */ +template +struct tree +{ + typedef db_ptr> node_type; + node_type root; +}; + +/** + * This is the interface that actually impliments the algorithms on + * the underlying memory. + */ +template +class tree_cache +{ + using node_cptr = std::shared_ptr>; + + public: + /** + * Constructed with a reference to the tree database record, + * the tree is a record that contains a single db handle to + * the current root of the tree. + */ + tree_cache(db_ref> s) : self(s) {} + + void insert(const T& v) const { set_root(insert(v, root())); } + + template + void visit(L&& v) const + { + visit(std::forward(v), root()); + } + + struct iterator + { + iterator() {} + const T& operator*() const { return pos->value(); } + iterator& operator++() + { + if (not pos) + return *this; + // std::cout << "start: " << **this << "\n"; + + auto& right = pos->right(); + if (right) + { + pos = right; + // std::cout << " go right to: " << **this << "\n"; + while (auto& left = pos->left()) + { + pos = left; + // std::cout << " go left to: " << **this << "\n"; + } + } + else + { + // std::cout << " go to parent while parent right != me\n"; + while (auto& parent = pos->parent()) + { + if (parent->right() == pos) + { + pos = parent; + // std::cout << " go to next parent: " << **this << "\n"; + } + else + { + // std::cout << " go to parent: " << **this << "\n"; + pos = parent; + return *this; + } + } + // std::cout << " no more parents! We are done\n"; + pos = node_cptr(); + } + return *this; + } + + friend bool operator==(const iterator& a, const iterator& b) { return a.pos == b.pos; } + friend bool operator!=(const iterator& a, const iterator& b) { return a.pos != b.pos; } + + private: + friend class tree_cache; + iterator(const node_cptr& p) : pos(p) {} + node_cptr pos; + }; + + iterator lower_bound(const T& v) const { return lower_bound(v, root()); } + iterator end() const { return iterator(node_cptr()); } + + private: + int height(const node_cptr& t) const { return t ? t->height() : -1; } + + template + void visit(L&& v, const node_cptr& t) const + { + if (not t) + return; + visit(std::forward(v), t->left()); + v(t->value()); + visit(std::forward(v), t->right()); + } + + iterator lower_bound(const T& v, const node_cptr& t) const + { + if (not t) + return iterator(); + std::cout << " t->value: " << t->value() << "\n"; + if (v < t->value()) + { + auto& left = t->left(); + if (left) + return lower_bound(v, left); + } + else if (v > t->value()) + { + auto& right = t->right(); + if (right) + return lower_bound(v, right); + } + return iterator(t); + } + + node_cptr insert(const T& v, node_cptr t) const + { + if (not t) + { + return make_node(v); + } + else if (v < t->value()) + { + t->set_left(insert(v, t->left())); + auto tl = t->left(); + auto tr = t->right(); + if (height(tl) - height(tr) == 2) + { + if (v < tl->value()) + { + t = single_right_rotate(t); + } + else + { + t = double_right_rotate(t); + } + } + } + else if (v > t->value()) + { + t->set_right(insert(v, t->right())); + auto tl = t->left(); + auto tr = t->right(); + if (height(tr) - height(tl) == 2) + { + if (v > tr->value()) + { + t = single_left_rotate(t); + } + else + { + t = double_left_rotate(t); + } + } + } + t->set_height(std::max(height(t->left()), height(t->right())) + 1); + return t; + } + + node_cptr single_right_rotate(const node_cptr& t) const + { + auto u = t->left(); + t->set_left(u->right()); + u->set_right(t); + t->set_height(std::max(height(t->left()), height(t->right())) + 1); + u->set_height(std::max(height(u->left()), height(u->right())) + 1); + return u; + } + node_cptr single_left_rotate(const node_cptr& t) const + { + auto u = t->right(); + t->set_right(u->left()); + u->set_left(t); + t->set_height(std::max(height(t->left()), height(t->right())) + 1); + u->set_height(std::max(height(u->left()), height(u->right())) + 1); + return u; + } + + node_cptr double_left_rotate(const node_cptr& t) const + { + t->set_right(single_right_rotate(t->right())); + return single_left_rotate(t); + } + + node_cptr double_right_rotate(const node_cptr& t) const + { + t->set_left(single_left_rotate(t->left())); + return single_right_rotate(t); + } + + const node_cptr& root() const + { + if (not _root and !!self->root) + { + set_root(std::make_shared>(*self->root)); + } + return _root; + } + + /** + * This function ensures the invariant that self.id == r->id, it + * is too easy change one without changing the other... therefore + * we use 'const' everywhere and this one function is given the + * right to cast it away and update the root. + */ + void set_root(const node_cptr& r) const { const_cast(this)->set_root(r); } + void set_root(const node_cptr& r) + { + _root = r; + if (not _root) + { + self->root = db_ptr>(); /// null id + } + else + { + std::cout << "_root->ptr.id(): " << _root->ptr().id << " self->root.id: " << self->root.id + << " value: " << _root->value() << "\n"; + /// TODO: make sure things really did change so we + /// don't trigger the dirty bit + // if( _root->handle().id != self->root ) { + + if (_root->ptr() != self->root) + { + std::cout << " current root: " << self->root.id; + std::cout << " setting root: " << _root->ptr().id << " \n"; + self->root = _root->ptr(); //.id = _root->handle().id; + } + _root->set_parent(node_cptr()); + } + } + + node_cptr make_node(const T& v) const + { + return std::make_shared>(make_db_ptr>(v)); + } + + db_ref> self; + node_cptr _root; +}; + +/* +template +struct base_index : { + T value; +} + +template +struct index : base_index { + index* left; + index* right; + index* parent; + int height; +} +*/ + +int main(int, char**) +{ + // auto root = make_db_ptr>( 5 ); + tree_cache tc(make_db_ptr>()); + /* + tc.insert( 6 ); + tc.insert( 8 ); + tc.insert( 9 ); + tc.insert( 2 ); + tc.insert( 41 ); + tc.insert( 42); + tc.insert( 43); + tc.insert( 44); + tc.insert( 45); + tc.insert( 46); + */ + for (uint32_t i = 0; i < 32; ++i) + tc.insert(i); + + auto itr = tc.lower_bound(10); + while (itr != tc.end()) + { + std::cout << "value: " << *itr << "\n"; + ++itr; + } + + /* + tc.visit( [&]( const auto& v ) { + std::cout << v << "\n"; + }); + */ + + // auto itr = tr->lower_bound(5); + + // tr->insert( 5 ); + + // std::cout << "root: " << root->data <<"\n"; + + // clio::flat_ptr> n = node(); + + return 0; +} diff --git a/libraries/clio/tests/db.cpp b/libraries/clio/tests/db.cpp new file mode 100644 index 000000000..8c8083ae8 --- /dev/null +++ b/libraries/clio/tests/db.cpp @@ -0,0 +1,222 @@ +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +class intrinsic +{ + public: + struct db_handle + { + uint32_t id; + }; + enum error_code + { + ok = 0, /// everything is ok + invalid_handle = 1, /// the handle is unknwon to us + freed_handle = 2, /// the handle requested has already been freed + invalid_buffer_pram = 3, + invalid_size_param = 4, + invalid_handle_param = 5 /// the user provided an invalid handle paramter ref + }; + + error_code db_create(size_t size, db_handle* new_handle) + { + if (size <= 0) + return invalid_size_param; + if (new_handle == nullptr) + return invalid_handle_param; + + _database.emplace_back(buffer, buffer + size); + + auto& rec = _database.back(); + rec.resize(size); + + new_handle->id = _database.size() - 1; + return ok; + } + + error_code db_set(db_handle h, const char* buffer, size_t size) + { + if (size <= 0) + return invalid_size_param; + if (h.id < _database.size()) + { + auto& rec = _database[h.id]; + if (rec.size() == 0) + return freed_handle; + rec.resize(size); + memcpy(rec.data(), buffer, size); + return ok; + } + else + return invalid_handle; + } + + size_t db_get_size(db_handle h) + { + if (h.id < _database.size()) + { + return _database[h.id].size(); + } + else + return invalid_handle; + } + + error_code db_get(db_handle h, char* buffer, size_t size_in) + { + if (h.id < _database.size()) + { + auto& rec = _database[h.id]; + if (buffer && size_in) + { + auto read_size = std::min(size_in, rec.size()); + memcpy(buffer, rec.data(), read_size); + } + else + { + return invalid_buffer_pram; + } + } + else + return invalid_handle; + } + + error_code db_free(db_handle h) + { + if (h.id >= _database.size()) + return invalid_handle_param; + _free_list.push_back(std::make_pair(h, std::move(_database[h.id]))); + return ok; + } + + private: + std::vector < std::pair > _free_list; + std::vector > _database; +}; + +class cache +{ + public: + class cache_record + { + public: + size_t size() const { return buffer.size(); } + const char* data() const { return buffer.data(); } + bool is_dirty() const { return _dirty; } + db_handle handle() const { return _handle; } + + template + cache_record& modify(M&& mod) + { + _dirty = true; + mod(_buffer); + return *this; + } + + cache_record(cache_record&& mv) + : _cache(mv._cache), _handle(mv._handle), _buffer(std::move(mv._buffer)) _dirty(mv._dirty) + { + _mv.handle.id = -1; + } + + private: + cache_record(const cache_record&) = delete; + cache_record(cache& c, db_handle h) : _cache(c), _handle(h) {} + + cache& _cache; + db_handle _handle; + std::vector _buffer; + bool _dirty = false; + }; + + template + cache_record& db_create(L&& constructor) + { + db_handle new_handle; + intrinsic::db_create(rec.size, &new_handle); + auto& r = _db_cache[new_handle.id] = + cache_record(*this, new_handle).modify(std::forward(constructor)); + _dirty.push_back(&r); + } + + cache_record& db_get(db_handle h) + { + auto itr = _db_cache.find(h.id); + if (itr != _db_cache.end()) + return itr->second; + + size_t s = db_get(h); + + if (s <= 0) + { + /// some kind of error happened, abort! + } + + auto& r = _db_cache[h] = cache_record(*this, h); + + r._buffer.resize(s); + if (auto err = intrinsic::db_get(h, r._buffer.data(), s)) + { + /// some kind of error happened, abort! + } + return r; + } + + private: + vector _dirty; + map _db_cache; +}; + +/** + * Can be serialized, and resets _record to nullptr when _handle changes + */ +class db_ptr +{ + public: + const char* data() const { return _get_record().data(); } + size_t size() const { return _get_record().size(); } + + template + void modify(L&& mod) + { + return _get_record().modify(std::forward(mod)); + } + + db_handle handle() const { return _handle; } + + private: + db_handle _handle; + cache::cache_record* _record = nullptr; +}; + +template +class view_ptr_impl : private db_ptr +{ + public: +}; + +template +struct tree_root +{ + T value; +}; + +int main(int argc, char** argv) +{ + return 0; +} diff --git a/libraries/clio/tests/flat_views.cpp b/libraries/clio/tests/flat_views.cpp new file mode 100644 index 000000000..09c1301fc --- /dev/null +++ b/libraries/clio/tests/flat_views.cpp @@ -0,0 +1,232 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +struct non_ref_nest +{ + float lite = 33; +}; +struct non_reflect_type +{ + int first = 0; + int second = 0; + double real = 3.14; + non_ref_nest nrn; +}; + +struct no_sub +{ + double myd = 3.14; + uint16_t my16 = 22; + ; +}; +CLIO_REFLECT(no_sub, myd, my16); + +struct sub_obj +{ + int a; + int b; + int c; + std::string substr; + no_sub ns; +}; +CLIO_REFLECT(sub_obj, a, b, c, substr, ns); + +struct flat_object +{ + non_reflect_type non_ref; + uint32_t x; + double y; + std::string z; + std::vector veci; + std::vector vecstr = {"aaaaaaaaa", "bbbbbbbbbbb", "ccccccccccccc"}; + std::vector vecns; + std::vector nested; + sub_obj sub; + clio::flat_ptr sub_view; +}; +CLIO_REFLECT(flat_object, non_ref, x, y, z, veci, vecstr, vecns, nested, sub, sub_view); + +struct fat +{ + uint16_t a; + uint16_t b; + std::string substr; +}; +CLIO_REFLECT(fat, a, b, substr); + +struct root +{ + std::string other; + clio::flat_ptr fatptr; +}; +CLIO_REFLECT(root, other, fatptr); + +TEST_CASE("flatptr") +{ + constexpr uint32_t offseta = + clio::get_tuple_offset<0, std::tuple>>::value; + constexpr uint32_t offsetb = + clio::get_tuple_offset<1, std::tuple>>::value; + std::cout << offseta << " " << offsetb << "\n"; + std::cout << "flatpacksize: " << clio::flatpack_size() << "\n"; + std::cout << "contains_offset_ptr: " << clio::contains_offset_ptr() << "\n"; + + /* + clio::flat_ptr r( root{ .other = "other", .fatptr = fat{ .a = 0x0A, .b = 0x0B, .substr = + "sub" } } ); std::cout << "r.size: " << r.size() << "\n"; std::vector d( r.data(), + r.data()+r.size() ); std::cout << clio::to_json(clio::bytes{d}) <<"\n"; + + std::cout << std::hex << r->fatptr->get()->a << "\n"; + std::cout << std::hex << r->fatptr->get()->b << "\n"; + std::cout << std::hex << r->fatptr->get()->substr << "\n"; + std::cout << std::dec << r->fatptr->get()->substr << "\n"; + */ + + flat_object tester; + tester.z = "my oh my"; + tester.x = 99; + tester.y = 99.99; + tester.veci = {1, 2, 3, 4, 6}; + tester.vecns = {{.myd = 33.33}, {}}; + tester.nested = {{.x = 11, .y = 3.21, .nested = {{.x = 88}}}, {.x = 33, .y = .123}}; + tester.sub.substr = "zzzzzzzzzz"; + tester.sub.ns.myd = .9876; + tester.sub.ns.my16 = 33; + tester.sub.a = 3; + tester.sub.b = 4; + tester.sub_view = sub_obj{.a = 0xdddd, .b = 0xbbbb, .substr = "yyyyyyyyyy"}; + + clio::flat_ptr me(tester); + std::cout << "fpsize: " << clio::flatpack_size() << "\n"; + // std::cerr<< "sub.a: " << std::hex << me->sub->a<<"\n"; + std::cerr << "sub_view->get()->a: \n"; + int x = me->sub_view->a; + std::cerr << "sub_view.a: " << std::hex << x << "\n"; + /* + + int y = me->sub->a.get(); + std::cerr<< "sub.a: " << std::hex << y <<"\n"; + + std::vector d( me.data(), me.data()+me.size() ); + std::cout << clio::to_json(clio::bytes{d}) <<"\n"; + std::cout << "start: " << std::dec << int64_t(me.data()) <<"\n"; + */ +} +TEST_CASE("flatview") +{ + flat_object tester; + tester.z = "my oh my"; + tester.x = 99; + tester.y = 99.99; + tester.veci = {1, 2, 3, 4, 6}; + tester.vecns = {{.myd = 33.33}, {}}; + tester.nested = {{.x = 11, .y = 3.21, .nested = {{.x = 88}}}, {.x = 33, .y = .123}}; + tester.sub.substr = "sub str"; + tester.sub.ns.myd = .9876; + tester.sub.ns.my16 = 33; + tester.sub.a = 3; + tester.sub.b = 4; + tester.sub_view = sub_obj{.a = 987, .b = 654, .substr = "subviewstr"}; + + clio::flat_ptr me(tester); + std::cerr << "me.size: " << me.size() << "\n"; + + auto start = std::chrono::steady_clock::now(); + uint64_t sum = 0; + for (uint32_t i = 0; i < 10000; ++i) + { + for (uint32_t x = 0; x < me->veci->size(); ++x) + sum += me->veci[x]; + } + auto end = std::chrono::steady_clock::now(); + auto delta = end - start; + + auto view_time = (delta).count(); + std::cout << "sum veci via view: " << std::chrono::duration(delta).count() + << " ms\n"; + + start = std::chrono::steady_clock::now(); + sum = 0; + for (uint32_t i = 0; i < 10000; ++i) + { + for (uint32_t x = 0; x < tester.veci.size(); ++x) + sum += tester.veci[x]; + } + end = std::chrono::steady_clock::now(); + delta = end - start; + std::cout << "sum: " << sum << "\n"; + std::cout << "sum veci via real: " << std::chrono::duration(delta).count() + << " ms\n"; + auto real_time = (delta).count(); + + std::cout << "view runtime: " << 100 * double(view_time) / real_time << "% of real\n"; + + std::cout << "sub.substr: " << me->sub->substr << "\n"; + + std::cout << "x: " << me->x << "\n"; + std::cout << "y: " << me->y << "\n"; + std::cout << "z: " << me->z << "\n"; + std::cout << "veci[0]: " << me->veci[0] << "\n"; + std::cout << "veci[1]: " << me->veci[1] << "\n"; + std::cout << "veci[2]: " << me->veci[2] << "\n"; + std::cout << "veci[3]: " << me->veci[3] << "\n"; + std::cout << "veci[4]: " << me->veci[4] << "\n"; + std::cout << "vecstr[0]: " << me->vecstr[0] << "\n"; + std::cout << "vecstr[1]: " << me->vecstr[1] << "\n"; + std::cout << "vecstr[2]: " << me->vecstr[2] << "\n"; + std::cout << "vecns[0].myd: " << me->vecns[0].myd << "\n"; + std::cout << "vecns[0].my16: " << me->vecns[0].my16 << "\n"; + std::cout << "vecns[1].myd: " << me->vecns[1].myd << "\n"; + std::cout << "vecns[1].my16: " << me->vecns[1].my16 << "\n"; + std::cout << "nested->size(): " << me->nested->size() << "\n"; + std::cout << "nested[0].x: " << me->nested[0].x << "\n"; + std::cout << "nested[0].y: " << me->nested[0].y << "\n"; + std::cout << "nested[0].z: " << me->nested[0].z << "\n"; + std::cout << "nested[0].nested[0].x: " << me->nested[0].nested[0].x << "\n"; + std::cout << "nested[0].nested[0].y: " << me->nested[0].nested[0].y << "\n"; + std::cout << "nested[1].z: " << me->nested[1].z << "\n"; + std::cout << "nested[1].x: " << me->nested[1].x << "\n"; + std::cout << "nested[1].y: " << me->nested[1].y << "\n"; + std::cout << "sub.a: " << me->sub->a << "\n"; + std::cout << "sub.b: " << me->sub->b << "\n"; + std::cout << "sub.ns.myd: " << me->sub->ns->myd << "\n"; + std::cout << "sub.ns.my16: " << me->sub->ns->my16 << "\n"; + + clio::input_stream in(me.data(), me.size()); + clio::flatcheck(in); + + std::cout << "sub_view.a: " << me->sub_view->a << "\n"; + // std::cout << "sub_view.b: " << me->sub_view->b<<"\n"; + + std::cout << "tester.sub.substr: " << tester.sub.substr << "\n"; + + flat_object copy = me; /// unpacks from buffer into full object + std::cout << "copy.sub.a: " << copy.sub.a << "\n"; + std::cout << "copy.sub.b: " << copy.sub.b << "\n"; + std::cout << "copy.sub.ns.myd: " << copy.sub.ns.myd << "\n"; + std::cout << "copy.sub.ns.my16: " << copy.sub.ns.my16 << "\n"; + std::cout << "copy.sub.substr: " << copy.sub.substr << "\n"; + std::cout << "copy.sub_view->a: " << copy.sub_view->a << "\n"; + std::cout << "copy.sub_view->b: " << copy.sub_view->b << "\n"; + std::cout << "copy.sub_view->b: " << copy.sub_view->substr << "\n"; + + std::cout << "non_ref->real: " << me->non_ref->real << "\n"; + std::cout << "non_ref->nrn.lite: " << me->non_ref->nrn.lite << "\n"; +} diff --git a/libraries/clio/tests/gql.cpp b/libraries/clio/tests/gql.cpp new file mode 100644 index 000000000..a887dfbbb --- /dev/null +++ b/libraries/clio/tests/gql.cpp @@ -0,0 +1,66 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +struct service +{ + int add(int a, int b) const + { + std::cout << a << " + " << b << "\n"; + return a + b; + } + int sub(int a, int b) + { + std::cout << a << " - " << b << "\n"; + return a - b; + } +}; + +CLIO_REFLECT_PB(service, (add, 0, a, b), (sub, 1, a, b)) + +TEST_CASE("try flatpack/unpack/view of std::varaint", "[variant]") +{ + using variant_type = std::variant; + + auto test_type = [](auto val) { + variant_type test_variant; + test_variant = val; + clio::flat_ptr packed = test_variant; + + variant_type unpacked = packed; + std::cout << "test_variant: " << clio::to_json(test_variant) << "\n"; + std::cout << "unpacked_variant: " << clio::to_json(unpacked) << "\n"; + + packed->visit([](const auto& v) { + std::cout << "visit value: " << v << "\n"; + /* + if constexpr ( std::is_same_v,std::decay_t> ) { + std::cout << "json value: " << clio::format_json( std::string_view(v) ) <<"\n"; + } else { + std::cout << "json value: " << clio::format_json(v) <<"\n"; + } + */ + }); + // clio::input_stream in( packed.data(), packed.size() ); + // clio::flatcheck( in ); + }; + test_type(33); + test_type(33.33); + test_type("hello world"); +} + +TEST_CASE("try flat pack/view of gql::query", "[gql]") +{ + clio::gql::query q; + + clio::schema service_schema; + service_schema.generate(); + // std::cout << "schema: " << clio::format_json( service_schema )<<"\n"; + + clio::flat_ptr fq(q); +} diff --git a/libraries/clio/tests/protobuf_query.cpp b/libraries/clio/tests/protobuf_query.cpp new file mode 100644 index 000000000..b927a2a33 --- /dev/null +++ b/libraries/clio/tests/protobuf_query.cpp @@ -0,0 +1,77 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +struct service +{ + int add(int a, int b) const + { + std::cout << a << " + " << b << "\n"; + return a + b; + } + int sub(int a, int b) + { + std::cout << a << " - " << b << "\n"; + return a - b; + } +}; + +CLIO_REFLECT(service, add, sub) + +TEST_CASE("try calling methods via protobuf", "[protobuf]") +{ + clio::schema service_schema; + service_schema.generate(); + service_schema.generate(); + std::cout << "schema: " << format_json(service_schema) << "\n"; + + clio::reflect::proxy query; + query.add(2, 3); + query.add(8, 5); + query.sub(2, 3); + + std::cout << clio::format_json(query->q) << "\n"; + auto protobuf_query = clio::to_protobuf(query->q); + + clio::input_stream protobuf_query_in(protobuf_query.data(), protobuf_query.size()); + std::string json_query; + clio::string_stream json_query_out(json_query); + + clio::translate_protobuf_to_json(service_schema, "query", protobuf_query_in, json_query_out); + + std::cout << "pbuf query: " << format_json(clio::bytes{protobuf_query}) << "\n"; + std::cout << "json pbuf query: " << json_query << "\n"; + std::cout << "json query: " + << clio::format_json(clio::protobuf::to_json_query(query->q)) << "\n"; + + service s; + auto result = clio::protobuf::dispatch(s, query->q); + + std::cout << clio::format_json(result) << "\n"; + + auto protobuf_result = clio::to_protobuf(result); + std::string json_result; + + clio::input_stream protobuf_in(protobuf_result.data(), protobuf_result.size()); + clio::string_stream json_out(json_result); + + clio::translate_protobuf_to_json(service_schema, "service", protobuf_in, json_out); + + std::cout << "json result: " << json_result << "\n"; +} diff --git a/libraries/eosiolib/CMakeLists.txt b/libraries/eosiolib/CMakeLists.txt index 28b14e04f..f3280015f 100644 --- a/libraries/eosiolib/CMakeLists.txt +++ b/libraries/eosiolib/CMakeLists.txt @@ -116,6 +116,27 @@ function(add_libs suffix) -Wl,--entry,_start -nostdlib ) + + add_library(basic-polyfill${suffix}) + target_sources(basic-polyfill${suffix} PRIVATE + tester/wasi_polyfill/__wasi_args_get.cpp + tester/wasi_polyfill/__wasi_args_sizes_get.cpp + tester/wasi_polyfill/__wasi_clock_time_get.cpp + tester/wasi_polyfill/__wasi_environ_get.cpp + tester/wasi_polyfill/__wasi_environ_sizes_get.cpp + tester/wasi_polyfill/__wasi_fd_close.cpp + tester/wasi_polyfill/__wasi_fd_fdstat_get.cpp + tester/wasi_polyfill/__wasi_fd_fdstat_set_flags.cpp + tester/wasi_polyfill/__wasi_fd_prestat_dir_name.cpp + tester/wasi_polyfill/__wasi_fd_prestat_get.cpp + tester/wasi_polyfill/__wasi_fd_read.cpp + tester/wasi_polyfill/__wasi_fd_seek.cpp + tester/wasi_polyfill/__wasi_fd_write.cpp + tester/wasi_polyfill/__wasi_path_open.cpp + tester/wasi_polyfill/__wasi_proc_exit.cpp + tester/wasi_polyfill/prints.cpp + ) + target_link_libraries(basic-polyfill${suffix} PUBLIC wasm-base${suffix}) endfunction(add_libs) add_libs("") diff --git a/libraries/eosiolib/tester/wasi_polyfill/prints.cpp b/libraries/eosiolib/tester/wasi_polyfill/prints.cpp new file mode 100644 index 000000000..940e701e2 --- /dev/null +++ b/libraries/eosiolib/tester/wasi_polyfill/prints.cpp @@ -0,0 +1,8 @@ +#include +#include + +extern "C" void prints(const char* cstr) +{ + [[clang::import_name("prints_l")]] void prints_l(const char*, uint32_t); + prints_l(cstr, strlen(cstr)); +} diff --git a/native/CMakeLists.txt b/native/CMakeLists.txt index ed4719b7f..4422ee055 100644 --- a/native/CMakeLists.txt +++ b/native/CMakeLists.txt @@ -25,6 +25,10 @@ function(native_test N) ) endfunction() +function(native_and_wasm_test N) + native_test(${N}) +endfunction() + add_subdirectory(../external external) add_subdirectory(../libraries libraries) add_subdirectory(../programs programs) diff --git a/wasm/CMakeLists.txt b/wasm/CMakeLists.txt index d23b2be43..de297556f 100644 --- a/wasm/CMakeLists.txt +++ b/wasm/CMakeLists.txt @@ -63,6 +63,11 @@ add_clsdk_base_lib("") add_clsdk_base_lib("-debug") add_subdirectory(../contracts contracts) + +function(native_and_wasm_test N) + eden_tester_test(${N}) +endfunction() + add_subdirectory(../libraries libraries) add_subdirectory(../external external) add_subdirectory(boost)